"Fossies" - the Fresh Open Source Software Archive 
Member "glance-20.0.1/glance/common/utils.py" (12 Aug 2020, 25550 Bytes) of package /linux/misc/openstack/glance-20.0.1.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style:
standard) with prefixed line numbers.
Alternatively you can here
view or
download the uninterpreted source code file.
For more information about "utils.py" see the
Fossies "Dox" file reference documentation and the latest
Fossies "Diffs" side-by-side code changes report:
20.0.0_vs_20.0.1.
1 # Copyright 2010 United States Government as represented by the
2 # Administrator of the National Aeronautics and Space Administration.
3 # Copyright 2014 SoftLayer Technologies, Inc.
4 # Copyright 2015 Mirantis, Inc
5 # All Rights Reserved.
6 #
7 # Licensed under the Apache License, Version 2.0 (the "License"); you may
8 # not use this file except in compliance with the License. You may obtain
9 # a copy of the License at
10 #
11 # http://www.apache.org/licenses/LICENSE-2.0
12 #
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16 # License for the specific language governing permissions and limitations
17 # under the License.
18
19 """
20 System-level utilities and helper functions.
21 """
22
23 import errno
24
25 try:
26 from eventlet import sleep
27 except ImportError:
28 from time import sleep
29 from eventlet.green import socket
30
31 import functools
32 import glance_store
33 import os
34 import re
35
36 from oslo_config import cfg
37 from oslo_log import log as logging
38 from oslo_utils import excutils
39 from oslo_utils import netutils
40 from oslo_utils import strutils
41 import six
42 from six.moves import urllib
43 from webob import exc
44
45 from glance.common import exception
46 from glance.common import timeutils
47 from glance.common import wsgi
48 from glance.i18n import _, _LE
49
50 CONF = cfg.CONF
51
52 LOG = logging.getLogger(__name__)
53
54 # Whitelist of v1 API headers of form x-image-meta-xxx
55 IMAGE_META_HEADERS = ['x-image-meta-location', 'x-image-meta-size',
56 'x-image-meta-is_public', 'x-image-meta-disk_format',
57 'x-image-meta-container_format', 'x-image-meta-name',
58 'x-image-meta-status', 'x-image-meta-copy_from',
59 'x-image-meta-uri', 'x-image-meta-checksum',
60 'x-image-meta-created_at', 'x-image-meta-updated_at',
61 'x-image-meta-deleted_at', 'x-image-meta-min_ram',
62 'x-image-meta-min_disk', 'x-image-meta-owner',
63 'x-image-meta-store', 'x-image-meta-id',
64 'x-image-meta-protected', 'x-image-meta-deleted',
65 'x-image-meta-virtual_size']
66
67 GLANCE_TEST_SOCKET_FD_STR = 'GLANCE_TEST_SOCKET_FD'
68
69
70 def chunkreadable(iter, chunk_size=65536):
71 """
72 Wrap a readable iterator with a reader yielding chunks of
73 a preferred size, otherwise leave iterator unchanged.
74
75 :param iter: an iter which may also be readable
76 :param chunk_size: maximum size of chunk
77 """
78 return chunkiter(iter, chunk_size) if hasattr(iter, 'read') else iter
79
80
81 def chunkiter(fp, chunk_size=65536):
82 """
83 Return an iterator to a file-like obj which yields fixed size chunks
84
85 :param fp: a file-like object
86 :param chunk_size: maximum size of chunk
87 """
88 while True:
89 chunk = fp.read(chunk_size)
90 if chunk:
91 yield chunk
92 else:
93 break
94
95
96 def cooperative_iter(iter):
97 """
98 Return an iterator which schedules after each
99 iteration. This can prevent eventlet thread starvation.
100
101 :param iter: an iterator to wrap
102 """
103 try:
104 for chunk in iter:
105 sleep(0)
106 yield chunk
107 except Exception as err:
108 with excutils.save_and_reraise_exception():
109 msg = _LE("Error: cooperative_iter exception %s") % err
110 LOG.error(msg)
111
112
113 def cooperative_read(fd):
114 """
115 Wrap a file descriptor's read with a partial function which schedules
116 after each read. This can prevent eventlet thread starvation.
117
118 :param fd: a file descriptor to wrap
119 """
120 def readfn(*args):
121 result = fd.read(*args)
122 sleep(0)
123 return result
124 return readfn
125
126
127 MAX_COOP_READER_BUFFER_SIZE = 134217728 # 128M seems like a sane buffer limit
128
129 CONF.import_group('import_filtering_opts',
130 'glance.async_.flows._internal_plugins')
131
132
133 def validate_import_uri(uri):
134 """Validate requested uri for Image Import web-download.
135
136 :param uri: target uri to be validated
137 """
138 if not uri:
139 return False
140
141 parsed_uri = urllib.parse.urlparse(uri)
142 scheme = parsed_uri.scheme
143 host = parsed_uri.hostname
144 port = parsed_uri.port
145 wl_schemes = CONF.import_filtering_opts.allowed_schemes
146 bl_schemes = CONF.import_filtering_opts.disallowed_schemes
147 wl_hosts = CONF.import_filtering_opts.allowed_hosts
148 bl_hosts = CONF.import_filtering_opts.disallowed_hosts
149 wl_ports = CONF.import_filtering_opts.allowed_ports
150 bl_ports = CONF.import_filtering_opts.disallowed_ports
151
152 # NOTE(jokke): Checking if both allowed and disallowed are defined and
153 # logging it to inform only allowed will be obeyed.
154 if wl_schemes and bl_schemes:
155 bl_schemes = []
156 LOG.debug("Both allowed and disallowed schemes has been configured. "
157 "Will only process allowed list.")
158 if wl_hosts and bl_hosts:
159 bl_hosts = []
160 LOG.debug("Both allowed and disallowed hosts has been configured. "
161 "Will only process allowed list.")
162 if wl_ports and bl_ports:
163 bl_ports = []
164 LOG.debug("Both allowed and disallowed ports has been configured. "
165 "Will only process allowed list.")
166
167 if not scheme or ((wl_schemes and scheme not in wl_schemes) or
168 parsed_uri.scheme in bl_schemes):
169 return False
170 if not host or ((wl_hosts and host not in wl_hosts) or
171 host in bl_hosts):
172 return False
173 if port and ((wl_ports and port not in wl_ports) or
174 port in bl_ports):
175 return False
176
177 return True
178
179
180 class CooperativeReader(object):
181 """
182 An eventlet thread friendly class for reading in image data.
183
184 When accessing data either through the iterator or the read method
185 we perform a sleep to allow a co-operative yield. When there is more than
186 one image being uploaded/downloaded this prevents eventlet thread
187 starvation, ie allows all threads to be scheduled periodically rather than
188 having the same thread be continuously active.
189 """
190 def __init__(self, fd):
191 """
192 :param fd: Underlying image file object
193 """
194 self.fd = fd
195 self.iterator = None
196 # NOTE(markwash): if the underlying supports read(), overwrite the
197 # default iterator-based implementation with cooperative_read which
198 # is more straightforward
199 if hasattr(fd, 'read'):
200 self.read = cooperative_read(fd)
201 else:
202 self.iterator = None
203 self.buffer = b''
204 self.position = 0
205
206 def read(self, length=None):
207 """Return the requested amount of bytes, fetching the next chunk of
208 the underlying iterator when needed.
209
210 This is replaced with cooperative_read in __init__ if the underlying
211 fd already supports read().
212 """
213 if length is None:
214 if len(self.buffer) - self.position > 0:
215 # if no length specified but some data exists in buffer,
216 # return that data and clear the buffer
217 result = self.buffer[self.position:]
218 self.buffer = b''
219 self.position = 0
220 return bytes(result)
221 else:
222 # otherwise read the next chunk from the underlying iterator
223 # and return it as a whole. Reset the buffer, as subsequent
224 # calls may specify the length
225 try:
226 if self.iterator is None:
227 self.iterator = self.__iter__()
228 return next(self.iterator)
229 except StopIteration:
230 return b''
231 finally:
232 self.buffer = b''
233 self.position = 0
234 else:
235 result = bytearray()
236 while len(result) < length:
237 if self.position < len(self.buffer):
238 to_read = length - len(result)
239 chunk = self.buffer[self.position:self.position + to_read]
240 result.extend(chunk)
241
242 # This check is here to prevent potential OOM issues if
243 # this code is called with unreasonably high values of read
244 # size. Currently it is only called from the HTTP clients
245 # of Glance backend stores, which use httplib for data
246 # streaming, which has readsize hardcoded to 8K, so this
247 # check should never fire. Regardless it still worths to
248 # make the check, as the code may be reused somewhere else.
249 if len(result) >= MAX_COOP_READER_BUFFER_SIZE:
250 raise exception.LimitExceeded()
251 self.position += len(chunk)
252 else:
253 try:
254 if self.iterator is None:
255 self.iterator = self.__iter__()
256 self.buffer = next(self.iterator)
257 self.position = 0
258 except StopIteration:
259 self.buffer = b''
260 self.position = 0
261 return bytes(result)
262 return bytes(result)
263
264 def __iter__(self):
265 return cooperative_iter(self.fd.__iter__())
266
267
268 class LimitingReader(object):
269 """
270 Reader designed to fail when reading image data past the configured
271 allowable amount.
272 """
273 def __init__(self, data, limit,
274 exception_class=exception.ImageSizeLimitExceeded):
275 """
276 :param data: Underlying image data object
277 :param limit: maximum number of bytes the reader should allow
278 :param exception_class: Type of exception to be raised
279 """
280 self.data = data
281 self.limit = limit
282 self.bytes_read = 0
283 self.exception_class = exception_class
284
285 def __iter__(self):
286 for chunk in self.data:
287 self.bytes_read += len(chunk)
288 if self.bytes_read > self.limit:
289 raise self.exception_class()
290 else:
291 yield chunk
292
293 def read(self, i):
294 result = self.data.read(i)
295 self.bytes_read += len(result)
296 if self.bytes_read > self.limit:
297 raise self.exception_class()
298 return result
299
300
301 def image_meta_to_http_headers(image_meta):
302 """
303 Returns a set of image metadata into a dict
304 of HTTP headers that can be fed to either a Webob
305 Request object or an httplib.HTTP(S)Connection object
306
307 :param image_meta: Mapping of image metadata
308 """
309 headers = {}
310 for k, v in image_meta.items():
311 if v is not None:
312 if k == 'properties':
313 for pk, pv in v.items():
314 if pv is not None:
315 headers["x-image-meta-property-%s"
316 % pk.lower()] = six.text_type(pv)
317 else:
318 headers["x-image-meta-%s" % k.lower()] = six.text_type(v)
319 return headers
320
321
322 def get_image_meta_from_headers(response):
323 """
324 Processes HTTP headers from a supplied response that
325 match the x-image-meta and x-image-meta-property and
326 returns a mapping of image metadata and properties
327
328 :param response: Response to process
329 """
330 result = {}
331 properties = {}
332
333 if hasattr(response, 'getheaders'): # httplib.HTTPResponse
334 headers = response.getheaders()
335 else: # webob.Response
336 headers = response.headers.items()
337
338 for key, value in headers:
339 key = str(key.lower())
340 if key.startswith('x-image-meta-property-'):
341 field_name = key[len('x-image-meta-property-'):].replace('-', '_')
342 properties[field_name] = value or None
343 elif key.startswith('x-image-meta-'):
344 field_name = key[len('x-image-meta-'):].replace('-', '_')
345 if 'x-image-meta-' + field_name not in IMAGE_META_HEADERS:
346 msg = _("Bad header: %(header_name)s") % {'header_name': key}
347 raise exc.HTTPBadRequest(msg, content_type="text/plain")
348 result[field_name] = value or None
349 result['properties'] = properties
350
351 for key, nullable in [('size', False), ('min_disk', False),
352 ('min_ram', False), ('virtual_size', True)]:
353 if key in result:
354 try:
355 result[key] = int(result[key])
356 except ValueError:
357 if nullable and result[key] == str(None):
358 result[key] = None
359 else:
360 extra = (_("Cannot convert image %(key)s '%(value)s' "
361 "to an integer.")
362 % {'key': key, 'value': result[key]})
363 raise exception.InvalidParameterValue(value=result[key],
364 param=key,
365 extra_msg=extra)
366 if result[key] is not None and result[key] < 0:
367 extra = _('Cannot be a negative value.')
368 raise exception.InvalidParameterValue(value=result[key],
369 param=key,
370 extra_msg=extra)
371
372 for key in ('is_public', 'deleted', 'protected'):
373 if key in result:
374 result[key] = strutils.bool_from_string(result[key])
375 return result
376
377
378 def create_mashup_dict(image_meta):
379 """
380 Returns a dictionary-like mashup of the image core properties
381 and the image custom properties from given image metadata.
382
383 :param image_meta: metadata of image with core and custom properties
384 """
385
386 d = {}
387 for key, value in six.iteritems(image_meta):
388 if isinstance(value, dict):
389 for subkey, subvalue in six.iteritems(
390 create_mashup_dict(value)):
391 if subkey not in image_meta:
392 d[subkey] = subvalue
393 else:
394 d[key] = value
395
396 return d
397
398
399 def safe_mkdirs(path):
400 try:
401 os.makedirs(path)
402 except OSError as e:
403 if e.errno != errno.EEXIST:
404 raise
405
406
407 def mutating(func):
408 """Decorator to enforce read-only logic"""
409 @functools.wraps(func)
410 def wrapped(self, req, *args, **kwargs):
411 if req.context.read_only:
412 msg = "Read-only access"
413 LOG.debug(msg)
414 raise exc.HTTPForbidden(msg, request=req,
415 content_type="text/plain")
416 return func(self, req, *args, **kwargs)
417 return wrapped
418
419
420 def setup_remote_pydev_debug(host, port):
421 error_msg = _LE('Error setting up the debug environment. Verify that the'
422 ' option pydev_worker_debug_host is pointing to a valid '
423 'hostname or IP on which a pydev server is listening on'
424 ' the port indicated by pydev_worker_debug_port.')
425
426 try:
427 try:
428 from pydev import pydevd
429 except ImportError:
430 import pydevd
431
432 pydevd.settrace(host,
433 port=port,
434 stdoutToServer=True,
435 stderrToServer=True)
436 return True
437 except Exception:
438 with excutils.save_and_reraise_exception():
439 LOG.exception(error_msg)
440
441
442 def get_test_suite_socket():
443 global GLANCE_TEST_SOCKET_FD_STR
444 if GLANCE_TEST_SOCKET_FD_STR in os.environ:
445 fd = int(os.environ[GLANCE_TEST_SOCKET_FD_STR])
446 sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
447 if six.PY2:
448 sock = socket.SocketType(_sock=sock)
449 sock.listen(CONF.backlog)
450 del os.environ[GLANCE_TEST_SOCKET_FD_STR]
451 os.close(fd)
452 return sock
453 return None
454
455
456 def is_valid_hostname(hostname):
457 """Verify whether a hostname (not an FQDN) is valid."""
458 return re.match('^[a-zA-Z0-9-]+$', hostname) is not None
459
460
461 def is_valid_fqdn(fqdn):
462 """Verify whether a host is a valid FQDN."""
463 return re.match(r'^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$', fqdn) is not None
464
465
466 def parse_valid_host_port(host_port):
467 """
468 Given a "host:port" string, attempts to parse it as intelligently as
469 possible to determine if it is valid. This includes IPv6 [host]:port form,
470 IPv4 ip:port form, and hostname:port or fqdn:port form.
471
472 Invalid inputs will raise a ValueError, while valid inputs will return
473 a (host, port) tuple where the port will always be of type int.
474 """
475
476 try:
477 try:
478 host, port = netutils.parse_host_port(host_port)
479 except Exception:
480 raise ValueError(_('Host and port "%s" is not valid.') % host_port)
481
482 if not netutils.is_valid_port(port):
483 raise ValueError(_('Port "%s" is not valid.') % port)
484
485 # First check for valid IPv6 and IPv4 addresses, then a generic
486 # hostname. Failing those, if the host includes a period, then this
487 # should pass a very generic FQDN check. The FQDN check for letters at
488 # the tail end will weed out any hilariously absurd IPv4 addresses.
489
490 if not (netutils.is_valid_ipv6(host) or netutils.is_valid_ipv4(host) or
491 is_valid_hostname(host) or is_valid_fqdn(host)):
492 raise ValueError(_('Host "%s" is not valid.') % host)
493
494 except Exception as ex:
495 raise ValueError(_('%s '
496 'Please specify a host:port pair, where host is an '
497 'IPv4 address, IPv6 address, hostname, or FQDN. If '
498 'using an IPv6 address, enclose it in brackets '
499 'separately from the port (i.e., '
500 '"[fe80::a:b:c]:9876").') % ex)
501
502 return (host, int(port))
503
504
505 try:
506 REGEX_4BYTE_UNICODE = re.compile(u'[\U00010000-\U0010ffff]')
507 except re.error:
508 # UCS-2 build case
509 REGEX_4BYTE_UNICODE = re.compile(u'[\uD800-\uDBFF][\uDC00-\uDFFF]')
510
511
512 def no_4byte_params(f):
513 """
514 Checks that no 4 byte unicode characters are allowed
515 in dicts' keys/values and string's parameters
516 """
517 def wrapper(*args, **kwargs):
518
519 def _is_match(some_str):
520 return (isinstance(some_str, six.text_type) and
521 REGEX_4BYTE_UNICODE.findall(some_str) != [])
522
523 def _check_dict(data_dict):
524 # a dict of dicts has to be checked recursively
525 for key, value in six.iteritems(data_dict):
526 if isinstance(value, dict):
527 _check_dict(value)
528 else:
529 if _is_match(key):
530 msg = _("Property names can't contain 4 byte unicode.")
531 raise exception.Invalid(msg)
532 if _is_match(value):
533 msg = (_("%s can't contain 4 byte unicode characters.")
534 % key.title())
535 raise exception.Invalid(msg)
536
537 for data_dict in [arg for arg in args if isinstance(arg, dict)]:
538 _check_dict(data_dict)
539 # now check args for str values
540 for arg in args:
541 if _is_match(arg):
542 msg = _("Param values can't contain 4 byte unicode.")
543 raise exception.Invalid(msg)
544 # check kwargs as well, as params are passed as kwargs via
545 # registry calls
546 _check_dict(kwargs)
547 return f(*args, **kwargs)
548 return wrapper
549
550
551 def stash_conf_values():
552 """
553 Make a copy of some of the current global CONF's settings.
554 Allows determining if any of these values have changed
555 when the config is reloaded.
556 """
557 conf = {
558 'bind_host': CONF.bind_host,
559 'bind_port': CONF.bind_port,
560 'backlog': CONF.backlog,
561 }
562
563 return conf
564
565
566 def split_filter_op(expression):
567 """Split operator from threshold in an expression.
568 Designed for use on a comparative-filtering query field.
569 When no operator is found, default to an equality comparison.
570
571 :param expression: the expression to parse
572
573 :returns: a tuple (operator, threshold) parsed from expression
574 """
575 left, sep, right = expression.partition(':')
576 if sep:
577 # If the expression is a date of the format ISO 8601 like
578 # CCYY-MM-DDThh:mm:ss+hh:mm and has no operator, it should
579 # not be partitioned, and a default operator of eq should be
580 # assumed.
581 try:
582 timeutils.parse_isotime(expression)
583 op = 'eq'
584 threshold = expression
585 except ValueError:
586 op = left
587 threshold = right
588 else:
589 op = 'eq' # default operator
590 threshold = left
591
592 # NOTE stevelle decoding escaped values may be needed later
593 return op, threshold
594
595
596 def validate_quotes(value):
597 """Validate filter values
598
599 Validation opening/closing quotes in the expression.
600 """
601 open_quotes = True
602 for i in range(len(value)):
603 if value[i] == '"':
604 if i and value[i - 1] == '\\':
605 continue
606 if open_quotes:
607 if i and value[i - 1] != ',':
608 msg = _("Invalid filter value %s. There is no comma "
609 "before opening quotation mark.") % value
610 raise exception.InvalidParameterValue(message=msg)
611 else:
612 if i + 1 != len(value) and value[i + 1] != ",":
613 msg = _("Invalid filter value %s. There is no comma "
614 "after closing quotation mark.") % value
615 raise exception.InvalidParameterValue(message=msg)
616 open_quotes = not open_quotes
617 if not open_quotes:
618 msg = _("Invalid filter value %s. The quote is not closed.") % value
619 raise exception.InvalidParameterValue(message=msg)
620
621
622 def split_filter_value_for_quotes(value):
623 """Split filter values
624
625 Split values by commas and quotes for 'in' operator, according api-wg.
626 """
627 validate_quotes(value)
628 tmp = re.compile(r'''
629 "( # if found a double-quote
630 [^\"\\]* # take characters either non-quotes or backslashes
631 (?:\\. # take backslashes and character after it
632 [^\"\\]*)* # take characters either non-quotes or backslashes
633 ) # before double-quote
634 ",? # a double-quote with comma maybe
635 | ([^,]+),? # if not found double-quote take any non-comma
636 # characters with comma maybe
637 | , # if we have only comma take empty string
638 ''', re.VERBOSE)
639 return [val[0] or val[1] for val in re.findall(tmp, value)]
640
641
642 def evaluate_filter_op(value, operator, threshold):
643 """Evaluate a comparison operator.
644 Designed for use on a comparative-filtering query field.
645
646 :param value: evaluated against the operator, as left side of expression
647 :param operator: any supported filter operation
648 :param threshold: to compare value against, as right side of expression
649
650 :raises InvalidFilterOperatorValue: if an unknown operator is provided
651
652 :returns: boolean result of applied comparison
653
654 """
655 if operator == 'gt':
656 return value > threshold
657 elif operator == 'gte':
658 return value >= threshold
659 elif operator == 'lt':
660 return value < threshold
661 elif operator == 'lte':
662 return value <= threshold
663 elif operator == 'neq':
664 return value != threshold
665 elif operator == 'eq':
666 return value == threshold
667
668 msg = _("Unable to filter on a unknown operator.")
669 raise exception.InvalidFilterOperatorValue(msg)
670
671
672 def _get_available_stores():
673 available_stores = CONF.enabled_backends
674 stores = []
675 # Remove reserved stores from the available stores list
676 for store in available_stores:
677 # NOTE (abhishekk): http store is readonly and should be
678 # excluded from the list.
679 if available_stores[store] == 'http':
680 continue
681 if store not in wsgi.RESERVED_STORES:
682 stores.append(store)
683
684 return stores
685
686
687 def get_stores_from_request(req, body):
688 """Processes a supplied request and extract stores from it
689
690 :param req: request to process
691 :param body: request body
692
693 :raises glance_store.UnknownScheme: if a store is not valid
694 :return: a list of stores
695 """
696 if body.get('all_stores', False):
697 if 'stores' in body or 'x-image-meta-store' in req.headers:
698 msg = _("All_stores parameter can't be used with "
699 "x-image-meta-store header or stores parameter")
700 raise exc.HTTPBadRequest(explanation=msg)
701 stores = _get_available_stores()
702 else:
703 try:
704 stores = body['stores']
705 except KeyError:
706 stores = [req.headers.get('x-image-meta-store',
707 CONF.glance_store.default_backend)]
708 else:
709 if 'x-image-meta-store' in req.headers:
710 msg = _("Stores parameter and x-image-meta-store header can't "
711 "be both specified")
712 raise exc.HTTPBadRequest(explanation=msg)
713 # Validate each store
714 for store in stores:
715 glance_store.get_store_from_store_identifier(store)
716 return stores