Coverage for gramex\config.py : 89%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1'''
2Manages YAML config files as layered configurations with imports.
4:class:PathConfig loads YAML files from a path::
6 pc = PathConfig('/path/to/file.yaml')
8This can be reloaded via the ``+`` operator. ``+pc`` reloads the YAML file
9(but only if it is newer than before.)
11:class:ChainConfig chains multiple YAML files into a single config. For example
12this merges ``base.yaml`` and ``next.yaml`` in sequence::
14 cc = ChainConfig()
15 cc['base'] = PathConfig('base.yaml')
16 cc['next'] = PathConfig('next.yaml')
18To get the merged file, use ``+cc``. This updates the PathConfig files and
19merges the YAMLs.
20'''
22import os
23import re
24import csv
25import six
26import sys
27import yaml
28import string
29import socket
30import inspect
31import logging
32import datetime
33import dateutil.tz
34import dateutil.parser
35from pathlib import Path
36from copy import deepcopy
37from random import choice
38from fnmatch import fnmatch
39from six import string_types
40from collections import OrderedDict
41from pydoc import locate as _locate, ErrorDuringImport
42from yaml import Loader, MappingNode
43from json import loads, JSONEncoder, JSONDecoder
44from yaml.constructor import ConstructorError
45from orderedattrdict import AttrDict, DefaultAttrDict
46from errno import EACCES, EPERM
48ERROR_SHARING_VIOLATION = 32 # from winerror.ERROR_SHARING_VIOLATION
50# gramex.config.app_log is the default logger used by all of gramex
51# If it's not there, create one.
52logging.basicConfig()
53app_log = logging.getLogger('gramex')
55# app_log_extra has additional parameters that may be used by the logger
56app_log_extra = {'port': 'PORT'}
57app_log = logging.LoggerAdapter(app_log, app_log_extra)
59# sqlalchemy.create_engine requires an encoding= that must be an str across
60# Python 2 and Python 3. Expose this for other modules to use
61str_utf8 = str('utf-8') # noqa
64def walk(node):
65 '''
66 Bottom-up recursive walk through a data structure yielding a (key, value,
67 node) tuple for every entry. ``node[key] == value`` is true in every entry.
69 For example::
71 >>> list(walk([{'x': 1}]))
72 [
73 ('x', 1, {'x': 1}), # leaf: key, value, node
74 (0, {'x': 1}, [{'x': 1}]) # parent: index, value, node
75 ]
77 Circular linkage can lead to a RuntimeError::
79 >>> x = {}
80 >>> x['x'] = x
81 >>> list(walk(x))
82 ...
83 RuntimeError: maximum recursion depth exceeded
84 '''
85 if hasattr(node, 'items'):
86 for key, value in list(node.items()):
87 for item in walk(value):
88 yield item
89 yield key, value, node
90 elif isinstance(node, list):
91 for index, value in enumerate(node):
92 for item in walk(value):
93 yield item
94 yield index, value, node
97def merge(old, new, mode='overwrite', warn=None, _path=''):
98 '''
99 Update old dict with new dict recursively.
101 >>> merge({'a': {'x': 1}}, {'a': {'y': 2}})
102 {'a': {'x': 1, 'y': 2}}
104 If ``new`` is a list, convert into a dict with random keys.
106 If ``mode='overwrite'``, the old dict is overwritten (default).
107 If ``mode='setdefault'``, the old dict values are updated only if missing.
109 ``warn=`` is an optional list of key paths. Any conflict on dictionaries
110 matching any of these paths is logged as a warning. For example,
111 ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key
112 has a conflict.
113 '''
114 for key in new:
115 if key in old and hasattr(old[key], 'items') and hasattr(new[key], 'items'):
116 path_key = _path + ('.' if _path else '') + six.text_type(key)
117 if warn is not None:
118 for pattern in warn:
119 if fnmatch(path_key, pattern): 119 ↛ 120line 119 didn't jump to line 120, because the condition on line 119 was never true
120 app_log.warning('Duplicate key: %s', path_key)
121 break
122 merge(old=old[key], new=new[key], mode=mode, warn=warn, _path=path_key)
123 elif mode == 'overwrite' or key not in old:
124 old[key] = deepcopy(new[key])
125 return old
128class ChainConfig(AttrDict):
129 '''
130 An AttrDict that manages multiple configurations as layers.
132 >>> config = ChainConfig([
133 ... ('base', PathConfig('gramex.yaml')),
134 ... ('app1', PathConfig('app.yaml')),
135 ... ('app2', AttrDict())
136 ... ])
138 Any dict-compatible values are allowed. ``+config`` returns the merged values.
139 '''
141 def __pos__(self):
142 '''+config returns layers merged in order, removing null keys'''
143 conf = AttrDict()
144 for name, config in self.items():
145 if hasattr(config, '__pos__'):
146 config.__pos__()
147 merge(old=conf, new=config, mode='overwrite')
149 # Remove keys where the value is None
150 for key, value, node in list(walk(conf)):
151 if value is None:
152 del node[key]
154 return conf
157# Paths that users have already been warned about. Don't warn them again
158_warned_paths = set()
159# Get the directory where gramex is located. This is the same as the directory
160# where this file (config.py) is located.
161_gramex_path = os.path.dirname(os.path.abspath(__file__))
164def setup_variables():
165 '''Initialise variables'''
166 variables = DefaultAttrDict(str)
167 # Load all environment variables
168 variables.update(os.environ)
169 # GRAMEXPATH is the Gramex root directory
170 variables['GRAMEXPATH'] = _gramex_path
171 # GRAMEXAPPS is the Gramex apps directory
172 variables['GRAMEXAPPS'] = os.path.join(_gramex_path, 'apps')
173 # GRAMEXHOST is the hostname
174 variables['GRAMEXHOST'] = socket.gethostname()
175 # GRAMEXDATA varies based on OS
176 if 'GRAMEXDATA' not in variables: 176 ↛ 189line 176 didn't jump to line 189, because the condition on line 176 was never false
177 if sys.platform.startswith('linux') or sys.platform == 'cygwin': 177 ↛ 178line 177 didn't jump to line 178, because the condition on line 177 was never true
178 variables['GRAMEXDATA'] = os.path.expanduser('~/.config/gramexdata')
179 elif sys.platform == 'win32': 179 ↛ 181line 179 didn't jump to line 181, because the condition on line 179 was never false
180 variables['GRAMEXDATA'] = os.path.join(variables['LOCALAPPDATA'], 'Gramex Data')
181 elif sys.platform == 'darwin':
182 variables['GRAMEXDATA'] = os.path.expanduser(
183 '~/Library/Application Support/Gramex Data')
184 else:
185 variables['GRAMEXDATA'] = os.path.abspath('.')
186 app_log.warning('$GRAMEXDATA set to %s for OS %s', variables['GRAMEXDATA'],
187 sys.platform)
189 return variables
192variables = setup_variables()
195def _substitute_variable(val):
196 '''
197 If val contains a ${VAR} or $VAR and VAR is in the variables global,
198 substitute it.
200 Direct variables are substituted as-is. For example, $x will return
201 variables['x'] without converting it to a string. Otherwise, treat it as a
202 string tempate. So "/$x/" will return "/1/" if x=1.
203 '''
204 if not isinstance(val, string_types):
205 return val
206 if val.startswith('$') and val[1:] in variables:
207 return variables[val[1:]]
208 else:
209 try:
210 return string.Template(val).substitute(variables)
211 except ValueError:
212 raise ValueError('Use $$ instead of $ in %s' % val)
215def _calc_value(val, key):
216 '''
217 Calculate the value to assign to this key.
219 If ``val`` is not a dictionary that has a ``function`` key, return it as-is.
221 If it has a function key, call that function (with specified args, kwargs,
222 etc) and allow the ``key`` parameter as an argument.
224 If the function is a generator, the first value is used.
225 '''
226 if hasattr(val, 'get') and val.get('function'):
227 from .transforms import build_transform
228 function = build_transform(val, vars={'key': None}, filename='config:%s' % key)
229 for result in function(key):
230 if result is not None:
231 return result
232 return val.get('default')
233 else:
234 return _substitute_variable(val)
237_valid_key_chars = string.ascii_letters + string.digits
240def random_string(size, chars=_valid_key_chars):
241 '''Return random string of length size using chars (which defaults to alphanumeric)'''
242 return ''.join(choice(chars) for index in range(size)) # nosec - ok for non-cryptographic use
245RANDOM_KEY = r'$*'
248def _from_yaml(loader, node):
249 '''
250 Load mapping as AttrDict, preserving order. Raise error on duplicate keys
251 '''
252 # Based on yaml.constructor.SafeConstructor.construct_mapping()
253 attrdict = AttrDict()
254 yield attrdict
255 if not isinstance(node, MappingNode): 255 ↛ 256line 255 didn't jump to line 256, because the condition on line 255 was never true
256 raise ConstructorError(
257 None, None, 'expected a mapping node, but found %s' % node.id, node.start_mark)
258 loader.flatten_mapping(node)
259 for key_node, value_node in node.value:
260 key = loader.construct_object(key_node, deep=False)
261 if isinstance(key, six.string_types) and RANDOM_KEY in key:
262 # With k=5 there's a <0.1% chance of collision even for 1mn uses.
263 # (1 - decimal.Decimal(62 ** -5)) ** 1000000 ~ 0.999
264 key = key.replace(RANDOM_KEY, random_string(5))
265 try:
266 hash(key)
267 except TypeError as exc:
268 raise ConstructorError(
269 'while constructing a mapping', node.start_mark,
270 'found unacceptable key (%s)' % exc, key_node.start_mark)
271 if key in attrdict:
272 raise ConstructorError(
273 'while constructing a mapping', node.start_mark,
274 'found duplicate key (%s)' % key, key_node.start_mark)
275 attrdict[key] = loader.construct_object(value_node, deep=False)
278class ConfigYAMLLoader(Loader):
279 '''
280 A YAML loader that loads a YAML file into an ordered AttrDict. Usage::
282 >>> attrdict = yaml.load(yaml_string, Loader=ConfigYAMLLoader)
284 If there are duplicate keys, this raises an error.
285 '''
286 def __init__(self, *args, **kwargs):
287 super(ConfigYAMLLoader, self).__init__(*args, **kwargs)
288 self.add_constructor(u'tag:yaml.org,2002:map', _from_yaml)
289 self.add_constructor(u'tag:yaml.org,2002:omap', _from_yaml)
292def _yaml_open(path, default=AttrDict(), **kwargs):
293 '''
294 Load a YAML path.Path as AttrDict. Replace ${VAR} or $VAR with variables.
295 Defines special variables $YAMLPATH as the absolute path of the YAML file,
296 and $YAMLURL as the path relative to current directory. These can be
297 overridden via keyward arguments (e.g. ``YAMLURL=...``)
299 If key has " if ", include it only if the condition (eval-ed in Python) is
300 true.
302 If the path is missing, or YAML has a parse error, or the YAML is not a
303 dict, returns the default value.
304 '''
305 path = path.absolute()
306 if not path.exists():
307 if path not in _warned_paths:
308 app_log.warning('Missing config: %s', path)
309 _warned_paths.add(path)
310 return default
311 app_log.debug('Loading config: %s', path)
312 with path.open(encoding='utf-8') as handle:
313 try:
314 result = yaml.load(handle, Loader=ConfigYAMLLoader) # nosec
315 except Exception:
316 app_log.exception('Config error: %s', path)
317 return default
318 if not isinstance(result, AttrDict):
319 if result is not None:
320 app_log.warning('Config is not a dict: %s', path)
321 return default
323 # Variables based on YAML file location
324 yaml_path = str(path.parent)
325 kwargs.setdefault('YAMLPATH', yaml_path) # Path to YAML folder
326 kwargs.setdefault('YAMLFILE', str(path)) # Path to YAML file
327 # $YAMLURL defaults to the relative URL from cwd to YAML folder.
328 try:
329 yamlurl = os.path.relpath(yaml_path)
330 except ValueError:
331 # If YAML is in a different drive, this fails. So don't set YAMLURL.
332 # Impact: $YAMLURL is undefined for imports from a different drive.
333 pass
334 else:
335 kwargs.setdefault('YAMLURL', yamlurl)
336 # Typically, we use /$YAMLURL/url - so strip the slashes. Replace backslashes
337 if isinstance(kwargs.get('YAMLURL'), string_types):
338 kwargs['YAMLURL'] = kwargs['YAMLURL'].replace('\\', '/').strip('/')
339 variables.update(kwargs)
341 # Update context with the variables section.
342 # key: value sets key = value
343 # key: {function: fn} sets key = fn(key)
344 # key: {default: value} sets key = value if it's not already set
345 # key: {default: {function: fn}} sets key = fn(key) if it's not already set
346 if 'variables' in result:
347 for key, val in result['variables'].items():
348 if hasattr(val, 'get') and 'default' in val and 'function' not in val:
349 variables.setdefault(key, _calc_value(val['default'], key))
350 else:
351 variables[key] = _calc_value(val, key)
352 del result['variables']
354 # Evaluate conditionals. "x if cond: y" becomes "x: y" if cond evals to True
355 remove, replace = [], []
356 frozen_vars = dict(variables)
357 for key, value, node in walk(result):
358 if isinstance(key, string_types) and ' if ' in key:
359 # Evaluate conditional
360 base, expr = key.split(' if ', 2)
361 try:
362 condition = eval(expr, globals(), frozen_vars) # nosec - any Python expr is OK
363 except Exception:
364 condition = False
365 app_log.exception('Failed condition evaluation: %s', key)
366 if condition:
367 replace.append((node, key, base))
368 else:
369 remove.append((node, key))
370 for node, key in remove:
371 del node[key]
372 for node, key, base in replace:
373 node[base] = node.pop(key)
375 # Substitute variables
376 for key, value, node in walk(result):
377 if isinstance(value, string_types):
378 # Backward compatibility: before v1.0.4, we used {.} for {YAMLPATH}
379 value = value.replace('{.}', '$YAMLPATH')
380 # Substitute with variables in context, defaulting to ''
381 node[key] = _substitute_variable(value)
382 return result
385def _pathstat(path):
386 '''
387 Return a path stat object, which has 2 attributes/keys: ``.path`` is the
388 same as the ``path`` parameter. ``stat`` is the result of ``os.stat``. If
389 path is missing, ``stat`` has ``st_mtime`` and ``st_size`` set to ``0``.
390 '''
391 # If path doesn't exist, create a dummy stat structure with
392 # safe defaults (old mtime, 0 filesize, etc)
393 stat = path.stat() if path.exists() else AttrDict(st_mtime=0, st_size=0)
394 return AttrDict(path=path, stat=stat)
397def _add_ns(config, namespace, prefix):
398 '''
399 Given a YAML config (basically a dict), add prefix to specified namespaces.
401 For example::
403 >>> _add_ns({'x': 1}, '*', 'a')
404 {'a.x': 1}
405 >>> _add_ns({'x': {'y': 1}}, ['*', 'x'], 'a')
406 {'a.x': {'a.y': 1}}
407 '''
408 if not isinstance(namespace, list):
409 namespace = [namespace]
410 # Sort in descending order of key depth. So "x.y" is before "x" is before "*"
411 namespace = sorted(namespace, key=lambda ns: -1 if ns == '*' else ns.count('.'), reverse=True)
412 prefix += ':'
413 for keypath in namespace:
414 if keypath == '*':
415 el = config
416 else:
417 el = objectpath(config, keypath, default={})
418 if isinstance(el, dict): 418 ↛ 413line 418 didn't jump to line 413, because the condition on line 418 was never false
419 for subkey in list(el.keys()):
420 if subkey not in {'import'}: 420 ↛ 419line 420 didn't jump to line 419, because the condition on line 420 was never false
421 el[prefix + subkey] = el.pop(subkey)
422 return config
425def load_imports(config, source, warn=None):
426 '''
427 Post-process a config for imports.
429 ``config`` is the data to process. ``source`` is the path where it was
430 loaded from.
432 If ``config`` has an ``import:`` key, treat all values below that as YAML
433 files (specified relative to ``source``) and import them in sequence.
435 Return a list of imported paths as :func:_pathstat objects. (This includes
436 ``source``.)
438 For example, if the ``source`` is ``base.yaml`` (which has the below
439 configuration) and is loaded into ``config``::
441 app:
442 port: 20
443 start: true
444 path: /
445 import: update*.yaml # Can be any glob, e.g. */gramex.yaml
447 ... and ``update.yaml`` looks like this::
449 app:
450 port: 30
451 new: yes
453 ... then after this function is called, ``config`` looks like this::
455 app:
456 port: 20 # From base.yaml. NOT updated by update.yaml
457 start: true # From base.yaml
458 new: yes # From update.yaml
459 path: / # From base.yaml
461 The ``import:`` keys are deleted. The return value contains :func:_pathstat
462 values for ``base.yaml`` and ``update.yaml`` in that order.
464 Multiple ``import:`` values can be specified as a dictionary::
466 import:
467 first-app: app1/*.yaml
468 next-app: app2/*.yaml
470 To import sub-keys as namespaces, use::
472 import:
473 app: {path: */gramex.yaml, namespace: 'url'}
475 This prefixes all keys under ``url:``. Here are more examples::
477 namespace: True # Add namespace to all top-level keys
478 namespace: url # Add namespace to url.*
479 namespace: log.loggers # Add namespace to log.loggers.*
480 namespace: [True, url] # Add namespace to top level keys and url.*
482 By default, the prefix is the relative path of the imported YAML file
483 (relative to the importer).
485 ``warn=`` is an optional list of key paths. Any conflict on dictionaries
486 matching any of these paths is logged as a warning. For example,
487 ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key
488 has a conflict.
489 '''
490 imported_paths = [_pathstat(source)]
491 root = source.absolute().parent
492 for key, value, node in list(walk(config)):
493 if isinstance(key, six.string_types) and key.startswith('import.merge'):
494 # Strip the top level key(s) from import.merge values
495 if isinstance(value, dict):
496 for name, conf in value.items():
497 node[name] = conf
498 elif value: 498 ↛ 499line 498 didn't jump to line 499, because the condition on line 498 was never true
499 raise ValueError('import.merge: must be dict, not %s at %s' % (
500 repr(value), source))
501 # Delete the import key
502 del node[key]
503 elif key == 'import':
504 # Convert "import: path" to "import: {app: path}"
505 if isinstance(value, six.string_types):
506 value = {'apps': value}
507 # Allow "import: [path, path]" to "import: {app0: path, app1: path}"
508 elif isinstance(value, list):
509 value = OrderedDict((('app%d' % i, conf) for i, conf in enumerate(value)))
510 # By now, import: should be a dict
511 elif not isinstance(value, dict): 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true
512 raise ValueError('import: must be string/list/dict, not %s at %s' % (
513 repr(value), source))
514 # If already a dict with a single import via 'path', convert to dict of apps
515 if 'path' in value:
516 value = {'app': value}
517 for name, conf in value.items():
518 if not isinstance(conf, dict):
519 conf = AttrDict(path=conf)
520 if 'path' not in conf: 520 ↛ 521line 520 didn't jump to line 521, because the condition on line 520 was never true
521 raise ValueError('import: has no conf at %s' % source)
522 paths = conf.pop('path')
523 paths = paths if isinstance(paths, list) else [paths]
524 globbed_paths = []
525 for path in paths:
526 globbed_paths += sorted(root.glob(path)) if '*' in path else [Path(path)]
527 ns = conf.pop('namespace', None)
528 for path in globbed_paths:
529 abspath = root.joinpath(path)
530 new_conf = _yaml_open(abspath, **conf)
531 if ns is not None:
532 prefix = Path(path).as_posix()
533 new_conf = _add_ns(new_conf, ns, name + ':' + prefix)
534 imported_paths += load_imports(new_conf, source=abspath)
535 merge(old=node, new=new_conf, mode='setdefault', warn=warn)
536 # Delete the import key
537 del node[key]
538 return imported_paths
541class PathConfig(AttrDict):
542 '''
543 An ``AttrDict`` that is loaded from a path as a YAML file. For e.g.,
544 ``conf = PathConfig(path)`` loads the YAML file at ``path`` as an AttrDict.
545 ``+conf`` reloads the path if required.
547 ``warn=`` is an optional list of key paths. Any conflict on dictionaries
548 matching any of these paths is logged as a warning. For example,
549 ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key
550 has a conflict.
552 Like http://configure.readthedocs.org/ but supports imports not inheritance.
553 This lets us import YAML files in the middle of a YAML structure::
555 key:
556 import:
557 conf1: file1.yaml # Import file1.yaml here
558 conf2: file2.yaml # Import file2.yaml here
560 Each ``PathConfig`` object has an ``__info__`` attribute with the following
561 keys:
563 __info__.path
564 The path that this instance syncs with, stored as a ``pathlib.Path``
565 __info__.warn
566 The keys to warn in case about in case of an import merge conflict
567 __info__.imports
568 A list of imported files, stored as an ``AttrDict`` with 2 attributes:
570 path
571 The path that was imported, stored as a ``pathlib.Path``
572 stat
573 The ``os.stat()`` information about this file (or ``None`` if the
574 file is missing.)
575 '''
576 duplicate_warn = None
578 def __init__(self, path, warn=None):
579 super(PathConfig, self).__init__()
580 if warn is None: 580 ↛ 582line 580 didn't jump to line 582, because the condition on line 580 was never false
581 warn = self.duplicate_warn
582 self.__info__ = AttrDict(path=Path(path), imports=[], warn=warn)
583 self.__pos__()
585 def __pos__(self):
586 '''+config reloads this config (if it has a path)'''
587 path = self.__info__.path
589 # We must reload the layer if nothing has been imported...
590 reload = not self.__info__.imports
591 # ... or if an imported file is deleted / updated
592 for imp in self.__info__.imports:
593 exists = imp.path.exists()
594 if not exists and imp.stat is not None:
595 reload = True
596 app_log.info('No config found: %s', imp.path)
597 break
598 if exists and (imp.path.stat().st_mtime > imp.stat.st_mtime or
599 imp.path.stat().st_size != imp.stat.st_size):
600 reload = True
601 app_log.info('Updated config: %s', imp.path)
602 break
603 if reload:
604 self.clear()
605 self.update(_yaml_open(path))
606 self.__info__.imports = load_imports(self, source=path, warn=self.__info__.warn)
607 return self
610def locate(path, modules=[], forceload=0):
611 '''
612 Locate an object by name or dotted path.
614 For example, ``locate('str')`` returns the ``str`` built-in.
615 ``locate('gramex.handlers.FileHandler')`` returns the class
616 ``gramex.handlers.FileHandler``.
618 ``modules`` is a list of modules to search for the path in first. So
619 ``locate('FileHandler', modules=[gramex.handlers])`` will return
620 ``gramex.handlers.FileHandler``.
622 If importing raises an Exception, log it and return None.
623 '''
624 try:
625 for module_name in modules:
626 module = _locate(module_name, forceload)
627 if hasattr(module, path):
628 return getattr(module, path)
629 return _locate(path, forceload)
630 except ErrorDuringImport:
631 app_log.exception('Exception when importing %s', path)
632 return None
635_checked_old_certs = []
638class CustomJSONEncoder(JSONEncoder):
639 '''
640 Encodes object to JSON, additionally converting datetime into ISO 8601 format
641 '''
642 def default(self, obj):
643 import numpy as np
645 if hasattr(obj, 'to_dict'):
646 # Slow but reliable. Handles conversion of numpy objects, mixed types, etc.
647 return loads(obj.to_json(orient='records', date_format='iso'),
648 object_pairs_hook=OrderedDict)
649 elif isinstance(obj, datetime.datetime):
650 # Use local timezone if no timezone is specified
651 if obj.tzinfo is None:
652 obj = obj.replace(tzinfo=dateutil.tz.tzlocal())
653 return obj.isoformat()
654 elif isinstance(obj, np.datetime64):
655 obj = obj.item()
656 if (isinstance(obj, datetime.datetime) and obj.tzinfo is None):
657 obj = obj.replace(tzinfo=dateutil.tz.tzlocal())
658 return obj.isoformat()
659 elif isinstance(obj, np.integer):
660 return int(obj)
661 elif isinstance(obj, np.floating):
662 return float(obj)
663 elif isinstance(obj, np.ndarray):
664 return obj.tolist()
665 elif isinstance(obj, np.bool_):
666 return bool(obj)
667 elif isinstance(obj, np.bytes_): 667 ↛ 669line 667 didn't jump to line 669, because the condition on line 667 was never false
668 return obj.decode('utf-8')
669 return super(CustomJSONEncoder, self).default(obj)
672class CustomJSONDecoder(JSONDecoder):
673 '''
674 Decodes JSON string, converting ISO 8601 datetime to datetime
675 '''
676 # Check if a string might be a datetime. Handles variants like:
677 # 2001-02-03T04:05:06Z
678 # 2001-02-03T04:05:06+000
679 # 2001-02-03T04:05:06.000+0000
680 re_datetimeval = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}')
681 re_datetimestr = re.compile(r'"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}')
683 def __init__(self, *args, **kwargs):
684 self.old_object_pairs_hook = kwargs.get('object_pairs_hook')
685 kwargs['object_pairs_hook'] = self.convert
686 super(CustomJSONDecoder, self).__init__(*args, **kwargs)
688 def decode(self, obj):
689 if self.re_datetimestr.match(obj):
690 return dateutil.parser.parse(obj[1:-1])
691 return super(CustomJSONDecoder, self).decode(obj)
693 def convert(self, obj):
694 for index, (key, val) in enumerate(obj):
695 if isinstance(val, six.string_types) and self.re_datetimeval.match(val):
696 obj[index] = (key, dateutil.parser.parse(val))
697 if callable(self.old_object_pairs_hook):
698 return self.old_object_pairs_hook(obj)
699 return dict(obj)
702def check_old_certs():
703 '''
704 The latest SSL certificates from certifi don't work for Google Auth. Do
705 a one-time check to access accounts.google.com. If it throws an SSL
706 error, switch to old SSL certificates. See
707 https://github.com/tornadoweb/tornado/issues/1534
708 '''
709 if not _checked_old_certs:
710 _checked_old_certs.append(True)
712 import ssl
713 from tornado.httpclient import HTTPClient, AsyncHTTPClient
715 # Use HTTPClient to check instead of AsyncHTTPClient because it's synchronous.
716 _client = HTTPClient()
717 try:
718 # Use accounts.google.com because we know it fails with new certifi certificates
719 # cdn.redhat.com is another site that fails.
720 _client.fetch("https://accounts.google.com/")
721 except ssl.SSLError:
722 try:
723 import certifi # noqa: late import to minimise dependencies
724 AsyncHTTPClient.configure(None, defaults=dict(ca_certs=certifi.old_where()))
725 app_log.warning('Using old SSL certificates for compatibility')
726 except ImportError:
727 pass
728 try:
729 _client.fetch("https://accounts.google.com/")
730 except ssl.SSLError:
731 app_log.error('Gramex cannot connect to HTTPS sites. Auth may fail')
732 except Exception:
733 # Ignore any other kind of exception
734 app_log.warning('Gramex has no direct Internet connection')
735 _client.close()
738def objectpath(node, keypath, default=None):
739 '''
740 Traverse down a dot-separated object path into dict items or object attrs.
741 For example, ``objectpath(handler, 'request.headers.User-Agent')`` returns
742 ``handler.request.headers['User-Agent']``. Dictionary access is preferred.
743 Returns ``None`` if the path is not found.
744 '''
745 for key in keypath.split('.'):
746 if hasattr(node, '__getitem__'): 746 ↛ 749line 746 didn't jump to line 749, because the condition on line 746 was never false
747 node = node.get(key)
748 else:
749 node = getattr(node, key, None)
750 if node is None:
751 return default
752 return node
755def recursive_encode(data, encoding='utf-8'):
756 '''
757 Convert all Unicode values into UTF-8 encoded byte strings in-place
758 '''
759 for key, value, node in walk(data):
760 if isinstance(key, six.text_type):
761 newkey = key.encode(encoding)
762 node[newkey] = node.pop(key)
763 key = newkey
764 if isinstance(value, six.text_type):
765 node[key] = value.encode(encoding)
768class TimedRotatingCSVHandler(logging.handlers.TimedRotatingFileHandler):
769 '''
770 Same as logging.handlers.TimedRotatingFileHandler, but writes to a CSV.
771 The constructor accepts an additional ``keys`` list as input that has
772 column keys. When ``.emit()`` is called, it expects an object with the
773 same keys as ``keys``.
774 '''
775 def __init__(self, *args, **kwargs):
776 self.keys = kwargs.pop('keys')
777 super(TimedRotatingCSVHandler, self).__init__(*args, **kwargs)
779 def _open(self):
780 stream = super(TimedRotatingCSVHandler, self)._open()
781 self.writer = csv.DictWriter(stream, fieldnames=self.keys, lineterminator='\n')
782 return stream
784 def emit(self, record):
785 try:
786 # From logging.handlers.BaseRotatingHandler
787 if self.shouldRollover(record):
788 self.doRollover()
789 # From logging.handlers.StreamHandler
790 if self.stream is None:
791 self.stream = self._open()
792 except (KeyboardInterrupt, SystemExit):
793 raise
794 except Exception as e:
795 # On Windows, multiple processes cannot rotate the same file.
796 # Ignore this, and just re-open the stream.
797 # On Linux, this needs to be tested.
798 if e.errno == EPERM or e.errno == EACCES:
799 if getattr(e, 'winerror', None) == ERROR_SHARING_VIOLATION:
800 self.stream = self._open()
801 else:
802 return self.handleError(record)
803 try:
804 # Write the CSV record instead of the formatted record
805 self.writer.writerow(record.msg)
806 self.stream.flush()
807 except Exception:
808 self.handleError(record)
811def ioloop_running(loop):
812 '''Returns whether the Tornado ioloop is running on not'''
813 # Python 2.7 and Tornado < 5.0 use this
814 if hasattr(loop, '_running'): 814 ↛ 815line 814 didn't jump to line 815, because the condition on line 814 was never true
815 return loop._running
816 # Python 3 on Tornado >= 5.0 delegates to asyncio
817 if hasattr(loop, 'asyncio_loop'): 817 ↛ 819line 817 didn't jump to line 819, because the condition on line 817 was never false
818 return loop.asyncio_loop.is_running()
819 raise NotImplementedError('Cannot determine tornado.ioloop is running')
822def used_kwargs(method, kwargs, ignore_keywords=False):
823 '''
824 Splits kwargs into those used by method, and those that are not.
826 Returns a tuple of (used, rest). *used* is a dict subset of kwargs with only
827 keys used by method. *rest* has the remaining kwargs keys.
829 If the method uses ``**kwargs`` (keywords), it uses all keys. To ignore this
830 and return only named arguments, use ``ignore_keywords=True``.
831 '''
832 argspec = inspect.getargspec(method)
833 # If method uses **kwargs, return all kwargs (unless you ignore **kwargs)
834 if argspec.keywords and not ignore_keywords:
835 used, rest = kwargs, {}
836 else:
837 # Split kwargs into 2 dicts -- used and rest
838 used, rest = {}, {}
839 for key, val in kwargs.items():
840 target = used if key in set(argspec.args) else rest
841 target[key] = val
842 return used, rest