Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1''' 

2Manages YAML config files as layered configurations with imports. 

3 

4:class:PathConfig loads YAML files from a path:: 

5 

6 pc = PathConfig('/path/to/file.yaml') 

7 

8This can be reloaded via the ``+`` operator. ``+pc`` reloads the YAML file 

9(but only if it is newer than before.) 

10 

11:class:ChainConfig chains multiple YAML files into a single config. For example 

12this merges ``base.yaml`` and ``next.yaml`` in sequence:: 

13 

14 cc = ChainConfig() 

15 cc['base'] = PathConfig('base.yaml') 

16 cc['next'] = PathConfig('next.yaml') 

17 

18To get the merged file, use ``+cc``. This updates the PathConfig files and 

19merges the YAMLs. 

20''' 

21 

22import os 

23import re 

24import csv 

25import six 

26import sys 

27import yaml 

28import string 

29import socket 

30import inspect 

31import logging 

32import datetime 

33import dateutil.tz 

34import dateutil.parser 

35from pathlib import Path 

36from copy import deepcopy 

37from random import choice 

38from fnmatch import fnmatch 

39from six import string_types 

40from collections import OrderedDict 

41from pydoc import locate as _locate, ErrorDuringImport 

42from yaml import Loader, MappingNode 

43from json import loads, JSONEncoder, JSONDecoder 

44from yaml.constructor import ConstructorError 

45from orderedattrdict import AttrDict, DefaultAttrDict 

46from errno import EACCES, EPERM 

47 

48ERROR_SHARING_VIOLATION = 32 # from winerror.ERROR_SHARING_VIOLATION 

49 

50# gramex.config.app_log is the default logger used by all of gramex 

51# If it's not there, create one. 

52logging.basicConfig() 

53app_log = logging.getLogger('gramex') 

54 

55# app_log_extra has additional parameters that may be used by the logger 

56app_log_extra = {'port': 'PORT'} 

57app_log = logging.LoggerAdapter(app_log, app_log_extra) 

58 

59# sqlalchemy.create_engine requires an encoding= that must be an str across 

60# Python 2 and Python 3. Expose this for other modules to use 

61str_utf8 = str('utf-8') # noqa 

62 

63 

64def walk(node): 

65 ''' 

66 Bottom-up recursive walk through a data structure yielding a (key, value, 

67 node) tuple for every entry. ``node[key] == value`` is true in every entry. 

68 

69 For example:: 

70 

71 >>> list(walk([{'x': 1}])) 

72 [ 

73 ('x', 1, {'x': 1}), # leaf: key, value, node 

74 (0, {'x': 1}, [{'x': 1}]) # parent: index, value, node 

75 ] 

76 

77 Circular linkage can lead to a RuntimeError:: 

78 

79 >>> x = {} 

80 >>> x['x'] = x 

81 >>> list(walk(x)) 

82 ... 

83 RuntimeError: maximum recursion depth exceeded 

84 ''' 

85 if hasattr(node, 'items'): 

86 for key, value in list(node.items()): 

87 for item in walk(value): 

88 yield item 

89 yield key, value, node 

90 elif isinstance(node, list): 

91 for index, value in enumerate(node): 

92 for item in walk(value): 

93 yield item 

94 yield index, value, node 

95 

96 

97def merge(old, new, mode='overwrite', warn=None, _path=''): 

98 ''' 

99 Update old dict with new dict recursively. 

100 

101 >>> merge({'a': {'x': 1}}, {'a': {'y': 2}}) 

102 {'a': {'x': 1, 'y': 2}} 

103 

104 If ``new`` is a list, convert into a dict with random keys. 

105 

106 If ``mode='overwrite'``, the old dict is overwritten (default). 

107 If ``mode='setdefault'``, the old dict values are updated only if missing. 

108 

109 ``warn=`` is an optional list of key paths. Any conflict on dictionaries 

110 matching any of these paths is logged as a warning. For example, 

111 ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key 

112 has a conflict. 

113 ''' 

114 for key in new: 

115 if key in old and hasattr(old[key], 'items') and hasattr(new[key], 'items'): 

116 path_key = _path + ('.' if _path else '') + six.text_type(key) 

117 if warn is not None: 

118 for pattern in warn: 

119 if fnmatch(path_key, pattern): 119 ↛ 120line 119 didn't jump to line 120, because the condition on line 119 was never true

120 app_log.warning('Duplicate key: %s', path_key) 

121 break 

122 merge(old=old[key], new=new[key], mode=mode, warn=warn, _path=path_key) 

123 elif mode == 'overwrite' or key not in old: 

124 old[key] = deepcopy(new[key]) 

125 return old 

126 

127 

128class ChainConfig(AttrDict): 

129 ''' 

130 An AttrDict that manages multiple configurations as layers. 

131 

132 >>> config = ChainConfig([ 

133 ... ('base', PathConfig('gramex.yaml')), 

134 ... ('app1', PathConfig('app.yaml')), 

135 ... ('app2', AttrDict()) 

136 ... ]) 

137 

138 Any dict-compatible values are allowed. ``+config`` returns the merged values. 

139 ''' 

140 

141 def __pos__(self): 

142 '''+config returns layers merged in order, removing null keys''' 

143 conf = AttrDict() 

144 for name, config in self.items(): 

145 if hasattr(config, '__pos__'): 

146 config.__pos__() 

147 merge(old=conf, new=config, mode='overwrite') 

148 

149 # Remove keys where the value is None 

150 for key, value, node in list(walk(conf)): 

151 if value is None: 

152 del node[key] 

153 

154 return conf 

155 

156 

157# Paths that users have already been warned about. Don't warn them again 

158_warned_paths = set() 

159# Get the directory where gramex is located. This is the same as the directory 

160# where this file (config.py) is located. 

161_gramex_path = os.path.dirname(os.path.abspath(__file__)) 

162 

163 

164def setup_variables(): 

165 '''Initialise variables''' 

166 variables = DefaultAttrDict(str) 

167 # Load all environment variables 

168 variables.update(os.environ) 

169 # GRAMEXPATH is the Gramex root directory 

170 variables['GRAMEXPATH'] = _gramex_path 

171 # GRAMEXAPPS is the Gramex apps directory 

172 variables['GRAMEXAPPS'] = os.path.join(_gramex_path, 'apps') 

173 # GRAMEXHOST is the hostname 

174 variables['GRAMEXHOST'] = socket.gethostname() 

175 # GRAMEXDATA varies based on OS 

176 if 'GRAMEXDATA' not in variables: 176 ↛ 189line 176 didn't jump to line 189, because the condition on line 176 was never false

177 if sys.platform.startswith('linux') or sys.platform == 'cygwin': 177 ↛ 178line 177 didn't jump to line 178, because the condition on line 177 was never true

178 variables['GRAMEXDATA'] = os.path.expanduser('~/.config/gramexdata') 

179 elif sys.platform == 'win32': 179 ↛ 181line 179 didn't jump to line 181, because the condition on line 179 was never false

180 variables['GRAMEXDATA'] = os.path.join(variables['LOCALAPPDATA'], 'Gramex Data') 

181 elif sys.platform == 'darwin': 

182 variables['GRAMEXDATA'] = os.path.expanduser( 

183 '~/Library/Application Support/Gramex Data') 

184 else: 

185 variables['GRAMEXDATA'] = os.path.abspath('.') 

186 app_log.warning('$GRAMEXDATA set to %s for OS %s', variables['GRAMEXDATA'], 

187 sys.platform) 

188 

189 return variables 

190 

191 

192variables = setup_variables() 

193 

194 

195def _substitute_variable(val): 

196 ''' 

197 If val contains a ${VAR} or $VAR and VAR is in the variables global, 

198 substitute it. 

199 

200 Direct variables are substituted as-is. For example, $x will return 

201 variables['x'] without converting it to a string. Otherwise, treat it as a 

202 string tempate. So "/$x/" will return "/1/" if x=1. 

203 ''' 

204 if not isinstance(val, string_types): 

205 return val 

206 if val.startswith('$') and val[1:] in variables: 

207 return variables[val[1:]] 

208 else: 

209 try: 

210 return string.Template(val).substitute(variables) 

211 except ValueError: 

212 raise ValueError('Use $$ instead of $ in %s' % val) 

213 

214 

215def _calc_value(val, key): 

216 ''' 

217 Calculate the value to assign to this key. 

218 

219 If ``val`` is not a dictionary that has a ``function`` key, return it as-is. 

220 

221 If it has a function key, call that function (with specified args, kwargs, 

222 etc) and allow the ``key`` parameter as an argument. 

223 

224 If the function is a generator, the first value is used. 

225 ''' 

226 if hasattr(val, 'get') and val.get('function'): 

227 from .transforms import build_transform 

228 function = build_transform(val, vars={'key': None}, filename='config:%s' % key) 

229 for result in function(key): 

230 if result is not None: 

231 return result 

232 return val.get('default') 

233 else: 

234 return _substitute_variable(val) 

235 

236 

237_valid_key_chars = string.ascii_letters + string.digits 

238 

239 

240def random_string(size, chars=_valid_key_chars): 

241 '''Return random string of length size using chars (which defaults to alphanumeric)''' 

242 return ''.join(choice(chars) for index in range(size)) # nosec - ok for non-cryptographic use 

243 

244 

245RANDOM_KEY = r'$*' 

246 

247 

248def _from_yaml(loader, node): 

249 ''' 

250 Load mapping as AttrDict, preserving order. Raise error on duplicate keys 

251 ''' 

252 # Based on yaml.constructor.SafeConstructor.construct_mapping() 

253 attrdict = AttrDict() 

254 yield attrdict 

255 if not isinstance(node, MappingNode): 255 ↛ 256line 255 didn't jump to line 256, because the condition on line 255 was never true

256 raise ConstructorError( 

257 None, None, 'expected a mapping node, but found %s' % node.id, node.start_mark) 

258 loader.flatten_mapping(node) 

259 for key_node, value_node in node.value: 

260 key = loader.construct_object(key_node, deep=False) 

261 if isinstance(key, six.string_types) and RANDOM_KEY in key: 

262 # With k=5 there's a <0.1% chance of collision even for 1mn uses. 

263 # (1 - decimal.Decimal(62 ** -5)) ** 1000000 ~ 0.999 

264 key = key.replace(RANDOM_KEY, random_string(5)) 

265 try: 

266 hash(key) 

267 except TypeError as exc: 

268 raise ConstructorError( 

269 'while constructing a mapping', node.start_mark, 

270 'found unacceptable key (%s)' % exc, key_node.start_mark) 

271 if key in attrdict: 

272 raise ConstructorError( 

273 'while constructing a mapping', node.start_mark, 

274 'found duplicate key (%s)' % key, key_node.start_mark) 

275 attrdict[key] = loader.construct_object(value_node, deep=False) 

276 

277 

278class ConfigYAMLLoader(Loader): 

279 ''' 

280 A YAML loader that loads a YAML file into an ordered AttrDict. Usage:: 

281 

282 >>> attrdict = yaml.load(yaml_string, Loader=ConfigYAMLLoader) 

283 

284 If there are duplicate keys, this raises an error. 

285 ''' 

286 def __init__(self, *args, **kwargs): 

287 super(ConfigYAMLLoader, self).__init__(*args, **kwargs) 

288 self.add_constructor(u'tag:yaml.org,2002:map', _from_yaml) 

289 self.add_constructor(u'tag:yaml.org,2002:omap', _from_yaml) 

290 

291 

292def _yaml_open(path, default=AttrDict(), **kwargs): 

293 ''' 

294 Load a YAML path.Path as AttrDict. Replace ${VAR} or $VAR with variables. 

295 Defines special variables $YAMLPATH as the absolute path of the YAML file, 

296 and $YAMLURL as the path relative to current directory. These can be 

297 overridden via keyward arguments (e.g. ``YAMLURL=...``) 

298 

299 If key has " if ", include it only if the condition (eval-ed in Python) is 

300 true. 

301 

302 If the path is missing, or YAML has a parse error, or the YAML is not a 

303 dict, returns the default value. 

304 ''' 

305 path = path.absolute() 

306 if not path.exists(): 

307 if path not in _warned_paths: 

308 app_log.warning('Missing config: %s', path) 

309 _warned_paths.add(path) 

310 return default 

311 app_log.debug('Loading config: %s', path) 

312 with path.open(encoding='utf-8') as handle: 

313 try: 

314 result = yaml.load(handle, Loader=ConfigYAMLLoader) # nosec 

315 except Exception: 

316 app_log.exception('Config error: %s', path) 

317 return default 

318 if not isinstance(result, AttrDict): 

319 if result is not None: 

320 app_log.warning('Config is not a dict: %s', path) 

321 return default 

322 

323 # Variables based on YAML file location 

324 yaml_path = str(path.parent) 

325 kwargs.setdefault('YAMLPATH', yaml_path) # Path to YAML folder 

326 kwargs.setdefault('YAMLFILE', str(path)) # Path to YAML file 

327 # $YAMLURL defaults to the relative URL from cwd to YAML folder. 

328 try: 

329 yamlurl = os.path.relpath(yaml_path) 

330 except ValueError: 

331 # If YAML is in a different drive, this fails. So don't set YAMLURL. 

332 # Impact: $YAMLURL is undefined for imports from a different drive. 

333 pass 

334 else: 

335 kwargs.setdefault('YAMLURL', yamlurl) 

336 # Typically, we use /$YAMLURL/url - so strip the slashes. Replace backslashes 

337 if isinstance(kwargs.get('YAMLURL'), string_types): 

338 kwargs['YAMLURL'] = kwargs['YAMLURL'].replace('\\', '/').strip('/') 

339 variables.update(kwargs) 

340 

341 # Update context with the variables section. 

342 # key: value sets key = value 

343 # key: {function: fn} sets key = fn(key) 

344 # key: {default: value} sets key = value if it's not already set 

345 # key: {default: {function: fn}} sets key = fn(key) if it's not already set 

346 if 'variables' in result: 

347 for key, val in result['variables'].items(): 

348 if hasattr(val, 'get') and 'default' in val and 'function' not in val: 

349 variables.setdefault(key, _calc_value(val['default'], key)) 

350 else: 

351 variables[key] = _calc_value(val, key) 

352 del result['variables'] 

353 

354 # Evaluate conditionals. "x if cond: y" becomes "x: y" if cond evals to True 

355 remove, replace = [], [] 

356 frozen_vars = dict(variables) 

357 for key, value, node in walk(result): 

358 if isinstance(key, string_types) and ' if ' in key: 

359 # Evaluate conditional 

360 base, expr = key.split(' if ', 2) 

361 try: 

362 condition = eval(expr, globals(), frozen_vars) # nosec - any Python expr is OK 

363 except Exception: 

364 condition = False 

365 app_log.exception('Failed condition evaluation: %s', key) 

366 if condition: 

367 replace.append((node, key, base)) 

368 else: 

369 remove.append((node, key)) 

370 for node, key in remove: 

371 del node[key] 

372 for node, key, base in replace: 

373 node[base] = node.pop(key) 

374 

375 # Substitute variables 

376 for key, value, node in walk(result): 

377 if isinstance(value, string_types): 

378 # Backward compatibility: before v1.0.4, we used {.} for {YAMLPATH} 

379 value = value.replace('{.}', '$YAMLPATH') 

380 # Substitute with variables in context, defaulting to '' 

381 node[key] = _substitute_variable(value) 

382 return result 

383 

384 

385def _pathstat(path): 

386 ''' 

387 Return a path stat object, which has 2 attributes/keys: ``.path`` is the 

388 same as the ``path`` parameter. ``stat`` is the result of ``os.stat``. If 

389 path is missing, ``stat`` has ``st_mtime`` and ``st_size`` set to ``0``. 

390 ''' 

391 # If path doesn't exist, create a dummy stat structure with 

392 # safe defaults (old mtime, 0 filesize, etc) 

393 stat = path.stat() if path.exists() else AttrDict(st_mtime=0, st_size=0) 

394 return AttrDict(path=path, stat=stat) 

395 

396 

397def _add_ns(config, namespace, prefix): 

398 ''' 

399 Given a YAML config (basically a dict), add prefix to specified namespaces. 

400 

401 For example:: 

402 

403 >>> _add_ns({'x': 1}, '*', 'a') 

404 {'a.x': 1} 

405 >>> _add_ns({'x': {'y': 1}}, ['*', 'x'], 'a') 

406 {'a.x': {'a.y': 1}} 

407 ''' 

408 if not isinstance(namespace, list): 

409 namespace = [namespace] 

410 # Sort in descending order of key depth. So "x.y" is before "x" is before "*" 

411 namespace = sorted(namespace, key=lambda ns: -1 if ns == '*' else ns.count('.'), reverse=True) 

412 prefix += ':' 

413 for keypath in namespace: 

414 if keypath == '*': 

415 el = config 

416 else: 

417 el = objectpath(config, keypath, default={}) 

418 if isinstance(el, dict): 418 ↛ 413line 418 didn't jump to line 413, because the condition on line 418 was never false

419 for subkey in list(el.keys()): 

420 if subkey not in {'import'}: 420 ↛ 419line 420 didn't jump to line 419, because the condition on line 420 was never false

421 el[prefix + subkey] = el.pop(subkey) 

422 return config 

423 

424 

425def load_imports(config, source, warn=None): 

426 ''' 

427 Post-process a config for imports. 

428 

429 ``config`` is the data to process. ``source`` is the path where it was 

430 loaded from. 

431 

432 If ``config`` has an ``import:`` key, treat all values below that as YAML 

433 files (specified relative to ``source``) and import them in sequence. 

434 

435 Return a list of imported paths as :func:_pathstat objects. (This includes 

436 ``source``.) 

437 

438 For example, if the ``source`` is ``base.yaml`` (which has the below 

439 configuration) and is loaded into ``config``:: 

440 

441 app: 

442 port: 20 

443 start: true 

444 path: / 

445 import: update*.yaml # Can be any glob, e.g. */gramex.yaml 

446 

447 ... and ``update.yaml`` looks like this:: 

448 

449 app: 

450 port: 30 

451 new: yes 

452 

453 ... then after this function is called, ``config`` looks like this:: 

454 

455 app: 

456 port: 20 # From base.yaml. NOT updated by update.yaml 

457 start: true # From base.yaml 

458 new: yes # From update.yaml 

459 path: / # From base.yaml 

460 

461 The ``import:`` keys are deleted. The return value contains :func:_pathstat 

462 values for ``base.yaml`` and ``update.yaml`` in that order. 

463 

464 Multiple ``import:`` values can be specified as a dictionary:: 

465 

466 import: 

467 first-app: app1/*.yaml 

468 next-app: app2/*.yaml 

469 

470 To import sub-keys as namespaces, use:: 

471 

472 import: 

473 app: {path: */gramex.yaml, namespace: 'url'} 

474 

475 This prefixes all keys under ``url:``. Here are more examples:: 

476 

477 namespace: True # Add namespace to all top-level keys 

478 namespace: url # Add namespace to url.* 

479 namespace: log.loggers # Add namespace to log.loggers.* 

480 namespace: [True, url] # Add namespace to top level keys and url.* 

481 

482 By default, the prefix is the relative path of the imported YAML file 

483 (relative to the importer). 

484 

485 ``warn=`` is an optional list of key paths. Any conflict on dictionaries 

486 matching any of these paths is logged as a warning. For example, 

487 ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key 

488 has a conflict. 

489 ''' 

490 imported_paths = [_pathstat(source)] 

491 root = source.absolute().parent 

492 for key, value, node in list(walk(config)): 

493 if isinstance(key, six.string_types) and key.startswith('import.merge'): 

494 # Strip the top level key(s) from import.merge values 

495 if isinstance(value, dict): 

496 for name, conf in value.items(): 

497 node[name] = conf 

498 elif value: 498 ↛ 499line 498 didn't jump to line 499, because the condition on line 498 was never true

499 raise ValueError('import.merge: must be dict, not %s at %s' % ( 

500 repr(value), source)) 

501 # Delete the import key 

502 del node[key] 

503 elif key == 'import': 

504 # Convert "import: path" to "import: {app: path}" 

505 if isinstance(value, six.string_types): 

506 value = {'apps': value} 

507 # Allow "import: [path, path]" to "import: {app0: path, app1: path}" 

508 elif isinstance(value, list): 

509 value = OrderedDict((('app%d' % i, conf) for i, conf in enumerate(value))) 

510 # By now, import: should be a dict 

511 elif not isinstance(value, dict): 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true

512 raise ValueError('import: must be string/list/dict, not %s at %s' % ( 

513 repr(value), source)) 

514 # If already a dict with a single import via 'path', convert to dict of apps 

515 if 'path' in value: 

516 value = {'app': value} 

517 for name, conf in value.items(): 

518 if not isinstance(conf, dict): 

519 conf = AttrDict(path=conf) 

520 if 'path' not in conf: 520 ↛ 521line 520 didn't jump to line 521, because the condition on line 520 was never true

521 raise ValueError('import: has no conf at %s' % source) 

522 paths = conf.pop('path') 

523 paths = paths if isinstance(paths, list) else [paths] 

524 globbed_paths = [] 

525 for path in paths: 

526 globbed_paths += sorted(root.glob(path)) if '*' in path else [Path(path)] 

527 ns = conf.pop('namespace', None) 

528 for path in globbed_paths: 

529 abspath = root.joinpath(path) 

530 new_conf = _yaml_open(abspath, **conf) 

531 if ns is not None: 

532 prefix = Path(path).as_posix() 

533 new_conf = _add_ns(new_conf, ns, name + ':' + prefix) 

534 imported_paths += load_imports(new_conf, source=abspath) 

535 merge(old=node, new=new_conf, mode='setdefault', warn=warn) 

536 # Delete the import key 

537 del node[key] 

538 return imported_paths 

539 

540 

541class PathConfig(AttrDict): 

542 ''' 

543 An ``AttrDict`` that is loaded from a path as a YAML file. For e.g., 

544 ``conf = PathConfig(path)`` loads the YAML file at ``path`` as an AttrDict. 

545 ``+conf`` reloads the path if required. 

546 

547 ``warn=`` is an optional list of key paths. Any conflict on dictionaries 

548 matching any of these paths is logged as a warning. For example, 

549 ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key 

550 has a conflict. 

551 

552 Like http://configure.readthedocs.org/ but supports imports not inheritance. 

553 This lets us import YAML files in the middle of a YAML structure:: 

554 

555 key: 

556 import: 

557 conf1: file1.yaml # Import file1.yaml here 

558 conf2: file2.yaml # Import file2.yaml here 

559 

560 Each ``PathConfig`` object has an ``__info__`` attribute with the following 

561 keys: 

562 

563 __info__.path 

564 The path that this instance syncs with, stored as a ``pathlib.Path`` 

565 __info__.warn 

566 The keys to warn in case about in case of an import merge conflict 

567 __info__.imports 

568 A list of imported files, stored as an ``AttrDict`` with 2 attributes: 

569 

570 path 

571 The path that was imported, stored as a ``pathlib.Path`` 

572 stat 

573 The ``os.stat()`` information about this file (or ``None`` if the 

574 file is missing.) 

575 ''' 

576 duplicate_warn = None 

577 

578 def __init__(self, path, warn=None): 

579 super(PathConfig, self).__init__() 

580 if warn is None: 580 ↛ 582line 580 didn't jump to line 582, because the condition on line 580 was never false

581 warn = self.duplicate_warn 

582 self.__info__ = AttrDict(path=Path(path), imports=[], warn=warn) 

583 self.__pos__() 

584 

585 def __pos__(self): 

586 '''+config reloads this config (if it has a path)''' 

587 path = self.__info__.path 

588 

589 # We must reload the layer if nothing has been imported... 

590 reload = not self.__info__.imports 

591 # ... or if an imported file is deleted / updated 

592 for imp in self.__info__.imports: 

593 exists = imp.path.exists() 

594 if not exists and imp.stat is not None: 

595 reload = True 

596 app_log.info('No config found: %s', imp.path) 

597 break 

598 if exists and (imp.path.stat().st_mtime > imp.stat.st_mtime or 

599 imp.path.stat().st_size != imp.stat.st_size): 

600 reload = True 

601 app_log.info('Updated config: %s', imp.path) 

602 break 

603 if reload: 

604 self.clear() 

605 self.update(_yaml_open(path)) 

606 self.__info__.imports = load_imports(self, source=path, warn=self.__info__.warn) 

607 return self 

608 

609 

610def locate(path, modules=[], forceload=0): 

611 ''' 

612 Locate an object by name or dotted path. 

613 

614 For example, ``locate('str')`` returns the ``str`` built-in. 

615 ``locate('gramex.handlers.FileHandler')`` returns the class 

616 ``gramex.handlers.FileHandler``. 

617 

618 ``modules`` is a list of modules to search for the path in first. So 

619 ``locate('FileHandler', modules=[gramex.handlers])`` will return 

620 ``gramex.handlers.FileHandler``. 

621 

622 If importing raises an Exception, log it and return None. 

623 ''' 

624 try: 

625 for module_name in modules: 

626 module = _locate(module_name, forceload) 

627 if hasattr(module, path): 

628 return getattr(module, path) 

629 return _locate(path, forceload) 

630 except ErrorDuringImport: 

631 app_log.exception('Exception when importing %s', path) 

632 return None 

633 

634 

635_checked_old_certs = [] 

636 

637 

638class CustomJSONEncoder(JSONEncoder): 

639 ''' 

640 Encodes object to JSON, additionally converting datetime into ISO 8601 format 

641 ''' 

642 def default(self, obj): 

643 import numpy as np 

644 

645 if hasattr(obj, 'to_dict'): 

646 # Slow but reliable. Handles conversion of numpy objects, mixed types, etc. 

647 return loads(obj.to_json(orient='records', date_format='iso'), 

648 object_pairs_hook=OrderedDict) 

649 elif isinstance(obj, datetime.datetime): 

650 # Use local timezone if no timezone is specified 

651 if obj.tzinfo is None: 

652 obj = obj.replace(tzinfo=dateutil.tz.tzlocal()) 

653 return obj.isoformat() 

654 elif isinstance(obj, np.datetime64): 

655 obj = obj.item() 

656 if (isinstance(obj, datetime.datetime) and obj.tzinfo is None): 

657 obj = obj.replace(tzinfo=dateutil.tz.tzlocal()) 

658 return obj.isoformat() 

659 elif isinstance(obj, np.integer): 

660 return int(obj) 

661 elif isinstance(obj, np.floating): 

662 return float(obj) 

663 elif isinstance(obj, np.ndarray): 

664 return obj.tolist() 

665 elif isinstance(obj, np.bool_): 

666 return bool(obj) 

667 elif isinstance(obj, np.bytes_): 667 ↛ 669line 667 didn't jump to line 669, because the condition on line 667 was never false

668 return obj.decode('utf-8') 

669 return super(CustomJSONEncoder, self).default(obj) 

670 

671 

672class CustomJSONDecoder(JSONDecoder): 

673 ''' 

674 Decodes JSON string, converting ISO 8601 datetime to datetime 

675 ''' 

676 # Check if a string might be a datetime. Handles variants like: 

677 # 2001-02-03T04:05:06Z 

678 # 2001-02-03T04:05:06+000 

679 # 2001-02-03T04:05:06.000+0000 

680 re_datetimeval = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}') 

681 re_datetimestr = re.compile(r'"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}') 

682 

683 def __init__(self, *args, **kwargs): 

684 self.old_object_pairs_hook = kwargs.get('object_pairs_hook') 

685 kwargs['object_pairs_hook'] = self.convert 

686 super(CustomJSONDecoder, self).__init__(*args, **kwargs) 

687 

688 def decode(self, obj): 

689 if self.re_datetimestr.match(obj): 

690 return dateutil.parser.parse(obj[1:-1]) 

691 return super(CustomJSONDecoder, self).decode(obj) 

692 

693 def convert(self, obj): 

694 for index, (key, val) in enumerate(obj): 

695 if isinstance(val, six.string_types) and self.re_datetimeval.match(val): 

696 obj[index] = (key, dateutil.parser.parse(val)) 

697 if callable(self.old_object_pairs_hook): 

698 return self.old_object_pairs_hook(obj) 

699 return dict(obj) 

700 

701 

702def check_old_certs(): 

703 ''' 

704 The latest SSL certificates from certifi don't work for Google Auth. Do 

705 a one-time check to access accounts.google.com. If it throws an SSL 

706 error, switch to old SSL certificates. See 

707 https://github.com/tornadoweb/tornado/issues/1534 

708 ''' 

709 if not _checked_old_certs: 

710 _checked_old_certs.append(True) 

711 

712 import ssl 

713 from tornado.httpclient import HTTPClient, AsyncHTTPClient 

714 

715 # Use HTTPClient to check instead of AsyncHTTPClient because it's synchronous. 

716 _client = HTTPClient() 

717 try: 

718 # Use accounts.google.com because we know it fails with new certifi certificates 

719 # cdn.redhat.com is another site that fails. 

720 _client.fetch("https://accounts.google.com/") 

721 except ssl.SSLError: 

722 try: 

723 import certifi # noqa: late import to minimise dependencies 

724 AsyncHTTPClient.configure(None, defaults=dict(ca_certs=certifi.old_where())) 

725 app_log.warning('Using old SSL certificates for compatibility') 

726 except ImportError: 

727 pass 

728 try: 

729 _client.fetch("https://accounts.google.com/") 

730 except ssl.SSLError: 

731 app_log.error('Gramex cannot connect to HTTPS sites. Auth may fail') 

732 except Exception: 

733 # Ignore any other kind of exception 

734 app_log.warning('Gramex has no direct Internet connection') 

735 _client.close() 

736 

737 

738def objectpath(node, keypath, default=None): 

739 ''' 

740 Traverse down a dot-separated object path into dict items or object attrs. 

741 For example, ``objectpath(handler, 'request.headers.User-Agent')`` returns 

742 ``handler.request.headers['User-Agent']``. Dictionary access is preferred. 

743 Returns ``None`` if the path is not found. 

744 ''' 

745 for key in keypath.split('.'): 

746 if hasattr(node, '__getitem__'): 746 ↛ 749line 746 didn't jump to line 749, because the condition on line 746 was never false

747 node = node.get(key) 

748 else: 

749 node = getattr(node, key, None) 

750 if node is None: 

751 return default 

752 return node 

753 

754 

755def recursive_encode(data, encoding='utf-8'): 

756 ''' 

757 Convert all Unicode values into UTF-8 encoded byte strings in-place 

758 ''' 

759 for key, value, node in walk(data): 

760 if isinstance(key, six.text_type): 

761 newkey = key.encode(encoding) 

762 node[newkey] = node.pop(key) 

763 key = newkey 

764 if isinstance(value, six.text_type): 

765 node[key] = value.encode(encoding) 

766 

767 

768class TimedRotatingCSVHandler(logging.handlers.TimedRotatingFileHandler): 

769 ''' 

770 Same as logging.handlers.TimedRotatingFileHandler, but writes to a CSV. 

771 The constructor accepts an additional ``keys`` list as input that has 

772 column keys. When ``.emit()`` is called, it expects an object with the 

773 same keys as ``keys``. 

774 ''' 

775 def __init__(self, *args, **kwargs): 

776 self.keys = kwargs.pop('keys') 

777 super(TimedRotatingCSVHandler, self).__init__(*args, **kwargs) 

778 

779 def _open(self): 

780 stream = super(TimedRotatingCSVHandler, self)._open() 

781 self.writer = csv.DictWriter(stream, fieldnames=self.keys, lineterminator='\n') 

782 return stream 

783 

784 def emit(self, record): 

785 try: 

786 # From logging.handlers.BaseRotatingHandler 

787 if self.shouldRollover(record): 

788 self.doRollover() 

789 # From logging.handlers.StreamHandler 

790 if self.stream is None: 

791 self.stream = self._open() 

792 except (KeyboardInterrupt, SystemExit): 

793 raise 

794 except Exception as e: 

795 # On Windows, multiple processes cannot rotate the same file. 

796 # Ignore this, and just re-open the stream. 

797 # On Linux, this needs to be tested. 

798 if e.errno == EPERM or e.errno == EACCES: 

799 if getattr(e, 'winerror', None) == ERROR_SHARING_VIOLATION: 

800 self.stream = self._open() 

801 else: 

802 return self.handleError(record) 

803 try: 

804 # Write the CSV record instead of the formatted record 

805 self.writer.writerow(record.msg) 

806 self.stream.flush() 

807 except Exception: 

808 self.handleError(record) 

809 

810 

811def ioloop_running(loop): 

812 '''Returns whether the Tornado ioloop is running on not''' 

813 # Python 2.7 and Tornado < 5.0 use this 

814 if hasattr(loop, '_running'): 814 ↛ 815line 814 didn't jump to line 815, because the condition on line 814 was never true

815 return loop._running 

816 # Python 3 on Tornado >= 5.0 delegates to asyncio 

817 if hasattr(loop, 'asyncio_loop'): 817 ↛ 819line 817 didn't jump to line 819, because the condition on line 817 was never false

818 return loop.asyncio_loop.is_running() 

819 raise NotImplementedError('Cannot determine tornado.ioloop is running') 

820 

821 

822def used_kwargs(method, kwargs, ignore_keywords=False): 

823 ''' 

824 Splits kwargs into those used by method, and those that are not. 

825 

826 Returns a tuple of (used, rest). *used* is a dict subset of kwargs with only 

827 keys used by method. *rest* has the remaining kwargs keys. 

828 

829 If the method uses ``**kwargs`` (keywords), it uses all keys. To ignore this 

830 and return only named arguments, use ``ignore_keywords=True``. 

831 ''' 

832 argspec = inspect.getargspec(method) 

833 # If method uses **kwargs, return all kwargs (unless you ignore **kwargs) 

834 if argspec.keywords and not ignore_keywords: 

835 used, rest = kwargs, {} 

836 else: 

837 # Split kwargs into 2 dicts -- used and rest 

838 used, rest = {}, {} 

839 for key, val in kwargs.items(): 

840 target = used if key in set(argspec.args) else rest 

841 target[key] = val 

842 return used, rest