Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import io 

2import os 

3import six 

4import time 

5import logging 

6import datetime 

7import mimetypes 

8import traceback 

9import tornado.gen 

10import gramex.cache 

11from binascii import b2a_base64, hexlify 

12from orderedattrdict import AttrDict 

13from six.moves.urllib_parse import urlparse, urlsplit, urljoin, urlencode 

14from tornado.web import RequestHandler, HTTPError, MissingArgumentError, decode_signed_value 

15from tornado.websocket import WebSocketHandler 

16from gramex import conf, __version__ 

17from gramex.config import merge, objectpath, app_log 

18from gramex.transforms import build_transform, CacheLoader 

19from gramex.http import UNAUTHORIZED, FORBIDDEN, BAD_REQUEST 

20from gramex.cache import get_store 

21# We don't use these, but these stores used to be defined here. Programs may import these 

22from gramex.cache import KeyStore, JSONStore, HDF5Store, SQLiteStore, RedisStore # noqa 

23 

24server_header = 'Gramex/%s' % __version__ 

25session_store_cache = {} 

26_missing = object() 

27_arg_default = object() 

28 

29 

30class BaseMixin(object): 

31 @classmethod 

32 def setup(cls, transform={}, redirect={}, auth=None, log=None, set_xsrf=None, 

33 error=None, xsrf_cookies=None, **kwargs): 

34 ''' 

35 One-time setup for all request handlers. This is called only when 

36 gramex.yaml is parsed / changed. 

37 ''' 

38 cls._on_init_methods = [] 

39 cls._on_finish_methods = [] 

40 cls._set_xsrf = set_xsrf 

41 

42 cls.kwargs = cls.conf.get('kwargs', AttrDict()) 

43 

44 cls.setup_transform(transform) 

45 cls.setup_redirect(redirect) 

46 # Note: call setup_session before setup_auth to ensure that 

47 # override_user is run before authorize 

48 cls.setup_session(conf.app.get('session')) 

49 cls.setup_auth(auth) 

50 cls.setup_error(error) 

51 cls.setup_xsrf(xsrf_cookies) 

52 cls.setup_log() 

53 

54 # app.settings.debug enables debugging exceptions using pdb 

55 if conf.app.settings.get('debug', False): 55 ↛ 56line 55 didn't jump to line 56, because the condition on line 55 was never true

56 cls.log_exception = cls.debug_exception 

57 

58 # A list of special keys for BaseHandler. Can be extended by other classes. 

59 special_keys = ['transform', 'redirect', 'auth', 'log', 'set_xsrf', 

60 'error', 'xsrf_cookies', 'headers'] 

61 

62 @classmethod 

63 def clear_special_keys(cls, kwargs, *args): 

64 ''' 

65 Remove keys handled by BaseHandler that may interfere with setup(). 

66 This should be called explicitly in setup() where required. 

67 ''' 

68 for special_key in cls.special_keys: 

69 kwargs.pop(special_key, None) 

70 for special_key in args: 

71 kwargs.pop(special_key, None) 

72 return kwargs 

73 

74 @classmethod 

75 def setup_default_kwargs(cls): 

76 ''' 

77 Use default config from handlers.<Class>.* and handlers.BaseHandler. 

78 Called by gramex.services.url(). 

79 ''' 

80 c = cls.conf.setdefault('kwargs', {}) 

81 merge(c, objectpath(conf, 'handlers.' + cls.conf.handler, {}), mode='setdefault') 

82 merge(c, objectpath(conf, 'handlers.BaseHandler', {}), mode='setdefault') 

83 

84 @classmethod 

85 def setup_transform(cls, transform): 

86 cls.transform = {} 

87 for pattern, trans in transform.items(): 

88 cls.transform[pattern] = { 

89 'function': build_transform( 

90 trans, vars=AttrDict((('content', None), ('handler', None))), 

91 filename='url:%s' % cls.name), 

92 'headers': trans.get('headers', {}), 

93 'encoding': trans.get('encoding'), 

94 } 

95 

96 @staticmethod 

97 def _purge_keys(data): 

98 ''' 

99 Returns session keys to be deleted. These are either None values or 

100 those with expired keys based on _t. 

101 setup_session makes the session store call this method. 

102 Until v1.20 (31 Jul 2017) no _t keys were set. 

103 From v1.23 (31 Oct 2017) these are cleared. 

104 ''' 

105 now = time.time() 

106 week = 7 * 24 * 60 * 60 

107 keys = [] 

108 # When using sqlitedict, fetching keys may fail if DB is locked. Try later 

109 try: 

110 items = list(data.items()) 

111 except Exception: 

112 items = [] 

113 for key, val in items: 

114 # Purge already cleared / removed sessions 

115 if val is None: 

116 keys.append(key) 

117 elif isinstance(val, dict): 117 ↛ 127line 117 didn't jump to line 127, because the condition on line 117 was never false

118 # If the session has expired, remove it 

119 if val.get('_t', 0) < now: 

120 keys.append(key) 

121 # If the session is inactive, remove it after a week. 

122 # If we remove immediately, then we may lose WIP sessions. 

123 # For example, people who opened a login page where _next_url was set 

124 elif '_i' in val and '_l' in val and val['_i'] + val['_l'] < now - week: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true

125 keys.append(key) 

126 else: 

127 app_log.warning('Store key: %s has value type %s (not dict)', key, type(val)) 

128 return keys 

129 

130 @classmethod 

131 def setup_session(cls, session_conf): 

132 '''handler.session returns the session object. It is saved on finish.''' 

133 if session_conf is None: 133 ↛ 134line 133 didn't jump to line 134, because the condition on line 133 was never true

134 return 

135 key = store_type, store_path = session_conf.get('type'), session_conf.get('path') 

136 if key not in session_store_cache: 

137 session_store_cache[key] = get_store( 

138 type=store_type, 

139 path=store_path, 

140 flush=session_conf.get('flush'), 

141 purge=session_conf.get('purge'), 

142 purge_keys=cls._purge_keys 

143 ) 

144 cls._session_store = session_store_cache[key] 

145 cls.session = property(cls.get_session) 

146 cls._session_expiry = session_conf.get('expiry') 

147 cls._session_cookie = { 

148 key: session_conf[key] for key in ('domain', 'httponly', 'secure') 

149 if key in session_conf 

150 } 

151 cls._on_finish_methods.append(cls.save_session) 

152 cls._on_init_methods.append(cls.override_user) 

153 cls._on_finish_methods.append(cls.set_last_visited) 

154 

155 @classmethod 

156 def setup_redirect(cls, redirect): 

157 ''' 

158 Any handler can have a ``redirect:`` kwarg that looks like this:: 

159 

160 redirect: 

161 query: next # If the URL has a ?next=..., redirect to that page next 

162 header: X-Next # Else if the header has an X-Next=... redirect to that 

163 url: ... # Else redirect to this URL 

164 

165 Only these 3 keys are allowed. All are optional, and checked in the 

166 order specified. So, for example:: 

167 

168 redirect: 

169 header: X-Next # Checks the X-Next header first 

170 query: next # If it's missing, uses the ?next= 

171 

172 You can also specify a string for redirect. ``redirect: ...`` is the same 

173 as ``redirect: {url: ...}``. 

174 

175 When any BaseHandler subclass calls ``self.save_redirect_page()``, it 

176 stores the redirect URL in ``session['_next_url']``. The URL is 

177 calculated relative to the handler's URL. 

178 

179 After that, when the subclass calls ``self.redirect_next()``, it 

180 redirects to ``session['_next_url']`` and clears the value. (If the 

181 ``_next_url`` was not stored, we redirect to the home page ``/``.) 

182 

183 Only some handlers implement redirection. But they all implement it in 

184 this same consistent way. 

185 ''' 

186 # Ensure that redirect is a dictionary before proceeding. 

187 if isinstance(redirect, six.string_types): 187 ↛ 188line 187 didn't jump to line 188, because the condition on line 187 was never true

188 redirect = {'url': redirect} 

189 if not isinstance(redirect, dict): 189 ↛ 190line 189 didn't jump to line 190, because the condition on line 189 was never true

190 app_log.error('url:%s.redirect must be a URL or a dict, not %s', 

191 cls.name, repr(redirect)) 

192 return 

193 

194 cls.redirects = [] 

195 add = cls.redirects.append 

196 for key, value in redirect.items(): 

197 if key == 'query': 

198 add(lambda h, v=value: h.get_argument(v, None)) 

199 elif key == 'header': 

200 add(lambda h, v=value: h.request.headers.get(v)) 

201 elif key == 'url': 201 ↛ 196line 201 didn't jump to line 196, because the condition on line 201 was never false

202 add(lambda h, v=value: v) 

203 

204 # redirect.external=False disallows external URLs 

205 if not redirect.get('external', False): 205 ↛ exitline 205 didn't return from function 'setup_redirect', because the condition on line 205 was never false

206 def no_external(method): 

207 def redirect_method(handler): 

208 next_uri = method(handler) 

209 if next_uri is not None: 

210 target = urlparse(next_uri) 

211 if not target.scheme and not target.netloc: 

212 return next_uri 

213 req = handler.request 

214 if req.protocol == target.scheme and req.host == target.netloc: 

215 return next_uri 

216 app_log.error('Not redirecting to external url: %s', next_uri) 

217 return redirect_method 

218 cls.redirects = [no_external(method) for method in cls.redirects] 

219 

220 @classmethod 

221 def setup_auth(cls, auth): 

222 # auth: if there's no auth: in handler, default to app.auth 

223 if auth is None: 

224 auth = conf.app.get('auth') 

225 # Treat True as an empty dict, i.e. auth: {} 

226 if auth is True: 

227 auth = AttrDict() 

228 # Set up the auth 

229 if isinstance(auth, dict): 

230 cls._login_url = auth.get('login_url', None) 

231 cls._on_init_methods.append(cls.authorize) 

232 cls.permissions = [] 

233 # Add check for condition 

234 if auth.get('condition'): 

235 cls.permissions.append( 

236 build_transform(auth['condition'], vars=AttrDict(handler=None), 

237 filename='url:%s.auth.permission' % cls.name)) 

238 # Add check for membership 

239 memberships = auth.get('membership', []) 

240 if not isinstance(memberships, list): 

241 memberships = [memberships] 

242 if len(memberships): 

243 cls.permissions.append(check_membership(memberships)) 

244 elif auth: 244 ↛ 245line 244 didn't jump to line 245, because the condition on line 244 was never true

245 app_log.error('url:%s.auth is not a dict', cls.name) 

246 

247 @classmethod 

248 def setup_log(cls): 

249 ''' 

250 Logs access requests to gramex.requests as a CSV file. 

251 ''' 

252 logger = logging.getLogger('gramex.requests') 

253 keys = objectpath(conf, 'log.handlers.requests.keys', []) 

254 log_info = build_log_info(keys) 

255 cls.log_request = lambda handler: logger.info(log_info(handler)) 

256 

257 @classmethod 

258 def _error_fn(cls, error_code, error_config): 

259 template_kwargs = {} 

260 if 'autoescape' in error_config: 

261 if not error_config['autoescape']: 261 ↛ 264line 261 didn't jump to line 264, because the condition on line 261 was never false

262 template_kwargs['autoescape'] = None 

263 else: 

264 app_log.error('url:%s.error.%d.autoescape can only be false', cls.name, error_code) 

265 if 'whitespace' in error_config: 

266 template_kwargs['whitespace'] = error_config['whitespace'] 

267 

268 def error(*args, **kwargs): 

269 tmpl = gramex.cache.open(error_config['path'], 'template', **template_kwargs) 

270 return tmpl.generate(*args, **kwargs) 

271 

272 return error 

273 

274 @classmethod 

275 def setup_error(cls, error): 

276 ''' 

277 Sample configuration:: 

278 

279 error: 

280 404: 

281 path: template.json # Use a template 

282 autoescape: false # with no autoescape 

283 whitespace: single # as a single line 

284 headers: 

285 Content-Type: application/json 

286 500: 

287 function: module.fn 

288 args: [=status_code, =kwargs, =handler] 

289 ''' 

290 if not error: 

291 return 

292 if not isinstance(error, dict): 292 ↛ 293line 292 didn't jump to line 293, because the condition on line 292 was never true

293 return app_log.error('url:%s.error is not a dict', cls.name) 

294 # Compile all errors handlers 

295 cls.error = {} 

296 for error_code, error_config in error.items(): 

297 try: 

298 error_code = int(error_code) 

299 if error_code < 100 or error_code > 1000: 299 ↛ 300line 299 didn't jump to line 300, because the condition on line 299 was never true

300 raise ValueError() 

301 except ValueError: 

302 app_log.error('url.%s.error code %s is not a number (100 - 1000)', 

303 cls.name, error_code) 

304 continue 

305 if not isinstance(error_config, dict): 305 ↛ 306line 305 didn't jump to line 306, because the condition on line 305 was never true

306 return app_log.error('url:%s.error.%d is not a dict', cls.name, error_code) 

307 # Make a copy of the original. When we add headers, etc, it shouldn't affect original 

308 error_config = AttrDict(error_config) 

309 error_path, error_function = error_config.get('path'), error_config.get('function') 

310 if error_function: 

311 if error_path: 311 ↛ 315line 311 didn't jump to line 315, because the condition on line 311 was never false

312 error_config.pop('path') 

313 app_log.warning('url.%s.error.%d has function: AND path:. Ignoring path:', 

314 cls.name, error_code) 

315 cls.error[error_code] = {'function': build_transform( 

316 error_config, 

317 vars=AttrDict((('status_code', None), ('kwargs', None), ('handler', None))), 

318 filename='url:%s.error.%d' % (cls.name, error_code) 

319 )} 

320 elif error_path: 320 ↛ 327line 320 didn't jump to line 327, because the condition on line 320 was never false

321 encoding = error_config.get('encoding', 'utf-8') 

322 cls.error[error_code] = {'function': cls._error_fn(error_code, error_config)} 

323 mime_type, encoding = mimetypes.guess_type(error_path, strict=False) 

324 if mime_type: 324 ↛ 330line 324 didn't jump to line 330, because the condition on line 324 was never false

325 error_config.setdefault('headers', {}).setdefault('Content-Type', mime_type) 

326 else: 

327 app_log.error('url.%s.error.%d must have a path or function key', 

328 cls.name, error_code) 

329 # Add the error configuration for reference 

330 if error_code in cls.error: 330 ↛ 296line 330 didn't jump to line 296, because the condition on line 330 was never false

331 cls.error[error_code]['conf'] = error_config 

332 cls._write_error, cls.write_error = cls.write_error, cls._write_custom_error 

333 

334 @classmethod 

335 def setup_xsrf(cls, xsrf_cookies): 

336 ''' 

337 Sample configuration:: 

338 

339 xsrf_cookies: false # Disables xsrf_cookies 

340 xsrf_cookies: true # or anything other than false keeps it enabled 

341 ''' 

342 cls.check_xsrf_cookie = cls.noop if xsrf_cookies is False else cls.xsrf_ajax 

343 

344 def xsrf_ajax(self): 

345 ''' 

346 TODO: explain things clearly. 

347 Same as Tornado's check_xsrf_cookie() -- but is ignored for AJAX requests 

348 ''' 

349 ajax = self.request.headers.get('X-Requested-With', '').lower() == 'xmlhttprequest' 

350 if not ajax: 

351 return super(BaseHandler, self).check_xsrf_cookie() 

352 

353 def noop(self): 

354 '''Does nothing. Used when overriding functions or providing a dummy operation''' 

355 pass 

356 

357 def save_redirect_page(self): 

358 ''' 

359 Loop through all redirect: methods and save the first available redirect 

360 page against the session. Defaults to previously set value, else ``/``. 

361 

362 See :py:func:`setup_redirect` 

363 ''' 

364 for method in self.redirects: 

365 next_url = method(self) 

366 if next_url: 

367 self.session['_next_url'] = urljoin(self.request.uri, next_url) 

368 return 

369 self.session.setdefault('_next_url', '/') 

370 

371 def redirect_next(self): 

372 ''' 

373 Redirect the user ``session['_next_url']``. If it does not exist, 

374 set it up first. Then redirect. 

375 

376 See :py:func:`setup_redirect` 

377 ''' 

378 if '_next_url' not in self.session: 

379 self.save_redirect_page() 

380 self.redirect(self.session.pop('_next_url', '/')) 

381 

382 @tornado.gen.coroutine 

383 def _cached_get(self, *args, **kwargs): 

384 cached = self.cachefile.get() 

385 if cached is not None: 

386 self.set_status(cached['status']) 

387 self._write_headers(cached['headers']) 

388 self.write(cached['body']) 

389 else: 

390 self.cachefile.wrap(self) 

391 yield self.original_get(*args, **kwargs) 

392 

393 def _write_headers(self, headers): 

394 '''Write headers from a list of pairs that may be duplicated''' 

395 headers_written = set() 

396 for name, value in headers: 

397 # If value is explicitly False or None, clear header. 

398 # This gives a way to clear pre-set headers like the Server header 

399 if value is False or value is None: 399 ↛ 400line 399 didn't jump to line 400, because the condition on line 399 was never true

400 self.clear_header(name) 

401 elif name in headers_written: 401 ↛ 402line 401 didn't jump to line 402, because the condition on line 401 was never true

402 self.add_header(name, value) 

403 else: 

404 self.set_header(name, value) 

405 headers_written.add(name) 

406 

407 def debug_exception(self, typ, value, tb): 

408 super(BaseHandler, self).log_exception(typ, value, tb) 

409 import ipdb as pdb # noqa 

410 pdb.post_mortem(tb) 

411 

412 def _write_custom_error(self, status_code, **kwargs): 

413 if status_code in self.error: 

414 try: 

415 result = self.error[status_code]['function']( 

416 status_code=status_code, kwargs=kwargs, handler=self) 

417 headers = self.error[status_code].get('conf', {}).get('headers', {}) 

418 self._write_headers(headers.items()) 

419 # result may be a generator / list from build_transform, 

420 # or a str/bytes/unicode from Template.generate. Handle both 

421 if isinstance(result, (six.string_types, six.binary_type)): 

422 self.write(result) 

423 else: 

424 for item in result: 

425 self.write(item) 

426 return 

427 except Exception: 

428 app_log.exception('url:%s.error.%d error handler raised an exception:', 

429 self.name, status_code) 

430 # If error was not written, use the default error 

431 self._write_error(status_code, **kwargs) 

432 

433 @property 

434 def session(self): 

435 ''' 

436 By default, session is not implemented. You need to specify a 

437 ``session:`` section in ``gramex.yaml`` to activate it. It is replaced by 

438 the ``get_session`` method as a property. 

439 ''' 

440 raise NotImplementedError('Specify a session: section in gramex.yaml') 

441 

442 def _set_new_session_id(self, expires_days): 

443 '''Sets a new random session ID as the sid: cookie. Returns a bytes object''' 

444 session_id = b2a_base64(os.urandom(24))[:-1] 

445 kwargs = dict(self._session_cookie) 

446 kwargs['expires_days'] = expires_days 

447 # Use Secure cookies on HTTPS to prevent leakage into HTTP 

448 if self.request.protocol == 'https': 448 ↛ 449line 448 didn't jump to line 449, because the condition on line 448 was never true

449 kwargs['secure'] = True 

450 # Websockets cannot set cookies. They raise a RuntimeError. Ignore those. 

451 try: 

452 self.set_secure_cookie('sid', session_id, **kwargs) 

453 except RuntimeError: 

454 pass 

455 return session_id 

456 

457 def get_session(self, expires_days=None, new=False): 

458 ''' 

459 Return the session object for the cookie "sid" value. 

460 If no "sid" cookie exists, set up a new one. 

461 If no session object exists for the sid, create it. 

462 By default, the session object contains a "id" holding the "sid" value. 

463 

464 The session is a dict. You must ensure that it is JSON serializable. 

465 

466 Sessions use these pre-defined timing keys (values are timestamps): 

467 

468 - ``_t`` is the expiry time of the session 

469 - ``_l`` is the last time the user accessed a page. Updated by 

470 :py:func:`BaseHandler.set_last_visited` 

471 - ``_i`` is the inactive expiry duration in seconds, i.e. if ``now > _l + 

472 _i``, the session has expired. 

473 

474 ``new=`` creates a new session to avoid session fixation. 

475 https://www.owasp.org/index.php/Session_fixation. 

476 :py:func:`gramex.handlers.authhandler.AuthHandler.set_user` uses it. 

477 When the user logs in: 

478 

479 - If no old session exists, it returns a new session object. 

480 - If an old session exists, it creates a new "sid" and new session 

481 object, copying all old contents, but updates the "id" and expiry (_t). 

482 ''' 

483 if expires_days is None: 

484 expires_days = self._session_expiry 

485 created_new_sid = False 

486 if getattr(self, '_session', None) is None: 

487 # Populate self._session based on the sid. If there's no sid cookie, 

488 # generate one and create an associated session object 

489 session_id = self.get_secure_cookie('sid', max_age_days=9999999) 

490 # If there's no session id cookie "sid", create a random 32-char cookie 

491 if session_id is None: 

492 session_id = self._set_new_session_id(expires_days) 

493 created_new_sid = True 

494 # Convert bytes session to unicode before using 

495 session_id = session_id.decode('ascii') 

496 # If there's no stored session associated with it, create it 

497 expires = time.time() + expires_days * 24 * 60 * 60 

498 self._session = self._session_store.load(session_id, {'_t': expires}) 

499 # Overwrite id to the session ID even if a handler has changed it 

500 self._session['id'] = session_id 

501 # At this point, the "sid" cookie and self._session exist and are synced 

502 s = self._session 

503 old_sid = s['id'] 

504 # If session has expiry keys _i and _l defined, check for expiry. Not otherwise 

505 if '_i' in s and '_l' in s and time.time() > s['_l'] + s['_i']: 

506 new = True 

507 s.clear() 

508 if new and not created_new_sid: 

509 new_sid = self._set_new_session_id(expires_days).decode('ascii') 

510 # Update expiry and new SID on session 

511 s.update(id=new_sid, _t=time.time() + expires_days * 24 * 60 * 60) 

512 # Delete old contents. No _t also means expired 

513 self._session_store.dump(old_sid, {}) 

514 

515 return s 

516 

517 def save_session(self): 

518 '''Persist the session object as a JSON''' 

519 if getattr(self, '_session', None) is not None: 519 ↛ exitline 519 didn't return from function 'save_session', because the condition on line 519 was never false

520 self._session_store.dump(self._session['id'], self._session) 

521 

522 def otp(self, expire=60): 

523 ''' 

524 Return a one-time password valid for ``expire`` seconds. When the 

525 X-Gramex-OTP header 

526 ''' 

527 user = self.current_user 

528 if not user: 528 ↛ 529line 528 didn't jump to line 529, because the condition on line 528 was never true

529 raise HTTPError(UNAUTHORIZED) 

530 nbits = 16 

531 otp = hexlify(os.urandom(nbits)).decode('ascii') 

532 self._session_store.dump('otp:' + otp, {'user': user, '_t': time.time() + expire}) 

533 return otp 

534 

535 def override_user(self): 

536 ''' 

537 Use ``X-Gramex-User`` HTTP header to override current user for the session. 

538 Use ``X-Gramex-OTP`` HTTP header to set user based on OTP. 

539 ``?gramex-otp=`` is a synonym for X-Gramex-OTP. 

540 ''' 

541 headers = self.request.headers 

542 cipher = headers.get('X-Gramex-User') 

543 if cipher: 

544 import json 

545 try: 

546 user = json.loads(decode_signed_value( 

547 conf.app.settings['cookie_secret'], 'user', cipher, 

548 max_age_days=self._session_expiry)) 

549 except Exception: 

550 reason = '%s: invalid X-Gramex-User: %s' % (self.name, cipher) 

551 raise HTTPError(BAD_REQUEST, reason=reason) 

552 else: 

553 app_log.debug('%s: Overriding user to %r', self.name, user) 

554 self.session['user'] = user 

555 return 

556 otp = headers.get('X-Gramex-OTP') or self.get_argument('gramex-otp', None) 

557 if otp: 

558 otp_data = self._session_store.load('otp:' + otp, None) 

559 if not isinstance(otp_data, dict) or '_t' not in otp_data or 'user' not in otp_data: 

560 reason = '%s: invalid X-Gramex-OTP: %s' % (self.name, otp) 

561 raise HTTPError(BAD_REQUEST, reason=reason) 

562 elif otp_data['_t'] < time.time(): 

563 reason = '%s: expired X-Gramex-OTP: %s' % (self.name, otp) 

564 raise HTTPError(BAD_REQUEST, reason=reason) 

565 self._session_store.dump('otp:' + otp, None) 

566 self.session['user'] = otp_data['user'] 

567 

568 def set_last_visited(self): 

569 ''' 

570 This method is called by :py:func:`BaseHandler.prepare` when any user 

571 accesses a page. It updates the last visited time in the ``_l`` session 

572 key. It does this only if the ``_i`` key exists. 

573 ''' 

574 # For efficiency reasons, don't call get_session every time. Check 

575 # session only if there's a valid sid cookie (with possibly long expiry) 

576 if self.get_secure_cookie('sid', max_age_days=9999999): 

577 session = self.get_session() 

578 if '_i' in session: 

579 session['_l'] = time.time() 

580 

581 

582class BaseHandler(RequestHandler, BaseMixin): 

583 ''' 

584 BaseHandler provides auth, caching and other services common to all request 

585 handlers. All RequestHandlers must inherit from BaseHandler. 

586 ''' 

587 def initialize(self, **kwargs): 

588 # self.request.arguments does not handle unicode keys well. 

589 # In Py2, it returns a str (not unicode). In Py3, it returns latin-1 unicode. 

590 # Convert this to proper unicode using UTF-8 and store in self.args 

591 self.args = {} 

592 for k in self.request.arguments: 

593 key = (k if isinstance(k, six.binary_type) else k.encode('latin-1')).decode('utf-8') 

594 # Invalid unicode (e.g. ?x=%f4) throws HTTPError. This disrupts even 

595 # error handlers. So if there's invalid unicode, log & continue. 

596 try: 

597 self.args[key] = self.get_arguments(k) 

598 except HTTPError: 

599 app_log.exception('Invalid URL argument %s' % k) 

600 

601 self._session, self._session_json = None, 'null' 

602 if self.cache: 

603 self.cachefile = self.cache() 

604 self.original_get = self.get 

605 self.get = self._cached_get 

606 if self._set_xsrf: 

607 self.xsrf_token 

608 

609 # Set the method to the ?x-http-method-overrride argument or the 

610 # X-HTTP-Method-Override header if they exist 

611 if 'x-http-method-override' in self.args: 611 ↛ 612line 611 didn't jump to line 612, because the condition on line 611 was never true

612 self.request.method = self.args.pop('x-http-method-override')[0].upper() 

613 elif 'X-HTTP-Method-Override' in self.request.headers: 613 ↛ 614line 613 didn't jump to line 614, because the condition on line 613 was never true

614 self.request.method = self.request.headers['X-HTTP-Method-Override'].upper() 

615 

616 def get_arg(self, name, default=_arg_default, first=False): 

617 ''' 

618 Returns the value of the argument with the given name. Similar to 

619 ``.get_argument`` but uses ``self.args`` instead. 

620 

621 If default is not provided, the argument is considered to be 

622 required, and we raise a `MissingArgumentError` if it is missing. 

623 

624 If the argument is repeated, we return the last value. If ``first=True`` 

625 is passed, we return the first value. 

626 

627 ``self.args`` is always UTF-8 decoded unicode. Whitespaces are stripped. 

628 ''' 

629 if name not in self.args: 

630 if default is _arg_default: 

631 raise MissingArgumentError(name) 

632 return default 

633 return self.args[name][0 if first else -1] 

634 

635 def prepare(self): 

636 for method in self._on_init_methods: 

637 method(self) 

638 

639 def set_default_headers(self): 

640 # Only set BaseHandler headers. 

641 # Don't set headers for the specific class. Those are overrides handled 

642 # by the respective classes, not the default headers. 

643 headers = [('Server', server_header)] 

644 headers += list(objectpath(conf, 'handlers.BaseHandler.headers', {}).items()) 

645 self._write_headers(headers) 

646 

647 def on_finish(self): 

648 # Loop through class-level callbacks 

649 for callback in self._on_finish_methods: 

650 callback(self) 

651 

652 def get_current_user(self): 

653 '''Return the ``user`` key from the session as an AttrDict if it exists.''' 

654 result = self.session.get('user') 

655 return AttrDict(result) if isinstance(result, dict) else result 

656 

657 def log_exception(self, typ, value, tb): 

658 '''Store the exception value for logging''' 

659 super(BaseHandler, self).log_exception(typ, value, tb) 

660 # _exception is stored for use by log_request. Sample error string: 

661 # ZeroDivisionError: integer division or modulo by zero 

662 self._exception = traceback.format_exception_only(typ, value)[0].strip() 

663 

664 def authorize(self): 

665 if not self.current_user: 

666 # Redirect non-AJAX requests GET/HEAD to login URL (if it's a string) 

667 ajax = self.request.headers.get('X-Requested-With', '').lower() == 'xmlhttprequest' 

668 if self.request.method in ('GET', 'HEAD') and not ajax: 

669 url = self.get_login_url() if self._login_url is None else self._login_url 

670 # If login_url is a string, redirect 

671 if isinstance(url, six.string_types): 

672 if '?' not in url: 

673 if urlsplit(url).scheme: 

674 # if login url is absolute, make next absolute too 

675 next_url = self.request.full_url() 

676 else: 

677 next_url = self.request.uri 

678 next_key = 'next' 

679 if isinstance(self.conf.kwargs.auth, dict): 

680 next_key = self.conf.kwargs.auth.get('query', 'next') 

681 url += '?' + urlencode({next_key: next_url}) 

682 self.redirect(url) 

683 return 

684 # Else, send a 401 header 

685 raise HTTPError(UNAUTHORIZED) 

686 

687 # If the user doesn't have permissions, show 403 (with template) 

688 for permit_generator in self.permissions: 

689 for result in permit_generator(self): 

690 if not result: 

691 template = self.conf.kwargs.auth.get('template') 

692 if template: 

693 self.set_status(FORBIDDEN) 

694 self.render(template) 

695 raise HTTPError(FORBIDDEN) 

696 

697 def argparse(self, *args, **kwargs): 

698 ''' 

699 Parse URL query parameters and return an AttrDict. For example:: 

700 

701 args = handler.argparse('x', 'y') 

702 args.x # is the last value of ?x=value 

703 args.y # is the last value of ?y=value 

704 

705 A missing ``?x=`` or ``?y=`` raises a HTTP 400 error mentioning the 

706 missing key. 

707 

708 For optional arguments, use:: 

709 

710 args = handler.argparse(z={'default': ''}) 

711 args.z # returns '' if ?z= is missing 

712 

713 You can convert the value to a type:: 

714 

715 args = handler.argparse(limit={'type': int, 'default': 100}) 

716 args.limit # returns ?limit= as an integer 

717 

718 You can restrict the choice of values. If the query parameter is not in 

719 choices, we raise a HTTP 400 error mentioning the invalid key & value:: 

720 

721 args = handler.argparse(gender={'choices': ['M', 'F']}) 

722 args.gender # returns ?gender= which will be 'M' or 'F' 

723 

724 You can retrieve multiple values as a list:: 

725 

726 args = handler.argparse(cols={'nargs': '*', 'default': []}) 

727 args.cols # returns an array with all ?col= values 

728 

729 ``type:`` conversion and ``choices:`` apply to each value in the list. 

730 

731 To return all arguments as a list, pass ``list`` as the first parameter:: 

732 

733 args = handler.argparse(list, 'x', 'y') 

734 args.x # ?x=1 sets args.x to ['1'], not '1' 

735 args.y # Similarly for ?y=1 

736 

737 To handle unicode arguments and return all arguments as ``str`` or 

738 ``unicode`` or ``bytes``, pass the type as the first parameter:: 

739 

740 args = handler.argparse(str, 'x', 'y') 

741 args = handler.argparse(bytes, 'x', 'y') 

742 args = handler.argparse(unicode, 'x', 'y') 

743 

744 By default, all arguments are added as str in PY3 and unicode in PY2. 

745 

746 There are the full list of parameters you can pass to each keyword 

747 argument: 

748 

749 - name: Name of the URL query parameter to read. Defaults to the key 

750 - required: Whether or not the query parameter may be omitted 

751 - default: The value produced if the argument is missing. Implies required=False 

752 - nargs: The number of parameters that should be returned. '*' or '+' 

753 return all values as a list. 

754 - type: Python type to which the parameter should be converted (e.g. `int`) 

755 - choices: A container of the allowable values for the argument (after type conversion) 

756 

757 You can combine all these options. For example:: 

758 

759 args = handler.argparse( 

760 'name', # Raise error if ?name= is missing 

761 department={'name': 'dept'}, # ?dept= is mapped to args.department 

762 org={'default': 'Gramener'}, # If ?org= is missing, defaults to Gramener 

763 age={'type': int}, # Convert ?age= to an integer 

764 married={'type': bool}, # Convert ?married to a boolean 

765 alias={'nargs': '*'}, # Convert all ?alias= to a list 

766 gender={'choices': ['M', 'F']}, # Raise error if gender is not M or F 

767 ) 

768 ''' 

769 result = AttrDict() 

770 

771 args_type = six.text_type 

772 if len(args) > 0 and args[0] in (six.text_type, six.binary_type, list, None): 

773 args_type, args = args[0], args[1:] 

774 

775 for key in args: 

776 result[key] = self.get_argument(key, None) 

777 if result[key] is None: 

778 raise HTTPError(BAD_REQUEST, reason='%s: missing ?%s=' % (key, key)) 

779 for key, config in kwargs.items(): 

780 name = config.get('name', key) 

781 val = self.args.get(name, []) 

782 

783 # default: set if query is missing 

784 # required: check if query is defined at all 

785 if len(val) == 0: 

786 if 'default' in config: 

787 result[key] = config['default'] 

788 continue 

789 if config.get('required', False): 

790 raise HTTPError(BAD_REQUEST, reason='%s: missing ?%s=' % (key, name)) 

791 

792 # nargs: select the subset of items 

793 nargs = config.get('nargs', None) 

794 if isinstance(nargs, int): 

795 val = val[:nargs] 

796 if len(val) < nargs: 

797 val += [''] * (nargs - len(val)) 

798 elif nargs not in ('*', '+', None): 798 ↛ 799line 798 didn't jump to line 799, because the condition on line 798 was never true

799 raise ValueError('%s: invalid nargs %s' % (key, nargs)) 

800 

801 # convert to specified type 

802 newtype = config.get('type', None) 

803 if newtype is not None: 

804 newval = [] 

805 for v in val: 

806 try: 

807 newval.append(newtype(v)) 

808 except ValueError: 

809 reason = "%s: type error ?%s=%s to %r" % (key, name, v, newtype) 

810 raise HTTPError(BAD_REQUEST, reason=reason) 

811 val = newval 

812 

813 # choices: check valid items 

814 choices = config.get('choices', None) 

815 if isinstance(choices, (list, dict, set)): 

816 choices = set(choices) 

817 for v in val: 

818 if v not in choices: 

819 reason = '%s: invalid choice ?%s=%s' % (key, name, v) 

820 raise HTTPError(BAD_REQUEST, reason=reason) 

821 

822 # Set the final value 

823 if nargs is None: 

824 if len(val) > 0: 

825 result[key] = val[-1] 

826 else: 

827 result[key] = val 

828 

829 # Parse remaining keys 

830 if args_type is list: 

831 for key, val in self.args.items(): 

832 if key not in args and key not in kwargs: 832 ↛ 831line 832 didn't jump to line 831, because the condition on line 832 was never false

833 result[key] = val 

834 elif args_type in (six.string_types, six.binary_type): 834 ↛ 835line 834 didn't jump to line 835, because the condition on line 834 was never true

835 for key, val in self.args.items(): 

836 if key not in args and key not in kwargs: 

837 result[key] = args_type(val[0]) 

838 

839 return result 

840 

841 def create_template_loader(self, template_path): 

842 settings = self.application.settings 

843 return CacheLoader(template_path, autoescape=settings['autoescape'], 

844 whitespace=settings.get('template_whitespace', None)) 

845 

846 

847class BaseWebSocketHandler(WebSocketHandler, BaseMixin): 

848 def initialize(self, **kwargs): 

849 self._session, self._session_json = None, 'null' 

850 if self.cache: 

851 self.cachefile = self.cache() 

852 self.original_get = self.get 

853 self.get = self._cached_get 

854 if self._set_xsrf: 

855 self.xsrf_token 

856 

857 @tornado.gen.coroutine 

858 def get(self, *args, **kwargs): 

859 for method in self._on_init_methods: 

860 method(self) 

861 super(BaseWebSocketHandler, self).get(*args, **kwargs) 

862 

863 def on_close(self): 

864 # Loop through class-level callbacks 

865 for callback in self._on_finish_methods: 

866 callback(self) 

867 

868 def get_current_user(self): 

869 '''Return the ``user`` key from the session as an AttrDict if it exists.''' 

870 result = self.session.get('user') 

871 return AttrDict(result) if isinstance(result, dict) else result 

872 

873 def authorize(self): 

874 '''If a valid user isn't logged in, send a message and close connection''' 

875 if not self.current_user: 

876 raise HTTPError(UNAUTHORIZED) 

877 for permit_generator in self.permissions: 

878 for result in permit_generator(self): 

879 if not result: 

880 raise HTTPError(FORBIDDEN) 

881 

882 

883class SetupFailedHandler(RequestHandler, BaseMixin): 

884 ''' 

885 Reports that the setup() operation has failed. 

886 

887 Used by gramex.services.init() when setting up URLs. If it's not able to set 

888 up a handler, it replaces it with this handler. 

889 ''' 

890 def get(self): 

891 six.reraise(*self.exc_info) 

892 

893 

894def check_membership(memberships): 

895 ''' 

896 Return a generator that checks all memberships for a user, and yields True if 

897 any membership is allowed, else False 

898 ''' 

899 # Pre-process memberships into an array of {objectpath: set(values)} 

900 conds = [{ 

901 keypath: set(values) if isinstance(values, list) else {values} 

902 for keypath, values in cond.items() 

903 } for cond in memberships] 

904 

905 def allowed(self): 

906 user = self.current_user 

907 for cond in conds: 

908 if _check_condition(cond, user): 

909 yield True 

910 break 

911 else: 

912 yield False 

913 

914 return allowed 

915 

916 

917def _check_condition(condition, user): 

918 ''' 

919 A condition is a dictionary of {keypath: values}. Extract the keypath from 

920 the user. Check if the value is in the values list. If not, this condition 

921 fails. 

922 ''' 

923 for keypath, values in condition.items(): 

924 node = objectpath(user, keypath) 

925 # If nothing exists at keypath, the check fails 

926 if node is None: 

927 return False 

928 # If the value is a list, it must overlap with values 

929 elif isinstance(node, list): 

930 if not set(node) & values: 

931 return False 

932 # If the value is not a list, it must be present in values 

933 elif node not in values: 

934 return False 

935 return True 

936 

937 

938handle_cache = {} 

939 

940 

941def _handle(path): 

942 '''Returns a cached append-binary handle to path''' 

943 if path not in handle_cache: 

944 # In Python 2, csv writerow writes byte string. In PY3, it's to a unicode string. 

945 # Open file handles accordingly 

946 handle_cache[path] = open(path, 'ab') if six.PY2 else io.open(path, 'a', encoding='utf-8') 

947 return handle_cache[path] 

948 

949 

950def build_log_info(keys, *vars): 

951 ''' 

952 Creates a ``handler.method(vars)`` that returns a dictionary of computed 

953 values. ``keys`` defines what keys are returned in the dictionary. The values 

954 are computed using the formulas in the code. 

955 ''' 

956 # Define direct keys. These can be used as-is 

957 direct_vars = { 

958 'name': 'handler.name', 

959 'class': 'handler.__class__.__name__', 

960 'time': 'round(time.time() * 1000, 0)', 

961 'datetime': 'datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ")', 

962 'method': 'handler.request.method', 

963 'uri': 'handler.request.uri', 

964 'ip': 'handler.request.remote_ip', 

965 'status': 'handler.get_status()', 

966 'duration': 'round(handler.request.request_time() * 1000, 0)', 

967 'port': 'conf.app.listen.port', 

968 # TODO: get_content_size() is not available in RequestHandler 

969 # 'size': 'handler.get_content_size()', 

970 'user': '(handler.current_user or {}).get("id", "")', 

971 'session': 'handler.session.get("id", "")', 

972 'error': 'getattr(handler, "_exception", "")', 

973 } 

974 # Define object keys for us as key.value. E.g. cookies.sid, user.email, etc 

975 object_vars = { 

976 'args': 'handler.get_argument("{val}", "")', 

977 'request': 'getattr(handler.request, "{val}", "")', 

978 'headers': 'handler.request.headers.get("{val}", "")', 

979 'cookies': 'handler.request.cookies["{val}"].value ' + 

980 'if "{val}" in handler.request.cookies else ""', 

981 'user': '(handler.current_user or {{}}).get("{val}", "")', 

982 'env': 'os.environ.get("{val}", "")', 

983 } 

984 vals = [] 

985 for key in keys: 

986 if key in vars: 

987 vals.append('"{}": {},'.format(key, key)) 

988 continue 

989 if key in direct_vars: 

990 vals.append('"{}": {},'.format(key, direct_vars[key])) 

991 continue 

992 if '.' in key: 992 ↛ 997line 992 didn't jump to line 997, because the condition on line 992 was never false

993 prefix, value = key.split('.', 2) 

994 if prefix in object_vars: 994 ↛ 997line 994 didn't jump to line 997, because the condition on line 994 was never false

995 vals.append('"{}": {},'.format(key, object_vars[prefix].format(val=value))) 

996 continue 

997 app_log.error('Skipping unknown key %s', key) 

998 code = compile('def fn(handler, %s):\n\treturn {%s}' % (', '.join(vars), ' '.join(vals)), 

999 filename='log', mode='exec') 

1000 context = {'os': os, 'time': time, 'datetime': datetime, 'conf': conf, 'AttrDict': AttrDict} 

1001 # The code is constructed entirely by this function. Using exec is safe 

1002 exec(code, context) # nosec 

1003 return context['fn']