Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import six 

2import json 

3import tornado.gen 

4import gramex.cache 

5import gramex.data 

6import pandas as pd 

7from orderedattrdict import AttrDict 

8from tornado.web import HTTPError 

9from gramex import conf as gramex_conf 

10from gramex.http import BAD_REQUEST, INTERNAL_SERVER_ERROR 

11from gramex.transforms import build_transform 

12from gramex.config import merge, app_log, objectpath, CustomJSONEncoder 

13from .basehandler import BaseHandler 

14 

15 

16def namespaced_args(args, namespace): 

17 ''' 

18 Filter from handler.args the keys relevant for namespace, i.e. those 

19 prefixed with the namespace, or those without a prefix. 

20 ''' 

21 if not namespace.endswith(':'): 21 ↛ 23line 21 didn't jump to line 23, because the condition on line 21 was never false

22 namespace += ':' 

23 result = {} 

24 for key, val in args.items(): 

25 if key.startswith(namespace): 25 ↛ 26line 25 didn't jump to line 26, because the condition on line 25 was never true

26 result[key[len(namespace):]] = val 

27 elif ':' not in key: 27 ↛ 24line 27 didn't jump to line 24, because the condition on line 27 was never false

28 result[key] = val 

29 return result 

30 

31 

32class FormHandler(BaseHandler): 

33 # Else there should be at least 1 key that has a url: sub-key. The data spec is at that level 

34 # Data spec is (url, engine, table, ext, ...) which goes directly to filter 

35 # It also has 

36 # default: which is interpreted as argument defaults 

37 # keys: defines the primary key columns 

38 

39 # FormHandler function kwargs and the parameters they accept: 

40 function_vars = { 

41 'modify': {'data': None, 'key': None, 'handler': None}, 

42 'prepare': {'args': None, 'key': None, 'handler': None}, 

43 'queryfunction': {'args': None, 'key': None, 'handler': None}, 

44 } 

45 data_filter_method = staticmethod(gramex.data.filter) 

46 

47 @classmethod 

48 def setup(cls, **kwargs): 

49 super(FormHandler, cls).setup(**kwargs) 

50 conf_kwargs = merge(AttrDict(kwargs), 

51 objectpath(gramex_conf, 'handlers.FormHandler', {}), 

52 'setdefault') 

53 cls.headers = conf_kwargs.pop('headers', {}) 

54 # Top level formats: key is special. Don't treat it as data 

55 cls.formats = conf_kwargs.pop('formats', {}) 

56 default_config = conf_kwargs.pop('default', None) 

57 # Remove other known special keys from dataset configuration 

58 cls.clear_special_keys(conf_kwargs) 

59 # If top level has url: then data spec is at top level. Else it's a set of sub-keys 

60 if 'url' in conf_kwargs: 

61 cls.datasets = AttrDict(data=conf_kwargs) 

62 cls.single = True 

63 else: 

64 if 'modify' in conf_kwargs: 

65 cls.modify_all = staticmethod(build_transform( 

66 conf={'function': conf_kwargs.pop('modify', None)}, 

67 vars=cls.function_vars['modify'], 

68 filename='%s.%s' % (cls.name, 'modify'), iter=False)) 

69 cls.datasets = conf_kwargs 

70 cls.single = False 

71 # Apply defaults to each key 

72 if isinstance(default_config, dict): 72 ↛ 77line 72 didn't jump to line 77, because the condition on line 72 was never false

73 for key in cls.datasets: 

74 config = cls.datasets[key].get('default', {}) 

75 cls.datasets[key]['default'] = merge(config, default_config, mode='setdefault') 

76 # Ensure that each dataset is a dict with a url: key at least 

77 for key, dataset in list(cls.datasets.items()): 

78 if not isinstance(dataset, dict): 78 ↛ 79line 78 didn't jump to line 79, because the condition on line 78 was never true

79 app_log.error('%s: %s: must be a dict, not %r' % (cls.name, key, dataset)) 

80 del cls.datasets[key] 

81 elif 'url' not in dataset: 81 ↛ 82line 81 didn't jump to line 82, because the condition on line 81 was never true

82 app_log.error('%s: %s: does not have a url: key' % (cls.name, key)) 

83 del cls.datasets[key] 

84 # Ensure that id: is a list -- if it exists 

85 if 'id' in dataset and not isinstance(dataset['id'], list): 

86 dataset['id'] = [dataset['id']] 

87 # Convert function: into a data = transform(data) function 

88 conf = { 

89 'function': dataset.pop('function', None), 

90 'args': dataset.pop('args', None), 

91 'kwargs': dataset.pop('kwargs', None) 

92 } 

93 if conf['function'] is not None: 

94 fn_name = '%s.%s.transform' % (cls.name, key) 

95 dataset['transform'] = build_transform( 

96 conf, vars={'data': None, 'handler': None}, filename=fn_name, iter=False) 

97 # Convert modify: and prepare: into a data = modify(data) function 

98 for fn, fn_vars in cls.function_vars.items(): 

99 if fn in dataset: 

100 dataset[fn] = build_transform( 

101 conf={'function': dataset[fn]}, 

102 vars=fn_vars, 

103 filename='%s.%s.%s' % (cls.name, key, fn), iter=False) 

104 

105 def _options(self, dataset, args, path_args, path_kwargs, key): 

106 """For each dataset, prepare the arguments.""" 

107 if self.request.body: 

108 content_type = self.request.headers.get('Content-Type', '') 

109 if content_type == 'application/json': 

110 args.update(json.loads(self.request.body)) 

111 filter_kwargs = AttrDict(dataset) 

112 filter_kwargs.pop('modify', None) 

113 prepare = filter_kwargs.pop('prepare', None) 

114 queryfunction = filter_kwargs.pop('queryfunction', None) 

115 filter_kwargs['transform_kwargs'] = {'handler': self} 

116 # Use default arguments 

117 defaults = { 

118 k: v if isinstance(v, list) else [v] 

119 for k, v in filter_kwargs.pop('default', {}).items() 

120 } 

121 # /(.*)/(.*) become 2 path arguments _0 and _1 

122 defaults.update({'_%d' % k: [v] for k, v in enumerate(path_args)}) 

123 # /(?P<x>\d+)/(?P<y>\d+) become 2 keyword arguments x and y 

124 defaults.update({k: [v] for k, v in path_kwargs.items()}) 

125 args = merge(namespaced_args(args, key), defaults, mode='setdefault') 

126 if callable(prepare): 

127 result = prepare(args=args, key=key, handler=self) 

128 if result is not None: 128 ↛ 129line 128 didn't jump to line 129, because the condition on line 128 was never true

129 args = result 

130 if callable(queryfunction): 

131 filter_kwargs['query'] = queryfunction(args=args, key=key, handler=self) 

132 return AttrDict( 

133 fmt=args.pop('_format', ['json'])[0], 

134 download=args.pop('_download', [''])[0], 

135 args=args, 

136 meta_header=args.pop('_meta', [''])[0], 

137 filter_kwargs=filter_kwargs, 

138 ) 

139 

140 @tornado.gen.coroutine 

141 def get(self, *path_args, **path_kwargs): 

142 meta, futures = AttrDict(), AttrDict() 

143 for key, dataset in self.datasets.items(): 

144 meta[key] = AttrDict() 

145 opt = self._options(dataset, self.args, path_args, path_kwargs, key) 

146 opt.filter_kwargs.pop('id', None) 

147 # Run query in a separate threadthread 

148 futures[key] = gramex.service.threadpool.submit( 

149 self.data_filter_method, args=opt.args, meta=meta[key], **opt.filter_kwargs) 

150 result = AttrDict() 

151 for key, val in futures.items(): 

152 try: 

153 result[key] = yield val 

154 except ValueError as e: 

155 app_log.exception('%s: filter failed' % self.name) 

156 raise HTTPError(BAD_REQUEST, reason=e.args[0]) 

157 except Exception as e: 

158 app_log.exception('%s: filter failed' % self.name) 

159 raise HTTPError(INTERNAL_SERVER_ERROR, reason=repr(e)) 

160 modify = self.datasets[key].get('modify', None) 

161 if callable(modify): 

162 result[key] = modify(data=result[key], key=key, handler=self) 

163 

164 # modify the result for multiple datasets 

165 if hasattr(self, 'modify_all'): 

166 result = self.modify_all(data=result, key=None, handler=self) 

167 

168 format_options = self.set_format(opt.fmt, meta) 

169 format_options['args'] = opt.args 

170 params = {k: v[0] for k, v in opt.args.items() if len(v) > 0} 

171 for key, val in format_options.items(): 

172 if isinstance(val, six.text_type): 

173 format_options[key] = val.format(**params) 

174 # In PY2, the values are binary. TODO: ensure that format values are in Unicode 

175 elif isinstance(val, six.binary_type): 175 ↛ 176line 175 didn't jump to line 176, because the condition on line 175 was never true

176 format_options[key] = val.decode('utf-8').format(**params) 

177 if opt.download: 

178 self.set_header('Content-Disposition', 'attachment;filename=%s' % opt.download) 

179 if opt.meta_header: 

180 self.set_meta_headers(meta) 

181 result = result['data'] if self.single else result 

182 # If modify has changed the content type from a dataframe, write it as-is 

183 if isinstance(result, (pd.DataFrame, dict)): 

184 self.write(gramex.data.download(result, **format_options)) 

185 else: 

186 self.write(result) 

187 

188 @tornado.gen.coroutine 

189 def update(self, method, *path_args, **path_kwargs): 

190 if self.redirects: 

191 self.save_redirect_page() 

192 meta, result = AttrDict(), AttrDict() 

193 # For each dataset 

194 for key, dataset in self.datasets.items(): 

195 meta[key] = AttrDict() 

196 opt = self._options(dataset, self.args, path_args, path_kwargs, key) 

197 if 'id' not in opt.filter_kwargs: 

198 raise HTTPError(BAD_REQUEST, reason='%s: missing id: <col> for %s' % ( 

199 self.name, self.request.method)) 

200 missing_args = [col for col in opt.filter_kwargs['id'] if col not in opt.args] 

201 if method != gramex.data.insert and len(missing_args) > 0: 

202 raise HTTPError(BAD_REQUEST, reason='%s: missing column(s) in URL query: %s' % ( 

203 self.name, ', '.join(missing_args))) 

204 # Execute the query. This returns the count of records updated 

205 result[key] = method(meta=meta[key], args=opt.args, **opt.filter_kwargs) 

206 for key, val in result.items(): 

207 modify = self.datasets[key].get('modify', None) 

208 if callable(modify): 

209 meta[key]['modify'] = modify(data=result[key], key=key, handler=self) 

210 self.set_header('Count-%s' % key, val) 

211 # modify the result for multiple datasets 

212 if hasattr(self, 'modify_all'): 212 ↛ 213line 212 didn't jump to line 213, because the condition on line 212 was never true

213 meta['modify'] = self.modify_all(data=result, key=None, handler=self) 

214 if opt.meta_header: 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true

215 self.set_meta_headers(meta) 

216 if self.redirects: 

217 self.redirect_next() 

218 else: 

219 self.set_format(opt.fmt, meta) 

220 self.set_header('Cache-Control', 'no-cache, no-store') 

221 self.write(json.dumps(meta, indent=2, cls=CustomJSONEncoder)) 

222 

223 @tornado.gen.coroutine 

224 def delete(self, *path_args, **path_kwargs): 

225 yield self.update(gramex.data.delete, *path_args, **path_kwargs) 

226 

227 @tornado.gen.coroutine 

228 def post(self, *path_args, **path_kwargs): 

229 yield self.update(gramex.data.insert, *path_args, **path_kwargs) 

230 

231 @tornado.gen.coroutine 

232 def put(self, *path_args, **path_kwargs): 

233 yield self.update(gramex.data.update, *path_args, **path_kwargs) 

234 

235 def set_format(self, fmt, meta): 

236 # Identify format to render in. The default format, json, is defined in 

237 # the base gramex.yaml under handlers.FormHandler.formats 

238 if fmt in self.formats: 238 ↛ 241line 238 didn't jump to line 241, because the condition on line 238 was never false

239 fmt = dict(self.formats[fmt]) 

240 else: 

241 app_log.error('%s: _format=%s unknown. Using _format=json' % (self.name, fmt)) 

242 fmt = dict(self.formats['json']) 

243 

244 # Set up default headers, and over-ride with headers for the format 

245 for key, val in self.headers.items(): 

246 self.set_header(key, val) 

247 for key, val in fmt.pop('headers', {}).items(): 

248 self.set_header(key, val) 

249 

250 if fmt['format'] in {'template', 'pptx', 'vega', 'vega-lite', 'vegam'}: 

251 fmt['handler'] = self 

252 if fmt['format'] in {'template'}: 252 ↛ 253line 252 didn't jump to line 253, because the condition on line 252 was never true

253 fmt['meta'] = meta['data'] if self.single else meta 

254 

255 return fmt 

256 

257 def set_meta_headers(self, meta): 

258 '''Add FH-<dataset>-<key>: JSON(value) for each key: value in meta''' 

259 prefix = 'FH-{}-{}' 

260 for dataset, metadata in meta.items(): 

261 for key, value in metadata.items(): 

262 string_value = json.dumps(value, separators=(',', ':'), 

263 ensure_ascii=True, cls=CustomJSONEncoder) 

264 self.set_header(prefix.format(dataset, key), string_value)