Coverage for gramex\handlers\formhandler.py : 91%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import six
2import json
3import tornado.gen
4import gramex.cache
5import gramex.data
6import pandas as pd
7from orderedattrdict import AttrDict
8from tornado.web import HTTPError
9from gramex import conf as gramex_conf
10from gramex.http import BAD_REQUEST, INTERNAL_SERVER_ERROR
11from gramex.transforms import build_transform
12from gramex.config import merge, app_log, objectpath, CustomJSONEncoder
13from .basehandler import BaseHandler
16def namespaced_args(args, namespace):
17 '''
18 Filter from handler.args the keys relevant for namespace, i.e. those
19 prefixed with the namespace, or those without a prefix.
20 '''
21 if not namespace.endswith(':'): 21 ↛ 23line 21 didn't jump to line 23, because the condition on line 21 was never false
22 namespace += ':'
23 result = {}
24 for key, val in args.items():
25 if key.startswith(namespace): 25 ↛ 26line 25 didn't jump to line 26, because the condition on line 25 was never true
26 result[key[len(namespace):]] = val
27 elif ':' not in key: 27 ↛ 24line 27 didn't jump to line 24, because the condition on line 27 was never false
28 result[key] = val
29 return result
32class FormHandler(BaseHandler):
33 # Else there should be at least 1 key that has a url: sub-key. The data spec is at that level
34 # Data spec is (url, engine, table, ext, ...) which goes directly to filter
35 # It also has
36 # default: which is interpreted as argument defaults
37 # keys: defines the primary key columns
39 # FormHandler function kwargs and the parameters they accept:
40 function_vars = {
41 'modify': {'data': None, 'key': None, 'handler': None},
42 'prepare': {'args': None, 'key': None, 'handler': None},
43 'queryfunction': {'args': None, 'key': None, 'handler': None},
44 }
45 data_filter_method = staticmethod(gramex.data.filter)
47 @classmethod
48 def setup(cls, **kwargs):
49 super(FormHandler, cls).setup(**kwargs)
50 conf_kwargs = merge(AttrDict(kwargs),
51 objectpath(gramex_conf, 'handlers.FormHandler', {}),
52 'setdefault')
53 cls.headers = conf_kwargs.pop('headers', {})
54 # Top level formats: key is special. Don't treat it as data
55 cls.formats = conf_kwargs.pop('formats', {})
56 default_config = conf_kwargs.pop('default', None)
57 # Remove other known special keys from dataset configuration
58 cls.clear_special_keys(conf_kwargs)
59 # If top level has url: then data spec is at top level. Else it's a set of sub-keys
60 if 'url' in conf_kwargs:
61 cls.datasets = AttrDict(data=conf_kwargs)
62 cls.single = True
63 else:
64 if 'modify' in conf_kwargs:
65 cls.modify_all = staticmethod(build_transform(
66 conf={'function': conf_kwargs.pop('modify', None)},
67 vars=cls.function_vars['modify'],
68 filename='%s.%s' % (cls.name, 'modify'), iter=False))
69 cls.datasets = conf_kwargs
70 cls.single = False
71 # Apply defaults to each key
72 if isinstance(default_config, dict): 72 ↛ 77line 72 didn't jump to line 77, because the condition on line 72 was never false
73 for key in cls.datasets:
74 config = cls.datasets[key].get('default', {})
75 cls.datasets[key]['default'] = merge(config, default_config, mode='setdefault')
76 # Ensure that each dataset is a dict with a url: key at least
77 for key, dataset in list(cls.datasets.items()):
78 if not isinstance(dataset, dict): 78 ↛ 79line 78 didn't jump to line 79, because the condition on line 78 was never true
79 app_log.error('%s: %s: must be a dict, not %r' % (cls.name, key, dataset))
80 del cls.datasets[key]
81 elif 'url' not in dataset: 81 ↛ 82line 81 didn't jump to line 82, because the condition on line 81 was never true
82 app_log.error('%s: %s: does not have a url: key' % (cls.name, key))
83 del cls.datasets[key]
84 # Ensure that id: is a list -- if it exists
85 if 'id' in dataset and not isinstance(dataset['id'], list):
86 dataset['id'] = [dataset['id']]
87 # Convert function: into a data = transform(data) function
88 conf = {
89 'function': dataset.pop('function', None),
90 'args': dataset.pop('args', None),
91 'kwargs': dataset.pop('kwargs', None)
92 }
93 if conf['function'] is not None:
94 fn_name = '%s.%s.transform' % (cls.name, key)
95 dataset['transform'] = build_transform(
96 conf, vars={'data': None, 'handler': None}, filename=fn_name, iter=False)
97 # Convert modify: and prepare: into a data = modify(data) function
98 for fn, fn_vars in cls.function_vars.items():
99 if fn in dataset:
100 dataset[fn] = build_transform(
101 conf={'function': dataset[fn]},
102 vars=fn_vars,
103 filename='%s.%s.%s' % (cls.name, key, fn), iter=False)
105 def _options(self, dataset, args, path_args, path_kwargs, key):
106 """For each dataset, prepare the arguments."""
107 if self.request.body:
108 content_type = self.request.headers.get('Content-Type', '')
109 if content_type == 'application/json':
110 args.update(json.loads(self.request.body))
111 filter_kwargs = AttrDict(dataset)
112 filter_kwargs.pop('modify', None)
113 prepare = filter_kwargs.pop('prepare', None)
114 queryfunction = filter_kwargs.pop('queryfunction', None)
115 filter_kwargs['transform_kwargs'] = {'handler': self}
116 # Use default arguments
117 defaults = {
118 k: v if isinstance(v, list) else [v]
119 for k, v in filter_kwargs.pop('default', {}).items()
120 }
121 # /(.*)/(.*) become 2 path arguments _0 and _1
122 defaults.update({'_%d' % k: [v] for k, v in enumerate(path_args)})
123 # /(?P<x>\d+)/(?P<y>\d+) become 2 keyword arguments x and y
124 defaults.update({k: [v] for k, v in path_kwargs.items()})
125 args = merge(namespaced_args(args, key), defaults, mode='setdefault')
126 if callable(prepare):
127 result = prepare(args=args, key=key, handler=self)
128 if result is not None: 128 ↛ 129line 128 didn't jump to line 129, because the condition on line 128 was never true
129 args = result
130 if callable(queryfunction):
131 filter_kwargs['query'] = queryfunction(args=args, key=key, handler=self)
132 return AttrDict(
133 fmt=args.pop('_format', ['json'])[0],
134 download=args.pop('_download', [''])[0],
135 args=args,
136 meta_header=args.pop('_meta', [''])[0],
137 filter_kwargs=filter_kwargs,
138 )
140 @tornado.gen.coroutine
141 def get(self, *path_args, **path_kwargs):
142 meta, futures = AttrDict(), AttrDict()
143 for key, dataset in self.datasets.items():
144 meta[key] = AttrDict()
145 opt = self._options(dataset, self.args, path_args, path_kwargs, key)
146 opt.filter_kwargs.pop('id', None)
147 # Run query in a separate threadthread
148 futures[key] = gramex.service.threadpool.submit(
149 self.data_filter_method, args=opt.args, meta=meta[key], **opt.filter_kwargs)
150 result = AttrDict()
151 for key, val in futures.items():
152 try:
153 result[key] = yield val
154 except ValueError as e:
155 app_log.exception('%s: filter failed' % self.name)
156 raise HTTPError(BAD_REQUEST, reason=e.args[0])
157 except Exception as e:
158 app_log.exception('%s: filter failed' % self.name)
159 raise HTTPError(INTERNAL_SERVER_ERROR, reason=repr(e))
160 modify = self.datasets[key].get('modify', None)
161 if callable(modify):
162 result[key] = modify(data=result[key], key=key, handler=self)
164 # modify the result for multiple datasets
165 if hasattr(self, 'modify_all'):
166 result = self.modify_all(data=result, key=None, handler=self)
168 format_options = self.set_format(opt.fmt, meta)
169 format_options['args'] = opt.args
170 params = {k: v[0] for k, v in opt.args.items() if len(v) > 0}
171 for key, val in format_options.items():
172 if isinstance(val, six.text_type):
173 format_options[key] = val.format(**params)
174 # In PY2, the values are binary. TODO: ensure that format values are in Unicode
175 elif isinstance(val, six.binary_type): 175 ↛ 176line 175 didn't jump to line 176, because the condition on line 175 was never true
176 format_options[key] = val.decode('utf-8').format(**params)
177 if opt.download:
178 self.set_header('Content-Disposition', 'attachment;filename=%s' % opt.download)
179 if opt.meta_header:
180 self.set_meta_headers(meta)
181 result = result['data'] if self.single else result
182 # If modify has changed the content type from a dataframe, write it as-is
183 if isinstance(result, (pd.DataFrame, dict)):
184 self.write(gramex.data.download(result, **format_options))
185 else:
186 self.write(result)
188 @tornado.gen.coroutine
189 def update(self, method, *path_args, **path_kwargs):
190 if self.redirects:
191 self.save_redirect_page()
192 meta, result = AttrDict(), AttrDict()
193 # For each dataset
194 for key, dataset in self.datasets.items():
195 meta[key] = AttrDict()
196 opt = self._options(dataset, self.args, path_args, path_kwargs, key)
197 if 'id' not in opt.filter_kwargs:
198 raise HTTPError(BAD_REQUEST, reason='%s: missing id: <col> for %s' % (
199 self.name, self.request.method))
200 missing_args = [col for col in opt.filter_kwargs['id'] if col not in opt.args]
201 if method != gramex.data.insert and len(missing_args) > 0:
202 raise HTTPError(BAD_REQUEST, reason='%s: missing column(s) in URL query: %s' % (
203 self.name, ', '.join(missing_args)))
204 # Execute the query. This returns the count of records updated
205 result[key] = method(meta=meta[key], args=opt.args, **opt.filter_kwargs)
206 for key, val in result.items():
207 modify = self.datasets[key].get('modify', None)
208 if callable(modify):
209 meta[key]['modify'] = modify(data=result[key], key=key, handler=self)
210 self.set_header('Count-%s' % key, val)
211 # modify the result for multiple datasets
212 if hasattr(self, 'modify_all'): 212 ↛ 213line 212 didn't jump to line 213, because the condition on line 212 was never true
213 meta['modify'] = self.modify_all(data=result, key=None, handler=self)
214 if opt.meta_header: 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true
215 self.set_meta_headers(meta)
216 if self.redirects:
217 self.redirect_next()
218 else:
219 self.set_format(opt.fmt, meta)
220 self.set_header('Cache-Control', 'no-cache, no-store')
221 self.write(json.dumps(meta, indent=2, cls=CustomJSONEncoder))
223 @tornado.gen.coroutine
224 def delete(self, *path_args, **path_kwargs):
225 yield self.update(gramex.data.delete, *path_args, **path_kwargs)
227 @tornado.gen.coroutine
228 def post(self, *path_args, **path_kwargs):
229 yield self.update(gramex.data.insert, *path_args, **path_kwargs)
231 @tornado.gen.coroutine
232 def put(self, *path_args, **path_kwargs):
233 yield self.update(gramex.data.update, *path_args, **path_kwargs)
235 def set_format(self, fmt, meta):
236 # Identify format to render in. The default format, json, is defined in
237 # the base gramex.yaml under handlers.FormHandler.formats
238 if fmt in self.formats: 238 ↛ 241line 238 didn't jump to line 241, because the condition on line 238 was never false
239 fmt = dict(self.formats[fmt])
240 else:
241 app_log.error('%s: _format=%s unknown. Using _format=json' % (self.name, fmt))
242 fmt = dict(self.formats['json'])
244 # Set up default headers, and over-ride with headers for the format
245 for key, val in self.headers.items():
246 self.set_header(key, val)
247 for key, val in fmt.pop('headers', {}).items():
248 self.set_header(key, val)
250 if fmt['format'] in {'template', 'pptx', 'vega', 'vega-lite', 'vegam'}:
251 fmt['handler'] = self
252 if fmt['format'] in {'template'}: 252 ↛ 253line 252 didn't jump to line 253, because the condition on line 252 was never true
253 fmt['meta'] = meta['data'] if self.single else meta
255 return fmt
257 def set_meta_headers(self, meta):
258 '''Add FH-<dataset>-<key>: JSON(value) for each key: value in meta'''
259 prefix = 'FH-{}-{}'
260 for dataset, metadata in meta.items():
261 for key, value in metadata.items():
262 string_value = json.dumps(value, separators=(',', ':'),
263 ensure_ascii=True, cls=CustomJSONEncoder)
264 self.set_header(prefix.format(dataset, key), string_value)