Coverage for gramex\apps\nlg\nlgapp.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# vim:fenc=utf-8
3"""
4Module for gramex exposure. This shouldn't be imported anywhere, only for use
5with gramex.
6"""
7import json
8import os
9import os.path as op
10from six.moves.urllib import parse
12import pandas as pd
13from tornado.template import Template
15from gramex.config import variables
16from gramex.apps.nlg import grammar
17from gramex.apps.nlg import templatize
18from gramex.apps.nlg import nlgutils as utils
20DATAFILE_EXTS = {'.csv', '.xls', '.xlsx', '.tsv'}
22grx_data_dir = variables['GRAMEXDATA']
23nlg_path = op.join(grx_data_dir, 'nlg')
25if not op.isdir(nlg_path): 25 ↛ 26line 25 didn't jump to line 26, because the condition on line 25 was never true
26 os.mkdir(nlg_path)
29def render_live_template(handler):
30 """Given a narrative ID and df records, render the template."""
31 orgdf = get_original_df(handler)
32 nrid = handler.args['nrid'][0]
33 if not nrid.endswith('.json'):
34 nrid += '.json'
35 data = json.loads(handler.args['data'][0])
36 df = pd.DataFrame.from_records(data)
37 nrpath = op.join(nlg_path, handler.current_user.email, nrid)
38 with open(nrpath, 'r') as fout: # noqa: No encoding for json
39 templates = json.load(fout)
40 narratives = []
41 style = json.loads(handler.args['style'][0])
42 for t in templates['config']:
43 tmpl = utils.add_html_styling(t['template'], style)
44 s = Template(tmpl).generate(df=df, fh_args=t.get('fh_args', {}),
45 G=grammar, U=utils, orgdf=orgdf)
46 rendered = s.decode('utf8')
47 narratives.append(rendered)
48 return '\n'.join(narratives)
51def get_original_df(handler):
52 """Get the original dataframe which was uploaded to the webapp."""
53 data_dir = op.join(nlg_path, handler.current_user.email)
54 with open(op.join(data_dir, 'meta.cfg'), 'r') as fout: # noqa: No encoding for json
55 meta = json.load(fout)
56 dataset_path = op.join(data_dir, meta['dsid'])
57 return pd.read_csv(dataset_path, encoding='utf-8')
60def render_template(handler):
61 """Render a set of templates against a dataframe and formhandler actions on it."""
62 orgdf = get_original_df(handler)
63 payload = parse.parse_qsl(handler.request.body.decode("utf8"))
64 if not payload:
65 payload = json.loads(handler.request.body.decode("utf8"))
66 fh_args = payload['args']
67 templates = payload['template']
68 df = pd.DataFrame.from_records(payload['data'])
69 else:
70 payload = dict(payload)
71 fh_args = json.loads(payload.get("args", {}))
72 templates = json.loads(payload["template"])
73 df = pd.read_json(payload["data"], orient="records")
74 # fh_args = {k: [x.lstrip('-') for x in v] for k, v in fh_args.items()}
75 resp = []
76 for t in templates:
77 rendered = Template(t).generate(
78 orgdf=orgdf, df=df, fh_args=fh_args, G=grammar, U=utils).decode('utf8')
79 rendered = rendered.replace('-', '')
80 # grmerr = utils.check_grammar(rendered)
81 resp.append({'text': rendered}) # , 'grmerr': grmerr})
82 return json.dumps(resp)
85def process_template(handler):
86 """Process English text in the context of a df and formhandler arguments
87 to templatize it."""
88 payload = parse.parse_qsl(handler.request.body.decode("utf8"))
89 payload = dict(payload)
90 text = json.loads(payload["text"])
91 df = pd.read_json(payload["data"], orient="records")
92 args = json.loads(payload.get("args", {}))
93 if args is None:
94 args = {}
95 resp = []
96 for t in text:
97 # grammar_errors = yield utils.check_grammar(t)
98 replacements, t, infl = templatize(t, args.copy(), df)
99 resp.append({
100 "text": t, "tokenmap": replacements, 'inflections': infl,
101 "fh_args": args, "setFHArgs": False,
102 # "grmerr": json.loads(grammar_errors.decode('utf8'))['matches']
103 })
104 return json.dumps(resp)
107def read_current_config(handler):
108 """Read the current data and narrative IDs written to the session file."""
109 meta_path = op.join(nlg_path, handler.current_user.email, 'meta.cfg')
110 with open(meta_path, 'r') as fout: # noqa: No encoding for json
111 meta = json.load(fout)
112 return meta
115def get_dataset_files(handler):
116 """Get all filenames uploaded by the user.
118 Parameters
119 ----------
120 handler : tornado.RequestHandler
122 Returns
123 -------
124 list
125 List of filenames.
126 """
128 user_dir = op.join(nlg_path, handler.current_user.email)
129 if op.isdir(user_dir):
130 files = [f for f in os.listdir(user_dir) if op.splitext(f)[-1].lower() in DATAFILE_EXTS]
131 else:
132 files = []
133 return files
136def get_narrative_config_files(handler):
137 """Get list of narrative config files generated by the user.
139 Parameters
140 ----------
141 handler : tornado.RequestHandler
143 Returns
144 -------
145 list
146 List of narrative configurations.
147 """
148 # TODO: current_user.email needs to be replaced with a more suitable user ID
149 user_dir = op.join(nlg_path, handler.current_user.email)
150 if op.isdir(user_dir):
151 return [f for f in os.listdir(user_dir) if f.endswith('.json')]
152 return []
155def download_config(handler):
156 """Download the current narrative config as JSON."""
157 payload = {}
158 payload['config'] = json.loads(parse.unquote(handler.args['config'][0]))
159 payload['data'] = json.loads(parse.unquote(handler.args.get('data', [None])[0]))
160 payload['name'] = parse.unquote(handler.args['name'][0])
161 return json.dumps(payload, indent=4)
164def save_config(handler):
165 """Save the current narrative config.
166 (to $GRAMEXDATA/{{ handler.current_user.email }})"""
167 payload = {}
168 payload['config'] = json.loads(parse.unquote(handler.args['config'][0]))
169 payload['name'] = parse.unquote(handler.args['name'][0])
170 nname = payload['name']
171 if not nname.endswith('.json'):
172 nname += '.json'
173 payload['dataset'] = parse.unquote(handler.args['dataset'][0])
174 fpath = op.join(nlg_path, handler.current_user.email, nname)
175 with open(fpath, 'w') as fout: # noqa: No encoding for json
176 json.dump(payload, fout, indent=4)
179def get_gramopts(handler):
180 """Find all Grammar and token inflection options from the NLG library.
182 Primarily used for creating the select box in the template settings dialog."""
183 funcs = {}
184 for attrname in dir(grammar):
185 obj = getattr(grammar, attrname)
186 if getattr(obj, 'gramopt', False):
187 funcs[obj.fe_name] = {'source': obj.source, 'func_name': attrname}
188 return funcs
191def init_form(handler):
192 """Process input from the landing page and write the current session config."""
193 meta = {}
194 # prioritize files first
195 data_dir = op.join(nlg_path, handler.current_user.email)
196 if not op.isdir(data_dir):
197 os.makedirs(data_dir)
199 # handle dataset
200 data_file = handler.request.files.get('data-file', [{}])[0]
201 if data_file:
202 # TODO: Unix filenames may not be valid Windows filenames.
203 outpath = op.join(data_dir, data_file['filename'])
204 with open(outpath, 'wb') as fout:
205 fout.write(data_file['body'])
206 else:
207 dataset = handler.args['dataset'][0]
208 outpath = op.join(data_dir, dataset)
209 # shutil.copy(outpath, fh_fpath)
210 meta['dsid'] = op.basename(outpath)
212 # handle config
213 config_name = handler.get_argument('narrative', '')
214 if config_name:
215 config_path = op.join(data_dir, config_name)
216 # shutil.copy(config_path, op.join(local_data_dir, 'config.json'))
217 meta['nrid'] = op.basename(config_path)
219 # write meta config
220 with open(op.join(data_dir, 'meta.cfg'), 'w') as fout: # NOQA: no encoding for JSON
221 json.dump(meta, fout, indent=4)
224def edit_narrative(handler):
225 """Set the handler's narrative and dataset ID to the current session."""
226 user_dir = op.join(nlg_path, handler.current_user.email)
227 dataset_name = handler.args.get('dsid', [''])[0]
228 narrative_name = handler.args.get('nrid', [''])[0] + '.json'
229 with open(op.join(user_dir, 'meta.cfg'), 'w') as fout: # NOQA: no encoding for JSON
230 json.dump({'dsid': dataset_name, 'nrid': narrative_name}, fout, indent=4)
233def get_init_config(handler):
234 """Get the initial default configuration for the current user."""
235 user_dir = op.join(nlg_path, handler.current_user.email)
236 metapath = op.join(user_dir, 'meta.cfg')
237 if op.isfile(metapath):
238 with open(metapath, 'r') as fout: # NOQA: no encoding for JSON
239 meta = json.load(fout)
240 config_file = op.join(user_dir, meta.get('nrid', ''))
241 if op.isfile(config_file):
242 with open(config_file, 'r') as fout: # NOQA: no encoding for JSON
243 meta['config'] = json.load(fout)
244 return meta
245 return {}