Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# vim:fenc=utf-8 

2 

3""" 

4Module for gramex exposure. This shouldn't be imported anywhere, only for use 

5with gramex. 

6""" 

7import json 

8import os 

9import os.path as op 

10from six.moves.urllib import parse 

11 

12import pandas as pd 

13from tornado.template import Template 

14 

15from gramex.config import variables 

16from gramex.apps.nlg import grammar 

17from gramex.apps.nlg import templatize 

18from gramex.apps.nlg import nlgutils as utils 

19 

20DATAFILE_EXTS = {'.csv', '.xls', '.xlsx', '.tsv'} 

21 

22grx_data_dir = variables['GRAMEXDATA'] 

23nlg_path = op.join(grx_data_dir, 'nlg') 

24 

25if not op.isdir(nlg_path): 25 ↛ 26line 25 didn't jump to line 26, because the condition on line 25 was never true

26 os.mkdir(nlg_path) 

27 

28 

29def render_live_template(handler): 

30 """Given a narrative ID and df records, render the template.""" 

31 orgdf = get_original_df(handler) 

32 nrid = handler.args['nrid'][0] 

33 if not nrid.endswith('.json'): 

34 nrid += '.json' 

35 data = json.loads(handler.args['data'][0]) 

36 df = pd.DataFrame.from_records(data) 

37 nrpath = op.join(nlg_path, handler.current_user.email, nrid) 

38 with open(nrpath, 'r') as fout: # noqa: No encoding for json 

39 templates = json.load(fout) 

40 narratives = [] 

41 style = json.loads(handler.args['style'][0]) 

42 for t in templates['config']: 

43 tmpl = utils.add_html_styling(t['template'], style) 

44 s = Template(tmpl).generate(df=df, fh_args=t.get('fh_args', {}), 

45 G=grammar, U=utils, orgdf=orgdf) 

46 rendered = s.decode('utf8') 

47 narratives.append(rendered) 

48 return '\n'.join(narratives) 

49 

50 

51def get_original_df(handler): 

52 """Get the original dataframe which was uploaded to the webapp.""" 

53 data_dir = op.join(nlg_path, handler.current_user.email) 

54 with open(op.join(data_dir, 'meta.cfg'), 'r') as fout: # noqa: No encoding for json 

55 meta = json.load(fout) 

56 dataset_path = op.join(data_dir, meta['dsid']) 

57 return pd.read_csv(dataset_path, encoding='utf-8') 

58 

59 

60def render_template(handler): 

61 """Render a set of templates against a dataframe and formhandler actions on it.""" 

62 orgdf = get_original_df(handler) 

63 payload = parse.parse_qsl(handler.request.body.decode("utf8")) 

64 if not payload: 

65 payload = json.loads(handler.request.body.decode("utf8")) 

66 fh_args = payload['args'] 

67 templates = payload['template'] 

68 df = pd.DataFrame.from_records(payload['data']) 

69 else: 

70 payload = dict(payload) 

71 fh_args = json.loads(payload.get("args", {})) 

72 templates = json.loads(payload["template"]) 

73 df = pd.read_json(payload["data"], orient="records") 

74 # fh_args = {k: [x.lstrip('-') for x in v] for k, v in fh_args.items()} 

75 resp = [] 

76 for t in templates: 

77 rendered = Template(t).generate( 

78 orgdf=orgdf, df=df, fh_args=fh_args, G=grammar, U=utils).decode('utf8') 

79 rendered = rendered.replace('-', '') 

80 # grmerr = utils.check_grammar(rendered) 

81 resp.append({'text': rendered}) # , 'grmerr': grmerr}) 

82 return json.dumps(resp) 

83 

84 

85def process_template(handler): 

86 """Process English text in the context of a df and formhandler arguments 

87 to templatize it.""" 

88 payload = parse.parse_qsl(handler.request.body.decode("utf8")) 

89 payload = dict(payload) 

90 text = json.loads(payload["text"]) 

91 df = pd.read_json(payload["data"], orient="records") 

92 args = json.loads(payload.get("args", {})) 

93 if args is None: 

94 args = {} 

95 resp = [] 

96 for t in text: 

97 # grammar_errors = yield utils.check_grammar(t) 

98 replacements, t, infl = templatize(t, args.copy(), df) 

99 resp.append({ 

100 "text": t, "tokenmap": replacements, 'inflections': infl, 

101 "fh_args": args, "setFHArgs": False, 

102 # "grmerr": json.loads(grammar_errors.decode('utf8'))['matches'] 

103 }) 

104 return json.dumps(resp) 

105 

106 

107def read_current_config(handler): 

108 """Read the current data and narrative IDs written to the session file.""" 

109 meta_path = op.join(nlg_path, handler.current_user.email, 'meta.cfg') 

110 with open(meta_path, 'r') as fout: # noqa: No encoding for json 

111 meta = json.load(fout) 

112 return meta 

113 

114 

115def get_dataset_files(handler): 

116 """Get all filenames uploaded by the user. 

117 

118 Parameters 

119 ---------- 

120 handler : tornado.RequestHandler 

121 

122 Returns 

123 ------- 

124 list 

125 List of filenames. 

126 """ 

127 

128 user_dir = op.join(nlg_path, handler.current_user.email) 

129 if op.isdir(user_dir): 

130 files = [f for f in os.listdir(user_dir) if op.splitext(f)[-1].lower() in DATAFILE_EXTS] 

131 else: 

132 files = [] 

133 return files 

134 

135 

136def get_narrative_config_files(handler): 

137 """Get list of narrative config files generated by the user. 

138 

139 Parameters 

140 ---------- 

141 handler : tornado.RequestHandler 

142 

143 Returns 

144 ------- 

145 list 

146 List of narrative configurations. 

147 """ 

148 # TODO: current_user.email needs to be replaced with a more suitable user ID 

149 user_dir = op.join(nlg_path, handler.current_user.email) 

150 if op.isdir(user_dir): 

151 return [f for f in os.listdir(user_dir) if f.endswith('.json')] 

152 return [] 

153 

154 

155def download_config(handler): 

156 """Download the current narrative config as JSON.""" 

157 payload = {} 

158 payload['config'] = json.loads(parse.unquote(handler.args['config'][0])) 

159 payload['data'] = json.loads(parse.unquote(handler.args.get('data', [None])[0])) 

160 payload['name'] = parse.unquote(handler.args['name'][0]) 

161 return json.dumps(payload, indent=4) 

162 

163 

164def save_config(handler): 

165 """Save the current narrative config. 

166 (to $GRAMEXDATA/{{ handler.current_user.email }})""" 

167 payload = {} 

168 payload['config'] = json.loads(parse.unquote(handler.args['config'][0])) 

169 payload['name'] = parse.unquote(handler.args['name'][0]) 

170 nname = payload['name'] 

171 if not nname.endswith('.json'): 

172 nname += '.json' 

173 payload['dataset'] = parse.unquote(handler.args['dataset'][0]) 

174 fpath = op.join(nlg_path, handler.current_user.email, nname) 

175 with open(fpath, 'w') as fout: # noqa: No encoding for json 

176 json.dump(payload, fout, indent=4) 

177 

178 

179def get_gramopts(handler): 

180 """Find all Grammar and token inflection options from the NLG library. 

181 

182 Primarily used for creating the select box in the template settings dialog.""" 

183 funcs = {} 

184 for attrname in dir(grammar): 

185 obj = getattr(grammar, attrname) 

186 if getattr(obj, 'gramopt', False): 

187 funcs[obj.fe_name] = {'source': obj.source, 'func_name': attrname} 

188 return funcs 

189 

190 

191def init_form(handler): 

192 """Process input from the landing page and write the current session config.""" 

193 meta = {} 

194 # prioritize files first 

195 data_dir = op.join(nlg_path, handler.current_user.email) 

196 if not op.isdir(data_dir): 

197 os.makedirs(data_dir) 

198 

199 # handle dataset 

200 data_file = handler.request.files.get('data-file', [{}])[0] 

201 if data_file: 

202 # TODO: Unix filenames may not be valid Windows filenames. 

203 outpath = op.join(data_dir, data_file['filename']) 

204 with open(outpath, 'wb') as fout: 

205 fout.write(data_file['body']) 

206 else: 

207 dataset = handler.args['dataset'][0] 

208 outpath = op.join(data_dir, dataset) 

209 # shutil.copy(outpath, fh_fpath) 

210 meta['dsid'] = op.basename(outpath) 

211 

212 # handle config 

213 config_name = handler.get_argument('narrative', '') 

214 if config_name: 

215 config_path = op.join(data_dir, config_name) 

216 # shutil.copy(config_path, op.join(local_data_dir, 'config.json')) 

217 meta['nrid'] = op.basename(config_path) 

218 

219 # write meta config 

220 with open(op.join(data_dir, 'meta.cfg'), 'w') as fout: # NOQA: no encoding for JSON 

221 json.dump(meta, fout, indent=4) 

222 

223 

224def edit_narrative(handler): 

225 """Set the handler's narrative and dataset ID to the current session.""" 

226 user_dir = op.join(nlg_path, handler.current_user.email) 

227 dataset_name = handler.args.get('dsid', [''])[0] 

228 narrative_name = handler.args.get('nrid', [''])[0] + '.json' 

229 with open(op.join(user_dir, 'meta.cfg'), 'w') as fout: # NOQA: no encoding for JSON 

230 json.dump({'dsid': dataset_name, 'nrid': narrative_name}, fout, indent=4) 

231 

232 

233def get_init_config(handler): 

234 """Get the initial default configuration for the current user.""" 

235 user_dir = op.join(nlg_path, handler.current_user.email) 

236 metapath = op.join(user_dir, 'meta.cfg') 

237 if op.isfile(metapath): 

238 with open(metapath, 'r') as fout: # NOQA: no encoding for JSON 

239 meta = json.load(fout) 

240 config_file = op.join(user_dir, meta.get('nrid', '')) 

241 if op.isfile(config_file): 

242 with open(config_file, 'r') as fout: # NOQA: no encoding for JSON 

243 meta['config'] = json.load(fout) 

244 return meta 

245 return {}