Coverage for phml\embedded\__init__.py: 99%

170 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-05 15:06 -0500

1""" 

2Embedded has all the logic for processing python elements, attributes, and text blocks. 

3""" 

4from __future__ import annotations 

5 

6import ast 

7import re 

8import types 

9from functools import cached_property 

10from pathlib import Path 

11from shutil import get_terminal_size 

12from traceback import FrameSummary, extract_tb 

13from typing import Any, Iterator, TypedDict 

14 

15from phml.embedded.built_in import built_in_funcs, built_in_types 

16from phml.helpers import normalize_indent 

17from phml.nodes import Element, Literal 

18 

19# Global cached imports 

20__IMPORTS__ = {} 

21__FROM_IMPORTS__ = {} 

22 

23# PERF: Only allow assignments, methods, imports, and classes? 

24class EmbeddedTryCatch: 

25 """Context manager around embedded python execution. Will parse the traceback 

26 and the content being executed to create a detailed error message. The final 

27 error message is raised in a custom EmbeddedPythonException. 

28 """ 

29 

30 def __init__( 

31 self, 

32 path: str | Path | None = None, 

33 content: str | None = None, 

34 pos: tuple[int, int] | None = None, 

35 ) -> None: 

36 self._path = str(path or "<python>") 

37 self._content = content or "" 

38 self._pos = pos or (0, 0) 

39 

40 def __enter__(self): 

41 pass 

42 

43 def __exit__(self, _, exc_val, exc_tb): 

44 if exc_val is not None and not isinstance(exc_val, SystemExit): 

45 raise EmbeddedPythonException( 

46 self._path, 

47 self._content, 

48 self._pos, 

49 exc_val, 

50 exc_tb, 

51 ) from exc_val 

52 

53 

54class EmbeddedPythonException(Exception): 

55 def __init__(self, path, content, pos, exc_val, exc_tb) -> None: 

56 self.max_width, _ = get_terminal_size((20, 0)) 

57 self.msg = exc_val.msg if hasattr(exc_val, "msg") else str(exc_val) 

58 if isinstance(exc_val, SyntaxError): 

59 self.l_slice = (exc_val.lineno or 0, exc_val.end_lineno or 0) 

60 self.c_slice = (exc_val.offset or 0, exc_val.end_offset or 0) 

61 else: 

62 fs: FrameSummary = extract_tb(exc_tb)[-1] 

63 self.l_slice = (fs.lineno or 0, fs.end_lineno or 0) 

64 self.c_slice = (fs.colno or 0, fs.end_colno or 0) 

65 

66 self._content = content 

67 self._path = path 

68 self._pos = pos 

69 

70 def format_line(self, line, c_width, leading: str = " "): 

71 return f"{leading.ljust(c_width, ' ')}│{line}" 

72 

73 def generate_exception_lines(self, lines: list[str], width: int): 

74 max_width = self.max_width - width - 3 

75 result = [] 

76 for i, line in enumerate(lines): 

77 if len(line) > max_width: 

78 parts = [ 

79 line[j : j + max_width] for j in range(0, len(line), max_width) 

80 ] 

81 result.append(self.format_line(parts[0], width, str(i + 1))) 

82 for part in parts[1:]: 

83 result.append(self.format_line(part, width)) 

84 else: 

85 result.append(self.format_line(line, width, str(i + 1))) 

86 return result 

87 

88 def __str__(self) -> str: 

89 message = "" 

90 if self._path != "": 

91 pos = ( 

92 self._pos[0] + (self.l_slice[0] or 0), 

93 self.c_slice[0] or self._pos[1], 

94 ) 

95 if pos[0] > self._content.count("\n"): 

96 message = f"{self._path} Failed to execute phml embedded python" 

97 else: 

98 message = f"[{pos[0]+1}:{pos[1]}] {self._path} Failed to execute phml embedded python" 

99 if self._content != "": 

100 lines = self._content.split("\n") 

101 target_lines = lines[self.l_slice[0] - 1 : self.l_slice[1]] 

102 if len(target_lines) > 0: 

103 if self.l_slice[0] == self.l_slice[1]: 

104 target_lines[0] = ( 

105 target_lines[0][: self.c_slice[0]] 

106 + "\x1b[31m" 

107 + target_lines[0][self.c_slice[0] : self.c_slice[1]] 

108 + "\x1b[0m" 

109 + target_lines[0][self.c_slice[1] :] 

110 ) 

111 else: 

112 target_lines[0] = ( 

113 target_lines[0][: self.c_slice[0] + 1] 

114 + "\x1b[31m" 

115 + target_lines[0][self.c_slice[0] + 1 :] 

116 + "\x1b[0m" 

117 ) 

118 for i, line in enumerate(target_lines[1:-1]): 

119 target_lines[i + 1] = "\x1b[31m" + line + "\x1b[0m" 

120 target_lines[-1] = ( 

121 "\x1b[31m" 

122 + target_lines[-1][: self.c_slice[-1] + 1] 

123 + "\x1b[0m" 

124 + target_lines[-1][self.c_slice[-1] + 1 :] 

125 ) 

126 

127 lines = [ 

128 *lines[: self.l_slice[0] - 1], 

129 *target_lines, 

130 *lines[self.l_slice[1] :], 

131 ] 

132 

133 w_fmt = len(f"{len(lines)}") 

134 content = "\n".join( 

135 self.generate_exception_lines(lines, w_fmt), 

136 ) 

137 line_width = self.max_width - w_fmt - 2 

138 

139 exception = f"{self.msg}" 

140 if len(target_lines) > 0: 

141 exception += f" at <{self.l_slice[0]}:{self.c_slice[0]}-{self.l_slice[1]}:{self.c_slice[1]}>" 

142 ls = [ 

143 exception[i : i + line_width] 

144 for i in range(0, len(exception), line_width) 

145 ] 

146 exception_line = self.format_line(ls[0], w_fmt, "#") 

147 for l in ls[1:]: 

148 exception_line += "\n" + self.format_line(l, w_fmt) 

149 

150 message += ( 

151 f"\n{'─'.ljust(w_fmt, '─')}┬─{'─'*(line_width)}\n" 

152 + exception_line 

153 + "\n" 

154 + f"{'═'.ljust(w_fmt, '═')}╪═{'═'*(line_width)}\n" 

155 + f"{content}" 

156 ) 

157 

158 return message 

159 

160 

161def parse_import_values(_import: str) -> list[str | tuple[str, str]]: 

162 values = [] 

163 for value in re.finditer(r"(?:([^,\s]+) as (.+)|([^,\s]+))(?=\s*,)?", _import): 

164 if value.group(1) is not None: 

165 values.append((value.group(1), value.group(2))) 

166 elif value.groups(3) is not None: 

167 values.append(value.group(3)) 

168 return values 

169 

170 

171class ImportStruct(TypedDict): 

172 key: str 

173 values: str | list[str] 

174 

175 

176class Module: 

177 """Object used to access the gobal imports. Readonly data.""" 

178 

179 def __init__(self, module: str, *, imports: list[str] | None = None) -> None: 

180 self.objects = imports or [] 

181 if imports is not None and len(imports) > 0: 

182 if module not in __FROM_IMPORTS__: 

183 raise ValueError(f"Unkown module {module!r}") 

184 try: 

185 imports = { 

186 _import: __FROM_IMPORTS__[module][_import] for _import in imports 

187 } 

188 except KeyError as kerr: 

189 back_frame = kerr.__traceback__.tb_frame.f_back 

190 back_tb = types.TracebackType( 

191 tb_next=None, 

192 tb_frame=back_frame, 

193 tb_lasti=back_frame.f_lasti, 

194 tb_lineno=back_frame.f_lineno, 

195 ) 

196 FrameSummary("", 2, "") 

197 raise ValueError( 

198 f"{', '.join(kerr.args)!r} {'arg' if len(kerr.args) > 1 else 'is'} not found in cached imported module {module!r}", 

199 ).with_traceback(back_tb) 

200 

201 globals().update(imports) 

202 locals().update(imports) 

203 self.module = module 

204 else: 

205 if module not in __IMPORTS__: 

206 raise ValueError(f"Unkown module {module!r}") 

207 

208 imports = {module: __IMPORTS__[module]} 

209 locals().update(imports) 

210 globals().update(imports) 

211 self.module = module 

212 

213 def collect(self) -> Any: 

214 """Collect the imports and return the single import or a tuple of multiple imports.""" 

215 if len(self.objects) > 0: 

216 if len(self.objects) == 1: 

217 return __FROM_IMPORTS__[self.module][self.objects[0]] 

218 return tuple( 

219 [__FROM_IMPORTS__[self.module][object] for object in self.objects] 

220 ) 

221 return __IMPORTS__[self.module] 

222 

223 

224class EmbeddedImport: 

225 """Data representation of an import.""" 

226 

227 module: str 

228 """Package where the import(s) are from.""" 

229 

230 objects: list[str] 

231 """The imported objects.""" 

232 

233 def __init__( 

234 self, module: str, values: str | list[str] | None = None, *, push: bool = False 

235 ) -> None: 

236 self.module = module 

237 

238 if isinstance(values, list): 

239 self.objects = values 

240 else: 

241 self.objects = parse_import_values(values or "") 

242 

243 if push: 

244 self.data 

245 

246 def _parse_from_import(self): 

247 if self.module in __FROM_IMPORTS__: 

248 values = list( 

249 filter( 

250 lambda v: (v if isinstance(v, str) else v[0]) 

251 not in __FROM_IMPORTS__[self.module], 

252 self.objects, 

253 ) 

254 ) 

255 else: 

256 values = self.objects 

257 

258 if len(values) > 0: 

259 local_env = {} 

260 exec_val = compile(str(self), "_embedded_import_", "exec") 

261 exec(exec_val, {}, local_env) 

262 

263 if self.module not in __FROM_IMPORTS__: 

264 __FROM_IMPORTS__[self.module] = {} 

265 __FROM_IMPORTS__[self.module].update(local_env) 

266 

267 keys = [key if isinstance(key, str) else key[1] for key in self.objects] 

268 return {key: __FROM_IMPORTS__[self.module][key] for key in keys} 

269 

270 def _parse_import(self): 

271 if self.module not in __IMPORTS__: 

272 local_env = {} 

273 exec_val = compile(str(self), "_embedded_import_", "exec") 

274 exec(exec_val, {}, local_env) 

275 __IMPORTS__.update(local_env) 

276 

277 return {self.module: __IMPORTS__[self.module]} 

278 

279 def __iter__(self) -> Iterator[tuple[str, Any]]: 

280 if len(self.objects) > 0: 

281 if self.module not in __FROM_IMPORTS__: 

282 raise KeyError(f"{self.module} is not a known exposed module") 

283 yield from __FROM_IMPORTS__[self.module].items() 

284 else: 

285 if self.module not in __IMPORTS__: 

286 raise KeyError(f"{self.module} is not a known exposed module") 

287 yield __IMPORTS__[self.module] 

288 

289 @cached_property 

290 def data(self) -> dict[str, Any]: 

291 """The actual imports stored by a name to value mapping.""" 

292 if len(self.objects) > 0: 

293 return self._parse_from_import() 

294 return self._parse_import() 

295 

296 def __getitem__(self, key: str) -> Any: 

297 self.data[key] 

298 

299 def __repr__(self) -> str: 

300 if len(self.objects) > 0: 

301 return f"FROM({self.module}).IMPORT({', '.join(self.objects)})" 

302 return f"IMPORT({self.module})" 

303 

304 def __str__(self) -> str: 

305 if len(self.objects) > 0: 

306 return f"from {self.module} import {', '.join(obj if isinstance(obj, str) else f'{obj[0]} as {obj[1]}' for obj in self.objects)}" 

307 return f"import {self.module}" 

308 

309 

310class Embedded: 

311 """Logic for parsing and storing locals and imports of dynamic python code.""" 

312 

313 context: dict[str, Any] 

314 """Variables and locals found in the python code block.""" 

315 

316 imports: list[EmbeddedImport] 

317 """Imports needed for the python in this scope. Imports are stored in the module globally 

318 to reduce duplicate imports. 

319 """ 

320 

321 def __init__(self, content: str | Element, path: str | None = None) -> None: 

322 self._path = path or "<python>" 

323 self._pos = (0, 0) 

324 if isinstance(content, Element): 

325 if len(content) > 1 or ( 

326 len(content) == 1 and not Literal.is_text(content[0]) 

327 ): 

328 # TODO: Custom error 

329 raise ValueError( 

330 "Expected python elements to contain one text node or nothing", 

331 ) 

332 if content.position is not None: 

333 start = content.position.start 

334 self._pos = (start.line, start.column) 

335 content = content[0].content 

336 content = normalize_indent(content) 

337 self.imports = [] 

338 self.context = {} 

339 if len(content) > 0: 

340 with EmbeddedTryCatch(path, content, self._pos): 

341 self.parse_data(content) 

342 

343 def __add__(self, _o) -> Embedded: 

344 self.imports.extend(_o.imports) 

345 self.context.update(_o.context) 

346 return self 

347 

348 def __contains__(self, key: str) -> bool: 

349 return key in self.context 

350 

351 def __getitem__(self, key: str) -> Any: 

352 if key in self.context: 

353 return self.context[key] 

354 elif key in self.imports: 

355 return __IMPORTS__[key] 

356 

357 raise KeyError(f"Key is not in Embedded context or imports: {key}") 

358 

359 def split_contexts(self, content: str) -> tuple[list[str], list[EmbeddedImport]]: 

360 re_context = re.compile(r"class.+|def.+") 

361 re_import = re.compile( 

362 r"from (?P<key>.+) import (?P<values>.+)|import (?P<value>.+)", 

363 ) 

364 

365 imports = [] 

366 blocks = [] 

367 current = [] 

368 

369 lines = content.split("\n") 

370 i = 0 

371 while i < len(lines): 

372 imp_match = re_import.match(lines[i]) 

373 if imp_match is not None: 

374 data = imp_match.groupdict() 

375 imports.append( 

376 EmbeddedImport(data["key"] or data["value"], data["values"]) 

377 ) 

378 elif re_context.match(lines[i]) is not None: 

379 blocks.append("\n".join(current)) 

380 current = [lines[i]] 

381 i += 1 

382 while i < len(lines) and lines[i].startswith(" "): 

383 current.append(lines[i]) 

384 i += 1 

385 blocks.append("\n".join(current)) 

386 current = [] 

387 else: 

388 current.append(lines[i]) 

389 if i < len(lines): 

390 i += 1 

391 

392 if len(current) > 0: 

393 blocks.append("\n".join(current)) 

394 

395 return blocks, imports 

396 

397 def parse_data(self, content: str): 

398 blocks, self.imports = self.split_contexts(content) 

399 

400 local_env = {} 

401 global_env = {key: value for _import in self.imports for key, value in _import} 

402 context = {**global_env} 

403 

404 for block in blocks: 

405 exec_val = compile(block, self._path, "exec") 

406 exec(exec_val, global_env, local_env) 

407 context.update(local_env) 

408 # update global env with found locals so they can be used inside methods and classes 

409 global_env.update(local_env) 

410 

411 self.context = context 

412 

413 

414def _validate_kwargs(code: ast.Module, kwargs: dict[str, Any]): 

415 exclude_list = [*built_in_funcs, *built_in_types] 

416 for var in ( 

417 name.id 

418 for name in ast.walk(code) 

419 if isinstance( 

420 name, 

421 ast.Name, 

422 ) # Get all variables/names used. This can be methods or values 

423 and name.id not in exclude_list 

424 ): 

425 if var not in kwargs: 

426 kwargs[var] = None 

427 

428 

429def update_ast_node_pos(dest, source): 

430 """Assign lineno, end_lineno, col_offset, and end_col_offset 

431 from a source python ast node to a destination python ast node. 

432 """ 

433 dest.lineno = source.lineno 

434 dest.end_lineno = source.end_lineno 

435 dest.col_offset = source.col_offset 

436 dest.end_col_offset = source.end_col_offset 

437 

438 

439RESULT = "_phml_embedded_result_" 

440 

441 

442def exec_embedded(code: str, _path: str | None = None, **context: Any) -> Any: 

443 """Execute embedded python and return the extracted value. This is the last 

444 assignment in the embedded python. The embedded python must have the last line as a value 

445 or an assignment. 

446 

447 Note: 

448 No local or global variables will be retained from the embedded python code. 

449 

450 Args: 

451 code (str): The embedded python code. 

452 **context (Any): The additional context to provide to the embedded python. 

453 

454 Returns: 

455 Any: The value of the last assignment or value defined 

456 """ 

457 from phml.utilities import blank 

458 

459 context = { 

460 "blank": blank, 

461 **context, 

462 } 

463 

464 # last line must be an assignment or the value to be used 

465 with EmbeddedTryCatch(_path, code): 

466 code = normalize_indent(code) 

467 AST = ast.parse(code) 

468 _validate_kwargs(AST, context) 

469 

470 last = AST.body[-1] 

471 returns = [ret for ret in AST.body if isinstance(ret, ast.Return)] 

472 

473 if len(returns) > 0: 

474 last = returns[0] 

475 idx = AST.body.index(last) 

476 

477 n_expr = ast.Name(id=RESULT, ctx=ast.Store()) 

478 n_assign = ast.Assign(targets=[n_expr], value=last.value) 

479 

480 update_ast_node_pos(dest=n_expr, source=last) 

481 update_ast_node_pos(dest=n_assign, source=last) 

482 

483 AST.body = [*AST.body[:idx], n_assign] 

484 elif isinstance(last, ast.Expr): 

485 n_expr = ast.Name(id=RESULT, ctx=ast.Store()) 

486 n_assign = ast.Assign(targets=[n_expr], value=last.value) 

487 

488 update_ast_node_pos(dest=n_expr, source=last) 

489 update_ast_node_pos(dest=n_assign, source=last) 

490 

491 AST.body[-1] = n_assign 

492 elif isinstance(last, ast.Assign): 

493 n_expr = ast.Name(id=RESULT, ctx=ast.Store()) 

494 update_ast_node_pos(dest=n_expr, source=last) 

495 last.targets.append(n_expr) 

496 

497 ccode = compile(AST, "_phml_embedded_", "exec") 

498 local_env = {} 

499 exec(ccode, {**context}, local_env) 

500 return local_env[RESULT] 

501 

502 

503def exec_embedded_blocks(code: str, _path: str = "", **context: dict[str, Any]): 

504 """Execute embedded python inside `{{}}` blocks. The resulting values are subsituted 

505 in for the found blocks. 

506 

507 Note: 

508 No local or global variables will be retained from the embedded python code. 

509 

510 Args: 

511 code (str): The embedded python code. 

512 **context (Any): The additional context to provide to the embedded python. 

513 

514 Returns: 

515 str: The value of the passed in string with the python blocks replaced. 

516 """ 

517 

518 result = "" 

519 data = [] 

520 next_block = re.search(r"\{\{", code) 

521 while next_block is not None: 

522 start = next_block.start() 

523 if start > 0: 

524 result += code[:start] 

525 code = code[start + 2 :] 

526 

527 balance = 2 

528 index = 0 

529 while balance > 0 and index < len(code): 

530 if code[index] == "}": 

531 balance -= 1 

532 elif code[index] == "{": 

533 balance += 1 

534 index += 1 

535 

536 result += "{}" 

537 data.append( 

538 str( 

539 exec_embedded( 

540 code[: index - 2], 

541 _path + f" block #{len(data)+1}", 

542 **context, 

543 ), 

544 ), 

545 ) 

546 code = code[index + 1 :] 

547 next_block = re.search(r"(?<!\\)\{\{", code) 

548 

549 if len(code) > 0: 

550 result += code 

551 

552 return result.format(*data)