Coverage for phml\embedded\__init__.py: 99%
170 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-05 15:06 -0500
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-05 15:06 -0500
1"""
2Embedded has all the logic for processing python elements, attributes, and text blocks.
3"""
4from __future__ import annotations
6import ast
7import re
8import types
9from functools import cached_property
10from pathlib import Path
11from shutil import get_terminal_size
12from traceback import FrameSummary, extract_tb
13from typing import Any, Iterator, TypedDict
15from phml.embedded.built_in import built_in_funcs, built_in_types
16from phml.helpers import normalize_indent
17from phml.nodes import Element, Literal
19# Global cached imports
20__IMPORTS__ = {}
21__FROM_IMPORTS__ = {}
23# PERF: Only allow assignments, methods, imports, and classes?
24class EmbeddedTryCatch:
25 """Context manager around embedded python execution. Will parse the traceback
26 and the content being executed to create a detailed error message. The final
27 error message is raised in a custom EmbeddedPythonException.
28 """
30 def __init__(
31 self,
32 path: str | Path | None = None,
33 content: str | None = None,
34 pos: tuple[int, int] | None = None,
35 ) -> None:
36 self._path = str(path or "<python>")
37 self._content = content or ""
38 self._pos = pos or (0, 0)
40 def __enter__(self):
41 pass
43 def __exit__(self, _, exc_val, exc_tb):
44 if exc_val is not None and not isinstance(exc_val, SystemExit):
45 raise EmbeddedPythonException(
46 self._path,
47 self._content,
48 self._pos,
49 exc_val,
50 exc_tb,
51 ) from exc_val
54class EmbeddedPythonException(Exception):
55 def __init__(self, path, content, pos, exc_val, exc_tb) -> None:
56 self.max_width, _ = get_terminal_size((20, 0))
57 self.msg = exc_val.msg if hasattr(exc_val, "msg") else str(exc_val)
58 if isinstance(exc_val, SyntaxError):
59 self.l_slice = (exc_val.lineno or 0, exc_val.end_lineno or 0)
60 self.c_slice = (exc_val.offset or 0, exc_val.end_offset or 0)
61 else:
62 fs: FrameSummary = extract_tb(exc_tb)[-1]
63 self.l_slice = (fs.lineno or 0, fs.end_lineno or 0)
64 self.c_slice = (fs.colno or 0, fs.end_colno or 0)
66 self._content = content
67 self._path = path
68 self._pos = pos
70 def format_line(self, line, c_width, leading: str = " "):
71 return f"{leading.ljust(c_width, ' ')}│{line}"
73 def generate_exception_lines(self, lines: list[str], width: int):
74 max_width = self.max_width - width - 3
75 result = []
76 for i, line in enumerate(lines):
77 if len(line) > max_width:
78 parts = [
79 line[j : j + max_width] for j in range(0, len(line), max_width)
80 ]
81 result.append(self.format_line(parts[0], width, str(i + 1)))
82 for part in parts[1:]:
83 result.append(self.format_line(part, width))
84 else:
85 result.append(self.format_line(line, width, str(i + 1)))
86 return result
88 def __str__(self) -> str:
89 message = ""
90 if self._path != "":
91 pos = (
92 self._pos[0] + (self.l_slice[0] or 0),
93 self.c_slice[0] or self._pos[1],
94 )
95 if pos[0] > self._content.count("\n"):
96 message = f"{self._path} Failed to execute phml embedded python"
97 else:
98 message = f"[{pos[0]+1}:{pos[1]}] {self._path} Failed to execute phml embedded python"
99 if self._content != "":
100 lines = self._content.split("\n")
101 target_lines = lines[self.l_slice[0] - 1 : self.l_slice[1]]
102 if len(target_lines) > 0:
103 if self.l_slice[0] == self.l_slice[1]:
104 target_lines[0] = (
105 target_lines[0][: self.c_slice[0]]
106 + "\x1b[31m"
107 + target_lines[0][self.c_slice[0] : self.c_slice[1]]
108 + "\x1b[0m"
109 + target_lines[0][self.c_slice[1] :]
110 )
111 else:
112 target_lines[0] = (
113 target_lines[0][: self.c_slice[0] + 1]
114 + "\x1b[31m"
115 + target_lines[0][self.c_slice[0] + 1 :]
116 + "\x1b[0m"
117 )
118 for i, line in enumerate(target_lines[1:-1]):
119 target_lines[i + 1] = "\x1b[31m" + line + "\x1b[0m"
120 target_lines[-1] = (
121 "\x1b[31m"
122 + target_lines[-1][: self.c_slice[-1] + 1]
123 + "\x1b[0m"
124 + target_lines[-1][self.c_slice[-1] + 1 :]
125 )
127 lines = [
128 *lines[: self.l_slice[0] - 1],
129 *target_lines,
130 *lines[self.l_slice[1] :],
131 ]
133 w_fmt = len(f"{len(lines)}")
134 content = "\n".join(
135 self.generate_exception_lines(lines, w_fmt),
136 )
137 line_width = self.max_width - w_fmt - 2
139 exception = f"{self.msg}"
140 if len(target_lines) > 0:
141 exception += f" at <{self.l_slice[0]}:{self.c_slice[0]}-{self.l_slice[1]}:{self.c_slice[1]}>"
142 ls = [
143 exception[i : i + line_width]
144 for i in range(0, len(exception), line_width)
145 ]
146 exception_line = self.format_line(ls[0], w_fmt, "#")
147 for l in ls[1:]:
148 exception_line += "\n" + self.format_line(l, w_fmt)
150 message += (
151 f"\n{'─'.ljust(w_fmt, '─')}┬─{'─'*(line_width)}\n"
152 + exception_line
153 + "\n"
154 + f"{'═'.ljust(w_fmt, '═')}╪═{'═'*(line_width)}\n"
155 + f"{content}"
156 )
158 return message
161def parse_import_values(_import: str) -> list[str | tuple[str, str]]:
162 values = []
163 for value in re.finditer(r"(?:([^,\s]+) as (.+)|([^,\s]+))(?=\s*,)?", _import):
164 if value.group(1) is not None:
165 values.append((value.group(1), value.group(2)))
166 elif value.groups(3) is not None:
167 values.append(value.group(3))
168 return values
171class ImportStruct(TypedDict):
172 key: str
173 values: str | list[str]
176class Module:
177 """Object used to access the gobal imports. Readonly data."""
179 def __init__(self, module: str, *, imports: list[str] | None = None) -> None:
180 self.objects = imports or []
181 if imports is not None and len(imports) > 0:
182 if module not in __FROM_IMPORTS__:
183 raise ValueError(f"Unkown module {module!r}")
184 try:
185 imports = {
186 _import: __FROM_IMPORTS__[module][_import] for _import in imports
187 }
188 except KeyError as kerr:
189 back_frame = kerr.__traceback__.tb_frame.f_back
190 back_tb = types.TracebackType(
191 tb_next=None,
192 tb_frame=back_frame,
193 tb_lasti=back_frame.f_lasti,
194 tb_lineno=back_frame.f_lineno,
195 )
196 FrameSummary("", 2, "")
197 raise ValueError(
198 f"{', '.join(kerr.args)!r} {'arg' if len(kerr.args) > 1 else 'is'} not found in cached imported module {module!r}",
199 ).with_traceback(back_tb)
201 globals().update(imports)
202 locals().update(imports)
203 self.module = module
204 else:
205 if module not in __IMPORTS__:
206 raise ValueError(f"Unkown module {module!r}")
208 imports = {module: __IMPORTS__[module]}
209 locals().update(imports)
210 globals().update(imports)
211 self.module = module
213 def collect(self) -> Any:
214 """Collect the imports and return the single import or a tuple of multiple imports."""
215 if len(self.objects) > 0:
216 if len(self.objects) == 1:
217 return __FROM_IMPORTS__[self.module][self.objects[0]]
218 return tuple(
219 [__FROM_IMPORTS__[self.module][object] for object in self.objects]
220 )
221 return __IMPORTS__[self.module]
224class EmbeddedImport:
225 """Data representation of an import."""
227 module: str
228 """Package where the import(s) are from."""
230 objects: list[str]
231 """The imported objects."""
233 def __init__(
234 self, module: str, values: str | list[str] | None = None, *, push: bool = False
235 ) -> None:
236 self.module = module
238 if isinstance(values, list):
239 self.objects = values
240 else:
241 self.objects = parse_import_values(values or "")
243 if push:
244 self.data
246 def _parse_from_import(self):
247 if self.module in __FROM_IMPORTS__:
248 values = list(
249 filter(
250 lambda v: (v if isinstance(v, str) else v[0])
251 not in __FROM_IMPORTS__[self.module],
252 self.objects,
253 )
254 )
255 else:
256 values = self.objects
258 if len(values) > 0:
259 local_env = {}
260 exec_val = compile(str(self), "_embedded_import_", "exec")
261 exec(exec_val, {}, local_env)
263 if self.module not in __FROM_IMPORTS__:
264 __FROM_IMPORTS__[self.module] = {}
265 __FROM_IMPORTS__[self.module].update(local_env)
267 keys = [key if isinstance(key, str) else key[1] for key in self.objects]
268 return {key: __FROM_IMPORTS__[self.module][key] for key in keys}
270 def _parse_import(self):
271 if self.module not in __IMPORTS__:
272 local_env = {}
273 exec_val = compile(str(self), "_embedded_import_", "exec")
274 exec(exec_val, {}, local_env)
275 __IMPORTS__.update(local_env)
277 return {self.module: __IMPORTS__[self.module]}
279 def __iter__(self) -> Iterator[tuple[str, Any]]:
280 if len(self.objects) > 0:
281 if self.module not in __FROM_IMPORTS__:
282 raise KeyError(f"{self.module} is not a known exposed module")
283 yield from __FROM_IMPORTS__[self.module].items()
284 else:
285 if self.module not in __IMPORTS__:
286 raise KeyError(f"{self.module} is not a known exposed module")
287 yield __IMPORTS__[self.module]
289 @cached_property
290 def data(self) -> dict[str, Any]:
291 """The actual imports stored by a name to value mapping."""
292 if len(self.objects) > 0:
293 return self._parse_from_import()
294 return self._parse_import()
296 def __getitem__(self, key: str) -> Any:
297 self.data[key]
299 def __repr__(self) -> str:
300 if len(self.objects) > 0:
301 return f"FROM({self.module}).IMPORT({', '.join(self.objects)})"
302 return f"IMPORT({self.module})"
304 def __str__(self) -> str:
305 if len(self.objects) > 0:
306 return f"from {self.module} import {', '.join(obj if isinstance(obj, str) else f'{obj[0]} as {obj[1]}' for obj in self.objects)}"
307 return f"import {self.module}"
310class Embedded:
311 """Logic for parsing and storing locals and imports of dynamic python code."""
313 context: dict[str, Any]
314 """Variables and locals found in the python code block."""
316 imports: list[EmbeddedImport]
317 """Imports needed for the python in this scope. Imports are stored in the module globally
318 to reduce duplicate imports.
319 """
321 def __init__(self, content: str | Element, path: str | None = None) -> None:
322 self._path = path or "<python>"
323 self._pos = (0, 0)
324 if isinstance(content, Element):
325 if len(content) > 1 or (
326 len(content) == 1 and not Literal.is_text(content[0])
327 ):
328 # TODO: Custom error
329 raise ValueError(
330 "Expected python elements to contain one text node or nothing",
331 )
332 if content.position is not None:
333 start = content.position.start
334 self._pos = (start.line, start.column)
335 content = content[0].content
336 content = normalize_indent(content)
337 self.imports = []
338 self.context = {}
339 if len(content) > 0:
340 with EmbeddedTryCatch(path, content, self._pos):
341 self.parse_data(content)
343 def __add__(self, _o) -> Embedded:
344 self.imports.extend(_o.imports)
345 self.context.update(_o.context)
346 return self
348 def __contains__(self, key: str) -> bool:
349 return key in self.context
351 def __getitem__(self, key: str) -> Any:
352 if key in self.context:
353 return self.context[key]
354 elif key in self.imports:
355 return __IMPORTS__[key]
357 raise KeyError(f"Key is not in Embedded context or imports: {key}")
359 def split_contexts(self, content: str) -> tuple[list[str], list[EmbeddedImport]]:
360 re_context = re.compile(r"class.+|def.+")
361 re_import = re.compile(
362 r"from (?P<key>.+) import (?P<values>.+)|import (?P<value>.+)",
363 )
365 imports = []
366 blocks = []
367 current = []
369 lines = content.split("\n")
370 i = 0
371 while i < len(lines):
372 imp_match = re_import.match(lines[i])
373 if imp_match is not None:
374 data = imp_match.groupdict()
375 imports.append(
376 EmbeddedImport(data["key"] or data["value"], data["values"])
377 )
378 elif re_context.match(lines[i]) is not None:
379 blocks.append("\n".join(current))
380 current = [lines[i]]
381 i += 1
382 while i < len(lines) and lines[i].startswith(" "):
383 current.append(lines[i])
384 i += 1
385 blocks.append("\n".join(current))
386 current = []
387 else:
388 current.append(lines[i])
389 if i < len(lines):
390 i += 1
392 if len(current) > 0:
393 blocks.append("\n".join(current))
395 return blocks, imports
397 def parse_data(self, content: str):
398 blocks, self.imports = self.split_contexts(content)
400 local_env = {}
401 global_env = {key: value for _import in self.imports for key, value in _import}
402 context = {**global_env}
404 for block in blocks:
405 exec_val = compile(block, self._path, "exec")
406 exec(exec_val, global_env, local_env)
407 context.update(local_env)
408 # update global env with found locals so they can be used inside methods and classes
409 global_env.update(local_env)
411 self.context = context
414def _validate_kwargs(code: ast.Module, kwargs: dict[str, Any]):
415 exclude_list = [*built_in_funcs, *built_in_types]
416 for var in (
417 name.id
418 for name in ast.walk(code)
419 if isinstance(
420 name,
421 ast.Name,
422 ) # Get all variables/names used. This can be methods or values
423 and name.id not in exclude_list
424 ):
425 if var not in kwargs:
426 kwargs[var] = None
429def update_ast_node_pos(dest, source):
430 """Assign lineno, end_lineno, col_offset, and end_col_offset
431 from a source python ast node to a destination python ast node.
432 """
433 dest.lineno = source.lineno
434 dest.end_lineno = source.end_lineno
435 dest.col_offset = source.col_offset
436 dest.end_col_offset = source.end_col_offset
439RESULT = "_phml_embedded_result_"
442def exec_embedded(code: str, _path: str | None = None, **context: Any) -> Any:
443 """Execute embedded python and return the extracted value. This is the last
444 assignment in the embedded python. The embedded python must have the last line as a value
445 or an assignment.
447 Note:
448 No local or global variables will be retained from the embedded python code.
450 Args:
451 code (str): The embedded python code.
452 **context (Any): The additional context to provide to the embedded python.
454 Returns:
455 Any: The value of the last assignment or value defined
456 """
457 from phml.utilities import blank
459 context = {
460 "blank": blank,
461 **context,
462 }
464 # last line must be an assignment or the value to be used
465 with EmbeddedTryCatch(_path, code):
466 code = normalize_indent(code)
467 AST = ast.parse(code)
468 _validate_kwargs(AST, context)
470 last = AST.body[-1]
471 returns = [ret for ret in AST.body if isinstance(ret, ast.Return)]
473 if len(returns) > 0:
474 last = returns[0]
475 idx = AST.body.index(last)
477 n_expr = ast.Name(id=RESULT, ctx=ast.Store())
478 n_assign = ast.Assign(targets=[n_expr], value=last.value)
480 update_ast_node_pos(dest=n_expr, source=last)
481 update_ast_node_pos(dest=n_assign, source=last)
483 AST.body = [*AST.body[:idx], n_assign]
484 elif isinstance(last, ast.Expr):
485 n_expr = ast.Name(id=RESULT, ctx=ast.Store())
486 n_assign = ast.Assign(targets=[n_expr], value=last.value)
488 update_ast_node_pos(dest=n_expr, source=last)
489 update_ast_node_pos(dest=n_assign, source=last)
491 AST.body[-1] = n_assign
492 elif isinstance(last, ast.Assign):
493 n_expr = ast.Name(id=RESULT, ctx=ast.Store())
494 update_ast_node_pos(dest=n_expr, source=last)
495 last.targets.append(n_expr)
497 ccode = compile(AST, "_phml_embedded_", "exec")
498 local_env = {}
499 exec(ccode, {**context}, local_env)
500 return local_env[RESULT]
503def exec_embedded_blocks(code: str, _path: str = "", **context: dict[str, Any]):
504 """Execute embedded python inside `{{}}` blocks. The resulting values are subsituted
505 in for the found blocks.
507 Note:
508 No local or global variables will be retained from the embedded python code.
510 Args:
511 code (str): The embedded python code.
512 **context (Any): The additional context to provide to the embedded python.
514 Returns:
515 str: The value of the passed in string with the python blocks replaced.
516 """
518 result = ""
519 data = []
520 next_block = re.search(r"\{\{", code)
521 while next_block is not None:
522 start = next_block.start()
523 if start > 0:
524 result += code[:start]
525 code = code[start + 2 :]
527 balance = 2
528 index = 0
529 while balance > 0 and index < len(code):
530 if code[index] == "}":
531 balance -= 1
532 elif code[index] == "{":
533 balance += 1
534 index += 1
536 result += "{}"
537 data.append(
538 str(
539 exec_embedded(
540 code[: index - 2],
541 _path + f" block #{len(data)+1}",
542 **context,
543 ),
544 ),
545 )
546 code = code[index + 1 :]
547 next_block = re.search(r"(?<!\\)\{\{", code)
549 if len(code) > 0:
550 result += code
552 return result.format(*data)