Coverage for /home/tbone/.local/share/hatch/env/virtual/importnb-aVRh-lqt/released.interactive/lib/python3.9/site-packages/importnb/loader.py: 93%
229 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-24 15:27 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-24 15:27 -0700
1# coding: utf-8
2"""# `loader`
4Combine the __import__ finder with the loader.
5"""
8import ast
9from contextlib import contextmanager
10import re
11import sys
12from telnetlib import DO
13import textwrap
14from dataclasses import asdict, dataclass, field
15from functools import partial
16from importlib import reload
17from importlib._bootstrap import _init_module_attrs, _requires_builtin
18from importlib._bootstrap_external import FileFinder, decode_source
19from importlib.machinery import ModuleSpec, SourceFileLoader
20from importlib.util import LazyLoader, find_spec
21from pathlib import Path
22from types import ModuleType
24from . import get_ipython
25from .decoder import LineCacheNotebookDecoder, quote
26from .docstrings import update_docstring
27from .finder import FuzzyFinder, get_loader_details, get_loader_index
29_GTE38 = sys.version_info.major == 3 and sys.version_info.minor >= 8
31try:
32 import IPython
33 from IPython.core.inputsplitter import IPythonInputSplitter
35 dedent = IPythonInputSplitter(
36 line_input_checker=False,
37 physical_line_transforms=[
38 IPython.core.inputsplitter.leading_indent(),
39 IPython.core.inputsplitter.ipy_prompt(),
40 IPython.core.inputsplitter.cellmagic(end_on_blank_line=False),
41 ],
42 ).transform_cell
43except ModuleNotFoundError:
45 def dedent(body):
46 from textwrap import dedent, indent
48 if MAGIC.match(body):
49 return indent(body, "# ")
50 return dedent(body)
53__all__ = "Notebook", "reload"
56MAGIC = re.compile("^\s*%{2}", re.MULTILINE)
59@dataclass
60class Interface:
61 name: str = None
62 path: str = None
63 lazy: bool = False
64 extensions: tuple = field(default_factory=[".ipy", ".ipynb"].copy)
65 include_fuzzy_finder: bool = True
67 include_markdown_docstring: bool = True
68 only_defs: bool = False
69 no_magic: bool = False
70 _loader_hook_position: int = field(default=0, repr=False)
72 def __new__(cls, name=None, path=None, **kwargs):
73 kwargs.update(name=name, path=path)
74 self = super().__new__(cls)
75 self.__init__(**kwargs)
76 return self
79class BaseLoader(Interface, SourceFileLoader):
80 """The simplest implementation of a Notebook Source File Loader."""
82 @property
83 def loader(self):
84 """Create a lazy loader source file loader."""
85 loader = type(self)
86 if self.lazy and (sys.version_info.major, sys.version_info.minor) != (3, 4):
87 loader = LazyLoader.factory(loader)
88 # Strip the leading underscore from slots
89 params = asdict(self)
90 params.pop("name")
91 params.pop("path")
92 return partial(loader, **params)
94 @property
95 def finder(self):
96 """Permit fuzzy finding of files with special characters."""
97 return self.include_fuzzy_finder and FuzzyFinder or FileFinder
99 def translate(self, source):
100 if self.path and self.path.endswith(".ipynb"):
101 return LineCacheNotebookDecoder(
102 code=self.code, raw=self.raw, markdown=self.markdown
103 ).decode(source, self.path)
104 return self.code(source)
106 def get_data(self, path):
107 """Needs to return the string source for the module."""
108 return self.translate(self.decode())
110 def create_module(self, spec):
111 module = ModuleType(str(spec.name))
112 _init_module_attrs(spec, module)
113 if self.name:
114 module.__name__ = self.name
115 if getattr(spec, "alias", None):
116 # put a fuzzy spec on the modules to avoid re importing it.
117 # there is a funky trick you do with the fuzzy finder where you
118 # load multiple versions with different finders.
120 sys.modules[spec.alias] = module
121 module.get_ipython = get_ipython
122 return module
124 def decode(self):
125 return decode_source(super().get_data(self.path))
127 def code(self, str):
128 return dedent(str)
130 def markdown(self, str):
131 return quote(str)
133 def raw(self, str):
134 return comment(str)
136 def visit(self, node):
137 return node
139 @classmethod
140 @_requires_builtin
141 def is_package(cls, fullname):
142 """Return False as built-in modules are never packages."""
143 if "." not in fullname:
144 return True
145 return super().is_package(fullname)
147 get_source = get_data
149 def __enter__(self):
150 path_id, loader_id, details = get_loader_index(".py")
151 for _, e in details:
152 if all(map(e.__contains__, self.extensions)):
153 self._loader_hook_position = None
154 return self
155 else:
156 self._loader_hook_position = loader_id + 1
157 details.insert(self._loader_hook_position, (self.loader, self.extensions))
158 sys.path_hooks[path_id] = self.finder.path_hook(*details)
159 sys.path_importer_cache.clear()
160 return self
162 def __exit__(self, *excepts):
163 if self._loader_hook_position is not None:
164 path_id, details = get_loader_details()
165 details.pop(self._loader_hook_position)
166 sys.path_hooks[path_id] = self.finder.path_hook(*details)
167 sys.path_importer_cache.clear()
170class FileModuleSpec(ModuleSpec):
171 def __init__(self, *args, **kwargs):
172 super().__init__(*args, **kwargs)
173 self._set_fileattr = True
176def comment(str):
177 return textwrap.indent(str, "# ")
180class DefsOnly(ast.NodeTransformer):
181 INCLUDE = ast.Import, ast.ImportFrom, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef
183 def visit_Module(self, node):
184 args = ([x for x in node.body if isinstance(x, self.INCLUDE)],)
185 if _GTE38:
186 args += (node.type_ignores,)
187 return ast.Module(*args)
190class Notebook(BaseLoader):
191 """Notebook is a user friendly file finder and module loader for notebook source code.
193 > Remember, restart and run all or it didn't happen.
195 Notebook provides several useful options.
197 * Lazy module loading. A module is executed the first time it is used in a script.
198 """
200 def parse(self, nodes):
201 return ast.parse(nodes, self.path)
203 def visit(self, nodes):
204 if self.only_defs:
205 nodes = DefsOnly().visit(nodes)
206 return nodes
208 def code(self, str):
209 if self.no_magic:
210 if MAGIC.match(str):
211 return comment(str)
212 return super().code(str)
214 def source_to_code(self, nodes, path, *, _optimize=-1):
215 """* Convert the current source to ast
216 * Apply ast transformers.
217 * Compile the code."""
218 if not isinstance(nodes, ast.Module):
219 nodes = self.parse(nodes)
220 if self.include_markdown_docstring:
221 nodes = update_docstring(nodes)
222 return super().source_to_code(
223 ast.fix_missing_locations(self.visit(nodes)), path, _optimize=_optimize
224 )
226 @classmethod
227 def load_file(cls, filename, main=True, **kwargs):
228 """Import a notebook as a module from a filename.
230 dir: The directory to load the file from.
231 main: Load the module in the __main__ context.
233 > assert Notebook.load('loader.ipynb')
234 """
235 name = main and "__main__" or filename
236 loader = cls(name, str(filename), **kwargs)
237 spec = FileModuleSpec(name, loader, origin=loader.path)
238 module = loader.create_module(spec)
239 loader.exec_module(module)
240 return module
242 @classmethod
243 def load_module(cls, module, main=False, **kwargs):
244 """Import a notebook as a module.
246 dir: The directory to load the file from.
247 main: Load the module in the __main__ context.
249 > assert Notebook.load('loader.ipynb')
250 """
251 from runpy import _run_module_as_main, run_module
253 with cls() as loader:
254 if main:
255 return _dict_module(_run_module_as_main(module))
256 else:
257 spec = find_spec(module)
259 m = spec.loader.create_module(spec)
260 spec.loader.exec_module(m)
261 return m
263 @classmethod
264 def load_argv(cls, argv=None, *, parser=None):
265 import sys
267 if parser is None:
268 parser = cls.get_argparser()
270 if argv is None:
271 from sys import argv
273 argv = argv[1:]
275 if isinstance(argv, str):
276 from shlex import split
278 argv = split(argv)
280 module = cls.load_ns(parser.parse_args(argv))
281 if module is None:
282 return parser.print_help()
284 return module
286 @classmethod
287 def load_ns(cls, ns):
288 from sys import path
290 if ns.tasks:
291 from doit.doit_cmd import DoitMain
292 from doit.cmd_base import ModuleTaskLoader
294 if ns.code:
295 with main_argv(sys.argv[0], ns.args):
296 result = cls.load_code(ns.code)
297 elif ns.module:
298 path.insert(0, ns.dir) if ns.dir else ... if "" in path else path.insert(0, "")
299 with main_argv(ns.module, ns.args):
300 result = cls.load_module(ns.module, main=True)
301 elif ns.file:
302 where = Path(ns.dir, ns.file) if ns.dir else Path(ns.file)
303 with main_argv(str(where), ns.args):
304 result = cls.load_file(ns.file)
305 else:
306 return
308 if ns.tasks:
309 DoitMain(ModuleTaskLoader(result)).run(ns.args)
310 return result
312 @classmethod
313 def load_code(cls, code, argv=None, mod_name=None, script_name=None, main=False):
314 from runpy import _run_module_code
316 self = cls()
317 name = main and "__main__" or mod_name or "<markdown code>"
319 return _dict_module(
320 _run_module_code(self.translate(code), mod_name=name, script_name=script_name)
321 )
323 @staticmethod
324 def get_argparser(parser=None):
325 from argparse import REMAINDER, ArgumentParser
327 if parser is None:
328 parser = ArgumentParser("importnb", description="run notebooks as python code")
329 parser.add_argument("file", nargs="?", help="run a file")
330 parser.add_argument("args", nargs=REMAINDER, help="arguments to pass to script")
331 parser.add_argument("-m", "--module", help="run a module")
332 parser.add_argument("-c", "--code", help="run raw code")
333 parser.add_argument("-d", "--dir", help="path to run script in")
334 parser.add_argument("-t", "--tasks", action="store_true", help="run doit tasks")
335 return parser
338def _dict_module(ns):
339 m = ModuleType(ns.get("__name__"), ns.get("__doc__"))
340 m.__dict__.update(ns)
341 return m
344@contextmanager
345def main_argv(prog, args=None):
346 if args is not None:
347 if isinstance(args, str):
348 from shlex import split
350 args = split(args)
351 args = [prog] + args
352 prior, sys.argv = sys.argv, args
353 yield
354 if args is not None:
355 sys.argv = prior