Coverage for /opt/homebrew/lib/python3.11/site-packages/_pytest/pathlib.py: 40%
378 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-05-04 13:14 +0700
« prev ^ index » next coverage.py v7.2.3, created at 2023-05-04 13:14 +0700
1import atexit
2import contextlib
3import fnmatch
4import importlib.util
5import itertools
6import os
7import shutil
8import sys
9import uuid
10import warnings
11from enum import Enum
12from errno import EBADF
13from errno import ELOOP
14from errno import ENOENT
15from errno import ENOTDIR
16from functools import partial
17from os.path import expanduser
18from os.path import expandvars
19from os.path import isabs
20from os.path import sep
21from pathlib import Path
22from pathlib import PurePath
23from posixpath import sep as posix_sep
24from types import ModuleType
25from typing import Callable
26from typing import Dict
27from typing import Iterable
28from typing import Iterator
29from typing import Optional
30from typing import Set
31from typing import TypeVar
32from typing import Union
34from _pytest.compat import assert_never
35from _pytest.outcomes import skip
36from _pytest.warning_types import PytestWarning
38LOCK_TIMEOUT = 60 * 60 * 24 * 3
41_AnyPurePath = TypeVar("_AnyPurePath", bound=PurePath)
43# The following function, variables and comments were
44# copied from cpython 3.9 Lib/pathlib.py file.
46# EBADF - guard against macOS `stat` throwing EBADF
47_IGNORED_ERRORS = (ENOENT, ENOTDIR, EBADF, ELOOP)
49_IGNORED_WINERRORS = (
50 21, # ERROR_NOT_READY - drive exists but is not accessible
51 1921, # ERROR_CANT_RESOLVE_FILENAME - fix for broken symlink pointing to itself
52)
55def _ignore_error(exception):
56 return (
57 getattr(exception, "errno", None) in _IGNORED_ERRORS
58 or getattr(exception, "winerror", None) in _IGNORED_WINERRORS
59 )
62def get_lock_path(path: _AnyPurePath) -> _AnyPurePath:
63 return path.joinpath(".lock")
66def on_rm_rf_error(func, path: str, exc, *, start_path: Path) -> bool:
67 """Handle known read-only errors during rmtree.
69 The returned value is used only by our own tests.
70 """
71 exctype, excvalue = exc[:2]
73 # Another process removed the file in the middle of the "rm_rf" (xdist for example).
74 # More context: https://github.com/pytest-dev/pytest/issues/5974#issuecomment-543799018
75 if isinstance(excvalue, FileNotFoundError):
76 return False
78 if not isinstance(excvalue, PermissionError):
79 warnings.warn(
80 PytestWarning(f"(rm_rf) error removing {path}\n{exctype}: {excvalue}")
81 )
82 return False
84 if func not in (os.rmdir, os.remove, os.unlink):
85 if func not in (os.open,):
86 warnings.warn(
87 PytestWarning(
88 "(rm_rf) unknown function {} when removing {}:\n{}: {}".format(
89 func, path, exctype, excvalue
90 )
91 )
92 )
93 return False
95 # Chmod + retry.
96 import stat
98 def chmod_rw(p: str) -> None:
99 mode = os.stat(p).st_mode
100 os.chmod(p, mode | stat.S_IRUSR | stat.S_IWUSR)
102 # For files, we need to recursively go upwards in the directories to
103 # ensure they all are also writable.
104 p = Path(path)
105 if p.is_file():
106 for parent in p.parents:
107 chmod_rw(str(parent))
108 # Stop when we reach the original path passed to rm_rf.
109 if parent == start_path:
110 break
111 chmod_rw(str(path))
113 func(path)
114 return True
117def ensure_extended_length_path(path: Path) -> Path:
118 """Get the extended-length version of a path (Windows).
120 On Windows, by default, the maximum length of a path (MAX_PATH) is 260
121 characters, and operations on paths longer than that fail. But it is possible
122 to overcome this by converting the path to "extended-length" form before
123 performing the operation:
124 https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation
126 On Windows, this function returns the extended-length absolute version of path.
127 On other platforms it returns path unchanged.
128 """
129 if sys.platform.startswith("win32"):
130 path = path.resolve()
131 path = Path(get_extended_length_path_str(str(path)))
132 return path
135def get_extended_length_path_str(path: str) -> str:
136 """Convert a path to a Windows extended length path."""
137 long_path_prefix = "\\\\?\\"
138 unc_long_path_prefix = "\\\\?\\UNC\\"
139 if path.startswith((long_path_prefix, unc_long_path_prefix)):
140 return path
141 # UNC
142 if path.startswith("\\\\"):
143 return unc_long_path_prefix + path[2:]
144 return long_path_prefix + path
147def rm_rf(path: Path) -> None:
148 """Remove the path contents recursively, even if some elements
149 are read-only."""
150 path = ensure_extended_length_path(path)
151 onerror = partial(on_rm_rf_error, start_path=path)
152 shutil.rmtree(str(path), onerror=onerror)
155def find_prefixed(root: Path, prefix: str) -> Iterator[Path]:
156 """Find all elements in root that begin with the prefix, case insensitive."""
157 l_prefix = prefix.lower()
158 for x in root.iterdir():
159 if x.name.lower().startswith(l_prefix):
160 yield x
163def extract_suffixes(iter: Iterable[PurePath], prefix: str) -> Iterator[str]:
164 """Return the parts of the paths following the prefix.
166 :param iter: Iterator over path names.
167 :param prefix: Expected prefix of the path names.
168 """
169 p_len = len(prefix)
170 for p in iter:
171 yield p.name[p_len:]
174def find_suffixes(root: Path, prefix: str) -> Iterator[str]:
175 """Combine find_prefixes and extract_suffixes."""
176 return extract_suffixes(find_prefixed(root, prefix), prefix)
179def parse_num(maybe_num) -> int:
180 """Parse number path suffixes, returns -1 on error."""
181 try:
182 return int(maybe_num)
183 except ValueError:
184 return -1
187def _force_symlink(
188 root: Path, target: Union[str, PurePath], link_to: Union[str, Path]
189) -> None:
190 """Helper to create the current symlink.
192 It's full of race conditions that are reasonably OK to ignore
193 for the context of best effort linking to the latest test run.
195 The presumption being that in case of much parallelism
196 the inaccuracy is going to be acceptable.
197 """
198 current_symlink = root.joinpath(target)
199 try:
200 current_symlink.unlink()
201 except OSError:
202 pass
203 try:
204 current_symlink.symlink_to(link_to)
205 except Exception:
206 pass
209def make_numbered_dir(root: Path, prefix: str, mode: int = 0o700) -> Path:
210 """Create a directory with an increased number as suffix for the given prefix."""
211 for i in range(10):
212 # try up to 10 times to create the folder
213 max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1)
214 new_number = max_existing + 1
215 new_path = root.joinpath(f"{prefix}{new_number}")
216 try:
217 new_path.mkdir(mode=mode)
218 except Exception:
219 pass
220 else:
221 _force_symlink(root, prefix + "current", new_path)
222 return new_path
223 else:
224 raise OSError(
225 "could not create numbered dir with prefix "
226 "{prefix} in {root} after 10 tries".format(prefix=prefix, root=root)
227 )
230def create_cleanup_lock(p: Path) -> Path:
231 """Create a lock to prevent premature folder cleanup."""
232 lock_path = get_lock_path(p)
233 try:
234 fd = os.open(str(lock_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
235 except FileExistsError as e:
236 raise OSError(f"cannot create lockfile in {p}") from e
237 else:
238 pid = os.getpid()
239 spid = str(pid).encode()
240 os.write(fd, spid)
241 os.close(fd)
242 if not lock_path.is_file():
243 raise OSError("lock path got renamed after successful creation")
244 return lock_path
247def register_cleanup_lock_removal(lock_path: Path, register=atexit.register):
248 """Register a cleanup function for removing a lock, by default on atexit."""
249 pid = os.getpid()
251 def cleanup_on_exit(lock_path: Path = lock_path, original_pid: int = pid) -> None:
252 current_pid = os.getpid()
253 if current_pid != original_pid:
254 # fork
255 return
256 try:
257 lock_path.unlink()
258 except OSError:
259 pass
261 return register(cleanup_on_exit)
264def maybe_delete_a_numbered_dir(path: Path) -> None:
265 """Remove a numbered directory if its lock can be obtained and it does
266 not seem to be in use."""
267 path = ensure_extended_length_path(path)
268 lock_path = None
269 try:
270 lock_path = create_cleanup_lock(path)
271 parent = path.parent
273 garbage = parent.joinpath(f"garbage-{uuid.uuid4()}")
274 path.rename(garbage)
275 rm_rf(garbage)
276 except OSError:
277 # known races:
278 # * other process did a cleanup at the same time
279 # * deletable folder was found
280 # * process cwd (Windows)
281 return
282 finally:
283 # If we created the lock, ensure we remove it even if we failed
284 # to properly remove the numbered dir.
285 if lock_path is not None:
286 try:
287 lock_path.unlink()
288 except OSError:
289 pass
292def ensure_deletable(path: Path, consider_lock_dead_if_created_before: float) -> bool:
293 """Check if `path` is deletable based on whether the lock file is expired."""
294 if path.is_symlink():
295 return False
296 lock = get_lock_path(path)
297 try:
298 if not lock.is_file():
299 return True
300 except OSError:
301 # we might not have access to the lock file at all, in this case assume
302 # we don't have access to the entire directory (#7491).
303 return False
304 try:
305 lock_time = lock.stat().st_mtime
306 except Exception:
307 return False
308 else:
309 if lock_time < consider_lock_dead_if_created_before:
310 # We want to ignore any errors while trying to remove the lock such as:
311 # - PermissionDenied, like the file permissions have changed since the lock creation;
312 # - FileNotFoundError, in case another pytest process got here first;
313 # and any other cause of failure.
314 with contextlib.suppress(OSError):
315 lock.unlink()
316 return True
317 return False
320def try_cleanup(path: Path, consider_lock_dead_if_created_before: float) -> None:
321 """Try to cleanup a folder if we can ensure it's deletable."""
322 if ensure_deletable(path, consider_lock_dead_if_created_before):
323 maybe_delete_a_numbered_dir(path)
326def cleanup_candidates(root: Path, prefix: str, keep: int) -> Iterator[Path]:
327 """List candidates for numbered directories to be removed - follows py.path."""
328 max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1)
329 max_delete = max_existing - keep
330 paths = find_prefixed(root, prefix)
331 paths, paths2 = itertools.tee(paths)
332 numbers = map(parse_num, extract_suffixes(paths2, prefix))
333 for path, number in zip(paths, numbers):
334 if number <= max_delete:
335 yield path
338def cleanup_numbered_dir(
339 root: Path, prefix: str, keep: int, consider_lock_dead_if_created_before: float
340) -> None:
341 """Cleanup for lock driven numbered directories."""
342 for path in cleanup_candidates(root, prefix, keep):
343 try_cleanup(path, consider_lock_dead_if_created_before)
344 for path in root.glob("garbage-*"):
345 try_cleanup(path, consider_lock_dead_if_created_before)
348def make_numbered_dir_with_cleanup(
349 root: Path,
350 prefix: str,
351 keep: int,
352 lock_timeout: float,
353 mode: int,
354) -> Path:
355 """Create a numbered dir with a cleanup lock and remove old ones."""
356 e = None
357 for i in range(10):
358 try:
359 p = make_numbered_dir(root, prefix, mode)
360 lock_path = create_cleanup_lock(p)
361 register_cleanup_lock_removal(lock_path)
362 except Exception as exc:
363 e = exc
364 else:
365 consider_lock_dead_if_created_before = p.stat().st_mtime - lock_timeout
366 # Register a cleanup for program exit
367 atexit.register(
368 cleanup_numbered_dir,
369 root,
370 prefix,
371 keep,
372 consider_lock_dead_if_created_before,
373 )
374 return p
375 assert e is not None
376 raise e
379def resolve_from_str(input: str, rootpath: Path) -> Path:
380 input = expanduser(input)
381 input = expandvars(input)
382 if isabs(input):
383 return Path(input)
384 else:
385 return rootpath.joinpath(input)
388def fnmatch_ex(pattern: str, path: Union[str, "os.PathLike[str]"]) -> bool:
389 """A port of FNMatcher from py.path.common which works with PurePath() instances.
391 The difference between this algorithm and PurePath.match() is that the
392 latter matches "**" glob expressions for each part of the path, while
393 this algorithm uses the whole path instead.
395 For example:
396 "tests/foo/bar/doc/test_foo.py" matches pattern "tests/**/doc/test*.py"
397 with this algorithm, but not with PurePath.match().
399 This algorithm was ported to keep backward-compatibility with existing
400 settings which assume paths match according this logic.
402 References:
403 * https://bugs.python.org/issue29249
404 * https://bugs.python.org/issue34731
405 """
406 path = PurePath(path)
407 iswin32 = sys.platform.startswith("win")
409 if iswin32 and sep not in pattern and posix_sep in pattern:
410 # Running on Windows, the pattern has no Windows path separators,
411 # and the pattern has one or more Posix path separators. Replace
412 # the Posix path separators with the Windows path separator.
413 pattern = pattern.replace(posix_sep, sep)
415 if sep not in pattern:
416 name = path.name
417 else:
418 name = str(path)
419 if path.is_absolute() and not os.path.isabs(pattern):
420 pattern = f"*{os.sep}{pattern}"
421 return fnmatch.fnmatch(name, pattern)
424def parts(s: str) -> Set[str]:
425 parts = s.split(sep)
426 return {sep.join(parts[: i + 1]) or sep for i in range(len(parts))}
429def symlink_or_skip(src, dst, **kwargs):
430 """Make a symlink, or skip the test in case symlinks are not supported."""
431 try:
432 os.symlink(str(src), str(dst), **kwargs)
433 except OSError as e:
434 skip(f"symlinks not supported: {e}")
437class ImportMode(Enum):
438 """Possible values for `mode` parameter of `import_path`."""
440 prepend = "prepend"
441 append = "append"
442 importlib = "importlib"
445class ImportPathMismatchError(ImportError):
446 """Raised on import_path() if there is a mismatch of __file__'s.
448 This can happen when `import_path` is called multiple times with different filenames that has
449 the same basename but reside in packages
450 (for example "/tests1/test_foo.py" and "/tests2/test_foo.py").
451 """
454def import_path(
455 p: Union[str, "os.PathLike[str]"],
456 *,
457 mode: Union[str, ImportMode] = ImportMode.prepend,
458 root: Path,
459) -> ModuleType:
460 """Import and return a module from the given path, which can be a file (a module) or
461 a directory (a package).
463 The import mechanism used is controlled by the `mode` parameter:
465 * `mode == ImportMode.prepend`: the directory containing the module (or package, taking
466 `__init__.py` files into account) will be put at the *start* of `sys.path` before
467 being imported with `importlib.import_module`.
469 * `mode == ImportMode.append`: same as `prepend`, but the directory will be appended
470 to the end of `sys.path`, if not already in `sys.path`.
472 * `mode == ImportMode.importlib`: uses more fine control mechanisms provided by `importlib`
473 to import the module, which avoids having to muck with `sys.path` at all. It effectively
474 allows having same-named test modules in different places.
476 :param root:
477 Used as an anchor when mode == ImportMode.importlib to obtain
478 a unique name for the module being imported so it can safely be stored
479 into ``sys.modules``.
481 :raises ImportPathMismatchError:
482 If after importing the given `path` and the module `__file__`
483 are different. Only raised in `prepend` and `append` modes.
484 """
485 mode = ImportMode(mode)
487 path = Path(p)
489 if not path.exists():
490 raise ImportError(path)
492 if mode is ImportMode.importlib:
493 module_name = module_name_from_path(path, root)
495 for meta_importer in sys.meta_path:
496 spec = meta_importer.find_spec(module_name, [str(path.parent)])
497 if spec is not None:
498 break
499 else:
500 spec = importlib.util.spec_from_file_location(module_name, str(path))
502 if spec is None:
503 raise ImportError(f"Can't find module {module_name} at location {path}")
504 mod = importlib.util.module_from_spec(spec)
505 sys.modules[module_name] = mod
506 spec.loader.exec_module(mod) # type: ignore[union-attr]
507 insert_missing_modules(sys.modules, module_name)
508 return mod
510 pkg_path = resolve_package_path(path)
511 if pkg_path is not None:
512 pkg_root = pkg_path.parent
513 names = list(path.with_suffix("").relative_to(pkg_root).parts)
514 if names[-1] == "__init__":
515 names.pop()
516 module_name = ".".join(names)
517 else:
518 pkg_root = path.parent
519 module_name = path.stem
521 # Change sys.path permanently: restoring it at the end of this function would cause surprising
522 # problems because of delayed imports: for example, a conftest.py file imported by this function
523 # might have local imports, which would fail at runtime if we restored sys.path.
524 if mode is ImportMode.append:
525 if str(pkg_root) not in sys.path:
526 sys.path.append(str(pkg_root))
527 elif mode is ImportMode.prepend:
528 if str(pkg_root) != sys.path[0]:
529 sys.path.insert(0, str(pkg_root))
530 else:
531 assert_never(mode)
533 importlib.import_module(module_name)
535 mod = sys.modules[module_name]
536 if path.name == "__init__.py":
537 return mod
539 ignore = os.environ.get("PY_IGNORE_IMPORTMISMATCH", "")
540 if ignore != "1":
541 module_file = mod.__file__
542 if module_file is None:
543 raise ImportPathMismatchError(module_name, module_file, path)
545 if module_file.endswith((".pyc", ".pyo")):
546 module_file = module_file[:-1]
547 if module_file.endswith(os.path.sep + "__init__.py"):
548 module_file = module_file[: -(len(os.path.sep + "__init__.py"))]
550 try:
551 is_same = _is_same(str(path), module_file)
552 except FileNotFoundError:
553 is_same = False
555 if not is_same:
556 raise ImportPathMismatchError(module_name, module_file, path)
558 return mod
561# Implement a special _is_same function on Windows which returns True if the two filenames
562# compare equal, to circumvent os.path.samefile returning False for mounts in UNC (#7678).
563if sys.platform.startswith("win"):
565 def _is_same(f1: str, f2: str) -> bool:
566 return Path(f1) == Path(f2) or os.path.samefile(f1, f2)
568else:
570 def _is_same(f1: str, f2: str) -> bool:
571 return os.path.samefile(f1, f2)
574def module_name_from_path(path: Path, root: Path) -> str:
575 """
576 Return a dotted module name based on the given path, anchored on root.
578 For example: path="projects/src/tests/test_foo.py" and root="/projects", the
579 resulting module name will be "src.tests.test_foo".
580 """
581 path = path.with_suffix("")
582 try:
583 relative_path = path.relative_to(root)
584 except ValueError:
585 # If we can't get a relative path to root, use the full path, except
586 # for the first part ("d:\\" or "/" depending on the platform, for example).
587 path_parts = path.parts[1:]
588 else:
589 # Use the parts for the relative path to the root path.
590 path_parts = relative_path.parts
592 return ".".join(path_parts)
595def insert_missing_modules(modules: Dict[str, ModuleType], module_name: str) -> None:
596 """
597 Used by ``import_path`` to create intermediate modules when using mode=importlib.
599 When we want to import a module as "src.tests.test_foo" for example, we need
600 to create empty modules "src" and "src.tests" after inserting "src.tests.test_foo",
601 otherwise "src.tests.test_foo" is not importable by ``__import__``.
602 """
603 module_parts = module_name.split(".")
604 while module_name:
605 if module_name not in modules:
606 try:
607 # If sys.meta_path is empty, calling import_module will issue
608 # a warning and raise ModuleNotFoundError. To avoid the
609 # warning, we check sys.meta_path explicitly and raise the error
610 # ourselves to fall back to creating a dummy module.
611 if not sys.meta_path:
612 raise ModuleNotFoundError
613 importlib.import_module(module_name)
614 except ModuleNotFoundError:
615 module = ModuleType(
616 module_name,
617 doc="Empty module created by pytest's importmode=importlib.",
618 )
619 modules[module_name] = module
620 module_parts.pop(-1)
621 module_name = ".".join(module_parts)
624def resolve_package_path(path: Path) -> Optional[Path]:
625 """Return the Python package path by looking for the last
626 directory upwards which still contains an __init__.py.
628 Returns None if it can not be determined.
629 """
630 result = None
631 for parent in itertools.chain((path,), path.parents):
632 if parent.is_dir():
633 if not parent.joinpath("__init__.py").is_file():
634 break
635 if not parent.name.isidentifier():
636 break
637 result = parent
638 return result
641def visit(
642 path: Union[str, "os.PathLike[str]"], recurse: Callable[["os.DirEntry[str]"], bool]
643) -> Iterator["os.DirEntry[str]"]:
644 """Walk a directory recursively, in breadth-first order.
646 Entries at each directory level are sorted.
647 """
649 # Skip entries with symlink loops and other brokenness, so the caller doesn't
650 # have to deal with it.
651 entries = []
652 for entry in os.scandir(path):
653 try:
654 entry.is_file()
655 except OSError as err:
656 if _ignore_error(err):
657 continue
658 raise
659 entries.append(entry)
661 entries.sort(key=lambda entry: entry.name)
663 yield from entries
665 for entry in entries:
666 if entry.is_dir() and recurse(entry):
667 yield from visit(entry.path, recurse)
670def absolutepath(path: Union[Path, str]) -> Path:
671 """Convert a path to an absolute path using os.path.abspath.
673 Prefer this over Path.resolve() (see #6523).
674 Prefer this over Path.absolute() (not public, doesn't normalize).
675 """
676 return Path(os.path.abspath(str(path)))
679def commonpath(path1: Path, path2: Path) -> Optional[Path]:
680 """Return the common part shared with the other path, or None if there is
681 no common part.
683 If one path is relative and one is absolute, returns None.
684 """
685 try:
686 return Path(os.path.commonpath((str(path1), str(path2))))
687 except ValueError:
688 return None
691def bestrelpath(directory: Path, dest: Path) -> str:
692 """Return a string which is a relative path from directory to dest such
693 that directory/bestrelpath == dest.
695 The paths must be either both absolute or both relative.
697 If no such path can be determined, returns dest.
698 """
699 assert isinstance(directory, Path)
700 assert isinstance(dest, Path)
701 if dest == directory:
702 return os.curdir
703 # Find the longest common directory.
704 base = commonpath(directory, dest)
705 # Can be the case on Windows for two absolute paths on different drives.
706 # Can be the case for two relative paths without common prefix.
707 # Can be the case for a relative path and an absolute path.
708 if not base:
709 return str(dest)
710 reldirectory = directory.relative_to(base)
711 reldest = dest.relative_to(base)
712 return os.path.join(
713 # Back from directory to base.
714 *([os.pardir] * len(reldirectory.parts)),
715 # Forward from base to dest.
716 *reldest.parts,
717 )
720# Originates from py. path.local.copy(), with siginficant trims and adjustments.
721# TODO(py38): Replace with shutil.copytree(..., symlinks=True, dirs_exist_ok=True)
722def copytree(source: Path, target: Path) -> None:
723 """Recursively copy a source directory to target."""
724 assert source.is_dir()
725 for entry in visit(source, recurse=lambda entry: not entry.is_symlink()):
726 x = Path(entry)
727 relpath = x.relative_to(source)
728 newx = target / relpath
729 newx.parent.mkdir(exist_ok=True)
730 if x.is_symlink():
731 newx.symlink_to(os.readlink(x))
732 elif x.is_file():
733 shutil.copyfile(x, newx)
734 elif x.is_dir():
735 newx.mkdir(exist_ok=True)