fix: include a .pth file in the distribution

nedbat · nedbat · commit 49a199289af6 · 2025-11-30T12:26:10.000-05:00
- metacov now has to run with -n0, I'm not sure why
- The extension has to be cleaned in a separate step, since it can be in use on
  Windows during the test step
diff --git a/.gitignore b/.gitignore
@@ -49,6 +49,7 @@ doc/sample_html_beta
 
 # Build intermediaries.
 tmp
+zzz_coverage.pth
 
 # OS junk
 .DS_Store
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -23,7 +23,11 @@ upgrading your version of coverage.py.
 Unreleased
 ----------
 
-Nothing yet.
+- Fix: coverage.py now includes a permanent .pth file in the distribution which
+  is installed with the code. This fixes `issue 2084`_: failure to patch for
+  subprocess measurement when site-packages is not writable.
+
+.. _issue 2084: https://github.com/coveragepy/coveragepy/issues/2084
 
 
 .. start-releases
diff --git a/Makefile b/Makefile
@@ -42,6 +42,7 @@ clean: debug_clean _clean_platform	## Remove artifacts of test execution, instal
 	@rm -f tests/covmain.zip tests/zipmods.zip tests/zip1.zip
 	@rm -rf doc/_build doc/_spell doc/sample_html_beta
 	@rm -rf tmp
+	@rm -rf zzz_coverage.pth
 	@rm -rf .*cache */.*cache */*/.*cache */*/*/.*cache .hypothesis
 	@rm -rf tests/actual
 	@-make -C tests/gold/html clean
@@ -215,7 +216,7 @@ relcommit2:				#: Commit the latest sample HTML report (see howto.txt).
 	git add doc/sample_html
 	git commit -am "docs: sample HTML for $$(python setup.py --version)"
 
-kit:					## Make the source distribution.
+kit:					## Make the source distribution and one wheel
 	python -m build
 
 pypi_upload:				## Upload the built distributions to PyPI.
@@ -235,7 +236,7 @@ kit_local:
 	# don't go crazy trying to figure out why our new code isn't installing.
 	find ~/Library/Caches/pip/wheels -name 'coverage-*' -delete
 
-build_kits:				## Trigger GitHub to build kits.
+build_kits:				## Trigger GitHub to build all the distributions.
 	python ci/trigger_action.py $(REPO_OWNER) build-kits
 
 tag:					#: Make a git tag with the version number (see howto.txt).
diff --git a/coverage/control.py b/coverage/control.py
@@ -288,9 +288,6 @@ def __init__(  # pylint: disable=too-many-arguments
         self._no_warn_slugs: set[str] = set()
         self._messages = messages
 
-        # If we're invoked from a .pth file, we shouldn't try to make another one.
-        self._make_pth_file = True
-
         # A record of all the warnings that have been issued.
         self._warnings: list[str] = []
 
@@ -719,7 +716,7 @@ def start(self) -> None:
         if self._auto_load:
             self.load()
 
-        apply_patches(self, self.config, self._debug, make_pth_file=self._make_pth_file)
+        apply_patches(self, self.config, self._debug)
 
         self._collector.start()
         self._started = True
@@ -1429,19 +1426,51 @@ def plugin_info(plugins: list[Any]) -> list[str]:
     )(Coverage)
 
 
-def process_startup(*, force: bool = False) -> Coverage | None:
+def process_startup(
+    *,
+    force: bool = False,
+    slug: str = "default",  # pylint: disable=unused-argument
+) -> Coverage | None:
     """Call this at Python start-up to perhaps measure coverage.
 
-    If the environment variable COVERAGE_PROCESS_START is defined, coverage
-    measurement is started.  The value of the variable is the config file
-    to use.
+    Coverage is started if one of these environment variables is defined:
+
+    - COVERAGE_PROCESS_START: the config file to use.
+    - COVERAGE_PROCESS_CONFIG: the config data to use, a string produced by
+      CoverageConfig.serialize, prefixed by ":data:".
+
+    If one of these is defined, it's used to get the coverage configuration,
+    and coverage is started.
 
     For details, see https://coverage.readthedocs.io/en/latest/subprocess.html.
 
     Returns the :class:`Coverage` instance that was started, or None if it was
     not started by this call.
 
     """
+    # This function can be called more than once in a process, for a few
+    # reasons.
+    #
+    # 1) We install a .pth file in multiple places reported by the site module,
+    #    so this function can be called more than once even in simple
+    #    situations.
+    #
+    # 2) In some virtualenv configurations the same directory is visible twice
+    #    in sys.path.  This means that the .pth file will be found twice and
+    #    executed twice, executing this function twice.
+    #    https://github.com/coveragepy/coveragepy/issues/340 has more details.
+    #
+    # We set a global flag (an attribute on this function) to indicate that
+    # coverage.py has already been started, so we can avoid starting it twice.
+
+    if not force and hasattr(process_startup, "coverage"):
+        # We've annotated this function before, so we must have already
+        # auto-started coverage.py in this process.  Nothing to do.
+        return None
+
+    # Now check for the environment variables that request coverage. If they
+    # aren't set, do nothing.
+
     config_data = os.getenv("COVERAGE_PROCESS_CONFIG")
     cps = os.getenv("COVERAGE_PROCESS_START")
     if config_data is not None:
@@ -1452,27 +1481,12 @@ def process_startup(*, force: bool = False) -> Coverage | None:
         # No request for coverage, nothing to do.
         return None
 
-    # This function can be called more than once in a process. This happens
-    # because some virtualenv configurations make the same directory visible
-    # twice in sys.path.  This means that the .pth file will be found twice,
-    # and executed twice, executing this function twice.  We set a global
-    # flag (an attribute on this function) to indicate that coverage.py has
-    # already been started, so we can avoid doing it twice.
-    #
-    # https://github.com/coveragepy/coveragepy/issues/340 has more details.
-
-    if not force and hasattr(process_startup, "coverage"):
-        # We've annotated this function before, so we must have already
-        # auto-started coverage.py in this process.  Nothing to do.
-        return None
-
     cov = Coverage(config_file=config_file)
     process_startup.coverage = cov  # type: ignore[attr-defined]
     cov._warn_no_data = False
     cov._warn_unimported_source = False
     cov._warn_preimported_source = False
     cov._auto_save = True
-    cov._make_pth_file = False
     cov.start()
 
     return cov
@@ -1482,7 +1496,7 @@ def _after_fork_in_child() -> None:
     """Used by patch=fork in the child process to restart coverage."""
     if cov := Coverage.current():
         cov.stop()
-    process_startup(force=True)
+    process_startup(force=True, slug="fork")
 
 
 def _prevent_sub_process_measurement() -> None:
diff --git a/coverage/patch.py b/coverage/patch.py
@@ -5,15 +5,12 @@
 
 from __future__ import annotations
 
-import atexit
 import contextlib
 import os
-import site
-from pathlib import Path
 from typing import TYPE_CHECKING, Any, NoReturn
 
 from coverage import env
-from coverage.debug import NoDebugging, DevNullDebug
+from coverage.debug import DevNullDebug
 from coverage.exceptions import ConfigError, CoverageException
 
 if TYPE_CHECKING:
@@ -26,8 +23,6 @@ def apply_patches(
     cov: Coverage,
     config: CoverageConfig,
     debug: TDebugCtl,
-    *,
-    make_pth_file: bool = True,
 ) -> None:
     """Apply invasive patches requested by `[run] patch=`."""
     debug = debug if debug.should("patch") else DevNullDebug()
@@ -43,7 +38,7 @@ def apply_patches(
                 _patch_fork(debug)
 
             case "subprocess":
-                _patch_subprocess(config, debug, make_pth_file)
+                _patch_subprocess(config, debug)
 
             case _:
                 raise ConfigError(f"Unknown patch {patch!r}")
@@ -116,55 +111,8 @@ def _patch_fork(debug: TDebugCtl) -> None:
     os.register_at_fork(after_in_child=_after_fork_in_child)
 
 
-def _patch_subprocess(config: CoverageConfig, debug: TDebugCtl, make_pth_file: bool) -> None:
+def _patch_subprocess(config: CoverageConfig, debug: TDebugCtl) -> None:
     """Write .pth files and set environment vars to measure subprocesses."""
     debug.write("Patching subprocess")
-
-    if make_pth_file:
-        pth_files = create_pth_files(debug)
-
-        def delete_pth_files() -> None:
-            for p in pth_files:
-                debug.write(f"Deleting subprocess .pth file: {str(p)!r}")
-                p.unlink(missing_ok=True)
-
-        atexit.register(delete_pth_files)
     assert config.config_file is not None
     os.environ["COVERAGE_PROCESS_CONFIG"] = config.serialize()
-
-
-# Writing .pth files is not obvious. On Windows, getsitepackages() returns two
-# directories.  A .pth file in the first will be run, but coverage isn't
-# importable yet.  We write into all the places we can, but with defensive
-# import code.
-
-PTH_CODE = """\
-try:
-    import coverage
-except:
-    pass
-else:
-    coverage.process_startup()
-"""
-
-PTH_TEXT = f"import sys; exec({PTH_CODE!r})\n"
-
-
-def create_pth_files(debug: TDebugCtl = NoDebugging()) -> list[Path]:
-    """Create .pth files for measuring subprocesses."""
-    pth_files = []
-    for pth_dir in site.getsitepackages():
-        pth_file = Path(pth_dir) / f"subcover_{os.getpid()}.pth"
-        try:
-            if debug.should("patch"):
-                debug.write(f"Writing subprocess .pth file: {str(pth_file)!r}")
-            pth_file.write_text(PTH_TEXT, encoding="utf-8")
-        except OSError as oserr:  # pragma: cant happen
-            if debug.should("patch"):
-                debug.write(f"Couldn't write subprocess .pth file: {oserr}")
-            continue
-        else:
-            pth_files.append(pth_file)
-    if debug.should("patch"):
-        debug.write(f"Subprocess .pth files created: {', '.join(map(str, pth_files)) or '-none-'}")
-    return pth_files
diff --git a/coverage/pth_file.py b/coverage/pth_file.py
@@ -0,0 +1,16 @@
+# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
+# For details: https://github.com/coveragepy/coveragepy/blob/main/NOTICE.txt
+
+# pylint: disable=missing-module-docstring
+# pragma: exclude file from coverage
+# This will become the .pth file for subprocesses.
+
+import os
+
+if os.getenv("COVERAGE_PROCESS_START") or os.getenv("COVERAGE_PROCESS_CONFIG"):
+    try:
+        import coverage
+    except:  # pylint: disable=bare-except
+        pass
+    else:
+        coverage.process_startup(slug="pth")
diff --git a/igor.py b/igor.py
@@ -52,7 +52,7 @@ def do_show_env():
         print(f"  {env} = {os.environ[env]!r}")
 
 
-def remove_extension(core):
+def do_clean_for_core(core):
     """Remove the compiled C extension, no matter what its name."""
 
     if core == "ctrace":
@@ -159,7 +159,6 @@ def make_env_id(core):
 
 def run_tests(core, *runner_args):
     """The actual running of tests."""
-    remove_extension(core)
     if "COVERAGE_TESTING" not in os.environ:
         os.environ["COVERAGE_TESTING"] = "True"
     print_banner(label_for_core(core))
@@ -183,9 +182,9 @@ def run_tests_with_coverage(core, *runner_args):
     # or the sys.path entries aren't created right?
     # There's an entry in "make clean" to get rid of this file.
     pth_dir = sysconfig.get_path("purelib")
-    pth_path = os.path.join(pth_dir, "zzz_metacov.pth")
+    pth_path = os.path.join(pth_dir, "zzy_metacov.pth")
     with open(pth_path, "w", encoding="utf-8") as pth_file:
-        pth_file.write("import coverage; coverage.process_startup()\n")
+        pth_file.write("import coverage; coverage.process_startup(slug='meta')\n")
 
     suffix = f"{make_env_id(core)}_{platform.platform()}"
     os.environ["COVERAGE_METAFILE"] = os.path.abspath(".metacov." + suffix)
@@ -211,7 +210,6 @@ def run_tests_with_coverage(core, *runner_args):
                 if getattr(mod, "__file__", "??").startswith(covdir):
                     covmods[name] = mod
                     del sys.modules[name]
-        remove_extension(core)
 
         import coverage  # pylint: disable=reimported
 
diff --git a/metacov.ini b/metacov.ini
@@ -11,9 +11,10 @@ branch = true
 data_file = ${COVERAGE_METAFILE-.metacov}
 parallel = true
 relative_files = true
-source =
-    ${COVERAGE_HOME-.}/coverage
-    ${COVERAGE_HOME-.}/tests
+include =
+    ${COVERAGE_HOME-.}/coverage/*
+    ${COVERAGE_HOME-.}/tests/*
+    **/coverage/*
 # $set_env.py: COVERAGE_DYNCTX - Set to 'test_function' for who-tests-what
 dynamic_context = ${COVERAGE_DYNCTX-none}
 # $set_env.py: COVERAGE_CONTEXT - Static context for this run (or $ENV_VAR like $TOX_ENV_NAME)
@@ -75,6 +76,9 @@ exclude_lines =
     # Lines that will never be called, but satisfy the type checker
     pragma: never called
 
+    # Exclude an entire file.
+    \A(?s:.*# pragma: exclude file from coverage.*)\Z
+
 partial_branches =
     pragma: part covered
     # A for-loop that always hits its break statement
diff --git a/setup.py b/setup.py
@@ -6,7 +6,10 @@
 # Setuptools setup for coverage.py
 # This file is used unchanged under all versions of Python.
 
+import re
 import os
+import os.path
+import site
 import sys
 
 from setuptools import Extension, errors, setup
@@ -71,6 +74,24 @@
     devstat = "5 - Production/Stable"
 classifier_list.append(f"Development Status :: {devstat}")
 
+
+def do_make_pth():
+    """Make the packaged .pth file used for measuring subprocess coverage."""
+
+    with open("coverage/pth_file.py", encoding="utf-8") as f:
+        code = f.read()
+
+    code = re.sub(r"\s*#.*\n", "\n", code)
+    code = code.replace("    ", " ")
+
+    # `import sys` is needed because .pth files are executed only if they start
+    # with `import `.
+    with open("zzz_coverage.pth", "w", encoding="utf-8") as pth_file:
+        pth_file.write(f"import sys; exec({code!r})\n")
+
+
+do_make_pth()
+
 # Create the keyword arguments for setup()
 
 setup_args = dict(
@@ -85,6 +106,15 @@
             "py.typed",
         ],
     },
+    data_files=[
+        # Write the .pth file into all site-packages directories. Different
+        # platforms read different directories for .pth files, so put it
+        # everywhere.  The process_startup function called in the .pth file
+        # does nothing the second time it's called, so it's fine to have
+        # multiple .pth files.
+        (os.path.relpath(sp, sys.prefix), ["zzz_coverage.pth"])
+        for sp in site.getsitepackages()
+    ],
     entry_points={
         # Install a script as "coverage", and as "coverage3", and as
         # "coverage-3.7" (or whatever).
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/test_process.py b/tests/test_process.py
diff --git a/tox.ini b/tox.ini
