From 44ede9e1ec4b272f67ac881e59fa66ae0d812b4c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Mon, 7 Sep 2026 23:26:32 +0000 Subject: [PATCH 1/3] gh-157146: Let linecache read sources from zip archives on sys.path linecache.getlines() and getline() returned nothing for a module imported from a zip archive on sys.path unless the caller passed module_globals: os.stat() fails on the archive-internal path, the loader lookup needs the globals, and the sys.path search only handles relative names. Callers that only have a file name, such as pdb, warnings and doctest, got no source. Read such files through the get_data() method of the path entry finder registered for the archive in sys.path_importer_cache instead. --- Lib/linecache.py | 48 +++++++++++ Lib/test/test_linecache.py | 85 +++++++++++++++++++ ...-09-07-23-40-00.gh-issue-157146.lczip1.rst | 3 + 3 files changed, 136 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-09-07-23-40-00.gh-issue-157146.lczip1.rst diff --git a/Lib/linecache.py b/Lib/linecache.py index b5bf9dbdd3cbc7..5f246448ebef08 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -163,6 +163,19 @@ def updatecache(filename, module_globals=None): cache[filename] = entry return entry[2] + # The file may be inside an archive on the module search path, such + # as a zip file. + data = _read_from_archive(fullname) + if data is not None: + entry = ( + len(data), + None, + [line + '\n' for line in data.splitlines()], + fullname + ) + cache[filename] = entry + return entry[2] + # Try looking through the module search path, which is only useful # when handling a relative filename. if os.path.isabs(filename): @@ -197,6 +210,41 @@ def updatecache(filename, module_globals=None): return lines +def _read_from_archive(filename): + """Return the decoded contents of a file inside an archive on sys.path. + + Path entry finders for archives, such as zipimport.zipimporter, have a + get_data() method that reads files by their path below the archive, + which is what __file__ and co_filename contain for modules imported + from it. The archive is one of the parent directories of the file, so + look for a finder registered for one of them. Return None if the file + is not in such an archive. + """ + import importlib.util + import os + import sys + if not os.path.isabs(filename): + return None + path = filename + while True: + parent = os.path.dirname(path) + if parent == path: + return None + path = parent + get_data = getattr(sys.path_importer_cache.get(path), 'get_data', + None) + if get_data is None: + continue + try: + data = get_data(filename) + except (ImportError, OSError): + continue + try: + return importlib.util.decode_source(data) + except (UnicodeDecodeError, SyntaxError): + return None + + def lazycache(filename, module_globals): """Seed the cache for filename with module_globals. diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index fcd94edc611fac..c496fef1cb7504 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -1,13 +1,18 @@ """ Tests for the linecache module """ +import importlib import linecache import unittest import os.path +import sys import tempfile import threading import tokenize +import zipfile +import zipimport from importlib.machinery import ModuleSpec from test import support +from test.support import import_helper from test.support import os_helper from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -398,6 +403,86 @@ def test_checkcache_with_no_parameter(self): self.assertIn(self.unchanged_file, linecache.cache) +class ZipArchiveTests(unittest.TestCase): + """Sources of modules imported from a zip archive on sys.path.""" + + MODULE_SOURCE = ( + '"""A module inside a zip archive."""\n' + '\n' + 'def f():\n' + ' return "from the zip"\n' + ) + PACKAGE_SOURCE = 'value = 42\n' + LATIN1_SOURCE = ( + '# -*- coding: latin-1 -*-\n' + 'value = "caf\xe9"\n' + ) + + def setUp(self): + linecache.clearcache() + self.addCleanup(linecache.clearcache) + tmpdir = self.enterContext(os_helper.temp_dir()) + self.zip_name = os.path.join(tmpdir, 'sources.zip') + with zipfile.ZipFile(self.zip_name, 'w') as zf: + zf.writestr('zipmod.py', self.MODULE_SOURCE) + zf.writestr('zippkg/__init__.py', self.PACKAGE_SOURCE) + zf.writestr('ziplatin1.py', self.LATIN1_SOURCE.encode('latin-1')) + self.enterContext(import_helper.DirsOnSysPath(self.zip_name)) + for name in 'zipmod', 'zippkg', 'ziplatin1': + self.addCleanup(import_helper.unload, name) + self.addCleanup(sys.path_importer_cache.pop, self.zip_name, None) + self.addCleanup(zipimport._zip_directory_cache.pop, + self.zip_name, None) + self.zipmod = importlib.import_module('zipmod') + + def test_getlines_without_module_globals(self): + filename = self.zipmod.__file__ + self.assertEqual(filename, os.path.join(self.zip_name, 'zipmod.py')) + self.assertFalse(os.path.exists(filename)) + lines = self.MODULE_SOURCE.splitlines(keepends=True) + self.assertEqual(linecache.getlines(filename), lines) + self.assertEqual(linecache.getline(filename, 4), + ' return "from the zip"\n') + self.assertEqual(linecache.getline(filename, 5), '') + + def test_getline_from_code_object(self): + code = self.zipmod.f.__code__ + self.assertEqual( + linecache.getline(code.co_filename, code.co_firstlineno), + 'def f():\n') + + def test_package(self): + zippkg = importlib.import_module('zippkg') + self.assertEqual(linecache.getlines(zippkg.__file__), + ['value = 42\n']) + + def test_encoding_declaration(self): + ziplatin1 = importlib.import_module('ziplatin1') + self.assertEqual(linecache.getlines(ziplatin1.__file__), + self.LATIN1_SOURCE.splitlines(keepends=True)) + + def test_missing_file(self): + filename = os.path.join(self.zip_name, 'missing.py') + self.assertEqual(linecache.getlines(filename), []) + self.assertEqual(linecache.getline(filename, 1), '') + self.assertNotIn(filename, linecache.cache) + + def test_checkcache_and_clearcache(self): + filename = self.zipmod.__file__ + lines = linecache.getlines(filename) + self.assertIn(filename, linecache.cache) + # A file inside an archive has no mtime of its own, so checkcache() + # keeps the entry, as it does for entries loaded through a loader. + self.assertIsNone(linecache.cache[filename][1]) + linecache.checkcache(filename) + linecache.checkcache() + self.assertIn(filename, linecache.cache) + self.assertEqual(linecache.getlines(filename), lines) + linecache.clearcache() + self.assertNotIn(filename, linecache.cache) + self.assertEqual(linecache.getlines(filename), lines) + + class MultiThreadingTest(unittest.TestCase): @threading_helper.reap_threads @threading_helper.requires_working_threading() diff --git a/Misc/NEWS.d/next/Library/2026-09-07-23-40-00.gh-issue-157146.lczip1.rst b/Misc/NEWS.d/next/Library/2026-09-07-23-40-00.gh-issue-157146.lczip1.rst new file mode 100644 index 00000000000000..d8b81798bd64be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-07-23-40-00.gh-issue-157146.lczip1.rst @@ -0,0 +1,3 @@ +:mod:`linecache` can now read the source of a module that was imported from +a zip archive on :data:`sys.path` when given only the file name, as +:mod:`pdb`, :mod:`warnings` and :mod:`doctest` do. From 0ecb0dfbafdc5e66f1e27cc792d6cb3ae22f8097 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 9 Sep 2026 22:05:11 -0700 Subject: [PATCH 2/3] gh-157146: Accept relative archive paths and tighten the lookup zipimporter does not absolutize its sys.path entry, so a relative entry gives its modules a relative __file__ and a relative key in sys.path_importer_cache; the isabs() guard made linecache skip exactly that case. Build the cache entry once for both the loader and the archive lookups, and only import importlib.util once there is data to decode. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016oQeaTzLMBouqehAe3W2EU --- Lib/linecache.py | 21 ++++++--------------- Lib/test/test_linecache.py | 22 +++++++++++++++++----- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Lib/linecache.py b/Lib/linecache.py index 5f246448ebef08..35bfaab19bc93c 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -144,6 +144,7 @@ def updatecache(filename, module_globals=None): lazy_entry = entry if entry is not None and len(entry) == 1 else None if lazy_entry is None: lazy_entry = _make_lazycache_entry(filename, module_globals) + data = None if lazy_entry is not None: try: data = lazy_entry[0]() @@ -154,18 +155,10 @@ def updatecache(filename, module_globals=None): # No luck, the PEP302 loader cannot find the source # for this module. return [] - entry = ( - len(data), - None, - [line + '\n' for line in data.splitlines()], - fullname - ) - cache[filename] = entry - return entry[2] - - # The file may be inside an archive on the module search path, such - # as a zip file. - data = _read_from_archive(fullname) + if data is None: + # The file may be inside an archive on the module search path, + # such as a zip file. + data = _read_from_archive(fullname) if data is not None: entry = ( len(data), @@ -220,11 +213,8 @@ def _read_from_archive(filename): look for a finder registered for one of them. Return None if the file is not in such an archive. """ - import importlib.util import os import sys - if not os.path.isabs(filename): - return None path = filename while True: parent = os.path.dirname(path) @@ -239,6 +229,7 @@ def _read_from_archive(filename): data = get_data(filename) except (ImportError, OSError): continue + import importlib.util try: return importlib.util.decode_source(data) except (UnicodeDecodeError, SyntaxError): diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index c496fef1cb7504..5fecf6440aebd5 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -444,12 +444,24 @@ def test_getlines_without_module_globals(self): self.assertEqual(linecache.getline(filename, 4), ' return "from the zip"\n') self.assertEqual(linecache.getline(filename, 5), '') - - def test_getline_from_code_object(self): code = self.zipmod.f.__code__ - self.assertEqual( - linecache.getline(code.co_filename, code.co_firstlineno), - 'def f():\n') + self.assertEqual(code.co_filename, filename) + self.assertEqual(linecache.getline(filename, code.co_firstlineno), + 'def f():\n') + + def test_relative_archive_path(self): + # A relative sys.path entry gives its modules a relative __file__. + tmpdir, zip_base = os.path.split(self.zip_name) + self.addCleanup(sys.path_importer_cache.pop, zip_base, None) + self.addCleanup(zipimport._zip_directory_cache.pop, zip_base, None) + sys.path.insert(0, zip_base) + self.addCleanup(sys.path.remove, zip_base) + with os_helper.change_cwd(tmpdir): + zippkg = importlib.import_module('zippkg') + self.assertEqual(zippkg.__file__, + os.path.join(zip_base, 'zippkg', '__init__.py')) + self.assertEqual(linecache.getlines(zippkg.__file__), + ['value = 42\n']) def test_package(self): zippkg = importlib.import_module('zippkg') From 8d5a2b4651cea84aca140730c69b1e11d4bb7f98 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 9 Sep 2026 22:47:27 -0700 Subject: [PATCH 3/3] gh-157146: Make the archive lookup safe during interpreter shutdown sys.path_importer_cache is set to None early in finalization, before the objects still referenced from sys attributes are released, so a __del__ can reach linecache while every module it imports is still available. Return early in that case, and treat an ImportError from the lookup like the other imports linecache makes lazily. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016oQeaTzLMBouqehAe3W2EU --- Lib/linecache.py | 13 ++++++++++--- Lib/test/test_linecache.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Lib/linecache.py b/Lib/linecache.py index 35bfaab19bc93c..d1391510473ae5 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -158,7 +158,11 @@ def updatecache(filename, module_globals=None): if data is None: # The file may be inside an archive on the module search path, # such as a zip file. - data = _read_from_archive(fullname) + try: + data = _read_from_archive(fullname) + except ImportError: + # Can happen if the interpreter is shutting down. + return [] if data is not None: entry = ( len(data), @@ -215,14 +219,17 @@ def _read_from_archive(filename): """ import os import sys + importers = sys.path_importer_cache + if importers is None: + # Cleared while the interpreter is shutting down. + return None path = filename while True: parent = os.path.dirname(path) if parent == path: return None path = parent - get_data = getattr(sys.path_importer_cache.get(path), 'get_data', - None) + get_data = getattr(importers.get(path), 'get_data', None) if get_data is None: continue try: diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 5fecf6440aebd5..202e30e6f6c07a 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -361,6 +361,17 @@ def test_linecache_python_string(self): self.assertEqual(stdout, b'') self.assertEqual(stderr, b'') + def test_path_importer_cache_None(self): + # sys.path_importer_cache is set to None while the interpreter is + # shutting down, before objects with a __del__ that may end up here + # are released. + filename = os.path.abspath(os_helper.TESTFN + '.py') + with support.swap_attr(sys, 'path_importer_cache', None): + self.assertEqual(linecache.getlines(filename), []) + self.assertEqual(linecache.getline(filename, 1), '') + self.assertNotIn(filename, linecache.cache) + + class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): super().setUp()