diff --git a/.gitignore b/.gitignore index 00813f9d1b6b00..45047d136bc5a0 100644 --- a/.gitignore +++ b/.gitignore @@ -146,6 +146,7 @@ Tools/unicode/data/ /dist/ /jit_stencils*.h /jit_unwind_info*.h +/trampoline_ehframe.h* .jit-stamp /platform /profile-clean-stamp diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 5565f99b244f11..b53a0182f12492 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -30,7 +30,8 @@ relationship between this piece of code and the associated Python function using samply support on macOS is available starting from Python 3.15. Check the output of the ``configure`` build step or check the output of ``python -m sysconfig | grep HAVE_PERF_TRAMPOLINE`` - to see if your system is supported. + to see if your system is supported. Building the perf trampoline needs a + Python interpreter, found by ``configure`` as ``PYTHON_FOR_REGEN``. For example, consider the following script: diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 88b5f35a796796..a0f1f51a1b41dd 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -25,6 +25,11 @@ To build CPython, you will need: * Support for threads. +* Optionally, a Python interpreter, found by ``configure`` as + ``PYTHON_FOR_REGEN``, to build the perf trampoline (see + :doc:`/howto/perf_profiling`). Without it, ``configure`` disables the perf + trampoline. + .. versionchanged:: 3.5 On Windows, Visual Studio 2015 or later is now required. diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index a2e67726e95959..896c6ac6e8dfe9 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -1,3 +1,4 @@ +import struct import unittest import string import subprocess @@ -11,6 +12,7 @@ assert_python_failure, assert_python_ok, ) +from test.support import import_helper from test.support.os_helper import temp_dir @@ -685,5 +687,175 @@ def tearDown(self) -> None: file.unlink() +JITDUMP_MAGIC = 0x4A695444 # "JiTD" +JITDUMP_VERSION = 1 +PERF_LOAD = 0 +PERF_UNWINDING_INFO = 4 +JITDUMP_ENDIAN = "<" if sys.byteorder == "little" else ">" +# Every jitdump record starts with event(u32), size(u32), timestamp(u64). +JITDUMP_RECORD_HEADER_SIZE = 16 +# CodeLoadEvent: record header, pid(u32), tid(u32), vma(u64), code_addr(u64), +# code_size(u64), code_id(u64), then the NUL-terminated name. +CODE_LOAD_CODE_SIZE_OFFSET = JITDUMP_RECORD_HEADER_SIZE + 4 + 4 + 8 + 8 +CODE_LOAD_NAME_OFFSET = CODE_LOAD_CODE_SIZE_OFFSET + 8 + 8 +# CodeUnwindingInfoEvent: record header, unwind_data_size(u64), +# eh_frame_hdr_size(u64), mapped_size(u64), then the .eh_frame bytes followed +# by perf's 20-byte eh_frame_hdr (EhFrameHeader in perf_jit_trampoline.c), +# whose "from" field is the signed distance back to the code. +UNWIND_DATA_SIZE_OFFSET = JITDUMP_RECORD_HEADER_SIZE +UNWIND_EH_FRAME_OFFSET = JITDUMP_RECORD_HEADER_SIZE + 3 * 8 +EH_FRAME_HDR_SIZE = 20 +EH_FRAME_HDR_FROM_OFFSET = 12 +# DWARF FDE pointer encodings: DW_EH_PE_pcrel | DW_EH_PE_sdata4 (ELF +# assemblers) and DW_EH_PE_pcrel | DW_EH_PE_absptr (Darwin assemblers). +DW_EH_PE_PCREL_SDATA4 = 0x1B +DW_EH_PE_PCREL_ABSPTR = 0x10 + + +def _jitdump_records(data): + """Yield (event, offset, size) for each record of a jitdump file.""" + header_size = struct.unpack_from(f"{JITDUMP_ENDIAN}I", data, 8)[0] + pos = header_size + while pos < len(data): + if pos + JITDUMP_RECORD_HEADER_SIZE > len(data): + raise ValueError(f"truncated record header at offset {pos}") + event, size = struct.unpack_from(f"{JITDUMP_ENDIAN}II", data, pos) + if size < JITDUMP_RECORD_HEADER_SIZE or pos + size > len(data): + raise ValueError(f"record at offset {pos} has a bad size {size}") + yield event, pos, size + pos += size + + +def _fde_pointer_encoding(eh_frame): + """Return the FDE pointer encoding byte of a version 1 "zR" CIE.""" + cie_length = struct.unpack_from(f"{JITDUMP_ENDIAN}I", eh_frame, 0)[0] + cie_total = 4 + cie_length + pos = 12 # past length, CIE_id, version and "zR\0" + for _ in range(2): # code and data alignment factors (LEB128) + while eh_frame[pos] & 0x80: + pos += 1 + pos += 1 + pos += 1 # return address column + while eh_frame[pos] & 0x80: # augmentation data length (LEB128) + pos += 1 + pos += 1 + if pos >= cie_total: + raise ValueError("truncated CIE augmentation data") + return eh_frame[pos] + + +@unittest.skipIf(support.check_bolt_optimized(), "fails on BOLT instrumented binaries") +class TestJitdumpFileFormat(unittest.TestCase): + """Validate the jitdump written by -Xperf_jit without requiring perf.""" + + def _run_and_get_jitdump(self, code): + # The child prints its pid so we open exactly its own jitdump file + # rather than whatever another test worker left in /tmp. + code = "import os, sys\nsys.stdout.write(str(os.getpid()))\n" + code + _, out, _ = assert_python_ok("-Xperf_jit", "-c", code, PYTHON_JIT="0") + path = pathlib.Path(f"/tmp/jit-{int(out)}.dump") + try: + data = path.read_bytes() + except FileNotFoundError: + # perf_map_jit_init() gives up silently when it cannot create + # the file (for example an unwritable /tmp). + self.skipTest("jitdump file was not created") + self.addCleanup(path.unlink) + if not data: + # The file is created before the executable mapping of the + # jitdump; if that mapping fails (for example a noexec /tmp) the + # backend gives up silently and never writes the header. + self.skipTest("jitdump could not be initialized") + return data + + def _check_code_load(self, data, pos, size): + """Validate a code load record and return its name and code size.""" + code_size = struct.unpack_from( + f"{JITDUMP_ENDIAN}Q", data, pos + CODE_LOAD_CODE_SIZE_OFFSET)[0] + name_start = pos + CODE_LOAD_NAME_OFFSET + name_end = data.find(b"\x00", name_start, pos + size) + self.assertGreater(name_end, 0) + name = data[name_start:name_end].decode("utf-8", errors="replace") + # The machine code follows the name inside the load record. + self.assertLessEqual(name_end + 1 + code_size, pos + size, name) + return name, code_size + + def _check_unwind_info(self, data, pos, size, name, code_size): + """Check the FDE and perf header against their code load record.""" + unwind_data_size, eh_frame_hdr_size = struct.unpack_from( + f"{JITDUMP_ENDIAN}QQ", data, pos + UNWIND_DATA_SIZE_OFFSET) + self.assertEqual(eh_frame_hdr_size, EH_FRAME_HDR_SIZE) + self.assertLessEqual(UNWIND_EH_FRAME_OFFSET + unwind_data_size, size) + eh_frame_size = unwind_data_size - eh_frame_hdr_size + self.assertGreater(eh_frame_size, 0) + start = pos + UNWIND_EH_FRAME_OFFSET + eh_frame = data[start:start + eh_frame_size] + + cie_length, cie_id = struct.unpack_from(f"{JITDUMP_ENDIAN}II", eh_frame, 0) + self.assertEqual(cie_id, 0, "first entry must be a CIE") + self.assertEqual(eh_frame[8], 1, "CIE version must be 1") + self.assertEqual(eh_frame[9:12], b"zR\x00") + encoding = _fde_pointer_encoding(eh_frame) + if encoding == DW_EH_PE_PCREL_SDATA4: + fields = "iI" + elif encoding == DW_EH_PE_PCREL_ABSPTR: + fields = "qQ" + else: + self.fail(f"unexpected FDE pointer encoding {encoding:#x}") + # jit_unwind.c patches initial_location and address_range for + # perf's DSO layout, where the .eh_frame follows the code at + # code_size rounded up to 8 bytes. + pc_offset = 4 + cie_length + 8 + initial_location, address_range = struct.unpack_from( + f"{JITDUMP_ENDIAN}{fields}", eh_frame, pc_offset) + self.assertEqual(address_range, code_size, name) + rounded_code_size = (code_size + 7) & ~7 + self.assertEqual(initial_location, -(rounded_code_size + pc_offset), name) + # perf's eh_frame_hdr must point back at the code with the same + # rounding as the FDE. + hdr_from = struct.unpack_from( + f"{JITDUMP_ENDIAN}i", data, + start + eh_frame_size + EH_FRAME_HDR_FROM_OFFSET)[0] + self.assertEqual(hdr_from, -(rounded_code_size + eh_frame_size), name) + + def _check_unwinding_records(self, data): + """Return {name: code_size} after checking each unwind/load pair.""" + records = list(_jitdump_records(data)) + regions = {} + for index, (event, pos, size) in enumerate(records): + if event != PERF_UNWINDING_INFO: + continue + # Unwinding info immediately precedes the code it describes. + self.assertLess(index + 1, len(records)) + load_event, load_pos, load_size = records[index + 1] + self.assertEqual(load_event, PERF_LOAD) + name, code_size = self._check_code_load(data, load_pos, load_size) + with self.subTest(region=name): + self._check_unwind_info(data, pos, size, name, code_size) + regions[name] = code_size + self.assertTrue(regions, "no CodeUnwindingInfoEvent found") + return regions + + def test_jitdump_unwinding_info(self): + """Each region's .eh_frame is patched for that region's size.""" + data = self._run_and_get_jitdump("def my_test_func(): pass\nmy_test_func()") + magic, version = struct.unpack_from(f"{JITDUMP_ENDIAN}II", data, 0) + self.assertEqual((magic, version), (JITDUMP_MAGIC, JITDUMP_VERSION)) + regions = self._check_unwinding_records(data) + self.assertTrue(any("my_test_func" in name for name in regions)) + + +class TestTrampolineEhframeHeader(unittest.TestCase): + """Structural checks on the generated trampoline_ehframe.h data.""" + + def test_generated_header_structure(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + check = getattr(_testinternalcapi, "test_trampoline_ehframe", None) + if check is None: + self.skipTest("_testinternalcapi built without the perf trampoline") + # Raises AssertionError describing the first failed check. + check() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tools/test_trampoline_ehframe.py b/Lib/test/test_tools/test_trampoline_ehframe.py new file mode 100644 index 00000000000000..04769dc1a094e1 --- /dev/null +++ b/Lib/test/test_tools/test_trampoline_ehframe.py @@ -0,0 +1,227 @@ +import pathlib +import struct +import sysconfig +import unittest +from unittest import mock + +from test.support.os_helper import temp_dir +from test.test_tools import imports_under_tool, skip_if_missing + + +skip_if_missing("jit") +with imports_under_tool("jit"): + import _trampoline_ehframe as ehframe + + +DW_EH_PE_PCREL_SDATA4 = 0x1B +DW_EH_PE_PCREL_ABSPTR = 0x10 + + +def _fake_cie(*, version=1, augmentation=b"zR", ra_column=16, + encoding=DW_EH_PE_PCREL_SDATA4, cie_id=0): + """A CIE like the assembler's: code align 1, data align -8, one + DW_CFA_def_cfa instruction, padded with DW_CFA_nop to 8 bytes.""" + body = bytes([version]) + augmentation + b"\x00" + body += bytes([1, 0x78, ra_column, 1, encoding]) + body += bytes([0x0C, 7, 8]) # DW_CFA_def_cfa: r7 (rsp) ofs 8 + body += b"\x00" * (-(8 + len(body)) % 8) + return struct.pack("IIQQII" if fat64 else ">IIIII") + offset = 8 + entry.size * len(blobs) + entries = b"" + body = b"" + for cputype, blob in blobs: + fields = (cputype, 0, offset + len(body), len(blob), 0) + if fat64: + fields += (0,) # reserved + entries += entry.pack(*fields) + body += blob + magic = ehframe._FAT_MAGIC_64 if fat64 else ehframe._FAT_MAGIC + return struct.pack(">II", magic, len(blobs)) + entries + body + + +class TestEhFrameParsing(unittest.TestCase): + def parse(self, data, text_size=8): + return ehframe.parse_ehframe(bytes(data), "<", text_size) + + def test_parse(self): + """Both FDE pointer encodings: ELF sdata4 and Darwin absptr.""" + cases = [ + (DW_EH_PE_PCREL_SDATA4, 4, 16, 8), + (DW_EH_PE_PCREL_ABSPTR, 8, 30, 20), + ] + for encoding, field_size, ra_column, text_size in cases: + with self.subTest(encoding=hex(encoding)): + cie = _fake_cie(encoding=encoding, ra_column=ra_column) + fde = _fake_fde( + len(cie), field_size=field_size, address_range=text_size + ) + result = self.parse(cie + fde, text_size) + self.assertEqual(result.field_size, field_size) + self.assertEqual(result.fde_pc_offset, len(cie) + 8) + self.assertEqual(result.fde_range_offset, len(cie) + 8 + field_size) + # Both patchable fields zeroed, everything else untouched. + expected = bytearray(cie + fde) + pc_offset = len(cie) + 8 + expected[pc_offset:pc_offset + 2 * field_size] = bytes(2 * field_size) + self.assertEqual(result.data, bytes(expected)) + + def test_parse_rejects_malformed(self): + cie = _fake_cie() + fde = _fake_fde(len(cie)) + cases = [ + ("no CIE", b"", 8), + ("CIE_id", _fake_cie(cie_id=1) + fde, 8), + ("bad CIE length", cie[:12], 8), + ("version", _fake_cie(version=3) + fde, 8), + ("augmentation", _fake_cie(augmentation=b"zPLR") + fde, 8), + ("encoding", _fake_cie(encoding=0x1A) + fde, 8), + ("exactly one FDE", cie + fde + fde, 8), + ("address_range", cie + fde, 12), + ("no FDE", cie, 8), + ] + for message, data, text_size in cases: + with self.subTest(message): + with self.assertRaisesRegex(ValueError, message): + self.parse(data, text_size) + + +class TestObjectLoading(unittest.TestCase): + def load_object(self, data): + with temp_dir() as tmp: + path = pathlib.Path(tmp) / "trampoline.o" + path.write_bytes(data) + return ehframe.load_object(path) + + def test_macho_thin(self): + text = b"\xc0\x03\x5f\xd6" + data = _fake_macho(ehframe._CPU_TYPE_ARM64, text, b"arm64 eh_frame") + (obj,) = self.load_object(data) + self.assertEqual(obj.arch_macro, "__aarch64__") + self.assertEqual(obj.sections, {".text": text, ".eh_frame": b"arm64 eh_frame"}) + + def test_macho_fat(self): + x86 = _fake_macho(ehframe._CPU_TYPE_X86_64, b"\x55\xc3", b"x86 eh_frame") + arm = _fake_macho( + ehframe._CPU_TYPE_ARM64, b"\xc0\x03\x5f\xd6", b"arm64 eh_frame" + ) + blobs = [ + (ehframe._CPU_TYPE_X86_64, x86), + (ehframe._CPU_TYPE_ARM64, arm), + ] + for fat64 in (False, True): + with self.subTest(fat64=fat64): + slices = self.load_object(_fake_fat_macho(blobs, fat64=fat64)) + self.assertEqual([s.arch_macro for s in slices], + ["__x86_64__", "__aarch64__"]) + for obj, (_, blob) in zip(slices, blobs, strict=True): + with self.subTest(arch=obj.arch_macro): + (thin,) = self.load_object(blob) + self.assertEqual(obj.sections, thin.sections) + + def test_macho_rejects_malformed_sections(self): + data = _fake_macho(ehframe._CPU_TYPE_ARM64, b"text", b"eh_frame") + cases = [ + ("truncated segment command", 32 + 4, "I", 16), + ("section table extends", 32 + 64, "I", 3), + ("section .text extends", 32 + 72 + 48, "I", len(data)), + ] + for message, offset, fmt, value in cases: + with self.subTest(message): + malformed = bytearray(data) + struct.pack_into("<" + fmt, malformed, offset, value) + with self.assertRaisesRegex(ValueError, message): + self.load_object(malformed) + + +class TestHeaderGeneration(unittest.TestCase): + def _build_trampoline_objects(self): + """The object(s) the Makefile fed to the generator.""" + builddir = pathlib.Path(sysconfig.get_config_var("abs_builddir") or ".") + universal2 = builddir / "Python" / "asm_trampoline_universal2.o" + if universal2.exists(): + return [universal2] + return sorted( + path for path in (builddir / "Python").glob("asm_trampoline_*.o") + if "apple-darwin" not in path.name + ) + + def test_failed_replace_preserves_header(self): + cie = _fake_cie() + obj = ehframe.ObjectSlice( + "trampoline.o", "__x86_64__", "<", + {".text": bytes(8), ".eh_frame": cie + _fake_fde(len(cie))}, + ) + entries = [(obj, ehframe.build_ehframe(obj))] + with temp_dir() as tmp: + path = pathlib.Path(tmp) / "trampoline_ehframe.h" + path.write_text("previous header") + with ( + mock.patch.object(pathlib.Path, "replace", side_effect=OSError), + self.assertRaises(OSError), + ): + ehframe.write_header(entries, path) + self.assertEqual(path.read_text(), "previous header") + self.assertEqual(list(path.parent.iterdir()), [path]) + + def test_generated_header_is_current(self): + """The header in the build directory matches a fresh generation.""" + objects = self._build_trampoline_objects() + builddir = pathlib.Path(sysconfig.get_config_var("abs_builddir") or ".") + header = builddir / "trampoline_ehframe.h" + if not objects or not header.exists(): + self.skipTest("trampoline object or generated header not found") + current = header.read_text() + with temp_dir() as tmp: + fresh_path = pathlib.Path(tmp) / "trampoline_ehframe.h" + ehframe.generate(objects, fresh_path) + fresh = fresh_path.read_text() + self.assertEqual(current, fresh) + + +if __name__ == "__main__": + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 78a486623181fa..f9f9a5db5d4ad7 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3185,9 +3185,16 @@ jit_shim-universal2-apple-darwin.o: jit_shim-aarch64-apple-darwin.o jit_shim-x86 Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< -Python/jit_unwind.o: $(srcdir)/Python/jit_unwind.c $(JIT_UNWIND_INFO_H) +TRAMPOLINE_EHFRAME_H = $(if @PERF_TRAMPOLINE_OBJ@,trampoline_ehframe.h) + +trampoline_ehframe.h: @PERF_TRAMPOLINE_OBJ@ $(srcdir)/Tools/jit/_trampoline_ehframe.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/jit/_trampoline_ehframe.py \ + -o $@ @PERF_TRAMPOLINE_OBJ@ + +Python/jit_unwind.o: $(srcdir)/Python/jit_unwind.c $(JIT_UNWIND_INFO_H) $(TRAMPOLINE_EHFRAME_H) $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + .PHONY: regen-jit regen-jit: $(JIT_TARGETS) @@ -3289,6 +3296,7 @@ clean-retain-profile: pycremoval -rm -f Python/frozen_modules/MANIFEST -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h + -rm -f trampoline_ehframe.h trampoline_ehframe.h.tmp -rm -f profile-gen-stamp -rm -rf Platforms/Apple/iOS/testbed/Python.xcframework/ios-*/bin -rm -rf Platforms/Apple/iOS/testbed/Python.xcframework/ios-*/lib @@ -3463,7 +3471,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo. MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h -MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h $(TRAMPOLINE_EHFRAME_H) MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h diff --git a/Misc/NEWS.d/next/Build/2026-09-09-14-23-23.gh-issue-149800.Ht8kaS.rst b/Misc/NEWS.d/next/Build/2026-09-09-14-23-23.gh-issue-149800.Ht8kaS.rst new file mode 100644 index 00000000000000..6c8ca4920c57ce --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-09-09-14-23-23.gh-issue-149800.Ht8kaS.rst @@ -0,0 +1,2 @@ +Generate the perf trampoline's unwind information from its compiled assembly +at build time. Building the trampoline now requires a Python interpreter. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 38e56ae7042098..f458652eb3805b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -30,6 +30,7 @@ #include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_interpframe.h" // _PyFrame_GetFunction() #include "pycore_jit.h" // _PyJIT_AddressInJitCode() +#include "pycore_jit_unwind.h" // DWRF_EH_PE_* constants #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_Executor_DependsOn #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() @@ -2718,6 +2719,121 @@ _testinternalcapi_test_long_numbits_impl(PyObject *module) Py_RETURN_NONE; } +#if defined(PY_HAVE_PERF_TRAMPOLINE) +#include "trampoline_ehframe.h" + +/* Structural checks on the generated perf trampoline .eh_frame and the + * field offsets Python/jit_unwind.c patches. Raises instead of assert() + * so the checks also run in release builds. */ +static PyObject * +test_trampoline_ehframe(PyObject *self, PyObject *Py_UNUSED(args)) +{ +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + PyErr_SetString(PyExc_AssertionError, \ + "trampoline_ehframe: " msg); \ + return NULL; \ + } \ + } while (0) + + const uint8_t *data = _trampoline_ehframe; + size_t size = TRAMPOLINE_EHFRAME_SIZE; + size_t field_size = TRAMPOLINE_EHFRAME_FDE_FIELD_SIZE; + + CHECK(field_size == 4 || field_size == 8, "unsupported FDE field size"); + CHECK(size >= 8 + 8 + 2 * field_size + 1, "data too small for a CIE and an FDE"); + + /* CIE: length, CIE_id == 0, version 1, augmentation "zR". */ + uint32_t cie_length; + memcpy(&cie_length, data, sizeof(cie_length)); + CHECK(cie_length > 0 && cie_length < size, "bad CIE length"); + + uint32_t cie_id; + memcpy(&cie_id, data + 4, sizeof(cie_id)); + CHECK(cie_id == 0, "first entry is not a CIE"); + + CHECK(data[8] == 1, "CIE version is not 1"); + CHECK(data[9] == 'z' && data[10] == 'R' && data[11] == '\0', + "CIE augmentation is not \"zR\""); + + /* Exactly one FDE must follow the CIE, and the walk below must stay + * inside the data. */ + size_t cie_total = 4 + cie_length; + CHECK(cie_total + 8 + 2 * field_size <= size, + "no room for an FDE after the CIE"); + + /* FDE pointer encoding byte: skip version(1), "zR\0"(3), code_align + * (ULEB128), data_align (SLEB128), RA column (1 byte in version 1), + * augmentation data length (ULEB128). */ + size_t pos = 12; + while (pos < cie_total && (data[pos] & 0x80)) { /* code alignment */ + pos++; + } + pos++; + while (pos < cie_total && (data[pos] & 0x80)) { /* data alignment */ + pos++; + } + pos++; + pos++; /* RA column */ + /* Augmentation data length (ULEB128) must be 1: the encoding byte. */ + uint32_t aug_length = 0; + int shift = 0; + while (pos < cie_total && (data[pos] & 0x80)) { + CHECK(shift < 28, "CIE augmentation data length is too long"); + aug_length |= (uint32_t)(data[pos] & 0x7f) << shift; + shift += 7; + pos++; + } + CHECK(pos < cie_total, "CIE augmentation data length is truncated"); + CHECK((uint32_t)(data[pos] & 0x7f) <= (UINT32_MAX >> shift), + "CIE augmentation data length overflows"); + aug_length |= (uint32_t)(data[pos] & 0x7f) << shift; + pos++; + CHECK(aug_length == 1 && pos < cie_total, + "CIE augmentation data length is not 1"); + uint8_t fde_enc = data[pos]; + CHECK(fde_enc == (DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4) + || fde_enc == (DWRF_EH_PE_pcrel | DWRF_EH_PE_absptr), + "unsupported FDE pointer encoding"); + size_t enc_field_size = (fde_enc & 0x0f) == DWRF_EH_PE_sdata4 ? 4 : 8; + CHECK(enc_field_size == field_size, + "FDE field size does not match the CIE pointer encoding"); + + /* The FDE ends exactly at the end of the data. */ + uint32_t fde_length; + memcpy(&fde_length, data + cie_total, sizeof(fde_length)); + CHECK(fde_length > 0, "FDE length is zero"); + CHECK(cie_total + 4 + fde_length == size, + "FDE does not end at the end of the data"); + + /* The CIE pointer is the distance from its own field back to the CIE. */ + uint32_t fde_cie_ptr; + memcpy(&fde_cie_ptr, data + cie_total + 4, sizeof(fde_cie_ptr)); + CHECK(fde_cie_ptr == cie_total + 4, "FDE CIE pointer does not point at the CIE"); + + /* The runtime patches initial_location and address_range at the + * recorded offsets. They must be the two fields after the CIE pointer + * and must be zeroed placeholders in the header. */ + CHECK(TRAMPOLINE_EHFRAME_FDE_PC_OFFSET == cie_total + 8, + "FDE initial_location offset is wrong"); + CHECK(TRAMPOLINE_EHFRAME_FDE_RANGE_OFFSET + == TRAMPOLINE_EHFRAME_FDE_PC_OFFSET + field_size, + "FDE address_range offset is wrong"); + for (size_t i = 0; i < 2 * field_size; i++) { + CHECK(data[TRAMPOLINE_EHFRAME_FDE_PC_OFFSET + i] == 0, + "FDE placeholder fields are not zero"); + } + /* The FDE's own augmentation data length must follow and be 0. */ + size_t fde_aug_offset = TRAMPOLINE_EHFRAME_FDE_RANGE_OFFSET + field_size; + CHECK(fde_aug_offset < size, "FDE augmentation data length byte is missing"); + CHECK(data[fde_aug_offset] == 0, "FDE augmentation data length is not 0"); + +#undef CHECK + Py_RETURN_NONE; +} +#endif /* PY_HAVE_PERF_TRAMPOLINE */ + static PyObject * compile_perf_trampoline_entry(PyObject *self, PyObject *args) { @@ -3353,6 +3469,9 @@ static PyMethodDef module_functions[] = { {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, +#if defined(PY_HAVE_PERF_TRAMPOLINE) + {"test_trampoline_ehframe", test_trampoline_ehframe, METH_NOARGS}, +#endif {"get_crossinterp_data", _PyCFunction_CAST(get_crossinterp_data), METH_VARARGS | METH_KEYWORDS}, {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, diff --git a/Python/asm_trampoline_aarch64.S b/Python/asm_trampoline_aarch64.S index b3aeb728de200c..883c3f1b0394a8 100644 --- a/Python/asm_trampoline_aarch64.S +++ b/Python/asm_trampoline_aarch64.S @@ -44,13 +44,28 @@ __Py_trampoline_func_start: .globl _Py_trampoline_func_start _Py_trampoline_func_start: #endif + .cfi_startproc SIGN_LR +#if defined(__ARM_FEATURE_PAC_DEFAULT) + .cfi_negate_ra_state +#endif stp x29, x30, [sp, -16]! + .cfi_def_cfa_offset 16 + .cfi_offset x29, -16 + .cfi_offset x30, -8 mov x29, sp + .cfi_def_cfa_register x29 blr x3 ldp x29, x30, [sp], 16 + .cfi_restore x30 + .cfi_restore x29 + .cfi_def_cfa sp, 0 VERIFY_LR +#if defined(__ARM_FEATURE_PAC_DEFAULT) + .cfi_negate_ra_state +#endif ret + .cfi_endproc #if defined(__APPLE__) .globl __Py_trampoline_func_end __Py_trampoline_func_end: diff --git a/Python/asm_trampoline_x86_64.S b/Python/asm_trampoline_x86_64.S index 0e6b11589eafc8..999728653aa534 100644 --- a/Python/asm_trampoline_x86_64.S +++ b/Python/asm_trampoline_x86_64.S @@ -7,14 +7,20 @@ __Py_trampoline_func_start: .globl _Py_trampoline_func_start _Py_trampoline_func_start: #endif + .cfi_startproc #if defined(__CET__) && (__CET__ & 1) endbr64 #endif push %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 mov %rsp, %rbp + .cfi_def_cfa_register %rbp call *%rcx pop %rbp + .cfi_def_cfa %rsp, 8 ret + .cfi_endproc #if defined(__APPLE__) .globl __Py_trampoline_func_end __Py_trampoline_func_end: diff --git a/Python/jit_unwind.c b/Python/jit_unwind.c index 0941ed593ff7d1..46660ed6839546 100644 --- a/Python/jit_unwind.c +++ b/Python/jit_unwind.c @@ -1,8 +1,8 @@ /* * Python JIT - DWARF .eh_frame builder * - * This file contains the DWARF CFI generator used to build .eh_frame - * data for JIT code (perf jitdump and other unwinders). + * Builds the .eh_frame data attached to JIT code regions, for the perf + * jitdump (from trampoline_ehframe.h) and for the GDB JIT interface. */ #include "Python.h" @@ -16,6 +16,10 @@ # endif #endif +#if defined(PY_HAVE_PERF_TRAMPOLINE) +# include "trampoline_ehframe.h" // generated by Tools/jit/_trampoline_ehframe.py +#endif + #if defined(PY_HAVE_PERF_TRAMPOLINE) \ || defined(PY_HAVE_JIT_GDB_UNWIND) \ || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) @@ -35,6 +39,33 @@ void __deregister_frame(const void *); #include #include + +// ============================================================================= +// ELF OBJECT CONTEXT +// ============================================================================= + +/* + * Context for building ELF/DWARF structures + * + * This structure maintains state while constructing DWARF unwind information. + * It acts as a simple buffer manager with pointers to track current position + * and important landmarks within the buffer. + */ +typedef struct ELFObjectContext { + uint8_t* p; // Current write position in buffer + uint8_t* startp; // Start of buffer (for offset calculations) + uintptr_t code_addr; // Address of the code section + size_t code_size; // Size of the code section +} ELFObjectContext; + +// ============================================================================= +// DWARF GENERATION UTILITIES +// ============================================================================= + +/* The perf path copies pre-built .eh_frame bytes (see elf_init_ehframe_perf), + * so the DWARF constants and writers below are only needed by the GDB path. */ +#if defined(PY_HAVE_JIT_GDB_UNWIND) + // ============================================================================= // DWARF CONSTANTS // ============================================================================= @@ -60,72 +91,9 @@ enum { DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter DWRF_CFA_offset = 0x80, // Simple offset instruction -#if defined(__aarch64__) - DWRF_CFA_AARCH64_negate_ra_state = 0x2d, // Toggle return address signing state -#endif DWRF_CFA_restore = 0xc0 // Restore register }; -/* - * Architecture-specific DWARF register numbers - * - * These constants define the register numbering scheme used by DWARF - * for each supported architecture. The numbers must match the ABI - * specification for proper stack unwinding. - */ -enum { -#ifdef __x86_64__ - /* x86_64 register numbering (note: order is defined by x86_64 ABI) */ - DWRF_REG_AX, // RAX - DWRF_REG_DX, // RDX - DWRF_REG_CX, // RCX - DWRF_REG_BX, // RBX - DWRF_REG_SI, // RSI - DWRF_REG_DI, // RDI - DWRF_REG_BP, // RBP - DWRF_REG_SP, // RSP - DWRF_REG_8, // R8 - DWRF_REG_9, // R9 - DWRF_REG_10, // R10 - DWRF_REG_11, // R11 - DWRF_REG_12, // R12 - DWRF_REG_13, // R13 - DWRF_REG_14, // R14 - DWRF_REG_15, // R15 - DWRF_REG_RA, // Return address (RIP) -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - /* AArch64 register numbering */ - DWRF_REG_FP = 29, // Frame Pointer - DWRF_REG_RA = 30, // Link register (return address) - DWRF_REG_SP = 31, // Stack pointer -#else -# error "Unsupported target architecture" -#endif -}; - -// ============================================================================= -// ELF OBJECT CONTEXT -// ============================================================================= - -/* - * Context for building ELF/DWARF structures - * - * This structure maintains state while constructing DWARF unwind information. - * It acts as a simple buffer manager with pointers to track current position - * and important landmarks within the buffer. - */ -typedef struct ELFObjectContext { - uint8_t* p; // Current write position in buffer - uint8_t* startp; // Start of buffer (for offset calculations) - uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) - uintptr_t code_addr; // Address of the code section - size_t code_size; // Size of the code section -} ELFObjectContext; - -// ============================================================================= -// DWARF GENERATION UTILITIES -// ============================================================================= - /* * Append a null-terminated string to the ELF context buffer. * @@ -221,6 +189,8 @@ static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ } +#endif /* PY_HAVE_JIT_GDB_UNWIND */ + // ============================================================================= // DWARF EH FRAME GENERATION // ============================================================================= @@ -246,7 +216,12 @@ static inline void elf_init_ehframe(ELFObjectContext* ctx, int absolute_addr) { size_t _PyJitUnwind_EhFrameSize(int absolute_addr) { - /* The .eh_frame we emit is small and bounded; keep a generous buffer. */ +#if defined(PY_HAVE_PERF_TRAMPOLINE) + if (!absolute_addr) { + return TRAMPOLINE_EHFRAME_SIZE; + } +#endif + /* GDB path: generate into scratch to learn the required size. */ uint8_t scratch[512]; _Static_assert(sizeof(scratch) >= 256, "scratch buffer may be too small for elf_init_ehframe"); @@ -254,8 +229,6 @@ _PyJitUnwind_EhFrameSize(int absolute_addr) ctx.code_size = 1; ctx.code_addr = 0; ctx.startp = ctx.p = scratch; - ctx.fde_p = NULL; - /* Generate once into scratch to learn the required size. */ elf_init_ehframe(&ctx, absolute_addr); ptrdiff_t size = ctx.p - ctx.startp; assert(size <= (ptrdiff_t)sizeof(scratch)); @@ -270,7 +243,17 @@ _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, if (buffer == NULL || code_addr == NULL || code_size == 0) { return 0; } - /* Generate the frame twice: once to size-check, once to write. */ +#if defined(PY_HAVE_PERF_TRAMPOLINE) + /* perf's EhFrameHeader stores the distance from the end of the frame + * back to the code as a signed 4-byte offset (see perf_jit_trampoline.c), + * and so does the FDE when its fields are 4 bytes wide. Refuse sizes + * those cannot hold rather than writing truncated offsets. */ + if (!absolute_addr + && code_size > (size_t)INT32_MAX - 8 - TRAMPOLINE_EHFRAME_SIZE) { + return 0; + } +#endif + /* Size the frame first (a constant for the perf path), then write it. */ size_t required = _PyJitUnwind_EhFrameSize(absolute_addr); if (required == 0 || required > buffer_size) { return 0; @@ -279,7 +262,6 @@ _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, ctx.code_size = code_size; ctx.code_addr = (uintptr_t)code_addr; ctx.startp = ctx.p = buffer; - ctx.fde_p = NULL; elf_init_ehframe(&ctx, absolute_addr); size_t written = (size_t)(ctx.p - ctx.startp); /* The frame size is independent of code_addr/code_size (fixed-width fields). */ @@ -301,10 +283,34 @@ _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, * Two flavors are emitted, dispatched on the absolute_addr flag: * * - absolute_addr == 0 (elf_init_ehframe_perf): PC-relative FDE address - * encoding for perf's synthesized DSO layout. The CIE describes the - * trampoline's entry state and the FDE walks through the prologue and - * epilogue with advance_loc instructions. This matches the pre-existing - * perf_jit_trampoline behavior byte-for-byte. + * encoding for perf's synthesized DSO layout. The bytes come from + * trampoline_ehframe.h, generated at build time from the compiled + * trampoline object. Only the FDE's initial_location and address_range + * are filled in here. + * + * To add an architecture, write Python/asm_trampoline_.S with + * .cfi directives describing the frame at every instruction (the + * x86_64 and AArch64 files are the reference), wire its object into + * configure.ac and Makefile.pre.in, and add its ELF machine type or + * Mach-O CPU type to the tables at the top of + * Tools/jit/_trampoline_ehframe.py. Compiling an equivalent C trampoline + * with "-O2 -fno-omit-frame-pointer -fno-optimize-sibling-calls" and + * running "readelf --debug-dump=frames" on it shows the CFI the + * directives must be equivalent to, for example with GCC on AArch64: + * + * CIE: Code alignment factor 4, Data alignment factor -8, + * Return address column 30, DW_CFA_def_cfa: r31 (sp) ofs 0 + * FDE: DW_CFA_advance_loc: 4 + * DW_CFA_def_cfa_offset: 16 + * DW_CFA_offset: r29 at cfa-16 + * DW_CFA_offset: r30 at cfa-8 + * DW_CFA_advance_loc: 12 + * DW_CFA_restore: r30 + * DW_CFA_restore: r29 + * DW_CFA_def_cfa_offset: 0 + * + * GCC keeps the CFA relative to sp there, the assembly moves it to the + * frame pointer after "mov x29, sp", which describes the same frame. * * - absolute_addr == 1 (elf_init_ehframe_gdb): absolute FDE address * encoding for the GDB JIT in-memory ELF. The CIE describes the @@ -316,336 +322,37 @@ _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, * for details. */ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { - int fde_ptr_enc = DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4; - uint8_t* p = ctx->p; - uint8_t* framep = p; // Remember start of frame data - - /* - * DWARF Unwind Table for Trampoline Function - * - * This section defines DWARF Call Frame Information (CFI) using encoded macros - * like `DWRF_U8`, `DWRF_UV`, and `DWRF_SECTION` to describe how the trampoline function - * preserves and restores registers. This is used by profiling tools (e.g., `perf`) - * and debuggers for stack unwinding in JIT-compiled code. - * - * ------------------------------------------------- - * TO REGENERATE THIS TABLE FROM GCC OBJECTS: - * ------------------------------------------------- - * - * 1. Create a trampoline source file (e.g., `trampoline.c`): - * - * #include - * typedef PyObject* (*py_evaluator)(void*, void*, int); - * PyObject* trampoline(void *ts, void *f, int throwflag, py_evaluator evaluator) { - * return evaluator(ts, f, throwflag); - * } - * - * 2. Compile to an object file with frame pointer preservation: - * - * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c - * - * 3. Extract DWARF unwind info from the object file: - * - * readelf -w trampoline.o - * - * Example output from `.eh_frame`: - * - * 00000000 CIE - * Version: 1 - * Augmentation: "zR" - * Code alignment factor: 4 - * Data alignment factor: -8 - * Return address column: 30 - * DW_CFA_def_cfa: r31 (sp) ofs 0 - * - * 00000014 FDE cie=00000000 pc=0..14 - * DW_CFA_advance_loc: 4 - * DW_CFA_def_cfa_offset: 16 - * DW_CFA_offset: r29 at cfa-16 - * DW_CFA_offset: r30 at cfa-8 - * DW_CFA_advance_loc: 12 - * DW_CFA_restore: r30 - * DW_CFA_restore: r29 - * DW_CFA_def_cfa_offset: 0 - * - * -- These values can be verified by comparing with `readelf -w` or `llvm-dwarfdump --eh-frame`. - * - * ---------------------------------- - * HOW TO TRANSLATE TO DWRF_* MACROS: - * ---------------------------------- - * - * After compiling your trampoline with: - * - * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c - * - * run: - * - * readelf -w trampoline.o - * - * to inspect the generated `.eh_frame` data. You will see two main components: - * - * 1. A CIE (Common Information Entry): shared configuration used by all FDEs. - * 2. An FDE (Frame Description Entry): function-specific unwind instructions. - * - * --------------------- - * Translating the CIE: - * --------------------- - * From `readelf -w`, you might see: - * - * 00000000 0000000000000010 00000000 CIE - * Version: 1 - * Augmentation: "zR" - * Code alignment factor: 4 - * Data alignment factor: -8 - * Return address column: 30 - * Augmentation data: 1b - * DW_CFA_def_cfa: r31 (sp) ofs 0 - * - * Map this to: - * - * DWRF_SECTION(CIE, - * DWRF_U32(0); // CIE ID (always 0 for CIEs) - * DWRF_U8(DWRF_CIE_VERSION); // Version: 1 - * DWRF_STR("zR"); // Augmentation string "zR" - * DWRF_UV(4); // Code alignment factor = 4 - * DWRF_SV(-8); // Data alignment factor = -8 - * DWRF_U8(DWRF_REG_RA); // Return address register (e.g., x30 = 30) - * DWRF_UV(1); // Augmentation data length = 1 - * DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // Encoding for FDE pointers - * - * DWRF_U8(DWRF_CFA_def_cfa); // DW_CFA_def_cfa - * DWRF_UV(DWRF_REG_SP); // Register: SP (r31) - * DWRF_UV(0); // Offset = 0 - * - * DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer size boundary - * ) - * - * Notes: - * - Use `DWRF_UV` for unsigned LEB128, `DWRF_SV` for signed LEB128. - * - `DWRF_REG_RA` and `DWRF_REG_SP` are architecture-defined constants. - * - * --------------------- - * Translating the FDE: - * --------------------- - * From `readelf -w`: - * - * 00000014 0000000000000020 00000018 FDE cie=00000000 pc=0000000000000000..0000000000000014 - * DW_CFA_advance_loc: 4 - * DW_CFA_def_cfa_offset: 16 - * DW_CFA_offset: r29 at cfa-16 - * DW_CFA_offset: r30 at cfa-8 - * DW_CFA_advance_loc: 12 - * DW_CFA_restore: r30 - * DW_CFA_restore: r29 - * DW_CFA_def_cfa_offset: 0 - * - * Map the FDE header and instructions to: - * - * DWRF_SECTION(FDE, - * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) - * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) - * DWRF_U32(ctx->code_size); // Code range covered by this FDE - * DWRF_U8(0); // Augmentation data length (none) - * - * DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 unit (1 * 4 = 4 bytes) - * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 - * DWRF_UV(16); - * - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Save x29 (frame pointer) - * DWRF_UV(2); // At offset 2 * 8 = 16 bytes - * - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Save x30 (return address) - * DWRF_UV(1); // At offset 1 * 8 = 8 bytes - * - * DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance location by 3 units (3 * 4 = 12 bytes) - * - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore x30 - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore x29 - * - * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP - * DWRF_UV(0); - * ) - * - * To regenerate: - * 1. Get the `code alignment factor`, `data alignment factor`, and `RA column` from the CIE. - * 2. Note the range of the function from the FDE's `pc=...` line and map it to the JIT code as - * the code is in a different address space every time. - * 3. For each `DW_CFA_*` entry, use the corresponding `DWRF_*` macro: - * - `DW_CFA_def_cfa_offset` → DWRF_U8(DWRF_CFA_def_cfa_offset), DWRF_UV(value) - * - `DW_CFA_offset: rX` → DWRF_U8(DWRF_CFA_offset | reg), DWRF_UV(offset) - * - `DW_CFA_restore: rX` → DWRF_U8(DWRF_CFA_offset | reg) // restore is same as reusing offset - * - `DW_CFA_advance_loc: N` → DWRF_U8(DWRF_CFA_advance_loc | (N / code_alignment_factor)) - * 4. Use `DWRF_REG_FP`, `DWRF_REG_RA`, etc., for register numbers. - * 5. Use `sizeof(uintptr_t)` (typically 8) for pointer size calculations and alignment. - */ - - /* - * Emit DWARF EH CIE (Common Information Entry) - * - * The CIE describes the calling conventions and basic unwinding rules - * that apply to all functions in this compilation unit. - */ - DWRF_SECTION(CIE, - DWRF_U32(0); // CIE ID (0 indicates this is a CIE) - DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) - DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) -#ifdef __x86_64__ - DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) -#endif - DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) - DWRF_U8(DWRF_REG_RA); // Return address register number - DWRF_UV(1); // Augmentation data length - DWRF_U8(fde_ptr_enc); // FDE pointer encoding - - /* Initial CFI instructions - describe default calling convention */ -#ifdef __x86_64__ - /* x86_64 initial CFI state */ - DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) - DWRF_UV(DWRF_REG_SP); // CFA = SP register - DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size - DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved - DWRF_UV(1); // At offset 1 from CFA -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - /* AArch64 initial CFI state */ - DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) - DWRF_UV(DWRF_REG_SP); // CFA = SP register - DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) - // No initial register saves in AArch64 CIE -#endif - DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary - ) - - /* - * Emit DWARF EH FDE (Frame Description Entry) - * - * The FDE describes unwinding information specific to this function. - * It references the CIE and provides function-specific CFI instructions. - * - * The PC-relative offset is calculated after the entire EH frame is built - * to ensure accurate positioning relative to the synthesized DSO layout. - */ - DWRF_SECTION(FDE, - DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) - /* - * In perf jitdump mode the FDE PC field is encoded PC-relative and - * points back to code_start. Record where that field lives so we can - * patch in the final offset after the rest of the synthetic DSO - * layout is known. - */ - ctx->fde_p = p; // Remember where PC offset field is located for later calculation - DWRF_U32(0); // Placeholder for PC-relative offset (calculated below) - DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) - DWRF_U8(0); // Augmentation data length (none) - - /* - * Architecture-specific CFI instructions - * - * These instructions describe how registers are saved and restored - * during function calls. Each architecture has different calling - * conventions and register usage patterns. - */ -#ifdef __x86_64__ - /* x86_64 calling convention unwinding rules */ -# if defined(__CET__) && (__CET__ & 1) - DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) -# endif - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) - DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 - DWRF_UV(16); // New offset: SP + 16 - DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 - DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) - DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 - DWRF_UV(DWRF_REG_BP); // Use base pointer register - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 - DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 - DWRF_UV(DWRF_REG_SP); // Use stack pointer register - DWRF_UV(8); // New offset: SP + 8 -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - /* AArch64 calling convention unwinding rules */ -#if defined(__ARM_FEATURE_PAC_DEFAULT) || \ - (defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1) - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past SIGN_LR (4 bytes) -#endif -#if defined(__ARM_FEATURE_PAC_DEFAULT) - DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // Saved LR is PAC-signed from here -#endif - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) - DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 - DWRF_UV(16); // Stack pointer moved by 16 bytes - DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved - DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved - DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) -#if defined(__ARM_FEATURE_PAC_DEFAULT) - DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // LR is authenticated, no longer PAC-signed -#endif - DWRF_U8(DWRF_CFA_def_cfa_register); // CFA = FP (x29) + 16 - DWRF_UV(DWRF_REG_FP); - DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! - DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! - DWRF_U8(DWRF_CFA_def_cfa); // CFA = SP + 0 (stack restored) - DWRF_UV(DWRF_REG_SP); - DWRF_UV(0); - +#if defined(PY_HAVE_PERF_TRAMPOLINE) + _Static_assert(TRAMPOLINE_EHFRAME_FDE_PC_OFFSET + TRAMPOLINE_EHFRAME_FDE_FIELD_SIZE + == TRAMPOLINE_EHFRAME_FDE_RANGE_OFFSET, + "trampoline_ehframe.h FDE fields are not adjacent"); + _Static_assert(TRAMPOLINE_EHFRAME_FDE_RANGE_OFFSET + + TRAMPOLINE_EHFRAME_FDE_FIELD_SIZE <= TRAMPOLINE_EHFRAME_SIZE, + "trampoline_ehframe.h patches fields outside its data"); + uint8_t *p = ctx->p; + memcpy(p, _trampoline_ehframe, TRAMPOLINE_EHFRAME_SIZE); + + /* perf maps this .eh_frame right after the code, at code_size rounded + * up to 8 bytes (see EhFrameHeader in perf_jit_trampoline.c), and + * initial_location is relative to its own field. */ + int64_t initial_location = -(int64_t)( + _Py_SIZE_ROUND_UP(ctx->code_size, 8) + TRAMPOLINE_EHFRAME_FDE_PC_OFFSET); + uint64_t address_range = ctx->code_size; +#if TRAMPOLINE_EHFRAME_FDE_FIELD_SIZE == 4 + int32_t pc_field = (int32_t)initial_location; + uint32_t range_field = (uint32_t)address_range; +#elif TRAMPOLINE_EHFRAME_FDE_FIELD_SIZE == 8 + int64_t pc_field = initial_location; + uint64_t range_field = address_range; #else -# error "Unsupported target architecture" +# error "unsupported FDE field size in trampoline_ehframe.h" #endif + memcpy(p + TRAMPOLINE_EHFRAME_FDE_PC_OFFSET, &pc_field, sizeof(pc_field)); + memcpy(p + TRAMPOLINE_EHFRAME_FDE_RANGE_OFFSET, &range_field, + sizeof(range_field)); - DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary - ) - - ctx->p = p; // Update context pointer to end of generated data - - /* Calculate and update the PC-relative offset in the FDE - * - * When perf processes the jitdump, it creates a synthesized DSO with this layout: - * - * Synthesized DSO Memory Layout: - * ┌─────────────────────────────────────────────────────────────┐ < code_start - * │ Code Section │ - * │ (round_up(code_size, 8) bytes) │ - * ├─────────────────────────────────────────────────────────────┤ < start of EH frame data - * │ EH Frame Data │ - * │ ┌─────────────────────────────────────────────────────┐ │ - * │ │ CIE data │ │ - * │ └─────────────────────────────────────────────────────┘ │ - * │ ┌─────────────────────────────────────────────────────┐ │ - * │ │ FDE Header: │ │ - * │ │ - CIE offset (4 bytes) │ │ - * │ │ - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start - * │ │ - address range (4 bytes) │ │ (this specific field) - * │ │ CFI Instructions... │ │ - * │ └─────────────────────────────────────────────────────┘ │ - * ├─────────────────────────────────────────────────────────────┤ < reference_point - * │ EhFrameHeader │ - * │ (navigation metadata) │ - * └─────────────────────────────────────────────────────────────┘ - * - * The PC offset field in the FDE must contain the distance from itself to code_start: - * - * distance = code_start - fde_pc_field - * - * Where: - * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame - * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) - * - * Therefore: - * distance = code_start_location - fde_pc_field_location - * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) - * = -rounded_code_size - fde_offset_in_frame - * = -(round_up(code_size, 8) + fde_offset_in_frame) - * - * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field. - * - */ - int32_t rounded_code_size = - (int32_t)_Py_SIZE_ROUND_UP(ctx->code_size, 8); - int32_t fde_offset_in_frame = (int32_t)(ctx->fde_p - framep); - *(int32_t *)ctx->fde_p = -(rounded_code_size + fde_offset_in_frame); + ctx->p = p + TRAMPOLINE_EHFRAME_SIZE; +#endif } /* diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index d90b789c2b5712..fa1cb785b4126e 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -172,10 +172,8 @@ typedef PyObject *(*py_evaluator)(PyThreadState *, _PyInterpreterFrame *, typedef PyObject *(*py_trampoline)(PyThreadState *, _PyInterpreterFrame *, int, py_evaluator); -extern void *_Py_trampoline_func_start; // Start of the template of the - // assembly trampoline -extern void * - _Py_trampoline_func_end; // End of the template of the assembly trampoline +extern char _Py_trampoline_func_start; // Start of the assembly trampoline template +extern char _Py_trampoline_func_end; // End of the assembly trampoline template struct code_arena_st { char *start_addr; // Start of the memory arena @@ -334,8 +332,8 @@ new_code_arena(void) return -1; } (void)_PyAnnotateMemoryMap(memory, mem_size, "cpython:perf_trampoline"); - void *start = &_Py_trampoline_func_start; - void *end = &_Py_trampoline_func_end; + char *start = &_Py_trampoline_func_start; + char *end = &_Py_trampoline_func_end; size_t code_size = end - start; size_t unaligned_size = code_size + trampoline_api.code_padding; size_t chunk_size = round_up(unaligned_size, trampoline_api.code_alignment); diff --git a/Tools/jit/_trampoline_ehframe.py b/Tools/jit/_trampoline_ehframe.py new file mode 100644 index 00000000000000..31fdf205c0dd1f --- /dev/null +++ b/Tools/jit/_trampoline_ehframe.py @@ -0,0 +1,534 @@ +"""Generate trampoline_ehframe.h from a compiled perf trampoline object. + +Copies the assembler-generated .eh_frame of the trampoline into a C header, +one block per architecture (a fat Mach-O file yields several). The FDE's +initial_location and address_range are left zeroed for Python/jit_unwind.c +to patch at runtime. + +This runs during bootstrap with Python 3.7 or newer, using only the standard +library. Keep annotations postponed so newer type syntax is not evaluated. +""" + +from __future__ import annotations + +import argparse +import struct +import sys +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +# ELF constants, see . +_ELF_MAGIC = b"\x7fELF" +_ELF64_HEADER_SIZE = 64 +_ELF64_SECTION_HEADER_SIZE = 64 +_ELFCLASS64 = 2 +_ELFDATA2LSB = 1 +_ELFDATA2MSB = 2 +_SHT_NOBITS = 8 +_EM_X86_64 = 62 +_EM_AARCH64 = 183 + +# Mach-O constants, see llvm/BinaryFormat/MachO.h. +_MH_MAGIC_64 = 0xFEEDFACF +_MH_CIGAM_64 = 0xCFFAEDFE +_MACHO64_HEADER_SIZE = 32 +_MACHO64_SEGMENT_COMMAND_SIZE = 72 +_MACHO64_SECTION_SIZE = 80 +_FAT_MAGIC = 0xCAFEBABE +_FAT_MAGIC_64 = 0xCAFEBABF +_FAT_HEADER_SIZE = 8 +_LC_SEGMENT_64 = 0x19 +_CPU_ARCH_ABI64 = 0x01000000 +_CPU_TYPE_X86_64 = 7 | _CPU_ARCH_ABI64 +_CPU_TYPE_ARM64 = 12 | _CPU_ARCH_ABI64 + +# DWARF exception header pointer encodings, see +# Include/internal/pycore_jit_unwind.h. +_DW_EH_PE_absptr = 0x00 +_DW_EH_PE_sdata4 = 0x0B +_DW_EH_PE_pcrel = 0x10 + +# Smallest CIE the parser accepts, in bytes after the length field: CIE_id, +# version, "zR\0", code and data alignment factors, return address column, +# augmentation data length and the FDE pointer encoding. +_CIE_MIN_LENGTH = 4 + 1 + 3 + 1 + 1 + 1 + 1 + 1 + +# Accepted FDE pointer encodings and the width of the initial_location and +# address_range fields they imply. GNU and LLVM ELF assemblers emit +# pcrel|sdata4, Darwin assemblers emit pcrel|absptr. +_FDE_FIELD_SIZES = { + _DW_EH_PE_pcrel | _DW_EH_PE_sdata4: 4, + _DW_EH_PE_pcrel | _DW_EH_PE_absptr: 8, +} + +# Compiler macro that selects each slice's block in the generated header. +_ELF_ARCH_MACROS = { + _EM_X86_64: "__x86_64__", + _EM_AARCH64: "__aarch64__", +} +_MACHO_ARCH_MACROS = { + _CPU_TYPE_X86_64: "__x86_64__", + _CPU_TYPE_ARM64: "__aarch64__", +} + +# The sections the generator needs, and their Mach-O names. +_WANTED_SECTIONS = (".eh_frame", ".text") +_MACHO_SECTION_NAMES = { + "__eh_frame": ".eh_frame", + "__text": ".text", +} + + +@dataclass(frozen=True) +class ObjectSlice: + """One architecture's worth of an object file.""" + + source: str + arch_macro: str + endian: str + sections: dict[str, bytes] + + +@dataclass(frozen=True) +class EhFrame: + """Parsed .eh_frame with the FDE fields zeroed for runtime patching.""" + + data: bytes + fde_pc_offset: int + fde_range_offset: int + field_size: int + + +def _elf_slice(data: bytes, source: str) -> ObjectSlice: + """Parse an ELF64 relocatable object.""" + if len(data) < _ELF64_HEADER_SIZE: + raise ValueError(f"{source}: truncated ELF header") + if data[4] != _ELFCLASS64: + raise ValueError(f"{source}: not an ELF64 object (class={data[4]})") + if data[5] == _ELFDATA2LSB: + endian = "<" + elif data[5] == _ELFDATA2MSB: + endian = ">" + else: + raise ValueError(f"{source}: unknown ELF byte order ({data[5]})") + + e_machine = struct.unpack_from(f"{endian}H", data, 18)[0] + try: + arch_macro = _ELF_ARCH_MACROS[e_machine] + except KeyError: + raise ValueError( + f"{source}: unsupported ELF machine type {e_machine}" + ) from None + + e_shoff = struct.unpack_from(f"{endian}Q", data, 40)[0] + e_shentsize, e_shnum, e_shstrndx = struct.unpack_from(f"{endian}HHH", data, 58) + if e_shoff == 0 or e_shnum == 0: + raise ValueError(f"{source}: no section headers") + if e_shentsize < _ELF64_SECTION_HEADER_SIZE: + raise ValueError(f"{source}: bad section header size {e_shentsize}") + if e_shoff + e_shnum * e_shentsize > len(data): + raise ValueError(f"{source}: section headers extend beyond the end of the file") + if e_shstrndx >= e_shnum: + raise ValueError(f"{source}: bad section name string table index") + + def section_header(index: int) -> tuple[int, int, int, int]: + base = e_shoff + index * e_shentsize + sh_name, sh_type = struct.unpack_from(f"{endian}II", data, base) + sh_offset, sh_size = struct.unpack_from(f"{endian}QQ", data, base + 24) + return sh_name, sh_type, sh_offset, sh_size + + def section_bytes(name: str, sh_type: int, sh_offset: int, sh_size: int) -> bytes: + if sh_type == _SHT_NOBITS: + raise ValueError(f"{source}: section {name} has no contents in the file") + if sh_offset + sh_size > len(data): + raise ValueError( + f"{source}: section {name} extends past the end of the file" + ) + return data[sh_offset : sh_offset + sh_size] + + _, shstr_type, shstr_offset, shstr_size = section_header(e_shstrndx) + shstrtab = section_bytes(".shstrtab", shstr_type, shstr_offset, shstr_size) + + sections: dict[str, bytes] = {} + for index in range(e_shnum): + sh_name, sh_type, sh_offset, sh_size = section_header(index) + end = shstrtab.find(b"\x00", sh_name) + if end < 0: + continue + name = shstrtab[sh_name:end].decode("ascii", errors="replace") + if name in _WANTED_SECTIONS: + if name in sections: + raise ValueError(f"{source}: more than one {name} section") + sections[name] = section_bytes(name, sh_type, sh_offset, sh_size) + + return ObjectSlice(source, arch_macro, endian, sections) + + +def _macho_sections( + data: bytes, source: str, endian: str, offset: int, cmdsize: int +) -> Iterator[tuple[str, bytes]]: + """Yield the wanted sections from one LC_SEGMENT_64 command.""" + # segment_command_64: cmd, cmdsize, segname[16], vmaddr, vmsize, + # fileoff, filesize, maxprot, initprot, nsects, flags (72 bytes), + # followed by nsects section_64 entries. + if cmdsize < _MACHO64_SEGMENT_COMMAND_SIZE: + raise ValueError(f"{source}: truncated segment command") + nsects = struct.unpack_from(f"{endian}I", data, offset + 64)[0] + if _MACHO64_SEGMENT_COMMAND_SIZE + nsects * _MACHO64_SECTION_SIZE > cmdsize: + raise ValueError( + f"{source}: section table extends beyond its segment command" + ) + sections_start = offset + _MACHO64_SEGMENT_COMMAND_SIZE + sections_end = sections_start + nsects * _MACHO64_SECTION_SIZE + for sect in range(sections_start, sections_end, _MACHO64_SECTION_SIZE): + # section_64: sectname[16], segname[16], addr, size, offset, + # align, reloff, nreloc, flags, reserved1-3 (80 bytes). + raw_name = data[sect : sect + 16].split(b"\x00", 1)[0] + segname = data[sect + 16 : sect + 32].split(b"\x00", 1)[0] + name = _MACHO_SECTION_NAMES.get( + raw_name.decode("ascii", errors="replace") + ) + if name is None or segname != b"__TEXT": + continue + size = struct.unpack_from(f"{endian}Q", data, sect + 40)[0] + file_offset = struct.unpack_from( + f"{endian}I", data, sect + 48 + )[0] + if file_offset + size > len(data): + raise ValueError( + f"{source}: section {name} extends past the end of the file" + ) + yield name, data[file_offset : file_offset + size] + + +def _macho_slice(data: bytes, source: str) -> ObjectSlice: + """Parse a thin Mach-O 64-bit object.""" + if len(data) < _MACHO64_HEADER_SIZE: + raise ValueError(f"{source}: truncated Mach-O header") + magic = struct.unpack_from(" len(data): + raise ValueError(f"{source}: load commands extend beyond the end of the file") + + sections: dict[str, bytes] = {} + offset = _MACHO64_HEADER_SIZE + for _ in range(ncmds): + if offset + 8 > commands_end: + raise ValueError(f"{source}: truncated load commands") + cmd, cmdsize = struct.unpack_from(f"{endian}II", data, offset) + if cmdsize < 8 or offset + cmdsize > commands_end: + raise ValueError(f"{source}: bad load command size {cmdsize}") + if cmd == _LC_SEGMENT_64: + for name, contents in _macho_sections( + data, source, endian, offset, cmdsize + ): + if name in sections: + raise ValueError(f"{source}: more than one {name} section") + sections[name] = contents + offset += cmdsize + + return ObjectSlice(source, arch_macro, endian, sections) + + +def _fat_slices(data: bytes, source: str) -> list[ObjectSlice]: + """Split a universal (fat) Mach-O file into its thin slices.""" + # The fat header and its fat_arch entries are always big-endian. + magic, nfat_arch = struct.unpack_from(">II", data, 0) + if magic == _FAT_MAGIC: + # fat_arch: cputype, cpusubtype, offset, size, align (20 bytes). + entry_struct = struct.Struct(">IIIII") + elif magic == _FAT_MAGIC_64: + # fat_arch_64: cputype, cpusubtype, offset, size, align, reserved. + entry_struct = struct.Struct(">IIQQII") + else: + raise ValueError(f"{source}: not a fat Mach-O file") + if nfat_arch == 0: + raise ValueError(f"{source}: fat Mach-O file with no architectures") + if _FAT_HEADER_SIZE + nfat_arch * entry_struct.size > len(data): + raise ValueError(f"{source}: truncated fat header") + + slices: list[ObjectSlice] = [] + for index in range(nfat_arch): + cputype, _, offset, size, *_ = entry_struct.unpack_from( + data, _FAT_HEADER_SIZE + index * entry_struct.size + ) + thin = data[offset : offset + size] + if len(thin) != size: + raise ValueError(f"{source}: fat slice {index} is truncated") + obj_slice = _macho_slice(thin, f"{source} (slice {cputype:#x})") + if _MACHO_ARCH_MACROS.get(cputype) != obj_slice.arch_macro: + raise ValueError( + f"{source}: fat slice {index} CPU type {cputype:#x} does not match " + "its Mach-O header" + ) + slices.append(obj_slice) + return slices + + +def load_object(path: str | Path) -> list[ObjectSlice]: + """Return the ObjectSlices (one per architecture) of an object file.""" + path = Path(path) + data = path.read_bytes() + source = path.name + if data[:4] == _ELF_MAGIC: + return [_elf_slice(data, source)] + if len(data) < _FAT_HEADER_SIZE: + raise ValueError(f"{source}: file too short to be an object file") + magic_be = struct.unpack_from(">I", data, 0)[0] + if magic_be in (_FAT_MAGIC, _FAT_MAGIC_64): + return _fat_slices(data, source) + magic_le = struct.unpack_from(" tuple[int, int]: + """Decode an unsigned LEB128 at pos; return (value, position after it).""" + value = 0 + shift = 0 + while True: + if pos >= limit: + raise ValueError("truncated LEB128 in CIE") + byte = data[pos] + pos += 1 + value |= (byte & 0x7F) << shift + shift += 7 + if not byte & 0x80: + return value, pos + + +def _parse_cie(data: bytes, endian: str) -> tuple[int, int]: + """Validate the CIE and return its end offset and FDE field size.""" + if len(data) < 8: + raise ValueError("no CIE found in .eh_frame") + + # CIE header: length, CIE_id (0), version, augmentation string. + cie_length, cie_id = struct.unpack_from(f"{endian}II", data, 0) + if cie_id != 0: + raise ValueError(f"expected a CIE at offset 0, got CIE_id={cie_id:#x}") + cie_total = 4 + cie_length + if cie_length < _CIE_MIN_LENGTH or cie_total > len(data): + raise ValueError(f"bad CIE length {cie_length}") + + version = data[8] + if version != 1: + raise ValueError(f"unexpected CIE version {version}") + + null_pos = data.find(b"\x00", 9, cie_total) + if null_pos < 0: + raise ValueError("CIE augmentation string not null-terminated") + augmentation = data[9:null_pos].decode("ascii", errors="replace") + if augmentation != "zR": + raise ValueError(f"CIE augmentation {augmentation!r} is not 'zR'") + + # After the augmentation string: code alignment factor (ULEB128), data + # alignment factor (SLEB128, skipped the same way), return address column + # (one byte in version 1), augmentation data length (ULEB128), then the + # augmentation data, which for "zR" is exactly the FDE pointer encoding. + pos = null_pos + 1 + _, pos = _read_uleb128(data, pos, cie_total) + _, pos = _read_uleb128(data, pos, cie_total) + pos += 1 + aug_length, pos = _read_uleb128(data, pos, cie_total) + if aug_length != 1 or pos + aug_length > cie_total: + raise ValueError(f"CIE augmentation data length {aug_length} is not 1") + fde_ptr_enc = data[pos] + try: + field_size = _FDE_FIELD_SIZES[fde_ptr_enc] + except KeyError: + raise ValueError(f"unsupported FDE pointer encoding {fde_ptr_enc:#x}") from None + + return cie_total, field_size + + +def parse_ehframe(eh_frame: bytes, endian: str, text_size: int) -> EhFrame: + """Validate a one-CIE, one-FDE .eh_frame and zero the FDE's fields. + + text_size is the size of the object's .text section; it must equal the + FDE's address_range, which catches a misplaced .cfi_endproc. + """ + fde_start, field_size = _parse_cie(eh_frame, endian) + data = bytearray(eh_frame) + + # FDE: length, CIE pointer, initial_location, address_range, augmentation + # data length (0 for a "zR" CIE), then the CFI instructions. + fde_min_length = 4 + 2 * field_size + 1 + if fde_start + 4 + fde_min_length > len(data): + raise ValueError("no FDE after the CIE") + fde_length, fde_cie_ptr = struct.unpack_from(f"{endian}II", data, fde_start) + if fde_length < fde_min_length: + raise ValueError(f"FDE too short ({fde_length} bytes)") + if fde_cie_ptr != fde_start + 4: + raise ValueError(f"FDE CIE pointer {fde_cie_ptr} does not point at the CIE") + if fde_start + 4 + fde_length != len(data): + raise ValueError("expected exactly one FDE ending at the end of .eh_frame") + + fde_pc_offset = fde_start + 8 + fde_range_offset = fde_pc_offset + field_size + fde_aug_length = data[fde_range_offset + field_size] + if fde_aug_length != 0: + raise ValueError(f"FDE augmentation data length {fde_aug_length} is not 0") + address_range = int.from_bytes( + data[fde_range_offset : fde_range_offset + field_size], + "little" if endian == "<" else "big", + ) + if address_range != text_size: + raise ValueError( + f"FDE address_range {address_range} != .text size {text_size}; " + "check the .cfi_startproc/.cfi_endproc placement" + ) + + # Zero the placeholders. The runtime fills in the real values. + data[fde_pc_offset : fde_range_offset + field_size] = bytes(2 * field_size) + return EhFrame(bytes(data), fde_pc_offset, fde_range_offset, field_size) + + +def build_ehframe(obj_slice: ObjectSlice) -> EhFrame: + """Extract and validate the .eh_frame of one object slice.""" + sections = obj_slice.sections + if ".eh_frame" not in sections: + raise ValueError( + f"{obj_slice.source}: no .eh_frame section; does the assembly " + "have .cfi_startproc/.cfi_endproc directives?" + ) + if ".text" not in sections: + raise ValueError(f"{obj_slice.source}: no .text section") + try: + return parse_ehframe( + sections[".eh_frame"], obj_slice.endian, len(sections[".text"]) + ) + except ValueError as exc: + raise ValueError(f"{obj_slice.source}: {exc}") from None + + +def _format_bytes(data: bytes, per_line: int = 12) -> str: + lines = [] + for start in range(0, len(data), per_line): + chunk = data[start : start + per_line] + lines.append(" " + ", ".join(f"0x{b:02x}" for b in chunk) + ",") + return "\n".join(lines) + + +def _format_header(entries: list[tuple[ObjectSlice, EhFrame]]) -> str: + """Render one conditional block per architecture in a C header.""" + if not entries: + raise ValueError("no architectures to write") + sources = sorted({obj_slice.source.split(" (")[0] for obj_slice, _ in entries}) + lines = [ + "/* Auto-generated by Tools/jit/_trampoline_ehframe.py" + f" from {', '.join(sources)}. Do not edit. */", + "", + "#include ", + "", + "/* .eh_frame of the perf trampoline. The FDE's initial_location and", + " * address_range are zeroed placeholders patched by Python/jit_unwind.c. */", + ] + entries = sorted(entries, key=lambda entry: entry[0].arch_macro) + for index, (obj_slice, eh_frame) in enumerate(entries): + keyword = "#elif" if index else "#if" + lines += [ + f"{keyword} defined({obj_slice.arch_macro})", + f"/* From {obj_slice.source}. */", + "static const uint8_t _trampoline_ehframe[] = {", + _format_bytes(eh_frame.data), + "};", + f"#define TRAMPOLINE_EHFRAME_FDE_PC_OFFSET {eh_frame.fde_pc_offset}", + f"#define TRAMPOLINE_EHFRAME_FDE_RANGE_OFFSET {eh_frame.fde_range_offset}", + f"#define TRAMPOLINE_EHFRAME_FDE_FIELD_SIZE {eh_frame.field_size}", + ] + lines += [ + "#else", + '# error "trampoline_ehframe.h was not generated for this architecture"', + "#endif", + "", + "#define TRAMPOLINE_EHFRAME_SIZE sizeof(_trampoline_ehframe)", + "", + ] + return "\n".join(lines) + + +def write_header( + entries: list[tuple[ObjectSlice, EhFrame]], output_path: str | Path +) -> None: + """Replace the header only after its complete contents have been written.""" + output = _format_header(entries) + output_path = Path(output_path) + tmp_path = output_path.with_name(output_path.name + ".tmp") + try: + tmp_path.write_text(output) + tmp_path.replace(output_path) + finally: + with suppress(FileNotFoundError): + tmp_path.unlink() + + +def generate( + object_paths: list[str | Path], output_path: str | Path +) -> list[tuple[ObjectSlice, EhFrame]]: + """Generate the header from the given objects; return what was written.""" + entries: list[tuple[ObjectSlice, EhFrame]] = [] + seen: dict[str, str] = {} + for path in object_paths: + for obj_slice in load_object(path): + if obj_slice.arch_macro in seen: + raise ValueError( + f"{obj_slice.source} and {seen[obj_slice.arch_macro]} " + f"are both {obj_slice.arch_macro}" + ) + seen[obj_slice.arch_macro] = obj_slice.source + entries.append((obj_slice, build_ehframe(obj_slice))) + write_header(entries, output_path) + return entries + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate trampoline_ehframe.h from trampoline objects" + ) + parser.add_argument( + "objects", + nargs="+", + type=Path, + metavar="OBJECT", + help="compiled trampoline object (ELF64, Mach-O 64, or fat Mach-O)", + ) + parser.add_argument( + "-o", "--output", type=Path, required=True, + help="path of the C header to write", + ) + args = parser.parse_args() + try: + entries = generate(args.objects, args.output) + except (OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + for obj_slice, eh_frame in entries: + print( + f"Generated {args.output}: {obj_slice.arch_macro} from " + f"{obj_slice.source}, {len(eh_frame.data)} bytes" + ) + + +if __name__ == "__main__": + main() diff --git a/configure b/configure index 1115be256afb49..5205f59431dbb6 100755 --- a/configure +++ b/configure @@ -14663,8 +14663,28 @@ esac ;; #( perf_trampoline=no ;; esac +perf_trampoline_missing_python=no +if test "x$perf_trampoline" = xyes +then : + + if $PYTHON_FOR_REGEN -c 'import sys; sys.exit(sys.version_info < (3, 7))' >/dev/null 2>&1 +then : + +else case e in #( + e) perf_trampoline=no + PERF_TRAMPOLINE_OBJ="" + perf_trampoline_missing_python=yes ;; +esac +fi + +fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $perf_trampoline" >&5 printf "%s\n" "$perf_trampoline" >&6; } +if test "x$perf_trampoline_missing_python" = xyes +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: perf trampoline disabled: PYTHON_FOR_REGEN ($PYTHON_FOR_REGEN) is missing or older than 3.7 (needed to generate trampoline_ehframe.h)" >&5 +printf "%s\n" "$as_me: WARNING: perf trampoline disabled: PYTHON_FOR_REGEN ($PYTHON_FOR_REGEN) is missing or older than 3.7 (needed to generate trampoline_ehframe.h)" >&2;} +fi if test "x$perf_trampoline" = xyes then : diff --git a/configure.ac b/configure.ac index 902a822e43a42f..5d7a59ed013e9d 100644 --- a/configure.ac +++ b/configure.ac @@ -3943,7 +3943,19 @@ AS_CASE([$PLATFORM_TRIPLET], )], [perf_trampoline=no] ) +dnl Generating the trampoline's unwind data (trampoline_ehframe.h) at build +dnl time needs a Python interpreter, 3.7 or newer (Tools/jit/_trampoline_ehframe.py). +perf_trampoline_missing_python=no +AS_VAR_IF([perf_trampoline], [yes], [ + AS_IF([$PYTHON_FOR_REGEN -c 'import sys; sys.exit(sys.version_info < (3, 7))' >/dev/null 2>&1], + [], + [perf_trampoline=no + PERF_TRAMPOLINE_OBJ="" + perf_trampoline_missing_python=yes]) +]) AC_MSG_RESULT([$perf_trampoline]) +AS_VAR_IF([perf_trampoline_missing_python], [yes], + [AC_MSG_WARN([perf trampoline disabled: PYTHON_FOR_REGEN ($PYTHON_FOR_REGEN) is missing or older than 3.7 (needed to generate trampoline_ehframe.h)])]) AS_VAR_IF([perf_trampoline], [yes], [ AC_DEFINE([PY_HAVE_PERF_TRAMPOLINE], [1], [Define to 1 if you have the perf trampoline.])