From 4994261ee654a511b0e9f6424b3f9e5bbac7ea07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5ns=20Nilsson?= Date: Tue, 23 Jun 2026 18:32:29 +0200 Subject: [PATCH] XNNPACK: Add optional performance stage to pytest/tester MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Force the ExecuTorch threadpool to one thread while timing, then restore the previous thread count after the run. This reduces noise and keeps results easier to compare across repeated local measurements. Store the complete measurement record in a json file for regression checks. Also write a short md file with only the key review fields. Deduce expected HW path based on host features. Use Resnet as first example with json result data. Change-Id: I2bfe57dfddcfeadb850594ef416774ba08c56292 Signed-off-by: Måns Nilsson Change-Id: I6977867a48daee3d6ab4073de8fe9702c6059840 --- backends/xnnpack/test/models/README.md | 85 ++ backends/xnnpack/test/models/resnet.py | 12 +- .../models/resnet_pytest_perf_results.json | 242 ++++++ .../test/models/resnet_pytest_perf_results.md | 8 + backends/xnnpack/test/tester/BUCK | 1 + backends/xnnpack/test/tester/performance.py | 749 ++++++++++++++++++ backends/xnnpack/test/tester/tester.py | 42 +- 7 files changed, 1134 insertions(+), 5 deletions(-) create mode 100644 backends/xnnpack/test/models/README.md create mode 100644 backends/xnnpack/test/models/resnet_pytest_perf_results.json create mode 100644 backends/xnnpack/test/models/resnet_pytest_perf_results.md create mode 100644 backends/xnnpack/test/tester/performance.py diff --git a/backends/xnnpack/test/models/README.md b/backends/xnnpack/test/models/README.md new file mode 100644 index 00000000000..ab83b62ee07 --- /dev/null +++ b/backends/xnnpack/test/models/README.md @@ -0,0 +1,85 @@ +# XNNPACK Model Tests + +## Optional Pytest Perf Stage + +Model tests can opt in to an optional latency check after correctness +comparison: + +```python +.run_method_and_compare_outputs(xnnpack_perf=True) +``` + +The perf stage is skipped by default. Enable it only for runs where local +latency comparison is useful. + +### Commands + +Run the normal correctness test with no perf stage: + +```bash +python3 -m pytest -q \ +backends/xnnpack/test/models/resnet.py::TestResNet18::test_fp32_resnet18 +``` + +Run correctness plus perf comparison against the recorded result: + +```bash +EXECUTORCH_XNNPACK_PYTEST_PERF=1 \ +python3 -m pytest -q \ +backends/xnnpack/test/models/resnet.py::TestResNet18::test_fp32_resnet18 +``` + +Record or update the local perf result: + +```bash +EXECUTORCH_XNNPACK_PYTEST_PERF=1 \ +EXECUTORCH_XNNPACK_PYTEST_PERF_UPDATE=1 \ +python3 -m pytest -q \ +backends/xnnpack/test/models/resnet.py::TestResNet18::test_fp32_resnet18 +``` + +Use more samples for a less noisy result: + +```bash +EXECUTORCH_XNNPACK_PYTEST_PERF=1 \ +EXECUTORCH_XNNPACK_PYTEST_PERF_RUNS=50 \ +EXECUTORCH_XNNPACK_PYTEST_PERF_WARMUP_RUNS=10 \ +python3 -m pytest -q \ +backends/xnnpack/test/models/resnet.py::TestResNet18::test_fp32_resnet18 +``` + +### Environment Variables + +| Variable | Default | Meaning | +| ---------------------------------------------- | ------: | ------------------------------------------------ | +| `EXECUTORCH_XNNPACK_PYTEST_PERF` | off | Enables the optional perf stage. | +| `EXECUTORCH_XNNPACK_PYTEST_PERF_UPDATE` | off | Records the current result instead of comparing. | +| `EXECUTORCH_XNNPACK_PYTEST_PERF_RUNS` | `10` | Number of timed runs. | +| `EXECUTORCH_XNNPACK_PYTEST_PERF_WARMUP_RUNS` | `2` | Number of warmup runs. | +| `EXECUTORCH_XNNPACK_PYTEST_PERF_THRESHOLD_PCT` | `10.0` | Allowed slowdown before comparison fails. | + +Boolean variables accept values such as `1`, `true`, `yes`, and `on`. + +The perf stage forces the ExecuTorch threadpool to one thread while timing, +then restores the previous thread count after the run. + +### Results + +Perf results are keyed by the pytest test name and host/runtime identity. For +example, the ResNet test writes: + +```text +backends/xnnpack/test/models/resnet_pytest_perf_results.json +backends/xnnpack/test/models/resnet_pytest_perf_results.md +``` + +The JSON file is the complete machine-readable record used for comparisons. +It includes samples, host details, runtime details, linked-symbol hints, and +the recorded latency metrics. + +The Markdown file is a compact human-readable summary for review. It includes +the host, processor, expected hardware path, thread count, mean latency, +median latency, coefficient of variation, and recorded time. + +`Expected HW path` is deduced from host ML features and linked runtime symbols. +It is not proof of the exact runtime microkernel selected for every operator. diff --git a/backends/xnnpack/test/models/resnet.py b/backends/xnnpack/test/models/resnet.py index 9f4989e1724..1b706cb7026 100644 --- a/backends/xnnpack/test/models/resnet.py +++ b/backends/xnnpack/test/models/resnet.py @@ -1,5 +1,6 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -39,7 +40,7 @@ def forward(self, x): ) return self.model(x) - def _test_exported_resnet(self, tester): + def _test_exported_resnet(self, tester, xnnpack_perf=False): ( tester.export() .to_edge_transform_and_lower() @@ -52,11 +53,16 @@ def _test_exported_resnet(self, tester): .check(["torch.ops.higher_order.executorch_call_delegate"]) .to_executorch() .serialize() - .run_method_and_compare_outputs() + .run_method_and_compare_outputs( + xnnpack_perf=xnnpack_perf, + ) ) def test_fp32_resnet18(self): - self._test_exported_resnet(Tester(torchvision.models.resnet18(), self.inputs)) + self._test_exported_resnet( + Tester(torchvision.models.resnet18(), self.inputs), + xnnpack_perf=True, + ) @unittest.skip("T187799178: Debugging Numerical Issues with Calibration") def _test_qs8_resnet18(self): diff --git a/backends/xnnpack/test/models/resnet_pytest_perf_results.json b/backends/xnnpack/test/models/resnet_pytest_perf_results.json new file mode 100644 index 00000000000..c6b12032361 --- /dev/null +++ b/backends/xnnpack/test/models/resnet_pytest_perf_results.json @@ -0,0 +1,242 @@ +{ + "entries": { + "xnnpack.models.resnet.test_fp32_resnet18": { + "darwin-arm64-apple-m3-pro-mac15-6-nosme2-pybinding-buffer-threads1-_portable_lib.cpython-313-darwin.so": { + "host": { + "cpu_id": "apple-m3-pro-mac15-6", + "cpu_model": "Apple M3 Pro", + "linux_features": [], + "machine": "arm64", + "macos_features": { + "hw.optional.arm.FEAT_BF16": "1", + "hw.optional.arm.FEAT_DotProd": "1", + "hw.optional.arm.FEAT_FP16": "1", + "hw.optional.arm.FEAT_I8MM": "1", + "hw.optional.arm.FEAT_SME": "0", + "hw.optional.arm.FEAT_SME2": "0" + }, + "macos_model": "Mac15,6", + "ml_features": [ + "fp", + "asimd", + "bf16", + "asimddp", + "fphp", + "i8mm" + ], + "processor": "arm", + "release": "25.5.0", + "sme2_available": false, + "system": "Darwin" + }, + "kernel_path": { + "backend": "xnnpack", + "basis": [ + "host.ml_features", + "runtime.linked_symbols" + ], + "confidence": "expected_not_proven", + "expected_hardware_path": "kleidiai+neon", + "host_ml_features": [ + "fp", + "asimd", + "bf16", + "asimddp", + "fphp", + "i8mm" + ], + "runtime_symbol_hints": { + "avx2_symbols_present": false, + "avx512_symbols_present": false, + "avx_symbols_present": false, + "kleidiai_symbols_present": true, + "neon_symbols_present": true, + "neonsme2_symbols_present": true, + "sme2_symbols_present": true, + "sme_symbols_present": true, + "sse_symbols_present": false, + "xnnpack_symbols_present": true + } + }, + "metrics": { + "cv_pct": 0.246957433434771, + "load_mode": "buffer", + "max_ms": 41.886459, + "mean_ms": 41.6864669, + "median_ms": 41.6786465, + "min_ms": 41.502291, + "p90_ms": 41.7885084, + "samples_ms": [ + 41.502291, + 41.560417, + 41.886459, + 41.729917, + 41.675084, + 41.651167, + 41.777625, + 41.682209, + 41.65375, + 41.74575 + ], + "thread_count_observed": 1, + "thread_count_requested": 1, + "timed_runs": 10, + "warmup_runs": 2 + }, + "recorded_at_utc": "2026-06-25T09:24:37.854788+00:00", + "test_id": "xnnpack.models.resnet.test_fp32_resnet18", + "threshold_pct": 10.0, + "timing_runtime": { + "avx2_symbols_present": false, + "avx512_symbols_present": false, + "avx_symbols_present": false, + "kind": "pybinding", + "kleidiai_symbols_present": true, + "native_mtime_utc": "2026-06-24T13:58:43+00:00", + "native_path": "extension/pybindings/_portable_lib.cpython-313-darwin.so", + "neon_symbols_present": true, + "neonsme2_symbols_present": true, + "pte_load_mode": "buffer", + "runtime_key": "darwin-arm64-apple-m3-pro-mac15-6-nosme2-pybinding-buffer-threads1-_portable_lib.cpython-313-darwin.so", + "sme2_symbols_present": true, + "sme_symbols_present": true, + "sse_symbols_present": false, + "thread_count": 1, + "xnnpack_symbols_present": true + } + }, + "linux-aarch64-aarch64-0x41-0x000-nosme2-pybinding-buffer-threads1-_portable_lib.cpython-312-aarch64-linux-gnu.so": { + "host": { + "cpu_id": "aarch64-0x41-0x000", + "cpu_model": "aarch64", + "linux_features": [ + "fp", + "asimd", + "evtstrm", + "aes", + "pmull", + "sha1", + "sha2", + "crc32", + "atomics", + "fphp", + "asimdhp", + "cpuid", + "asimdrdm", + "jscvt", + "fcma", + "lrcpc", + "dcpop", + "sha3", + "asimddp", + "sha512", + "asimdfhm", + "dit", + "uscat", + "ilrcpc", + "flagm", + "sb", + "paca", + "pacg", + "dcpodp", + "flagm2", + "frint", + "bf16", + "afp" + ], + "machine": "aarch64", + "macos_features": {}, + "macos_model": null, + "ml_features": [ + "fp", + "asimd", + "asimddp", + "asimdhp", + "bf16", + "fphp" + ], + "processor": "aarch64", + "release": "6.8.0-117-generic", + "sme2_available": false, + "system": "Linux" + }, + "kernel_path": { + "backend": "xnnpack", + "basis": [ + "host.ml_features", + "runtime.linked_symbols" + ], + "confidence": "expected_not_proven", + "expected_hardware_path": "kleidiai+neon", + "host_ml_features": [ + "fp", + "asimd", + "asimddp", + "asimdhp", + "bf16", + "fphp" + ], + "runtime_symbol_hints": { + "avx2_symbols_present": false, + "avx512_symbols_present": false, + "avx_symbols_present": false, + "kleidiai_symbols_present": true, + "neon_symbols_present": true, + "neonsme2_symbols_present": true, + "sme2_symbols_present": true, + "sme_symbols_present": true, + "sse_symbols_present": false, + "xnnpack_symbols_present": true + } + }, + "metrics": { + "cv_pct": 1.1082648754607403, + "load_mode": "buffer", + "max_ms": 43.026209, + "mean_ms": 41.900096, + "median_ms": 41.9061045, + "min_ms": 41.338542, + "p90_ms": 42.3321956, + "samples_ms": [ + 41.604084, + 41.338542, + 41.700709, + 41.923334, + 42.255083, + 43.026209, + 41.34325, + 41.987333, + 41.888875, + 41.933541 + ], + "thread_count_observed": 1, + "thread_count_requested": 1, + "timed_runs": 10, + "warmup_runs": 2 + }, + "recorded_at_utc": "2026-06-26T13:41:33.202559+00:00", + "test_id": "xnnpack.models.resnet.test_fp32_resnet18", + "threshold_pct": 10.0, + "timing_runtime": { + "avx2_symbols_present": false, + "avx512_symbols_present": false, + "avx_symbols_present": false, + "kind": "pybinding", + "kleidiai_symbols_present": true, + "native_mtime_utc": "2026-06-12T13:39:04+00:00", + "native_path": "extension/pybindings/_portable_lib.cpython-312-aarch64-linux-gnu.so", + "neon_symbols_present": true, + "neonsme2_symbols_present": true, + "pte_load_mode": "buffer", + "runtime_key": "linux-aarch64-aarch64-0x41-0x000-nosme2-pybinding-buffer-threads1-_portable_lib.cpython-312-aarch64-linux-gnu.so", + "sme2_symbols_present": true, + "sme_symbols_present": true, + "sse_symbols_present": false, + "thread_count": 1, + "xnnpack_symbols_present": true + } + } + } + }, + "version": 1 +} diff --git a/backends/xnnpack/test/models/resnet_pytest_perf_results.md b/backends/xnnpack/test/models/resnet_pytest_perf_results.md new file mode 100644 index 00000000000..e5f28f705b7 --- /dev/null +++ b/backends/xnnpack/test/models/resnet_pytest_perf_results.md @@ -0,0 +1,8 @@ +# XNNPACK Pytest Perf Results + +Generated from `resnet_pytest_perf_results.json`. + +| Test | Host | Processor | ML features | Expected HW path | Threads | Mean ms | Median ms | CV % | Recorded | +| ---------------------------------------- | -------------------- | ------------ | --------------------------------------- | ---------------- | ------: | ------: | --------: | ----: | -------------------- | +| xnnpack.models.resnet.test_fp32_resnet18 | Darwin arm64 Mac15,6 | Apple M3 Pro | fp, asimd, bf16, asimddp, fphp, i8mm | kleidiai+neon | 1 | 41.686 | 41.679 | 0.247 | 2026-06-25 09:24 UTC | +| xnnpack.models.resnet.test_fp32_resnet18 | Linux aarch64 | aarch64 | fp, asimd, asimddp, asimdhp, bf16, fphp | kleidiai+neon | 1 | 41.900 | 41.906 | 1.108 | 2026-06-26 13:41 UTC | diff --git a/backends/xnnpack/test/tester/BUCK b/backends/xnnpack/test/tester/BUCK index c9b0239de02..02c89a05a25 100644 --- a/backends/xnnpack/test/tester/BUCK +++ b/backends/xnnpack/test/tester/BUCK @@ -7,6 +7,7 @@ fbcode_target(_kind = runtime.python_library, name = "tester", srcs = [ "__init__.py", + "performance.py", "tester.py", ], base_module = "executorch.backends.xnnpack.test.tester", diff --git a/backends/xnnpack/test/tester/performance.py b/backends/xnnpack/test/tester/performance.py new file mode 100644 index 00000000000..009cbfac873 --- /dev/null +++ b/backends/xnnpack/test/tester/performance.py @@ -0,0 +1,749 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import json +import logging +import math +import os +import platform +import re +import statistics +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +from torch.utils._pytree import tree_flatten + + +logger = logging.getLogger(__name__) + +PERF_ENABLE_ENV = "EXECUTORCH_XNNPACK_PYTEST_PERF" +PERF_UPDATE_ENV = "EXECUTORCH_XNNPACK_PYTEST_PERF_UPDATE" +PERF_RUNS_ENV = "EXECUTORCH_XNNPACK_PYTEST_PERF_RUNS" +PERF_WARMUP_ENV = "EXECUTORCH_XNNPACK_PYTEST_PERF_WARMUP_RUNS" +PERF_THRESHOLD_ENV = "EXECUTORCH_XNNPACK_PYTEST_PERF_THRESHOLD_PCT" + +_DEFAULT_RUNS = 10 +_DEFAULT_WARMUP_RUNS = 2 +_DEFAULT_THRESHOLD_PCT = 10.0 +_FORCED_THREAD_COUNT = 1 +_LINUX_ML_FEATURES = ( + "fp", + "asimd", + "asimddp", + "asimdhp", + "avx", + "avx2", + "avx512_bf16", + "avx512_vnni", + "avx512bw", + "avx512f", + "bf16", + "fphp", + "i8mm", + "sme", + "sme2", + "sse", + "sse2", + "sse4_1", + "sse4_2", + "sve", + "sve2", +) +_MACOS_ML_FEATURES = ( + ("hw.optional.arm.FEAT_BF16", "bf16"), + ("hw.optional.arm.FEAT_DotProd", "asimddp"), + ("hw.optional.arm.FEAT_FP16", "fphp"), + ("hw.optional.arm.FEAT_I8MM", "i8mm"), + ("hw.optional.arm.FEAT_SME", "sme"), + ("hw.optional.arm.FEAT_SME2", "sme2"), +) +_MACOS_ML_FEATURE_KEYS = tuple(key for key, _ in _MACOS_ML_FEATURES) + + +def maybe_run_performance_test( + *, + serialized_buffer: bytes, + inputs: Tuple[Any, ...], + results_path: Optional[str], +) -> None: + """Run the optional XNNPACK pytest performance stage. + + The stage is skipped unless ``EXECUTORCH_XNNPACK_PYTEST_PERF`` is set. + In update mode it records a result entry and Markdown summary; otherwise + it compares the current median latency against the recorded result. + """ + if not _env_flag(PERF_ENABLE_ENV): + return + if not isinstance(serialized_buffer, bytes): + raise RuntimeError("XNNPACK pytest perf requires a serialized PTE buffer.") + + test_id = _current_pytest_test_id() + timed_runs = _env_int(PERF_RUNS_ENV, _DEFAULT_RUNS) + warmup_runs = _env_int(PERF_WARMUP_ENV, _DEFAULT_WARMUP_RUNS) + threshold_pct = _env_float(PERF_THRESHOLD_ENV, _DEFAULT_THRESHOLD_PCT) + + record = _measure_latency( + serialized_buffer=serialized_buffer, + inputs=inputs, + test_id=test_id, + timed_runs=timed_runs, + warmup_runs=warmup_runs, + threshold_pct=threshold_pct, + ) + + path = Path(results_path) if results_path else _default_results_path(test_id) + results = _load_results(path) + runtime_key = record["timing_runtime"]["runtime_key"] + test_entries = results.setdefault("entries", {}).setdefault(test_id, {}) + recorded_entry = test_entries.get(runtime_key) + + if _env_flag(PERF_UPDATE_ENV): + test_entries[runtime_key] = record + _write_results(path, results) + _write_results_summary(path.with_suffix(".md"), results, path.name) + logger.info("Updated XNNPACK pytest perf results for %s", runtime_key) + return + + if recorded_entry is None: + logger.warning( + "No XNNPACK pytest perf result for %s / %s. Set %s=1 to record one.", + test_id, + runtime_key, + PERF_UPDATE_ENV, + ) + return + + _assert_within_threshold( + test_id=test_id, + runtime_key=runtime_key, + current=record, + recorded=recorded_entry, + threshold_pct=threshold_pct, + ) + + +def _measure_latency( + *, + serialized_buffer: bytes, + inputs: Tuple[Any, ...], + test_id: str, + timed_runs: int, + warmup_runs: int, + threshold_pct: float, +) -> Dict[str, Any]: + """Measure in-process pybinding runtime latency for a serialized PTE.""" + from executorch.extension.pybindings import _portable_lib as portable_native + from executorch.extension.pybindings.portable_lib import ( + _load_for_executorch_from_buffer, + _threadpool_get_thread_count, + _unsafe_reset_threadpool, + Verification, + ) + + native_path = Path(portable_native.__file__).resolve() + host = _host_identity() + timing_runtime = _timing_runtime_identity(native_path, host) + inputs_flattened, _ = tree_flatten(inputs) + + original_thread_count = _threadpool_get_thread_count() + _unsafe_reset_threadpool(_FORCED_THREAD_COUNT) + observed_thread_count = _threadpool_get_thread_count() + + try: + module = _load_for_executorch_from_buffer( + serialized_buffer, program_verification=Verification.Minimal + ) + for _ in range(warmup_runs): + module.run_method("forward", tuple(inputs_flattened)) + + elapsed_ms = [] + for _ in range(timed_runs): + start_ns = time.perf_counter_ns() + module.run_method("forward", tuple(inputs_flattened)) + elapsed_ms.append((time.perf_counter_ns() - start_ns) / 1_000_000.0) + finally: + _unsafe_reset_threadpool(original_thread_count) + + mean_ms = statistics.fmean(elapsed_ms) + stdev_ms = statistics.pstdev(elapsed_ms) if len(elapsed_ms) > 1 else 0.0 + + return { + "test_id": test_id, + "recorded_at_utc": datetime.now(timezone.utc).isoformat(), + "host": host, + "timing_runtime": timing_runtime, + "kernel_path": _kernel_path_identity(host, timing_runtime), + "metrics": { + "load_mode": "buffer", + "warmup_runs": warmup_runs, + "timed_runs": timed_runs, + "thread_count_requested": _FORCED_THREAD_COUNT, + "thread_count_observed": observed_thread_count, + "median_ms": statistics.median(elapsed_ms), + "mean_ms": mean_ms, + "min_ms": min(elapsed_ms), + "max_ms": max(elapsed_ms), + "p90_ms": _percentile(elapsed_ms, 0.90), + "cv_pct": 0.0 if mean_ms == 0.0 else stdev_ms / mean_ms * 100.0, + "samples_ms": elapsed_ms, + }, + "threshold_pct": threshold_pct, + } + + +def _assert_within_threshold( + *, + test_id: str, + runtime_key: str, + current: Dict[str, Any], + recorded: Dict[str, Any], + threshold_pct: float, +) -> None: + """Assert the current median latency is not a thresholded regression.""" + current_median = current["metrics"]["median_ms"] + recorded_median = recorded["metrics"]["median_ms"] + if recorded_median == 0.0: + logger.warning( + "Skipping XNNPACK pytest perf comparison against zero recorded median" + ) + return + + delta_pct = (current_median - recorded_median) / recorded_median * 100.0 + if delta_pct <= threshold_pct: + logger.info( + "XNNPACK pytest perf %s / %s: %.3f ms vs %.3f ms recorded (%+.2f%%)", + test_id, + runtime_key, + current_median, + recorded_median, + delta_pct, + ) + return + + raise AssertionError( + f"XNNPACK pytest perf regression for {test_id} / {runtime_key} exceeded " + f"{threshold_pct:.2f}%: current median {current_median:.3f} ms, " + f"recorded median {recorded_median:.3f} ms, delta {delta_pct:+.2f}%." + ) + + +def _default_results_path(test_id: str) -> Path: + """Return the default JSON result path for a derived XNNPACK test id.""" + parts = test_id.split(".") + if len(parts) < 3 or parts[0] != "xnnpack": + raise RuntimeError( + "XNNPACK pytest perf test ids must look like " + "'xnnpack...'." + ) + + suite = parts[1] + name = parts[2] + return ( + Path(__file__).resolve().parents[1] + / suite + / (f"{name}_pytest_perf_results.json") + ) + + +def _current_pytest_test_id() -> str: + """Derive a stable perf record id from ``PYTEST_CURRENT_TEST``.""" + current_test = os.environ.get("PYTEST_CURRENT_TEST", "").strip() + current_test = re.sub(r"\s+\([^)]*\)$", "", current_test) + if not current_test: + raise RuntimeError( + "XNNPACK pytest perf requires PYTEST_CURRENT_TEST to derive the " + "perf record id." + ) + + test_path, *node_parts = current_test.split("::") + if not node_parts: + raise RuntimeError( + f"Could not derive XNNPACK pytest perf id from {current_test}" + ) + + namespace = _pytest_module_namespace(Path(test_path)) + test_name = node_parts[-1] + return ".".join(_slug(part) for part in ("xnnpack", *namespace, test_name)) + + +def _pytest_module_namespace(path: Path) -> list[str]: + """Map a pytest file path under XNNPACK tests to an id namespace.""" + parts = path.with_suffix("").parts + marker = ("backends", "xnnpack", "test") + for index in range(len(parts) - len(marker) + 1): + if tuple(parts[index : index + len(marker)]) == marker: + return list(parts[index + len(marker) :]) + return [path.stem] + + +def _timing_runtime_identity(native_path: Path, host: Dict[str, Any]) -> Dict[str, Any]: + """Build the runtime identity used to separate recorded perf results.""" + return { + "kind": "pybinding", + "runtime_key": _runtime_key(native_path, host), + "native_path": _display_path(native_path), + "native_mtime_utc": datetime.fromtimestamp( + native_path.stat().st_mtime, timezone.utc + ).isoformat(), + "pte_load_mode": "buffer", + "thread_count": _FORCED_THREAD_COUNT, + **_runtime_symbol_hints(native_path), + } + + +def _runtime_symbol_hints(native_path: Path) -> Dict[str, bool]: + """Inspect linked native symbols that hint at available kernel families.""" + return { + "xnnpack_symbols_present": _binary_contains( + native_path, (b"XnnpackBackend", b"xnn_") + ), + "neon_symbols_present": _binary_contains(native_path, (b"neon", b"NEON")), + "sme_symbols_present": _binary_contains( + native_path, (b"neonsme", b"_sme", b"-sme") + ), + "sme2_symbols_present": _binary_contains( + native_path, (b"neonsme2", b"_sme2", b"-sme2") + ), + "kleidiai_symbols_present": _binary_contains(native_path, (b"kai_", b"Kleidi")), + "neonsme2_symbols_present": _binary_contains(native_path, (b"neonsme2",)), + "sse_symbols_present": _binary_contains( + native_path, (b"_sse", b"-sse", b"sse2", b"sse4", b"SSE2", b"SSE4") + ), + "avx_symbols_present": _binary_contains( + native_path, (b"_avx", b"-avx", b"AVX") + ), + "avx2_symbols_present": _binary_contains( + native_path, (b"_avx2", b"-avx2", b"avx2", b"AVX2") + ), + "avx512_symbols_present": _binary_contains( + native_path, (b"avx512", b"avx-512", b"AVX512", b"AVX-512") + ), + } + + +def _kernel_path_identity( + host: Dict[str, Any], timing_runtime: Dict[str, Any] +) -> Dict[str, Any]: + """Describe the expected XNNPACK hardware path for this host/runtime.""" + symbol_hints = { + key: value + for key, value in timing_runtime.items() + if key.endswith("_symbols_present") + } + backend = "xnnpack" if symbol_hints["xnnpack_symbols_present"] else "unknown" + expected_hardware_path = _expected_hardware_path(host, symbol_hints) + return { + "backend": backend, + "expected_hardware_path": expected_hardware_path, + "confidence": "expected_not_proven", + "basis": [ + "host.ml_features", + "runtime.linked_symbols", + ], + "host_ml_features": host["ml_features"], + "runtime_symbol_hints": symbol_hints, + } + + +def _expected_hardware_path(host: Dict[str, Any], symbol_hints: Dict[str, bool]) -> str: + """Deduce the expected hardware path from host features and symbols.""" + features = set(host["ml_features"]) + arm_path = _expected_arm_hardware_path(features, host, symbol_hints) + if arm_path != "unknown": + return arm_path + + return _expected_x86_hardware_path(features, symbol_hints) + + +def _expected_arm_hardware_path( + features: set[str], + host: Dict[str, Any], + symbol_hints: Dict[str, bool], +) -> str: + """Deduce the expected Arm path such as NEON, SME, or SME2.""" + if _host_has_sme2(features, host): + return _expected_sme2_path(symbol_hints) + + if _host_has_sme(features): + return _expected_sme_path(symbol_hints) + + if "asimd" in features and symbol_hints["neon_symbols_present"]: + return "kleidiai+neon" if symbol_hints["kleidiai_symbols_present"] else "neon" + + return "unknown" + + +def _host_has_sme2(features: set[str], host: Dict[str, Any]) -> bool: + """Return whether the host feature set reports SME2 support.""" + return ( + "sme2" in features + or host["sme2_available"] + or "hw.optional.arm.FEAT_SME2" in features + ) + + +def _host_has_sme(features: set[str]) -> bool: + """Return whether the host feature set reports SME support.""" + return "sme" in features or "hw.optional.arm.FEAT_SME" in features + + +def _expected_sme2_path(symbol_hints: Dict[str, bool]) -> str: + """Choose the SME2 path name from linked runtime symbol hints.""" + if symbol_hints["kleidiai_symbols_present"]: + return "kleidiai+sme2" + if symbol_hints["sme2_symbols_present"] or symbol_hints["neonsme2_symbols_present"]: + return "sme2" + return "unknown" + + +def _expected_sme_path(symbol_hints: Dict[str, bool]) -> str: + """Choose the SME path name from linked runtime symbol hints.""" + if symbol_hints["kleidiai_symbols_present"]: + return "kleidiai+sme" + if symbol_hints["sme_symbols_present"]: + return "sme" + return "unknown" + + +def _expected_x86_hardware_path( + features: set[str], symbol_hints: Dict[str, bool] +) -> str: + """Deduce the expected x86 SIMD path from features and symbols.""" + candidates = ( + ("avx512f", "avx512_symbols_present", "avx512"), + ("avx2", "avx2_symbols_present", "avx2"), + ("avx", "avx_symbols_present", "avx"), + ) + for feature, symbol_key, hardware_path in candidates: + if feature in features and symbol_hints[symbol_key]: + return hardware_path + + if {"sse", "sse2", "sse4_1", "sse4_2"} & features and symbol_hints[ + "sse_symbols_present" + ]: + return "sse" + + return "unknown" + + +def _runtime_key(native_path: Path, host: Dict[str, Any]) -> str: + """Build a key that separates host, runtime, load mode, and threads.""" + stem = f"{host['system']}-{host['machine']}-{host['cpu_id']}" + sme2 = "sme2" if host["sme2_available"] else "nosme2" + runtime = f"pybinding-buffer-threads{_FORCED_THREAD_COUNT}" + return _slug(f"{stem}-{sme2}-{runtime}-{native_path.name}") + + +def _host_identity() -> Dict[str, Any]: + """Collect host details that affect perf comparability and path choice.""" + cpu_info = _read_linux_cpuinfo() + mac_brand = _sysctl("machdep.cpu.brand_string") + mac_model = _sysctl("hw.model") + mac_features = {key: _sysctl(key) for key in _MACOS_ML_FEATURE_KEYS} + mac_features = {key: value for key, value in mac_features.items() if value} + + linux_features = ( + cpu_info.get("Features") + or cpu_info.get("flags") + or cpu_info.get("Features".lower()) + or "" + ).split() + system = platform.system() + machine = platform.machine() + ml_features = _ml_features(linux_features, mac_features, system, machine) + cpu_model = ( + cpu_info.get("model name") + or cpu_info.get("Hardware") + or cpu_info.get("Processor") + or mac_brand + or mac_model + or platform.processor() + or "unknown-cpu" + ) + cpu_id = _slug( + " ".join( + filter( + None, + ( + cpu_model, + mac_model, + cpu_info.get("CPU implementer"), + cpu_info.get("CPU part"), + ), + ) + ) + ) + + return { + "system": system, + "release": platform.release(), + "machine": machine, + "processor": platform.processor(), + "cpu_model": cpu_model, + "cpu_id": cpu_id, + "linux_features": linux_features, + "ml_features": ml_features, + "macos_model": mac_model, + "macos_features": mac_features, + "sme2_available": "sme2" in ml_features, + } + + +def _read_linux_cpuinfo() -> Dict[str, str]: + """Read the first processor entry from Linux ``/proc/cpuinfo``.""" + path = Path("/proc/cpuinfo") + if not path.exists(): + return {} + + result = {} + for line in path.read_text(errors="ignore").splitlines(): + if ":" not in line: + if result: + break + continue + key, value = line.split(":", 1) + result[key.strip()] = value.strip() + return result + + +def _ml_features( + linux_features: list[str], + mac_features: Dict[str, str], + system: str, + machine: str, +) -> list[str]: + """Normalize Linux and macOS CPU features into one ML feature list.""" + relevant = [feature for feature in _LINUX_ML_FEATURES if feature in linux_features] + if system == "Darwin" and machine in ("arm64", "aarch64"): + relevant.extend(("fp", "asimd")) + relevant.extend( + feature + for key, feature in _MACOS_ML_FEATURES + if mac_features.get(key) in ("1", "true") + ) + return list(dict.fromkeys(relevant)) + + +def _display_path(path: Path) -> str: + """Display paths relative to the ExecuTorch repository when possible.""" + try: + return str(path.relative_to(Path(__file__).resolve().parents[4])) + except ValueError: + return str(path) + + +def _sysctl(key: str) -> Optional[str]: + """Read a macOS sysctl value, returning ``None`` when unavailable.""" + try: + completed = subprocess.run( + ["/usr/sbin/sysctl", "-n", key], + check=False, + capture_output=True, + text=True, + timeout=1.0, + ) + except (FileNotFoundError, subprocess.SubprocessError): + return None + if completed.returncode != 0: + return None + return completed.stdout.strip() or None + + +def _binary_contains(path: Path, needles: Tuple[bytes, ...]) -> bool: + """Return whether a binary contains any of the given byte sequences.""" + with path.open("rb") as handle: + overlap = b"" + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + return False + haystack = overlap + chunk + if any(needle in haystack for needle in needles): + return True + overlap = haystack[-64:] + + +def _load_results(path: Path) -> Dict[str, Any]: + """Load a perf results JSON file, or return an empty schema.""" + if not path.exists(): + return {"version": 1, "entries": {}} + with path.open("r") as handle: + return json.load(handle) + + +def _write_results(path: Path, results: Dict[str, Any]) -> None: + """Write the complete machine-readable perf results JSON file.""" + path.parent.mkdir(parents=True, exist_ok=True) + results["version"] = 1 + with path.open("w") as handle: + json.dump(results, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def _write_results_summary( + path: Path, results: Dict[str, Any], source_name: str +) -> None: + """Write a compact Markdown summary of the perf results JSON.""" + headers = [ + "Test", + "Host", + "Processor", + "ML features", + "Expected HW path", + "Threads", + "Mean ms", + "Median ms", + "CV %", + "Recorded", + ] + rows = _results_summary_rows(results) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w") as handle: + handle.write("# XNNPACK Pytest Perf Results\n\n") + handle.write(f"Generated from `{source_name}`.\n\n") + _write_markdown_table(handle, headers, rows, {5, 6, 7, 8}) + + +def _write_markdown_table( + handle: Any, + headers: list[str], + rows: list[list[str]], + right_aligned_columns: set[int], +) -> None: + """Write a GitHub-flavored Markdown table with raw column alignment.""" + widths = [ + max(len(row[index]) for row in [headers] + rows) + for index in range(len(headers)) + ] + + def _format_row(row: list[str]) -> str: + cells = [] + for index, cell in enumerate(row): + if index in right_aligned_columns: + cells.append(cell.rjust(widths[index])) + else: + cells.append(cell.ljust(widths[index])) + return "| " + " | ".join(cells) + " |\n" + + separators = [] + for index, width in enumerate(widths): + if index in right_aligned_columns: + separators.append("-" * (width - 1) + ":") + else: + separators.append("-" * width) + + handle.write(_format_row(headers)) + handle.write("| " + " | ".join(separators) + " |\n") + for row in rows: + handle.write(_format_row(row)) + + +def _results_summary_rows(results: Dict[str, Any]) -> list[list[str]]: + """Build Markdown table rows from the nested perf results schema.""" + rows = [] + entries = results.get("entries", {}) + for test_id in sorted(entries): + for runtime_key in sorted(entries[test_id]): + record = entries[test_id][runtime_key] + host = record.get("host", {}) + kernel_path = record.get("kernel_path", {}) + metrics = record.get("metrics", {}) + rows.append( + [ + _markdown_table_cell(test_id), + _markdown_table_cell(_host_summary(host)), + _markdown_table_cell(_processor_summary(host)), + _markdown_table_cell(", ".join(host.get("ml_features", []))), + _markdown_table_cell( + kernel_path.get("expected_hardware_path", "unknown") + ), + str(metrics.get("thread_count_observed", "unknown")), + _format_metric(metrics.get("mean_ms")), + _format_metric(metrics.get("median_ms")), + _format_metric(metrics.get("cv_pct")), + _markdown_table_cell(_recorded_date(record)), + ] + ) + return rows + + +def _host_summary(host: Dict[str, Any]) -> str: + """Return a compact host label for the Markdown summary.""" + parts = [ + host.get("system"), + host.get("machine"), + host.get("macos_model"), + ] + return " ".join(part for part in parts if part) or "unknown" + + +def _processor_summary(host: Dict[str, Any]) -> str: + """Return the most readable processor label available in host details.""" + return ( + host.get("cpu_model") + or host.get("processor") + or host.get("cpu_id") + or "unknown" + ) + + +def _recorded_date(record: Dict[str, Any]) -> str: + """Format a recorded UTC timestamp for the Markdown summary.""" + recorded_at = record.get("recorded_at_utc", "") + match = re.match(r"(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})", recorded_at) + return f"{match.group(1)} {match.group(2)} UTC" if match else recorded_at + + +def _format_metric(value: Any) -> str: + """Format numeric summary metrics to three decimal places.""" + return f"{value:.3f}" if isinstance(value, (float, int)) else "unknown" + + +def _markdown_table_cell(value: Any) -> str: + """Escape a value for use inside a Markdown table cell.""" + return str(value).replace("|", "\\|") if value is not None else "" + + +def _percentile(values: list[float], fraction: float) -> float: + """Compute an interpolated percentile for a non-empty list.""" + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + index = (len(ordered) - 1) * fraction + lower = math.floor(index) + upper = math.ceil(index) + if lower == upper: + return ordered[int(index)] + weight = index - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def _env_flag(name: str) -> bool: + """Return whether an environment variable is set to a truthy value.""" + return os.environ.get(name, "").strip().lower() in ("1", "true", "yes", "on") + + +def _env_int(name: str, default: int) -> int: + """Read a positive integer environment variable with a default.""" + value = os.environ.get(name) + if value is None: + return default + return max(1, int(value)) + + +def _env_float(name: str, default: float) -> float: + """Read a float environment variable with a default.""" + value = os.environ.get(name) + if value is None: + return default + return float(value) + + +def _slug(value: str) -> str: + """Convert a string to a stable lowercase identifier component.""" + return re.sub(r"[^a-zA-Z0-9_.-]+", "-", value.strip().lower()).strip("-") diff --git a/backends/xnnpack/test/tester/tester.py b/backends/xnnpack/test/tester/tester.py index fc12da231c0..9228fa678f0 100644 --- a/backends/xnnpack/test/tester/tester.py +++ b/backends/xnnpack/test/tester/tester.py @@ -1,6 +1,6 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. -# Copyright 2024-2025 Arm Limited and/or its affiliates. +# Copyright 2024-2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -12,10 +12,10 @@ import torch from executorch.backends.test.harness import Tester as TesterBase +from executorch.backends.test.harness.error_statistics import ErrorStatistics from executorch.backends.test.harness.stages import StageType from executorch.backends.xnnpack._passes import XNNPACKPassManager from executorch.backends.xnnpack.partition.xnnpack_partitioner import XnnpackPartitioner - from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( get_symmetric_quantization_config, XNNPACKQuantizer, @@ -23,6 +23,9 @@ from executorch.backends.xnnpack.quantizer.xnnpack_quantizer_utils import ( QuantizationConfig, ) +from executorch.backends.xnnpack.test.tester.performance import ( + maybe_run_performance_test, +) from executorch.backends.xnnpack.utils.configs import get_xnnpack_edge_compile_config from executorch.exir import EdgeCompileConfig from executorch.exir.backend.partitioner import Partitioner @@ -132,3 +135,38 @@ def __init__( dynamic_shapes=dynamic_shapes, **kwargs, ) + + def run_method_and_compare_outputs( + self, + stage: Optional[StageType] = None, + inputs: Optional[Tuple[torch.Tensor]] = None, + num_runs=1, + atol=1e-03, + rtol=1e-03, + qtol=0, + statistics_callback: Callable[[ErrorStatistics], None] | None = None, + artifact_dir: Optional[str] = None, + artifact_name: Optional[str] = None, + xnnpack_perf: bool = False, + xnnpack_perf_results_path: Optional[str] = None, + ): + super().run_method_and_compare_outputs( + stage=stage, + inputs=inputs, + num_runs=num_runs, + atol=atol, + rtol=rtol, + qtol=qtol, + statistics_callback=statistics_callback, + artifact_dir=artifact_dir, + artifact_name=artifact_name, + ) + + if xnnpack_perf: + stage = stage or self.cur + maybe_run_performance_test( + serialized_buffer=self.stages[stage].artifact, + inputs=inputs if inputs is not None else self.example_inputs, + results_path=xnnpack_perf_results_path, + ) + return self