From 5085e9142b001fc528a4fe5b25dddbad7d07519e Mon Sep 17 00:00:00 2001 From: Francois Farquet Date: Thu, 3 Sep 2026 11:31:09 +0200 Subject: [PATCH 1/4] Schedule micro-small GraalPy benchmarks after merges --- ci.jsonnet | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ci.jsonnet b/ci.jsonnet index de2077d15e..2aa7d89610 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -390,13 +390,18 @@ } + { // "small" benchmarks have their argument set such that they run in a resonable // time in the interpreter and they are used for interpreter benchmarking - [bench]: bench_task(bench) + platform_spec(no_jobs) + bench_variants({ + "micro_small": bench_task("micro_small") + platform_spec(no_jobs) + bench_variants({ + "vm_name:graalvm_ee_default_interpreter" : {"linux:amd64:jdk-latest" : post_merge + t("02:00:00") + need_pgo}, + "vm_name:graalvm_ee_default_interpreter_uncached" : {"linux:amd64:jdk-latest" : daily + t("02:00:00") + need_pgo}, + "vm_name:graalpython_enterprise_interpreter" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")}, + "vm_name:cpython" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")}, + }), + "meso_small": bench_task("meso_small") + platform_spec(no_jobs) + bench_variants({ "vm_name:graalvm_ee_default_interpreter" : {"linux:amd64:jdk-latest" : daily + t("02:00:00") + need_pgo}, "vm_name:graalvm_ee_default_interpreter_uncached" : {"linux:amd64:jdk-latest" : daily + t("02:00:00") + need_pgo}, "vm_name:graalpython_enterprise_interpreter" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")}, "vm_name:cpython" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")}, }), - for bench in ["micro_small", "meso_small"] } + { // benchmarks executed via Java embedding driver [bench]: bench_task(bench) + platform_spec(no_jobs) + bench_variants({ From 510fa00b7e70cb1b462d39b49f0c83ec1f14cffd Mon Sep 17 00:00:00 2001 From: Francois Farquet Date: Thu, 3 Sep 2026 11:31:43 +0200 Subject: [PATCH 2/4] Configure GraalPy PR-bench data sources --- ci.jsonnet | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/ci.jsonnet b/ci.jsonnet index 2aa7d89610..34bb157303 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -345,6 +345,28 @@ "raw_results.json", ], }), + // Existing GraalPy post-merge benchmark jobs provide master history for + // PR-bench. The report uses that history as a best-effort baseline and + // does not run separate baseline benchmarking. + local pr_bench_graalpy = task_spec({ + unicorn_pull_request_benchmarking:: { + name: "graalpy", + benchmarks: [ + "meso.nbody3", + "macro.c-pydantic-validate", + "macro.c-pydantic-serialize-json", + "micro-small.attribute-bool", + ], + metrics: ["time"], + baseline_benchmarking: false, + _extra_unicorn_args: [ + "--config-key", "dataserver/fetch/where/bench-suite-version/enum[]=unknown", + "--config-key", "dataserver/fetch/where/bench-suite-version/allow-absent=true", + "--config-key", "dataserver/fetch/where/platform.jdk-major-version/enum[]=0", + "--config-key", "dataserver/fetch/where/platform.jdk-major-version/allow-absent=true", + ], + }, + }), // ----------------------------------------------------------------------------------------------------------------- // benchmarks @@ -352,7 +374,7 @@ // [info]: when adding a benchmark, the key in the `bench_task_dict` is taken as the name of the benchmark if it is // not specified as the first arg to `bench_task`. local bench_task_dict = { - [bench]: bench_task(bench) + platform_spec(no_jobs) + bench_variants({ + "micro": bench_task("micro") + platform_spec(no_jobs) + bench_variants({ "vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : post_merge + t("08:00:00") + need_pgo}, "vm_name:graalpython_enterprise" : {"linux:amd64:jdk-latest" : daily + t("08:00:00"), "job_type:checkup" : {"linux:amd64:jdk-latest" : on_demand + t("08:00:00")} @@ -361,10 +383,18 @@ "vm_name:cpython" : {"linux:amd64:jdk-latest" : monthly + t("04:00:00")}, "vm_name:pypy" : {"linux:amd64:jdk-latest" : on_demand + t("04:00:00")}, }), - for bench in ["micro", "meso"] + "meso": bench_task("meso") + platform_spec(no_jobs) + bench_variants({ + "vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : post_merge + t("08:00:00") + need_pgo + pr_bench_graalpy}, + "vm_name:graalpython_enterprise" : {"linux:amd64:jdk-latest" : daily + t("08:00:00"), + "job_type:checkup" : {"linux:amd64:jdk-latest" : on_demand + t("08:00:00")} + }, + "vm_name:graalpython_enterprise_multi" : {"linux:amd64:jdk-latest" : weekly + t("08:00:00")}, + "vm_name:cpython" : {"linux:amd64:jdk-latest" : monthly + t("04:00:00")}, + "vm_name:pypy" : {"linux:amd64:jdk-latest" : on_demand + t("04:00:00")}, + }), } + { "macro": bench_task("macro") + internet_access_env + platform_spec(no_jobs) + bench_variants({ - "vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : post_merge + t("08:00:00") + need_pgo}, + "vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : post_merge + t("08:00:00") + need_pgo + pr_bench_graalpy}, "vm_name:graalpython_enterprise" : {"linux:amd64:jdk-latest" : daily + t("08:00:00"), "job_type:checkup" : {"linux:amd64:jdk-latest" : on_demand + t("08:00:00")} }, @@ -391,7 +421,7 @@ // "small" benchmarks have their argument set such that they run in a resonable // time in the interpreter and they are used for interpreter benchmarking "micro_small": bench_task("micro_small") + platform_spec(no_jobs) + bench_variants({ - "vm_name:graalvm_ee_default_interpreter" : {"linux:amd64:jdk-latest" : post_merge + t("02:00:00") + need_pgo}, + "vm_name:graalvm_ee_default_interpreter" : {"linux:amd64:jdk-latest" : post_merge + t("02:00:00") + need_pgo + pr_bench_graalpy}, "vm_name:graalvm_ee_default_interpreter_uncached" : {"linux:amd64:jdk-latest" : daily + t("02:00:00") + need_pgo}, "vm_name:graalpython_enterprise_interpreter" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")}, "vm_name:cpython" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")}, From dd035d7c1fca2c5cc7bc829e9974541696e4c261 Mon Sep 17 00:00:00 2001 From: Francois Farquet Date: Thu, 10 Sep 2026 12:01:19 +0200 Subject: [PATCH 3/4] Preserve string-valued Unicorn dimension filters --- ci.jsonnet | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci.jsonnet b/ci.jsonnet index 34bb157303..8018709771 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -360,10 +360,10 @@ metrics: ["time"], baseline_benchmarking: false, _extra_unicorn_args: [ - "--config-key", "dataserver/fetch/where/bench-suite-version/enum[]=unknown", - "--config-key", "dataserver/fetch/where/bench-suite-version/allow-absent=true", - "--config-key", "dataserver/fetch/where/platform.jdk-major-version/enum[]=0", - "--config-key", "dataserver/fetch/where/platform.jdk-major-version/allow-absent=true", + // Use JSON to preserve the string types expected by Bench Server; + // --config-key would coerce numeric-looking values such as "0". + "--config-json", + "{\"dataserver\":{\"fetch\":{\"where\":{\"bench-suite-version\":{\"enum\":[\"unknown\"],\"allow-absent\":true},\"platform.jdk-major-version\":{\"enum\":[\"0\"],\"allow-absent\":true}}}}}", ], }, }), From 358948fcab0d6b8d7c3a70cb7dac2fc6719ee5dc Mon Sep 17 00:00:00 2001 From: Francois Farquet Date: Thu, 10 Sep 2026 12:01:33 +0200 Subject: [PATCH 4/4] Pin GraalPy CI to rebased PR-bench overlay --- ci.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.jsonnet b/ci.jsonnet index 8018709771..913dfebc61 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -5,7 +5,7 @@ (import "ci/python-gate.libsonnet") + (import "ci/python-bench.libsonnet") + { - overlay: "12367561e6a2b54df8d3b1bd400e431de01eef0f", + overlay: "acd356580b18f9d67b0213c12da04076e6a4d493", specVersion: "8", // Until buildbot issues around CI tiers are resolved, we cannot use them // tierConfig: self.tierConfig,