Skip to content

[GR-79181] Add GraalPy PR-bench. #1146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
graalvmbot merged 4 commits into from
Sep 15, 2026
+41 −6
Merged

[GR-79181] Add GraalPy PR-bench. #1146

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5085e91
Schedule micro-small GraalPy benchmarks after merges
farquet Sep 3, 2026
510fa00
Configure GraalPy PR-bench data sources
farquet Sep 3, 2026
dd035d7
Preserve string-valued Unicorn dimension filters
farquet Sep 10, 2026
358948f
Pin GraalPy CI to rebased PR-bench overlay
farquet Sep 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 41 additions & 6 deletions ci.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
(import "ci/python-gate.libsonnet") +
(import "ci/python-bench.libsonnet") +
{
overlay: "12367561e6a2b54df8d3b1bd400e431de01eef0f",
overlay: "acd356580b18f9d67b0213c12da04076e6a4d493",
specVersion: "8",
// Until buildbot issues around CI tiers are resolved, we cannot use them
// tierConfig: self.tierConfig,
Expand Down Expand Up @@ -345,14 +345,36 @@
"raw_results.json",
],
}),
// Existing GraalPy post-merge benchmark jobs provide master history for
// PR-bench. The report uses that history as a best-effort baseline and
// does not run separate baseline benchmarking.
local pr_bench_graalpy = task_spec({
unicorn_pull_request_benchmarking:: {
name: "graalpy",
benchmarks: [
"meso.nbody3",
"macro.c-pydantic-validate",
"macro.c-pydantic-serialize-json",
"micro-small.attribute-bool",
],
metrics: ["time"],
baseline_benchmarking: false,
_extra_unicorn_args: [
// Use JSON to preserve the string types expected by Bench Server;
// --config-key would coerce numeric-looking values such as "0".
"--config-json",
"{\"dataserver\":{\"fetch\":{\"where\":{\"bench-suite-version\":{\"enum\":[\"unknown\"],\"allow-absent\":true},\"platform.jdk-major-version\":{\"enum\":[\"0\"],\"allow-absent\":true}}}}}",
],
},
}),

// -----------------------------------------------------------------------------------------------------------------
// benchmarks
// -----------------------------------------------------------------------------------------------------------------
// [info]: when adding a benchmark, the key in the `bench_task_dict` is taken as the name of the benchmark if it is
// not specified as the first arg to `bench_task`.
local bench_task_dict = {
[bench]: bench_task(bench) + platform_spec(no_jobs) + bench_variants({
"micro": bench_task("micro") + platform_spec(no_jobs) + bench_variants({
"vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : post_merge + t("08:00:00") + need_pgo},
"vm_name:graalpython_enterprise" : {"linux:amd64:jdk-latest" : daily + t("08:00:00"),
"job_type:checkup" : {"linux:amd64:jdk-latest" : on_demand + t("08:00:00")}
Expand All @@ -361,10 +383,18 @@
"vm_name:cpython" : {"linux:amd64:jdk-latest" : monthly + t("04:00:00")},
"vm_name:pypy" : {"linux:amd64:jdk-latest" : on_demand + t("04:00:00")},
}),
for bench in ["micro", "meso"]
"meso": bench_task("meso") + platform_spec(no_jobs) + bench_variants({
"vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : post_merge + t("08:00:00") + need_pgo + pr_bench_graalpy},
"vm_name:graalpython_enterprise" : {"linux:amd64:jdk-latest" : daily + t("08:00:00"),
"job_type:checkup" : {"linux:amd64:jdk-latest" : on_demand + t("08:00:00")}
},
"vm_name:graalpython_enterprise_multi" : {"linux:amd64:jdk-latest" : weekly + t("08:00:00")},
"vm_name:cpython" : {"linux:amd64:jdk-latest" : monthly + t("04:00:00")},
"vm_name:pypy" : {"linux:amd64:jdk-latest" : on_demand + t("04:00:00")},
}),
} + {
"macro": bench_task("macro") + internet_access_env + platform_spec(no_jobs) + bench_variants({
"vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : post_merge + t("08:00:00") + need_pgo},
"vm_name:graalvm_ee_default" : {"linux:amd64:jdk-latest" : post_merge + t("08:00:00") + need_pgo + pr_bench_graalpy},
"vm_name:graalpython_enterprise" : {"linux:amd64:jdk-latest" : daily + t("08:00:00"),
"job_type:checkup" : {"linux:amd64:jdk-latest" : on_demand + t("08:00:00")}
},
Expand All @@ -390,13 +420,18 @@
} + {
// "small" benchmarks have their argument set such that they run in a resonable
// time in the interpreter and they are used for interpreter benchmarking
[bench]: bench_task(bench) + platform_spec(no_jobs) + bench_variants({
"micro_small": bench_task("micro_small") + platform_spec(no_jobs) + bench_variants({
"vm_name:graalvm_ee_default_interpreter" : {"linux:amd64:jdk-latest" : post_merge + t("02:00:00") + need_pgo + pr_bench_graalpy},
"vm_name:graalvm_ee_default_interpreter_uncached" : {"linux:amd64:jdk-latest" : daily + t("02:00:00") + need_pgo},
"vm_name:graalpython_enterprise_interpreter" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")},
"vm_name:cpython" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")},
}),
"meso_small": bench_task("meso_small") + platform_spec(no_jobs) + bench_variants({
"vm_name:graalvm_ee_default_interpreter" : {"linux:amd64:jdk-latest" : daily + t("02:00:00") + need_pgo},
"vm_name:graalvm_ee_default_interpreter_uncached" : {"linux:amd64:jdk-latest" : daily + t("02:00:00") + need_pgo},
"vm_name:graalpython_enterprise_interpreter" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")},
"vm_name:cpython" : {"linux:amd64:jdk-latest" : weekly + t("02:00:00")},
}),
for bench in ["micro_small", "meso_small"]
} + {
// benchmarks executed via Java embedding driver
[bench]: bench_task(bench) + platform_spec(no_jobs) + bench_variants({
Expand Down
Loading