From 9705ceb904ac83cf0458fbe2893c4ffd3d295054 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 19 Aug 2026 14:26:04 +0200 Subject: [PATCH 1/2] update to op-rs 0.116, re-work using frameowrk labels and env-vars function, add and test constants! --- Cargo.lock | 251 +++++++------- Cargo.nix | 307 +++++++++--------- Cargo.toml | 2 +- crate-hashes.json | 18 +- extra/crds.yaml | 30 +- rust/operator-binary/src/connect/common.rs | 54 ++- .../src/connect/controller/apply.rs | 8 +- .../src/connect/controller/build/executor.rs | 113 +++++-- .../src/connect/controller/build/mod.rs | 47 ++- .../src/connect/controller/build/rbac.rs | 30 +- .../src/connect/controller/build/server.rs | 131 +++++--- .../src/connect/controller/build/service.rs | 13 +- .../src/connect/controller/update_status.rs | 4 +- .../src/connect/controller/validate.rs | 110 ++----- rust/operator-binary/src/connect/crd.rs | 7 +- rust/operator-binary/src/crd/constants.rs | 61 +++- rust/operator-binary/src/crd/history.rs | 4 +- rust/operator-binary/src/crd/mod.rs | 129 +++++--- rust/operator-binary/src/crd/roles.rs | 45 ++- .../src/crd/template_merger.rs | 60 +++- .../src/history/controller/apply.rs | 8 +- .../src/history/controller/build/mod.rs | 113 ++++++- .../controller/build/resource/config_map.rs | 10 +- .../controller/build/resource/listener.rs | 37 ++- .../history/controller/build/resource/pdb.rs | 10 +- .../history/controller/build/resource/rbac.rs | 28 +- .../controller/build/resource/service.rs | 7 +- .../controller/build/resource/statefulset.rs | 146 ++++++--- .../src/history/controller/validate.rs | 106 ++---- rust/operator-binary/src/main.rs | 8 +- .../src/spark_k8s_controller/build/mod.rs | 86 ++++- .../src/spark_k8s_controller/build/pod.rs | 158 +++++++-- .../build/resource/config_map.rs | 14 +- .../build/resource/job.rs | 160 ++++++++- .../build/resource/serviceaccount.rs | 36 +- .../src/spark_k8s_controller/validate.rs | 80 ++--- .../pyspark-pi-driver-pod-template-data.json | 2 +- ...pyspark-pi-executor-pod-template-data.json | 2 +- .../pyspark-pi-job-template-spec.json | 8 +- .../fixtures/spark-connect-server-data.json | 2 +- tests/templates/kuttl/smoke/42-assert.yaml | 1 - tests/templates/kuttl/smoke/52-assert.yaml | 7 - .../spark-connect-kerberos/12-assert.yaml | 7 - .../kuttl/spark-connect/12-assert.yaml | 7 - .../kuttl/spark-connect/14-assert.yaml.j2 | 1 - 45 files changed, 1524 insertions(+), 944 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1319a647..950d5d71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -163,9 +163,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", @@ -316,9 +316,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.4.1" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9066c49992464636f92905fa096ec58baaa4d57ec19a5c096c68d3e25ef3d136" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -631,7 +631,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -910,9 +910,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "flagset" @@ -959,9 +959,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -974,9 +974,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -984,15 +984,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -1001,32 +1001,32 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -1036,9 +1036,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures 0.1.31", "futures-channel", @@ -1154,9 +1154,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -1236,9 +1236,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -1366,9 +1366,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -1380,9 +1380,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1393,9 +1393,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1407,16 +1407,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1427,15 +1428,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" dependencies = [ "displaydoc", "icu_locale_core", @@ -1604,7 +1605,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -1653,9 +1654,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -1672,7 +1673,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -1685,7 +1686,7 @@ dependencies = [ "pest_derive", "regex", "serde_json", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -1714,7 +1715,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#fb2d86579f4e3df008f78f0e527a012243483a2d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#7b9f9ac9a76fa425ab27f2821377ef86571ca121" dependencies = [ "darling 0.24.0", "regex", @@ -1758,7 +1759,7 @@ dependencies = [ "base64 0.22.1", "bytes", "either", - "futures 0.3.33", + "futures 0.3.34", "http", "http-body", "http-body-util", @@ -1777,7 +1778,7 @@ dependencies = [ "serde", "serde-saphyr", "serde_json", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-util", "tower", @@ -1801,7 +1802,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -1829,7 +1830,7 @@ dependencies = [ "async-stream", "backon", "educe 0.6.0", - "futures 0.3.33", + "futures 0.3.34", "hashbrown 0.16.1", "hostname", "json-patch", @@ -1839,7 +1840,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-util", "tracing", @@ -1892,9 +1893,9 @@ dependencies = [ [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -1998,9 +1999,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -2053,7 +2054,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.19", + "thiserror 2.0.20", "tracing", ] @@ -2095,7 +2096,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tonic", "tonic-types", @@ -2133,7 +2134,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-stream", ] @@ -2215,9 +2216,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.8" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" +checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" dependencies = [ "memchr", "ucd-trie", @@ -2225,9 +2226,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.8" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" +checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" dependencies = [ "pest", "pest_generator", @@ -2235,9 +2236,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.8" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" +checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" dependencies = [ "pest", "pest_meta", @@ -2248,9 +2249,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.8" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" +checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" dependencies = [ "pest", ] @@ -2304,15 +2305,15 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" @@ -2325,9 +2326,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -2513,18 +2514,18 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", @@ -2745,9 +2746,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" dependencies = [ "ring", "rustls-pki-types", @@ -3155,7 +3156,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#fb2d86579f4e3df008f78f0e527a012243483a2d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#7b9f9ac9a76fa425ab27f2821377ef86571ca121" dependencies = [ "const-oid", "ecdsa", @@ -3178,8 +3179,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.115.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#fb2d86579f4e3df008f78f0e527a012243483a2d" +version = "0.116.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#7b9f9ac9a76fa425ab27f2821377ef86571ca121" dependencies = [ "base64 0.23.1", "clap", @@ -3188,7 +3189,7 @@ dependencies = [ "dockerfile-parser", "educe 0.7.6", "either", - "futures 0.3.33", + "futures 0.3.34", "http", "indexmap", "java-properties", @@ -3223,7 +3224,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#fb2d86579f4e3df008f78f0e527a012243483a2d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#7b9f9ac9a76fa425ab27f2821377ef86571ca121" dependencies = [ "darling 0.24.0", "proc-macro2", @@ -3234,7 +3235,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#fb2d86579f4e3df008f78f0e527a012243483a2d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#7b9f9ac9a76fa425ab27f2821377ef86571ca121" dependencies = [ "jiff", "k8s-openapi", @@ -3256,7 +3257,7 @@ dependencies = [ "built", "clap", "const_format", - "futures 0.3.33", + "futures 0.3.34", "indoc", "regex", "rstest", @@ -3275,7 +3276,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#fb2d86579f4e3df008f78f0e527a012243483a2d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#7b9f9ac9a76fa425ab27f2821377ef86571ca121" dependencies = [ "axum", "clap", @@ -3299,7 +3300,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#fb2d86579f4e3df008f78f0e527a012243483a2d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#7b9f9ac9a76fa425ab27f2821377ef86571ca121" dependencies = [ "kube", "schemars", @@ -3313,7 +3314,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#fb2d86579f4e3df008f78f0e527a012243483a2d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#7b9f9ac9a76fa425ab27f2821377ef86571ca121" dependencies = [ "convert_case", "convert_case_extras", @@ -3331,7 +3332,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#fb2d86579f4e3df008f78f0e527a012243483a2d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#7b9f9ac9a76fa425ab27f2821377ef86571ca121" dependencies = [ "arc-swap", "async-trait", @@ -3463,11 +3464,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -3483,9 +3484,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", @@ -3533,9 +3534,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -3794,7 +3795,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", "tracing-subscriber", ] @@ -3826,7 +3827,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "futures 0.3.33", + "futures 0.3.34", "futures-task", "pin-project", "tracing", @@ -3971,9 +3972,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" dependencies = [ "js-sys", "wasm-bindgen", @@ -4033,9 +4034,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -4046,9 +4047,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ "js-sys", "wasm-bindgen", @@ -4056,9 +4057,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4066,9 +4067,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -4079,18 +4080,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -4282,9 +4283,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x509-cert" @@ -4392,9 +4393,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -4403,9 +4404,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" dependencies = [ "yoke", "zerofrom", @@ -4414,13 +4415,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/Cargo.nix b/Cargo.nix index b2800298..772b4f39 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -489,9 +489,9 @@ rec { }; "async-trait" = rec { crateName = "async-trait"; - version = "0.1.91"; + version = "0.1.92"; edition = "2021"; - sha256 = "1v3cm8mzg66037wm392p1vsdx0lq8bid6y2ivr7z03lpfx0xqdmf"; + sha256 = "0rqn5iga1hlv2lm8xzav1zhar46jb4dvx89i6kfv93kb53maxxl2"; procMacro = true; libName = "async_trait"; authors = [ @@ -989,9 +989,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.4.1"; + version = "1.4.3"; edition = "2021"; - sha256 = "0dniydgf5lv8dh4mr6n1gvas9albqmp0kyh557wkcij6jacw8rlh"; + sha256 = "0v9b5arr047vbihfbh3fmbd3aj9vf1i7dbdgfpvlwzynpjvr35ah"; dependencies = [ { name = "find-msvc-tools"; @@ -1903,7 +1903,7 @@ rec { dependencies = [ { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; } ]; features = { @@ -2794,9 +2794,9 @@ rec { }; "find-msvc-tools" = rec { crateName = "find-msvc-tools"; - version = "0.1.10"; + version = "0.1.11"; edition = "2021"; - sha256 = "1pp1612g5k6im9732g16j6a87czhb35xcyzlrpq2mkgdwrrkbdr6"; + sha256 = "145qpfb9r4ml2klr8v4byvrkikp61qyiks9n69b8z0vbscbb0pfl"; libName = "find_msvc_tools"; }; @@ -2923,11 +2923,11 @@ rec { }; resolvedDefaultFeatures = [ "default" "use_std" "with-deprecated" ]; }; - "futures 0.3.33" = rec { + "futures 0.3.34" = rec { crateName = "futures"; - version = "0.3.33"; + version = "0.3.34"; edition = "2018"; - sha256 = "066j5aqz8an05xh4hn5ljdnjn80z3g335v4grx4gaifr57wg3358"; + sha256 = "18yhwmbdalhz2z9i1vm10hy2v0cfm82dkgcb6vr2msxazfix4ccs"; dependencies = [ { name = "futures-channel"; @@ -2987,9 +2987,9 @@ rec { }; "futures-channel" = rec { crateName = "futures-channel"; - version = "0.3.33"; + version = "0.3.34"; edition = "2018"; - sha256 = "1bn5hlhfkl1sgypmiachaqcgwmr6wmjal7dyhfyb1zkazvs90996"; + sha256 = "1i4kwcanpaphn1ax62ci3nx176kglxqx0gnhzqpqdr1rkpbf7ydi"; libName = "futures_channel"; dependencies = [ { @@ -3015,9 +3015,9 @@ rec { }; "futures-core" = rec { crateName = "futures-core"; - version = "0.3.33"; + version = "0.3.34"; edition = "2018"; - sha256 = "1iqdbvcdlplfr2g43h7xrfkv2sg5p1a26x8acz1xgxl07i3hrm9c"; + sha256 = "0pjgv4fx0np6hrs5sz5a2phabwv0z70yr51v03injbi44bjrkmlj"; libName = "futures_core"; features = { "default" = [ "std" ]; @@ -3028,9 +3028,9 @@ rec { }; "futures-executor" = rec { crateName = "futures-executor"; - version = "0.3.33"; + version = "0.3.34"; edition = "2018"; - sha256 = "0n3lpkmcfrsnh40i4armn040gnqbpd257hz5qs46zipjr6f8fm37"; + sha256 = "0cjl3y7jgg60wwb96ikxj23r6q91ylvx8v675yychv1w3b7lf6q3"; libName = "futures_executor"; dependencies = [ { @@ -3058,9 +3058,9 @@ rec { }; "futures-io" = rec { crateName = "futures-io"; - version = "0.3.33"; + version = "0.3.34"; edition = "2018"; - sha256 = "0yjx13qdm9b2p4w00ddw85k6yccnnmqrlrrz8yfmi5jg7jmfqxs5"; + sha256 = "1v9z6wj92ra18kpv0xig21hgpzrvcwmcr8fszyzh64yyay0zmh2k"; libName = "futures_io"; features = { "default" = [ "std" ]; @@ -3069,9 +3069,9 @@ rec { }; "futures-macro" = rec { crateName = "futures-macro"; - version = "0.3.33"; + version = "0.3.34"; edition = "2018"; - sha256 = "02xiyd5y1nk9b805aympj4wq2czgvxnhcml9w9xkc665d3g3qv9d"; + sha256 = "0i0czvcvsqq4hrccibq2f23004si5z34zjwdxfmqhlrmm15nbfcz"; procMacro = true; libName = "futures_macro"; dependencies = [ @@ -3085,7 +3085,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.119"; + packageId = "syn 3.0.3"; features = [ "full" ]; } ]; @@ -3093,9 +3093,9 @@ rec { }; "futures-sink" = rec { crateName = "futures-sink"; - version = "0.3.33"; + version = "0.3.34"; edition = "2018"; - sha256 = "01z38z344hpryw84b6r0rbwcb669d8pyvl2szg10aqwx96n1hi73"; + sha256 = "07cfvrgc3vxk6sw5g8a8dnrm1mzg6d5mwy08ywa1sgyhyxml4i0r"; libName = "futures_sink"; features = { "default" = [ "std" ]; @@ -3105,9 +3105,9 @@ rec { }; "futures-task" = rec { crateName = "futures-task"; - version = "0.3.33"; + version = "0.3.34"; edition = "2018"; - sha256 = "02f1y1yvjg1cv998zkgl1706pi9y4fyc9045l1hlmyqyhclfscdj"; + sha256 = "1zfilqs8nwlfqz4prk7ihvpp5avvzins87ibzlxzq5fhs7ipshfd"; libName = "futures_task"; features = { "default" = [ "std" ]; @@ -3132,9 +3132,9 @@ rec { }; "futures-util" = rec { crateName = "futures-util"; - version = "0.3.33"; + version = "0.3.34"; edition = "2018"; - sha256 = "1anyg40j5www5l22r2jbn1birsafz4q1w9qmcjk4vqzwasi90ym7"; + sha256 = "1g3r9ghzq7c2fh34lis43i72xavk9p84npgfwgb5vfpqcwjajl0d"; libName = "futures_util"; dependencies = [ { @@ -3621,9 +3621,9 @@ rec { }; "h2" = rec { crateName = "h2"; - version = "0.4.15"; + version = "0.4.16"; edition = "2021"; - sha256 = "0mgilh1g8gydcchqi6acs5l6j0gwg5jwpa64sj4b3ncb9v497c3c"; + sha256 = "09syqqhvh36b3rwyn8vjhiz597hfki1hcz3hwagb3cs1ifapmwx9"; authors = [ "Carl Lerche " "Sean McArthur " @@ -3861,9 +3861,9 @@ rec { }; "http-body-util" = rec { crateName = "http-body-util"; - version = "0.1.4"; + version = "0.1.5"; edition = "2018"; - sha256 = "1wizkqx9a75x8v5lm7cawpammz8sfvd7cngnkp34wkcfl3b1zx79"; + sha256 = "07773iilap808wjp6vywlq15zkgwnswqzrv270zxvg2z9biry5i3"; libName = "http_body_util"; authors = [ "Carl Lerche " @@ -4371,9 +4371,9 @@ rec { }; "icu_collections" = rec { crateName = "icu_collections"; - version = "2.2.0"; - edition = "2021"; - sha256 = "070r7xd0pynm0hnc1v2jzlbxka6wf50f81wybf9xg0y82v6x3119"; + version = "2.3.0"; + edition = "2024"; + sha256 = "04x59h6vdq0cnpippim1nr471ivlsnnn470sj1d5v864h48d4s7s"; authors = [ "The ICU4X Project Developers" ]; @@ -4421,9 +4421,9 @@ rec { }; "icu_locale_core" = rec { crateName = "icu_locale_core"; - version = "2.2.0"; - edition = "2021"; - sha256 = "0a9cmin5w1x3bg941dlmgszn33qgq428k7qiqn5did72ndi9n8cj"; + version = "2.3.0"; + edition = "2024"; + sha256 = "1sqdj16wwl7h9y6r7j394av4kpdb7zryz9h169ffwbm9imc2hvnm"; authors = [ "The ICU4X Project Developers" ]; @@ -4473,9 +4473,9 @@ rec { }; "icu_normalizer" = rec { crateName = "icu_normalizer"; - version = "2.2.0"; - edition = "2021"; - sha256 = "1d7krxr0xpc4x9635k1100a24nh0nrc59n65j6yk6gbfkplmwvn5"; + version = "2.3.0"; + edition = "2024"; + sha256 = "0vv43ixk2wmbxrx7kl33cwkhx1wdyb1q3pa18qkyshan4dgwzy8j"; authors = [ "The ICU4X Project Developers" ]; @@ -4527,9 +4527,9 @@ rec { }; "icu_normalizer_data" = rec { crateName = "icu_normalizer_data"; - version = "2.2.0"; - edition = "2021"; - sha256 = "0f5d5d5fhhr9937m2z6z38fzh6agf14z24kwlr6lyczafypf0fys"; + version = "2.3.0"; + edition = "2024"; + sha256 = "1811h0ppb7lwq1q2492p5x6lcmlwmhbmkf69fhyvzcz0scgdlqqm"; authors = [ "The ICU4X Project Developers" ]; @@ -4537,13 +4537,18 @@ rec { }; "icu_properties" = rec { crateName = "icu_properties"; - version = "2.2.0"; - edition = "2021"; - sha256 = "1pkh3s837808cbwxvfagwc28cvwrz2d9h5rl02jwrhm51ryvdqxy"; + version = "2.3.0"; + edition = "2024"; + sha256 = "0j51hi8qgf0l6a7qzvnwsc61598w2fpnsklicld6ci9immva4z3y"; authors = [ "The ICU4X Project Developers" ]; dependencies = [ + { + name = "displaydoc"; + packageId = "displaydoc"; + usesDefaultFeatures = false; + } { name = "icu_collections"; packageId = "icu_collections"; @@ -4585,6 +4590,7 @@ rec { "datagen" = [ "serde" "dep:databake" "zerovec/databake" "icu_collections/databake" "icu_locale_core/databake" "zerotrie/databake" "icu_provider/export" ]; "default" = [ "compiled_data" ]; "harfbuzz_traits" = [ "dep:harfbuzz-traits" ]; + "log" = [ "dep:log" ]; "serde" = [ "dep:serde" "icu_locale_core/serde" "zerovec/serde" "icu_collections/serde" "icu_provider/serde" "zerotrie/serde" ]; "unicode_bidi" = [ "dep:unicode-bidi" ]; }; @@ -4592,9 +4598,9 @@ rec { }; "icu_properties_data" = rec { crateName = "icu_properties_data"; - version = "2.2.0"; - edition = "2021"; - sha256 = "052awny0qwkbcbpd5jg2cd7vl5ry26pq4hz1nfsgf10c3qhbnawf"; + version = "2.3.0"; + edition = "2024"; + sha256 = "1akw1gp5rcaiz377xzsnkx8f92qax4kh3lfn9y4rcjj6q4wg1475"; authors = [ "The ICU4X Project Developers" ]; @@ -4602,9 +4608,9 @@ rec { }; "icu_provider" = rec { crateName = "icu_provider"; - version = "2.2.0"; - edition = "2021"; - sha256 = "08dl8pxbwr8zsz4c5vphqb7xw0hykkznwi4rw7bk6pwb3krlr70k"; + version = "2.3.0"; + edition = "2024"; + sha256 = "0a343jlrb7jlb20xv1airslzv63559wf38jihrx81bba39kyv9wj"; authors = [ "The ICU4X Project Developers" ]; @@ -5103,7 +5109,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; } { name = "windows-link"; @@ -5221,9 +5227,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.103"; + version = "0.3.104"; edition = "2021"; - sha256 = "00lib0b6hqmw56r2hjp7xrv730qacslirbkdlhvmi39zvgy4pd2k"; + sha256 = "0fjsgady7wbv7bbyy6c8qhrd93bnx11qbl83l1g7bb9a4601030f"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -5283,7 +5289,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; } ]; devDependencies = [ @@ -5332,7 +5338,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; } ]; @@ -5430,8 +5436,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "fb2d86579f4e3df008f78f0e527a012243483a2d"; - sha256 = "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb"; + rev = "7b9f9ac9a76fa425ab27f2821377ef86571ca121"; + sha256 = "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9"; }; libName = "k8s_version"; authors = [ @@ -5609,7 +5615,7 @@ rec { } { name = "futures"; - packageId = "futures 0.3.33"; + packageId = "futures 0.3.34"; optional = true; usesDefaultFeatures = false; features = [ "std" ]; @@ -5709,7 +5715,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; } { name = "tokio"; @@ -5745,7 +5751,7 @@ rec { devDependencies = [ { name = "futures"; - packageId = "futures 0.3.33"; + packageId = "futures 0.3.34"; usesDefaultFeatures = false; features = [ "async-await" ]; } @@ -5875,7 +5881,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; } ]; devDependencies = [ @@ -5982,7 +5988,7 @@ rec { } { name = "futures"; - packageId = "futures 0.3.33"; + packageId = "futures 0.3.34"; usesDefaultFeatures = false; features = [ "async-await" ]; } @@ -6027,7 +6033,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; } { name = "tokio"; @@ -6206,9 +6212,9 @@ rec { }; "litemap" = rec { crateName = "litemap"; - version = "0.8.2"; + version = "0.8.3"; edition = "2021"; - sha256 = "1w7628bc7wwcxc4n4s5kw0610xk06710nh2hn5kwwk2wa91z9nlj"; + sha256 = "1bpgpj87560hmckh3875fbahpmfxbk4g8pzns84h3ykf3nfx3na7"; authors = [ "The ICU4X Project Developers" ]; @@ -6526,9 +6532,9 @@ rec { }; "num-integer" = rec { crateName = "num-integer"; - version = "0.1.46"; + version = "0.1.47"; edition = "2018"; - sha256 = "13w5g54a9184cqlbsq80rnxw4jj4s0d8wv75jsq5r2lms8gncsbr"; + sha256 = "02z1p3azy6p10n99skrab4a6hhfd4amf2i9gm8sxqd1p9dfxkqkw"; libName = "num_integer"; authors = [ "The Rust Project Developers" @@ -6670,7 +6676,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; optional = true; usesDefaultFeatures = false; } @@ -6840,7 +6846,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; usesDefaultFeatures = false; } { @@ -7047,7 +7053,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; usesDefaultFeatures = false; } { @@ -7348,9 +7354,9 @@ rec { }; "pest" = rec { crateName = "pest"; - version = "2.8.8"; + version = "2.9.0"; edition = "2021"; - sha256 = "18jhl2zpxvl6kikc0jgp7gi7i7cy9s634z5bnvx70w1whjz2ixvx"; + sha256 = "1kwvhc5hyrfpxpmp0jw0wr891xyjs44mcs2wz68hrl54qw6ac1ss"; authors = [ "Dragoș Tiselice " ]; @@ -7377,9 +7383,9 @@ rec { }; "pest_derive" = rec { crateName = "pest_derive"; - version = "2.8.8"; + version = "2.9.0"; edition = "2021"; - sha256 = "1zcijlfdf6sk2s6l1qnm3j7kj7d4ymqcial8w4p4dcr67gydcbcy"; + sha256 = "13f8ihi8928s9mc13pcbhxy382kqc89h7i8d7s5mnif8lm23ga5k"; procMacro = true; authors = [ "Dragoș Tiselice " @@ -7405,9 +7411,9 @@ rec { }; "pest_generator" = rec { crateName = "pest_generator"; - version = "2.8.8"; + version = "2.9.0"; edition = "2021"; - sha256 = "1dkmk6r6bb2hh5wayymfmwd7mswwbyhw12dnx2lrdxdnrw2r4yka"; + sha256 = "0jihcdnmdban4bqjd02r2wbb79nywj2nhwqyk95hvrixm98k9kg0"; authors = [ "Dragoș Tiselice " ]; @@ -7443,9 +7449,9 @@ rec { }; "pest_meta" = rec { crateName = "pest_meta"; - version = "2.8.8"; + version = "2.9.0"; edition = "2021"; - sha256 = "0z7m54jc3nj3nxbbk4kyjfa06d8s24vhf6krzj2867nyq18x7aw5"; + sha256 = "15jly0r7r4m15fhm3pg814h65j7dqnqq6k43rvgxfhg29443lkg0"; authors = [ "Dragoș Tiselice " ]; @@ -7576,9 +7582,9 @@ rec { }; "pkg-config" = rec { crateName = "pkg-config"; - version = "0.3.33"; - edition = "2018"; - sha256 = "17jnqmcbxsnwhg9gjf0nh6dj5k0x3hgwi3mb9krjnmfa9v435w8r"; + version = "0.3.34"; + edition = "2021"; + sha256 = "0j05h08nzg0q8rf6lzw7nry0b7kn7x97vc9n4hwrl52fqzxn9d7n"; libName = "pkg_config"; authors = [ "Alex Crichton " @@ -7587,9 +7593,9 @@ rec { }; "portable-atomic" = rec { crateName = "portable-atomic"; - version = "1.14.0"; + version = "1.15.0"; edition = "2018"; - sha256 = "1hyfma9n2cs2ibazpfwrbv61zwg7cv86g0pr5yjkg07qgr4xa81x"; + sha256 = "11csag858ndk5w4yz17h91vy53ynh67r2903gwwdn2cnilzbdj05"; libName = "portable_atomic"; features = { "critical-section" = [ "dep:critical-section" ]; @@ -7620,9 +7626,9 @@ rec { }; "potential_utf" = rec { crateName = "potential_utf"; - version = "0.1.5"; + version = "0.1.6"; edition = "2021"; - sha256 = "0r0518fr32xbkgzqap509s3r60cr0iancsg9j1jgf37cyz7b20q1"; + sha256 = "0qbndl2fpphq7mph41m11vaixs05xrh1s451wxlgap4fdnybjgnq"; authors = [ "The ICU4X Project Developers" ]; @@ -8133,9 +8139,9 @@ rec { }; "ref-cast" = rec { crateName = "ref-cast"; - version = "1.0.26"; + version = "1.0.27"; edition = "2021"; - sha256 = "0vdra0766jcc2czzqwhql41kkfyajdnai1pbkjxbq8vr7mvqyvi1"; + sha256 = "1hv5sf0j7b65gz2g57c3wp0fzr5r3807dywf6fap455lwjs0yi3y"; libName = "ref_cast"; authors = [ "David Tolnay " @@ -8150,9 +8156,9 @@ rec { }; "ref-cast-impl" = rec { crateName = "ref-cast-impl"; - version = "1.0.26"; + version = "1.0.27"; edition = "2021"; - sha256 = "0g70ff9an5i97cw9kijgzqrqydz7smcfic2zyydddizfbxl874ic"; + sha256 = "0fnzgkvddgl9xs3884x5ypi9rd0dgc1p5vd1k4b74lw49ybdiv4j"; procMacro = true; libName = "ref_cast_impl"; authors = [ @@ -9090,9 +9096,9 @@ rec { }; "rustls-webpki" = rec { crateName = "rustls-webpki"; - version = "0.103.13"; + version = "0.103.14"; edition = "2021"; - sha256 = "0vkm7z9pnxz5qz66p2kmyy2pwx0g4jnsbqk5xzfhs4czcjl2ki31"; + sha256 = "0njk28gvbqrsfg1b5r35y4f80n37kcjylj72fpc0k0g60n3529q5"; libName = "webpki"; dependencies = [ { @@ -9116,7 +9122,7 @@ rec { "alloc" = [ "ring?/alloc" "pki-types/alloc" ]; "aws-lc-rs" = [ "dep:aws-lc-rs" "aws-lc-rs/aws-lc-sys" "aws-lc-rs/prebuilt-nasm" ]; "aws-lc-rs-fips" = [ "dep:aws-lc-rs" "aws-lc-rs/fips" ]; - "aws-lc-rs-unstable" = [ "aws-lc-rs" "aws-lc-rs/unstable" ]; + "aws-lc-rs-unstable" = [ "aws-lc-rs" ]; "default" = [ "std" ]; "ring" = [ "dep:ring" ]; "std" = [ "alloc" "pki-types/std" ]; @@ -10328,8 +10334,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "fb2d86579f4e3df008f78f0e527a012243483a2d"; - sha256 = "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb"; + rev = "7b9f9ac9a76fa425ab27f2821377ef86571ca121"; + sha256 = "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9"; }; libName = "stackable_certs"; authors = [ @@ -10426,13 +10432,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.115.0"; + version = "0.116.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "fb2d86579f4e3df008f78f0e527a012243483a2d"; - sha256 = "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb"; + rev = "7b9f9ac9a76fa425ab27f2821377ef86571ca121"; + sha256 = "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9"; }; libName = "stackable_operator"; authors = [ @@ -10472,7 +10478,7 @@ rec { } { name = "futures"; - packageId = "futures 0.3.33"; + packageId = "futures 0.3.34"; } { name = "http"; @@ -10612,7 +10618,8 @@ rec { "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "client-feature-gates" "crds" "certs" "test-support" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "test-support" "time" "webhook" "kube-ws" "kube-cel" ]; + "kube-cel" = [ "kube/cel" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -10626,8 +10633,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "fb2d86579f4e3df008f78f0e527a012243483a2d"; - sha256 = "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb"; + rev = "7b9f9ac9a76fa425ab27f2821377ef86571ca121"; + sha256 = "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10661,8 +10668,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "fb2d86579f4e3df008f78f0e527a012243483a2d"; - sha256 = "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb"; + rev = "7b9f9ac9a76fa425ab27f2821377ef86571ca121"; + sha256 = "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9"; }; libName = "stackable_shared"; authors = [ @@ -10765,7 +10772,7 @@ rec { } { name = "futures"; - packageId = "futures 0.3.33"; + packageId = "futures 0.3.34"; features = [ "compat" ]; } { @@ -10848,8 +10855,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "fb2d86579f4e3df008f78f0e527a012243483a2d"; - sha256 = "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb"; + rev = "7b9f9ac9a76fa425ab27f2821377ef86571ca121"; + sha256 = "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9"; }; libName = "stackable_telemetry"; authors = [ @@ -10958,8 +10965,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "fb2d86579f4e3df008f78f0e527a012243483a2d"; - sha256 = "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb"; + rev = "7b9f9ac9a76fa425ab27f2821377ef86571ca121"; + sha256 = "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9"; }; libName = "stackable_versioned"; authors = [ @@ -11008,8 +11015,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "fb2d86579f4e3df008f78f0e527a012243483a2d"; - sha256 = "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb"; + rev = "7b9f9ac9a76fa425ab27f2821377ef86571ca121"; + sha256 = "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11076,8 +11083,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "fb2d86579f4e3df008f78f0e527a012243483a2d"; - sha256 = "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb"; + rev = "7b9f9ac9a76fa425ab27f2821377ef86571ca121"; + sha256 = "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9"; }; libName = "stackable_webhook"; authors = [ @@ -11459,18 +11466,18 @@ rec { ]; }; - "thiserror 2.0.19" = rec { + "thiserror 2.0.20" = rec { crateName = "thiserror"; - version = "2.0.19"; + version = "2.0.20"; edition = "2021"; - sha256 = "1ngwxsjsa64v1n7vb90h2b0i3fqk1piwaf0z6fqdacqfhjc3b909"; + sha256 = "0kxs6p295jffxhzaxpxv1dwaaf5iqlm6sx8h0djp6ancbxgj71pc"; authors = [ "David Tolnay " ]; dependencies = [ { name = "thiserror-impl"; - packageId = "thiserror-impl 2.0.19"; + packageId = "thiserror-impl 2.0.20"; } ]; features = { @@ -11504,11 +11511,11 @@ rec { ]; }; - "thiserror-impl 2.0.19" = rec { + "thiserror-impl 2.0.20" = rec { crateName = "thiserror-impl"; - version = "2.0.19"; + version = "2.0.20"; edition = "2021"; - sha256 = "1ka10pqy1g8zy5al9m8yadg30jp8hx0q80j8awmd8131yw6gxjs3"; + sha256 = "1bwjc94gi0xn5jz26h1a8bjj1wdkvvr6jifamyc4mp9n28zcs15w"; procMacro = true; libName = "thiserror_impl"; authors = [ @@ -11657,9 +11664,9 @@ rec { }; "tinystr" = rec { crateName = "tinystr"; - version = "0.8.3"; + version = "0.8.4"; edition = "2021"; - sha256 = "0vfr8x285w6zsqhna0a9jyhylwiafb2kc8pj2qaqaahw48236cn8"; + sha256 = "0hzncw8rgk4syla79qscfml46jm7ll1zdp7kdacc42cj8n8prqmi"; authors = [ "The ICU4X Project Developers" ]; @@ -12740,7 +12747,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.19"; + packageId = "thiserror 2.0.20"; } { name = "time"; @@ -12834,7 +12841,7 @@ rec { dependencies = [ { name = "futures"; - packageId = "futures 0.3.33"; + packageId = "futures 0.3.34"; optional = true; } { @@ -13322,9 +13329,9 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.24.0"; + version = "1.24.1"; edition = "2021"; - sha256 = "0faj5x0zgri8m3i8dv9qgyhiwqwdyhbl2g351cp3iin4ynk26fdz"; + sha256 = "1n8b7fg7dbx6ws64387l2i0qq900rw9b7qax63acdh37sczw1vrc"; authors = [ "Ashley Mannix" "Dylan DPC" @@ -13488,9 +13495,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.126"; + version = "0.2.127"; edition = "2021"; - sha256 = "197rma4qg1kb8l4bl7857pgszzval8s1w740g9myyjh92467q1jb"; + sha256 = "0w6fa1mkbb6qlkffgy4qaz0hdf496zbjkyiyvs4lvmpd8xbr6w0v"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -13539,9 +13546,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.76"; + version = "0.4.77"; edition = "2021"; - sha256 = "0799v92cpaprapnmpaflc51sdnz362q2fsjdqnwiq8ij1wsg2bf6"; + sha256 = "0l3r8m335kb2p8yj65kb0biwlypcx3ay4g750hafkl13rkapfxvb"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -13567,9 +13574,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.126"; + version = "0.2.127"; edition = "2021"; - sha256 = "1cda6wl5zyiy7777cfgrix7fhpaqba55l5zpqj4zig7ng7jyaz0n"; + sha256 = "1hcvlb6bv771fvgifd367wd0cm4giyar8fq5i4h705vj7y7myxvp"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -13591,9 +13598,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.126"; + version = "0.2.127"; edition = "2021"; - sha256 = "03iq412frl2py55skwb3ya08xha0cf6q22zr5kqlwbr675w7r6gk"; + sha256 = "112j4d7dv8y2sk9yy9czrl9fpjx9388ywnn7icdv2bywazw367g1"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -13627,10 +13634,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.126"; + version = "0.2.127"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "097a3kbjls447s1lwr41l21x5crrh5vq3h6zsxccz7slrjq4q6yw"; + sha256 = "1gywp6xv8a27fvm3ga9xby93xyic3hc2s626b9z9rw2xqny4vxky"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -13645,9 +13652,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.103"; + version = "0.3.104"; edition = "2021"; - sha256 = "0hb1zdnrp99p5r5q66jagsddmwha460yv2wklvzrzk0b3jvdq8l6"; + sha256 = "0c0acbvaqzqf21q5vdff2g74fvb7afi91xjplmclybq4d24k6df4"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -13903,6 +13910,10 @@ rec { "MouseEvent" = [ "Event" "UiEvent" ]; "MouseScrollEvent" = [ "Event" "MouseEvent" "UiEvent" ]; "MutationEvent" = [ "Event" ]; + "NavigateEvent" = [ "Event" ]; + "Navigation" = [ "EventTarget" ]; + "NavigationCurrentEntryChangeEvent" = [ "Event" ]; + "NavigationHistoryEntry" = [ "EventTarget" ]; "NetworkInformation" = [ "EventTarget" ]; "Node" = [ "EventTarget" ]; "Notification" = [ "EventTarget" ]; @@ -15025,9 +15036,9 @@ rec { }; "writeable" = rec { crateName = "writeable"; - version = "0.6.3"; + version = "0.6.4"; edition = "2021"; - sha256 = "1i54d13h9bpap2hf13xcry1s4lxh7ap3923g8f3c0grd7c9fbyhz"; + sha256 = "1p3r4s4wbf3dksfpj3xyrn7id5p0f7r74mj6qx6ngjfd6cm2vn1s"; authors = [ "The ICU4X Project Developers" ]; @@ -15337,9 +15348,9 @@ rec { }; "zerotrie" = rec { crateName = "zerotrie"; - version = "0.2.4"; + version = "0.2.5"; edition = "2021"; - sha256 = "1gr0pkcn3qsr6in6iixqyp0vbzwf2j1jzyvh7yl2yydh3p9m548g"; + sha256 = "0gss16krjzk22m57dz5hkdjg99ibj6pa41qr68na7w1jpp1nk8jf"; authors = [ "The ICU4X Project Developers" ]; @@ -15377,9 +15388,9 @@ rec { }; "zerovec" = rec { crateName = "zerovec"; - version = "0.11.6"; + version = "0.11.7"; edition = "2021"; - sha256 = "0fdjsy6b31q9i0d73sl7xjd12xadbwi45lkpfgqnmasrqg5i3ych"; + sha256 = "1n4n109wgbbin5hljq6gmbnqqg74yp9igzf40gbw2rkdjyswddcl"; authors = [ "The ICU4X Project Developers" ]; @@ -15423,9 +15434,9 @@ rec { }; "zerovec-derive" = rec { crateName = "zerovec-derive"; - version = "0.11.3"; + version = "0.11.5"; edition = "2021"; - sha256 = "0m85qj92mmfvhjra6ziqky5b1p4kcmp5069k7kfadp5hr8jw8pb2"; + sha256 = "1a8pz516ddcgxvxq3j1xgprac5wnprlrbyzsgzarj0423la2l8cz"; procMacro = true; libName = "zerovec_derive"; authors = [ @@ -15442,7 +15453,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.119"; + packageId = "syn 3.0.3"; features = [ "extra-traits" ]; } ]; diff --git a/Cargo.toml b/Cargo.toml index 052c126b..298802c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ edition = "2024" repository = "https://github.com/stackabletech/spark-k8s-operator" [workspace.dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.115.0", features = ["webhook"] } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.116.0", features = ["webhook"] } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } diff --git a/crate-hashes.json b/crate-hashes.json index 43cd4b72..3fae0f9f 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,11 +1,11 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#k8s-version@0.1.3": "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#stackable-certs@0.4.1": "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#stackable-operator-derive@0.3.1": "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#stackable-operator@0.115.0": "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#stackable-shared@0.1.2": "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#stackable-telemetry@0.6.5": "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#stackable-versioned-macros@0.11.1": "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#stackable-versioned@0.11.1": "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.115.0#stackable-webhook@0.9.2": "1w57n5xx0ik63r252l1v5ymm51jlsf7v4pj682b902k8vinlhyqb" + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#k8s-version@0.1.3": "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#stackable-certs@0.4.1": "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#stackable-operator-derive@0.3.1": "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#stackable-operator@0.116.0": "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#stackable-shared@0.1.2": "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#stackable-telemetry@0.6.5": "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#stackable-versioned-macros@0.11.1": "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#stackable-versioned@0.11.1": "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.116.0#stackable-webhook@0.9.2": "1p3744fxgvs12sqwvi8hhainwrgvhdfwmbyqf0sp0aq3awq3q1v9" } \ No newline at end of file diff --git a/extra/crds.yaml b/extra/crds.yaml index 48bdb6d6..30719a06 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -706,7 +706,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about @@ -1495,7 +1496,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about @@ -1703,7 +1705,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about @@ -2823,7 +2826,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about @@ -3233,7 +3237,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about @@ -4057,7 +4062,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about @@ -4476,7 +4482,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about @@ -5305,7 +5312,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about @@ -6094,7 +6102,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about @@ -6302,7 +6311,8 @@ spec: default: {} description: |- `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. + It is a map from environment variable names to their values. The names are validated to be + valid environment variable names. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about diff --git a/rust/operator-binary/src/connect/common.rs b/rust/operator-binary/src/connect/common.rs index 2cba2620..426c80ac 100644 --- a/rust/operator-binary/src/connect/common.rs +++ b/rust/operator-binary/src/connect/common.rs @@ -1,12 +1,14 @@ -use std::{collections::BTreeMap, str::FromStr}; +use std::{collections::BTreeMap, ops::Deref, str::FromStr}; use snafu::{ResultExt, Snafu}; -use stackable_operator::v2::{ - config_file_writer::{PropertiesWriterError, to_java_properties_string}, - role_utils::JavaCommonConfig, - types::operator::RoleName, +use stackable_operator::{ + constant, + v2::{ + config_file_writer::{PropertiesWriterError, to_java_properties_string}, + role_utils::JavaCommonConfig, + types::operator::RoleName, + }, }; -use strum::{Display, EnumIter}; use super::crd::CONNECT_EXECUTOR_ROLE_NAME; use crate::{ @@ -30,26 +32,22 @@ pub enum Error { MetricsProperties { source: PropertiesWriterError }, } -#[derive(Clone, Debug, Display, EnumIter)] -#[strum(serialize_all = "lowercase")] +constant!(SERVER_ROLE_NAME: RoleName = CONNECT_SERVER_ROLE_NAME); +constant!(EXECUTOR_ROLE_NAME: RoleName = CONNECT_EXECUTOR_ROLE_NAME); + +#[derive(Clone, Debug)] pub(crate) enum SparkConnectRole { Server, Executor, } -impl From for RoleName { - fn from(value: SparkConnectRole) -> Self { - (&value).into() - } -} +impl Deref for SparkConnectRole { + type Target = RoleName; -impl From<&SparkConnectRole> for RoleName { - fn from(value: &SparkConnectRole) -> Self { - match value { - SparkConnectRole::Server => RoleName::from_str(CONNECT_SERVER_ROLE_NAME) - .expect("CONNECT_SERVER_ROLE_NAME is a valid role name"), - SparkConnectRole::Executor => RoleName::from_str(CONNECT_EXECUTOR_ROLE_NAME) - .expect("CONNECT_EXECUTOR_ROLE_NAME is a valid role name"), + fn deref(&self) -> &Self::Target { + match self { + SparkConnectRole::Server => &SERVER_ROLE_NAME, + SparkConnectRole::Executor => &EXECUTOR_ROLE_NAME, } } } @@ -131,18 +129,12 @@ pub(crate) fn metrics_properties( #[cfg(test)] mod tests { - use stackable_operator::v2::types::operator::RoleName; - use strum::IntoEnumIterator; - - use super::SparkConnectRole; + use super::*; - /// Locks the invariant behind the `expect` in the `From for RoleName` - /// impls: every variant (present and future) must map to a valid `RoleName`. #[test] - fn every_spark_connect_role_maps_to_a_valid_role_name() { - for role in SparkConnectRole::iter() { - let _: RoleName = (&role).into(); - let _: RoleName = role.into(); - } + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *EXECUTOR_ROLE_NAME; + let _ = *SERVER_ROLE_NAME; } } diff --git a/rust/operator-binary/src/connect/controller/apply.rs b/rust/operator-binary/src/connect/controller/apply.rs index 95ad48e9..73ab19c8 100644 --- a/rust/operator-binary/src/connect/controller/apply.rs +++ b/rust/operator-binary/src/connect/controller/apply.rs @@ -13,7 +13,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::connect::controller::{ Applied, Prepared, SparkConnectResources, - validate::{ValidatedSparkConnectServer, controller_name, operator_name, product_name}, + validate::{CONTROLLER_NAME, OPERATOR_NAME, PRODUCT_NAME, ValidatedSparkConnectServer}, }; #[derive(Snafu, Debug, EnumDiscriminants)] @@ -49,9 +49,9 @@ impl<'a> Applier<'a> { object_overrides: &'a ObjectOverrides, ) -> Applier<'a> { let cluster_resources = cluster_resources_new( - &product_name(), - &operator_name(), - &controller_name(), + &PRODUCT_NAME, + &OPERATOR_NAME, + &CONTROLLER_NAME, &cluster.name, &cluster.namespace, &cluster.uid, diff --git a/rust/operator-binary/src/connect/controller/build/executor.rs b/rust/operator-binary/src/connect/controller/build/executor.rs index 181132d8..ffbd29dd 100644 --- a/rust/operator-binary/src/connect/controller/build/executor.rs +++ b/rust/operator-binary/src/connect/controller/build/executor.rs @@ -1,7 +1,4 @@ -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, -}; +use std::{collections::BTreeMap, str::FromStr}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -14,7 +11,7 @@ use stackable_operator::{ commons::resources::{CpuLimits, MemoryLimits, Resources}, k8s_openapi::{ DeepMerge, - api::core::v1::{ConfigMap, EnvVar, PodTemplateSpec}, + api::core::v1::{ConfigMap, PodTemplateSpec}, }, kube::ResourceExt, product_logging::framework::{VECTOR_CONFIG_FILE, calculate_log_volume_size_limit}, @@ -30,7 +27,10 @@ use stackable_operator::{ use crate::{ connect::{ common::{self, SparkConnectRole, object_name}, - controller::{build::object_meta, validate::ValidatedSparkConnectServer}, + controller::{ + build::{object_meta, recommended_labels_for_role_resources}, + validate::ValidatedSparkConnectServer, + }, crd::{ CONNECT_EXECUTOR_ROLE_NAME, DEFAULT_SPARK_CONNECT_GROUP_NAME, SparkConnectContainer, v1alpha1, @@ -38,10 +38,10 @@ use crate::{ s3, }, crd::constants::{ - JVM_SECURITY_PROPERTIES_FILE, LOG4J2_CONFIG_FILE, MAX_SPARK_LOG_FILES_SIZE, - METRICS_PROPERTIES_FILE, POD_TEMPLATE_FILE, VOLUME_MOUNT_NAME_CONFIG, - VOLUME_MOUNT_NAME_LOG, VOLUME_MOUNT_NAME_LOG_CONFIG, VOLUME_MOUNT_PATH_CONFIG, - VOLUME_MOUNT_PATH_LOG, VOLUME_MOUNT_PATH_LOG_CONFIG, + CONTAINERDEBUG_LOG_DIRECTORY, JVM_SECURITY_PROPERTIES_FILE, LOG4J2_CONFIG_FILE, + MAX_SPARK_LOG_FILES_SIZE, METRICS_PROPERTIES_FILE, POD_TEMPLATE_FILE, + VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_NAME_LOG, VOLUME_MOUNT_NAME_LOG_CONFIG, + VOLUME_MOUNT_PATH_CONFIG, VOLUME_MOUNT_PATH_LOG, VOLUME_MOUNT_PATH_LOG_CONFIG, }, product_logging, }; @@ -93,7 +93,7 @@ pub fn executor_pod_template( let config = &validated.executor_config; let resolved_product_image = &validated.resolved_product_image; let resolved_s3 = &validated.cluster_config.resolved_s3; - let container_env = executor_env(Some(&validated.executor_overrides.env_overrides))?; + let container_env = executor_env(&validated.executor_overrides.env_overrides); let (s3_volumes, s3_volume_mounts) = resolved_s3 .volumes_and_mounts() @@ -110,7 +110,10 @@ pub fn executor_pod_template( .context(AddVolumeMountSnafu)?; let metadata = ObjectMetaBuilder::new() - .with_labels(validated.recommended_labels(SparkConnectRole::Executor)) + .with_labels(recommended_labels_for_role_resources( + validated, + &SparkConnectRole::Executor, + )) .build(); let mut template = PodBuilder::new(); @@ -202,29 +205,17 @@ pub fn executor_pod_template( Ok(result) } -fn executor_env(env_overrides: Option<&HashMap>) -> Result, Error> { - let mut envs = BTreeMap::from([ - // Needed by the `containerdebug` running in the background of the connect container - // to log its tracing information to. - ( - "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), +/// The environment variables of the executor container. +/// +/// The user's `envOverrides` are merged in last so that they override any operator-set +/// environment variable. +fn executor_env(env_overrides: &EnvVarSet) -> EnvVarSet { + EnvVarSet::new() + .with_value( + &CONTAINERDEBUG_LOG_DIRECTORY, format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug"), - ), - ]); - - // Add env overrides - if let Some(user_env) = env_overrides { - envs.extend(user_env.clone()); - } - - Ok(envs - .into_iter() - .map(|(name, value)| EnvVar { - name: name.to_owned(), - value: Some(value.to_owned()), - value_from: None, - }) - .collect()) + ) + .merge(env_overrides.clone()) } pub(crate) fn executor_properties( @@ -372,3 +363,57 @@ pub(crate) fn executor_config_map( .build() .context(InvalidConfigMapSnafu { cm_name }) } + +#[cfg(test)] +mod tests { + use stackable_operator::k8s_openapi::{ + api::core::v1::EnvVar, apimachinery::pkg::apis::meta::v1::ObjectMeta, + }; + + use super::*; + use crate::connect::controller::build::test_support::minimal_validated_cluster; + + /// `envOverrides` must be applied after all operator-set environment variables, so a user + /// override replaces the operator-set value instead of duplicating it or being ignored. + #[test] + fn env_overrides_override_operator_set_env_vars() { + let mut validated = minimal_validated_cluster(); + validated.executor_overrides.env_overrides = EnvVarSet::new().with_value( + &"CONTAINERDEBUG_LOG_DIRECTORY" + .parse() + .expect("valid env var name"), + "/custom/log/dir", + ); + + let config_map = ConfigMap { + metadata: ObjectMeta { + name: Some("my-connect-executor".to_string()), + ..ObjectMeta::default() + }, + ..ConfigMap::default() + }; + + let pod_template = executor_pod_template(&validated, &config_map) + .expect("the executor pod template can be built"); + + let env: Vec = pod_template + .spec + .expect("the pod template has a spec") + .containers + .iter() + .find(|container| container.name == "spark") + .expect("the spark container exists") + .env + .clone() + .expect("the spark container has env vars"); + + let matching: Vec<&EnvVar> = env + .iter() + .filter(|env_var| env_var.name == "CONTAINERDEBUG_LOG_DIRECTORY") + .collect(); + + // The override must replace the operator-set value, not duplicate it. + assert_eq!(matching.len(), 1); + assert_eq!(matching[0].value.as_deref(), Some("/custom/log/dir")); + } +} diff --git a/rust/operator-binary/src/connect/controller/build/mod.rs b/rust/operator-binary/src/connect/controller/build/mod.rs index e337c6e0..5dffa384 100644 --- a/rust/operator-binary/src/connect/controller/build/mod.rs +++ b/rust/operator-binary/src/connect/controller/build/mod.rs @@ -14,8 +14,10 @@ use std::marker::PhantomData; use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::meta::ObjectMetaBuilder, kube::ResourceExt, - v2::builder::meta::ownerreference_from_resource, + builder::meta::ObjectMetaBuilder, + kube::ResourceExt, + kvp::Labels, + v2::{builder::meta::ownerreference_from_resource, kvp::label, types::operator::RoleName}, }; use crate::connect::{ @@ -23,7 +25,7 @@ use crate::connect::{ controller::{ Prepared, SparkConnectResources, build::rbac::{build_role_binding, build_service_account}, - validate::ValidatedSparkConnectServer, + validate::{CONTROLLER_NAME, OPERATOR_NAME, PRODUCT_NAME, ValidatedSparkConnectServer}, }, }; @@ -128,10 +130,47 @@ pub(crate) fn object_meta( .name_and_namespace(validated) .name(name) .ownerreference(ownerreference_from_resource(validated, None, Some(true))) - .with_labels(validated.recommended_labels(role)); + .with_labels(recommended_labels_for_role_resources(validated, &role)); builder } +/// Recommended labels for resources shared by the whole Spark Connect server, like the RBAC +/// resources. +pub(crate) fn recommended_labels_for_cluster_resources( + server: &ValidatedSparkConnectServer, +) -> Labels { + label::recommended_labels_for_cluster_resources( + &server.name, + &PRODUCT_NAME, + &server.product_version, + &OPERATOR_NAME, + &CONTROLLER_NAME, + ) +} + +/// Recommended labels for resources of the given role. +/// +/// Spark Connect has no role groups, so all role resources use these labels (without a role +/// group label). +pub(crate) fn recommended_labels_for_role_resources( + server: &ValidatedSparkConnectServer, + role_name: &RoleName, +) -> Labels { + label::recommended_labels_for_role_resources( + &server.name, + &PRODUCT_NAME, + &server.product_version, + &OPERATOR_NAME, + &CONTROLLER_NAME, + role_name, + ) +} + +/// Selector labels matching the pods of the given role. +pub(crate) fn role_selector(server: &ValidatedSparkConnectServer, role_name: &RoleName) -> Labels { + label::role_selector(&server.name, &PRODUCT_NAME, role_name) +} + #[cfg(test)] pub(crate) mod test_support { use indoc::indoc; diff --git a/rust/operator-binary/src/connect/controller/build/rbac.rs b/rust/operator-binary/src/connect/controller/build/rbac.rs index 78ff5cd3..dac48f52 100644 --- a/rust/operator-binary/src/connect/controller/build/rbac.rs +++ b/rust/operator-binary/src/connect/controller/build/rbac.rs @@ -1,26 +1,19 @@ -//! Builds the RBAC resources (ServiceAccount + RoleBinding) shared by all role groups. - -use std::str::FromStr; +//! Builds the RBAC resources (ServiceAccount + RoleBinding) shared by the whole cluster. use stackable_operator::{ k8s_openapi::api::{core::v1::ServiceAccount, rbac::v1::RoleBinding}, - kvp::Labels, - v2::{ - rbac, - types::operator::{RoleGroupName, RoleName}, - }, + v2::rbac, }; -use crate::connect::controller::validate::ValidatedSparkConnectServer; - -stackable_operator::constant!(NONE_ROLE_NAME: RoleName = "none"); -stackable_operator::constant!(NONE_ROLE_GROUP_NAME: RoleGroupName = "none"); +use crate::connect::controller::{ + build::recommended_labels_for_cluster_resources, validate::ValidatedSparkConnectServer, +}; pub fn build_service_account(server: &ValidatedSparkConnectServer) -> ServiceAccount { rbac::build_service_account( server, &server.cluster_resource_names(), - rbac_labels(server), + recommended_labels_for_cluster_resources(server), ) } @@ -28,14 +21,10 @@ pub fn build_role_binding(server: &ValidatedSparkConnectServer) -> RoleBinding { rbac::build_role_binding( server, &server.cluster_resource_names(), - rbac_labels(server), + recommended_labels_for_cluster_resources(server), ) } -fn rbac_labels(server: &ValidatedSparkConnectServer) -> Labels { - server.recommended_labels_for(&NONE_ROLE_NAME, &NONE_ROLE_GROUP_NAME) -} - #[cfg(test)] mod tests { use serde_json::json; @@ -57,13 +46,10 @@ mod tests { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { - // The RBAC resources are cluster-shared, so role and role group are `none`. "labels": { - "app.kubernetes.io/component": "none", "app.kubernetes.io/instance": "my-connect", "app.kubernetes.io/managed-by": "spark.stackable.tech_connect", "app.kubernetes.io/name": "spark-connect", - "app.kubernetes.io/role-group": "none", "app.kubernetes.io/version": app_version_label("4.1.2"), "stackable.tech/vendor": "Stackable" }, @@ -94,11 +80,9 @@ mod tests { "kind": "RoleBinding", "metadata": { "labels": { - "app.kubernetes.io/component": "none", "app.kubernetes.io/instance": "my-connect", "app.kubernetes.io/managed-by": "spark.stackable.tech_connect", "app.kubernetes.io/name": "spark-connect", - "app.kubernetes.io/role-group": "none", "app.kubernetes.io/version": app_version_label("4.1.2"), "stackable.tech/vendor": "Stackable" }, diff --git a/rust/operator-binary/src/connect/controller/build/server.rs b/rust/operator-binary/src/connect/controller/build/server.rs index 0135985c..419bfefc 100644 --- a/rust/operator-binary/src/connect/controller/build/server.rs +++ b/rust/operator-binary/src/connect/controller/build/server.rs @@ -1,7 +1,4 @@ -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, -}; +use std::{collections::BTreeMap, str::FromStr}; use indoc::formatdoc; use snafu::{OptionExt, ResultExt, Snafu}; @@ -24,7 +21,7 @@ use stackable_operator::{ DeepMerge, api::{ apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{ConfigMap, EnvVar, HTTPGetAction, Probe, Service}, + core::v1::{ConfigMap, HTTPGetAction, Probe, Service}, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, @@ -46,7 +43,10 @@ use crate::{ connect::{ GRPC, HTTP, common::{self, SparkConnectRole, object_name}, - controller::{build::object_meta, validate::ValidatedSparkConnectServer}, + controller::{ + build::{object_meta, recommended_labels_for_role_resources, role_selector}, + validate::ValidatedSparkConnectServer, + }, crd::{ CONNECT_GRPC_PORT, CONNECT_SERVER_ROLE_NAME, CONNECT_UI_PORT, DEFAULT_SPARK_CONNECT_GROUP_NAME, SparkConnectContainer, v1alpha1, @@ -55,11 +55,12 @@ use crate::{ }, crd::{ constants::{ - JVM_SECURITY_PROPERTIES_FILE, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, - LOG4J2_CONFIG_FILE, MAX_SPARK_LOG_FILES_SIZE, METRICS_PROPERTIES_FILE, - POD_TEMPLATE_FILE, SPARK_DEFAULTS_FILE_NAME, VOLUME_MOUNT_NAME_CONFIG, - VOLUME_MOUNT_NAME_LOG, VOLUME_MOUNT_NAME_LOG_CONFIG, VOLUME_MOUNT_PATH_CONFIG, - VOLUME_MOUNT_PATH_LOG, VOLUME_MOUNT_PATH_LOG_CONFIG, + CONTAINERDEBUG_LOG_DIRECTORY, JVM_SECURITY_PROPERTIES_FILE, LISTENER_VOLUME_DIR, + LISTENER_VOLUME_NAME, LOG4J2_CONFIG_FILE, MAX_SPARK_LOG_FILES_SIZE, + METRICS_PROPERTIES_FILE, POD_TEMPLATE_FILE, SPARK_DEFAULTS_FILE_NAME, + SPARK_NO_DAEMONIZE, VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_NAME_LOG, + VOLUME_MOUNT_NAME_LOG_CONFIG, VOLUME_MOUNT_PATH_CONFIG, VOLUME_MOUNT_PATH_LOG, + VOLUME_MOUNT_PATH_LOG_CONFIG, }, listener_ext, }, @@ -178,7 +179,8 @@ pub(crate) fn build_stateful_set( let resolved_product_image = &validated.resolved_product_image; let resolved_s3 = &validated.cluster_config.resolved_s3; - let recommended_labels = validated.recommended_labels(SparkConnectRole::Server); + let recommended_labels = + recommended_labels_for_role_resources(validated, &SparkConnectRole::Server); let metadata = ObjectMetaBuilder::new() .with_labels(recommended_labels.clone()) @@ -213,7 +215,7 @@ pub(crate) fn build_stateful_set( .build(), ); - let container_env = env(Some(&validated.server_overrides.env_overrides))?; + let container_env = env(&validated.server_overrides.env_overrides); let (s3_volumes, s3_volume_mounts) = resolved_s3 .volumes_and_mounts() @@ -328,11 +330,7 @@ pub(crate) fn build_stateful_set( replicas: Some(1), volume_claim_templates, selector: LabelSelector { - match_labels: Some( - validated - .role_group_selector(SparkConnectRole::Server) - .into(), - ), + match_labels: Some(role_selector(validated, &SparkConnectRole::Server).into()), ..LabelSelector::default() }, ..StatefulSetSpec::default() @@ -361,33 +359,18 @@ pub(crate) fn command_args(user_args: &[String]) -> Vec { vec![command] } -#[allow(clippy::result_large_err)] -fn env(env_overrides: Option<&HashMap>) -> Result, Error> { - let mut envs = BTreeMap::from([ - // Needed by the `containerdebug` running in the background of the connect container - // to log its tracing information to. - ( - "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), +/// The environment variables of the server container. +/// +/// The user's `envOverrides` are merged in last so that they override any operator-set +/// environment variable. +fn env(env_overrides: &EnvVarSet) -> EnvVarSet { + EnvVarSet::new() + .with_value( + &CONTAINERDEBUG_LOG_DIRECTORY, format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug"), - ), - // This env var prevents the connect server from detaching itself from the - // start script because this leads to the Pod terminating immediately. - ("SPARK_NO_DAEMONIZE".to_string(), "true".to_string()), - ]); - - // Add env overrides - if let Some(user_env) = env_overrides { - envs.extend(user_env.clone()); - } - - Ok(envs - .into_iter() - .map(|(name, value)| EnvVar { - name: name.to_owned(), - value: Some(value.to_owned()), - value_from: None, - }) - .collect()) + ) + .with_value(&SPARK_NO_DAEMONIZE, "true") + .merge(env_overrides.clone()) } // Returns the contents of the spark properties file. @@ -501,11 +484,12 @@ pub(crate) fn build_listener( let listener_name = format!( "{cluster}-{role}", cluster = validated.name_any(), - role = SparkConnectRole::Server + role = SparkConnectRole::Server.as_ref() ); let listener_class = validated.role_config.listener_class.clone(); - let recommended_object_labels = validated.recommended_labels(SparkConnectRole::Server); + let recommended_object_labels = + recommended_labels_for_role_resources(validated, &SparkConnectRole::Server); let listener_ports = [ listener::v1alpha1::ListenerPort { @@ -528,3 +512,58 @@ pub(crate) fn build_listener( &listener_ports, ) } + +#[cfg(test)] +mod tests { + use stackable_operator::k8s_openapi::{ + api::core::v1::EnvVar, apimachinery::pkg::apis::meta::v1::ObjectMeta, + }; + + use super::*; + use crate::connect::controller::build::test_support::minimal_validated_cluster; + + /// `envOverrides` must be applied after all operator-set environment variables, so a user + /// override replaces the operator-set value instead of duplicating it or being ignored. + #[test] + fn env_overrides_override_operator_set_env_vars() { + let mut validated = minimal_validated_cluster(); + validated.server_overrides.env_overrides = EnvVarSet::new().with_value( + &"SPARK_NO_DAEMONIZE".parse().expect("valid env var name"), + "overridden", + ); + + let config_map = ConfigMap { + metadata: ObjectMeta { + name: Some("my-connect-server".to_string()), + ..ObjectMeta::default() + }, + ..ConfigMap::default() + }; + + let stateful_set = build_stateful_set(&validated, &config_map, "my-connect-server", vec![]) + .expect("the StatefulSet can be built"); + + let env: Vec = stateful_set + .spec + .expect("the StatefulSet has a spec") + .template + .spec + .expect("the StatefulSet has a pod spec") + .containers + .iter() + .find(|container| container.name == "spark") + .expect("the spark container exists") + .env + .clone() + .expect("the spark container has env vars"); + + let matching: Vec<&EnvVar> = env + .iter() + .filter(|env_var| env_var.name == "SPARK_NO_DAEMONIZE") + .collect(); + + // The override must replace the operator-set value, not duplicate it. + assert_eq!(matching.len(), 1); + assert_eq!(matching[0].value.as_deref(), Some("overridden")); + } +} diff --git a/rust/operator-binary/src/connect/controller/build/service.rs b/rust/operator-binary/src/connect/controller/build/service.rs index da01a657..3e85fa4b 100644 --- a/rust/operator-binary/src/connect/controller/build/service.rs +++ b/rust/operator-binary/src/connect/controller/build/service.rs @@ -7,7 +7,10 @@ use stackable_operator::{ use crate::connect::{ GRPC, HTTP, common::SparkConnectRole, - controller::{build::object_meta, validate::ValidatedSparkConnectServer}, + controller::{ + build::{object_meta, role_selector}, + validate::ValidatedSparkConnectServer, + }, crd::{CONNECT_GRPC_PORT, CONNECT_UI_PORT}, }; @@ -17,10 +20,10 @@ pub(crate) fn build_headless_service(validated: &ValidatedSparkConnectServer) -> let service_name = format!( "{cluster}-{role}-headless", cluster = validated.name_any(), - role = SparkConnectRole::Server + role = SparkConnectRole::Server.as_ref() ); - let selector = validated.role_selector(SparkConnectRole::Server).into(); + let selector = role_selector(validated, &SparkConnectRole::Server).into(); Service { metadata: object_meta(validated, service_name, SparkConnectRole::Server).build(), @@ -55,10 +58,10 @@ pub(crate) fn build_metrics_service(validated: &ValidatedSparkConnectServer) -> let service_name = format!( "{cluster}-{role}-metrics", cluster = validated.name_any(), - role = SparkConnectRole::Server + role = SparkConnectRole::Server.as_ref() ); - let selector = validated.role_selector(SparkConnectRole::Server).into(); + let selector = role_selector(validated, &SparkConnectRole::Server).into(); Service { metadata: object_meta(validated, service_name, SparkConnectRole::Server) diff --git a/rust/operator-binary/src/connect/controller/update_status.rs b/rust/operator-binary/src/connect/controller/update_status.rs index c56044a6..f86db21a 100644 --- a/rust/operator-binary/src/connect/controller/update_status.rs +++ b/rust/operator-binary/src/connect/controller/update_status.rs @@ -15,7 +15,7 @@ use crate::{ controller::{Applied, SparkConnectResources}, crd::{SparkConnectServerStatus, v1alpha1::SparkConnectServer}, }, - crd::constants::OPERATOR_NAME, + crd::constants::SPARK_OPERATOR_NAME, }; #[derive(Snafu, Debug, EnumDiscriminants)] @@ -60,7 +60,7 @@ pub async fn update_status( }; client - .apply_patch_status(OPERATOR_NAME, connect_server, &status) + .apply_patch_status(SPARK_OPERATOR_NAME, connect_server, &status) .await .context(ApplyStatusSnafu)?; diff --git a/rust/operator-binary/src/connect/controller/validate.rs b/rust/operator-binary/src/connect/controller/validate.rs index feb83e09..aa761dc7 100644 --- a/rust/operator-binary/src/connect/controller/validate.rs +++ b/rust/operator-binary/src/connect/controller/validate.rs @@ -3,45 +3,38 @@ //! Resolves the product image and the server/executor configs. //! Does not touch the Kubernetes API. -use std::{borrow::Cow, collections::HashMap, str::FromStr}; +use std::{borrow::Cow, str::FromStr}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, + constant, k8s_openapi::{api::core::v1::PodTemplateSpec, apimachinery::pkg::apis::meta::v1::ObjectMeta}, kube::Resource, - kvp::Labels, product_logging::spec::Logging, v2::{ HasName, HasUid, NameIsValidLabelValue, + builder::pod::container::EnvVarSet, controller_utils::{get_cluster_name, get_namespace, get_uid}, - kvp::label::{recommended_labels, role_group_selector, role_selector}, product_logging::framework::{ VectorContainerLogConfig, validate_logging_configuration_for_container, }, role_utils::{self, JavaCommonConfig}, types::{ kubernetes::{ConfigMapName, ListenerClassName, NamespaceName, Uid}, - operator::{ - ClusterName, ControllerName, OperatorName, ProductName, ProductVersion, - RoleGroupName, RoleName, - }, + operator::{ClusterName, ControllerName, OperatorName, ProductName, ProductVersion}, }, }, }; use crate::{ connect::{ - common::SparkConnectRole, controller::dereference::DereferencedSparkConnectServer, - crd::{ - self, CONNECT_APP_NAME, CONNECT_CONTROLLER_NAME, DEFAULT_SPARK_CONNECT_GROUP_NAME, - SparkConnectContainer, v1alpha1, - }, + crd::{self, CONNECT_APP_NAME, CONNECT_CONTROLLER_NAME, SparkConnectContainer, v1alpha1}, s3::ResolvedS3, }, - crd::constants::{CONTAINER_IMAGE_BASE_NAME, OPERATOR_NAME}, + crd::constants::{CONTAINER_IMAGE_BASE_NAME, SPARK_OPERATOR_NAME}, }; #[derive(Snafu, Debug)] @@ -115,9 +108,12 @@ fn validate_logging( type Result = std::result::Result; -stackable_operator::constant!( - DEFAULT_SPARK_CONNECT_ROLE_GROUP: RoleGroupName = DEFAULT_SPARK_CONNECT_GROUP_NAME -); +// The product name (`spark-connect`) as a type-safe label value. +constant!(pub(crate) PRODUCT_NAME: ProductName = CONNECT_APP_NAME); +// The operator name as a type-safe label value. +constant!(pub(crate) OPERATOR_NAME: OperatorName = SPARK_OPERATOR_NAME); +// The controller name as a type-safe label value. +constant!(pub(crate) CONTROLLER_NAME: ControllerName = CONNECT_CONTROLLER_NAME); /// Validated logging configuration for the (optional) Vector container. /// @@ -152,7 +148,7 @@ pub struct ValidatedSparkConnectServer { #[derive(Clone, Debug, Default)] pub struct ValidatedOverrides { pub config_overrides: v1alpha1::ConfigOverrides, - pub env_overrides: HashMap, + pub env_overrides: EnvVarSet, pub pod_overrides: PodTemplateSpec, pub jvm_config: Option, } @@ -169,79 +165,16 @@ pub struct ValidatedRoleConfig { } impl ValidatedSparkConnectServer { - /// Recommended labels for a resource that is not tied to a concrete [`SparkConnectRole`] - /// (e.g. the cluster-shared RBAC resources), using a free-form role/role-group label value. - pub fn recommended_labels_for( - &self, - role_name: &RoleName, - role_group_name: &RoleGroupName, - ) -> Labels { - self.recommended_labels_with(&self.product_version, role_name, role_group_name) - } - - /// Recommended labels for a resource of the given role. - pub fn recommended_labels(&self, role: SparkConnectRole) -> Labels { - self.recommended_labels_for(&role.into(), &DEFAULT_SPARK_CONNECT_ROLE_GROUP) - } - - fn recommended_labels_with( - &self, - product_version: &ProductVersion, - role_name: &RoleName, - role_group_name: &RoleGroupName, - ) -> Labels { - recommended_labels( - self, - &product_name(), - product_version, - &operator_name(), - &controller_name(), - role_name, - role_group_name, - ) - } - - /// Selector labels matching the pods of the given role. - pub fn role_selector(&self, role: SparkConnectRole) -> Labels { - role_selector(self, &product_name(), &role.into()) - } - - /// Selector labels matching the pods of the given role's (single) role group. - pub fn role_group_selector(&self, role: SparkConnectRole) -> Labels { - role_group_selector( - self, - &product_name(), - &role.into(), - &DEFAULT_SPARK_CONNECT_ROLE_GROUP, - ) - } - /// Type-safe names for the per-cluster RBAC resources: the ServiceAccount, /// its (namespaced) RoleBinding, and the operator-deployed ClusterRole it binds. pub fn cluster_resource_names(&self) -> role_utils::ResourceNames { role_utils::ResourceNames { cluster_name: self.name.clone(), - product_name: product_name(), + product_name: PRODUCT_NAME.clone(), } } } -/// The product name (`spark-connect`) as a type-safe label value. -pub fn product_name() -> ProductName { - ProductName::from_str(CONNECT_APP_NAME).expect("CONNECT_APP_NAME is a valid product name") -} - -/// The operator name as a type-safe label value. -pub fn operator_name() -> OperatorName { - OperatorName::from_str(OPERATOR_NAME).expect("the operator name is a valid label value") -} - -/// The controller name as a type-safe label value. -pub fn controller_name() -> ControllerName { - ControllerName::from_str(CONNECT_CONTROLLER_NAME) - .expect("the controller name is a valid label value") -} - impl NameIsValidLabelValue for ValidatedSparkConnectServer { fn to_label_value(&self) -> String { self.name.to_label_value() @@ -318,7 +251,7 @@ pub fn validate( .as_ref() .map(|cc| ValidatedOverrides { config_overrides: cc.config_overrides.clone(), - env_overrides: cc.env_overrides.clone(), + env_overrides: cc.env_overrides.clone().into(), pod_overrides: cc.pod_overrides.clone(), jvm_config: Some(cc.product_specific_common_config.clone()), }) @@ -329,7 +262,7 @@ pub fn validate( .as_ref() .map(|cc| ValidatedOverrides { config_overrides: cc.config_overrides.clone(), - env_overrides: cc.env_overrides.clone(), + env_overrides: cc.env_overrides.clone().into(), pod_overrides: cc.pod_overrides.clone(), jvm_config: Some(cc.product_specific_common_config.clone()), }) @@ -372,11 +305,20 @@ pub fn validate( #[cfg(test)] mod tests { + use super::*; use crate::{ connect::controller::build::test_support::minimal_validated_cluster, test_support::app_version_label, }; + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *CONTROLLER_NAME; + let _ = *OPERATOR_NAME; + let _ = *PRODUCT_NAME; + } + /// Locks every value the validate step itself derives from the minimal fixture — so a /// validation regression fails here, with a validate-shaped message, instead of surfacing as /// a confusing build-test failure downstream. @@ -411,7 +353,7 @@ mod tests { // The minimal fixture has no overrides and no Vector agent for either role. for overrides in [&validated.server_overrides, &validated.executor_overrides] { - assert!(overrides.env_overrides.is_empty()); + assert!(overrides.env_overrides.iter().next().is_none()); assert_eq!(overrides.jvm_config, None); } for logging in [&validated.server_logging, &validated.executor_logging] { diff --git a/rust/operator-binary/src/connect/crd.rs b/rust/operator-binary/src/connect/crd.rs index 8fa951be..18508d08 100644 --- a/rust/operator-binary/src/connect/crd.rs +++ b/rust/operator-binary/src/connect/crd.rs @@ -29,12 +29,11 @@ use stackable_operator::{ CustomContainerLogConfig, Logging, }, }, - role_utils::CommonConfiguration, schemars::{self, JsonSchema}, shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, v2::{ - role_utils::JavaCommonConfig, + role_utils::{CommonConfiguration, JavaCommonConfig}, types::{ common::Port, kubernetes::{ConfigMapName, ContainerName, ListenerClassName}, @@ -50,7 +49,7 @@ pub const CONNECT_CONTROLLER_NAME: &str = "connect"; pub const CONNECT_FULL_CONTROLLER_NAME: &str = concatcp!( CONNECT_CONTROLLER_NAME, '.', - crate::crd::constants::OPERATOR_NAME + crate::crd::constants::SPARK_OPERATOR_NAME ); pub const CONNECT_SERVER_ROLE_NAME: &str = "server"; pub const CONNECT_EXECUTOR_ROLE_NAME: &str = "executor"; @@ -430,7 +429,7 @@ impl v1alpha1::ExecutorConfig { let affinity_between_role_pods = affinity_between_role_pods( CONNECT_APP_NAME, cluster_name, - &SparkConnectRole::Executor.to_string(), + SparkConnectRole::Executor.as_ref(), 70, ); diff --git a/rust/operator-binary/src/crd/constants.rs b/rust/operator-binary/src/crd/constants.rs index efef0f14..ca12f300 100644 --- a/rust/operator-binary/src/crd/constants.rs +++ b/rust/operator-binary/src/crd/constants.rs @@ -2,39 +2,50 @@ use std::{collections::BTreeMap, str::FromStr}; use const_format::concatcp; use stackable_operator::{ + constant, memory::{BinaryMultiple, MemoryQuantity}, - v2::types::{common::Port, kubernetes::VolumeName}, + v2::{ + builder::pod::container::EnvVarName, + types::{common::Port, kubernetes::VolumeName}, + }, }; pub const APP_NAME: &str = "spark-k8s"; -stackable_operator::constant!(pub VOLUME_MOUNT_NAME_IVY2: VolumeName = "ivy2"); +constant!(pub VOLUME_MOUNT_NAME_IVY2: VolumeName = "ivy2"); pub const VOLUME_MOUNT_PATH_IVY2: &str = "/ivy2"; -stackable_operator::constant!(pub VOLUME_MOUNT_NAME_DRIVER_POD_TEMPLATES: VolumeName = "driver-pod-template"); +constant!(pub VOLUME_MOUNT_NAME_DRIVER_POD_TEMPLATES: VolumeName = "driver-pod-template"); pub const VOLUME_MOUNT_PATH_DRIVER_POD_TEMPLATES: &str = "/stackable/spark/driver-pod-templates"; -stackable_operator::constant!(pub VOLUME_MOUNT_NAME_EXECUTOR_POD_TEMPLATES: VolumeName = "executor-pod-template"); +constant!(pub VOLUME_MOUNT_NAME_EXECUTOR_POD_TEMPLATES: VolumeName = "executor-pod-template"); pub const VOLUME_MOUNT_PATH_EXECUTOR_POD_TEMPLATES: &str = "/stackable/spark/executor-pod-templates"; pub const POD_TEMPLATE_FILE: &str = "template.yaml"; -stackable_operator::constant!(pub VOLUME_MOUNT_NAME_CONFIG: VolumeName = "config"); +constant!(pub VOLUME_MOUNT_NAME_CONFIG: VolumeName = "config"); pub const VOLUME_MOUNT_PATH_CONFIG: &str = "/stackable/spark/conf"; -stackable_operator::constant!(pub VOLUME_MOUNT_NAME_JOB: VolumeName = "job-files"); +constant!(pub VOLUME_MOUNT_NAME_JOB: VolumeName = "job-files"); pub const VOLUME_MOUNT_PATH_JOB: &str = "/stackable/spark/jobs"; -stackable_operator::constant!(pub VOLUME_MOUNT_NAME_REQ: VolumeName = "req-files"); +constant!(pub VOLUME_MOUNT_NAME_REQ: VolumeName = "req-files"); pub const VOLUME_MOUNT_PATH_REQ: &str = "/stackable/spark/requirements"; -stackable_operator::constant!(pub VOLUME_MOUNT_NAME_LOG_CONFIG: VolumeName = "log-config"); +constant!(pub VOLUME_MOUNT_NAME_LOG_CONFIG: VolumeName = "log-config"); pub const VOLUME_MOUNT_PATH_LOG_CONFIG: &str = "/stackable/log_config"; -stackable_operator::constant!(pub VOLUME_MOUNT_NAME_LOG: VolumeName = "log"); +constant!(pub VOLUME_MOUNT_NAME_LOG: VolumeName = "log"); pub const VOLUME_MOUNT_PATH_LOG: &str = "/stackable/log"; +// Tells the `containerdebug` process running in the background of a product container where to +// write its tracing information. +constant!(pub CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY"); +// Prevents the history/connect server from detaching itself from the start script, which would +// terminate the Pod immediately. +constant!(pub SPARK_NO_DAEMONIZE: EnvVarName = "SPARK_NO_DAEMONIZE"); + pub const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; @@ -72,19 +83,20 @@ pub const MAX_INIT_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { unit: BinaryMultiple::Mebi, }; -pub const OPERATOR_NAME: &str = "spark.stackable.tech"; +pub const SPARK_OPERATOR_NAME: &str = "spark.stackable.tech"; pub const FIELD_MANAGER: &str = "spark-operator"; pub const SPARK_CONTROLLER_NAME: &str = "sparkapplication"; -pub const SPARK_FULL_CONTROLLER_NAME: &str = concatcp!(SPARK_CONTROLLER_NAME, '.', OPERATOR_NAME); +pub const SPARK_FULL_CONTROLLER_NAME: &str = + concatcp!(SPARK_CONTROLLER_NAME, '.', SPARK_OPERATOR_NAME); pub const POD_DRIVER_CONTROLLER_NAME: &str = "pod-driver"; pub const POD_DRIVER_FULL_CONTROLLER_NAME: &str = - concatcp!(POD_DRIVER_CONTROLLER_NAME, '.', OPERATOR_NAME); + concatcp!(POD_DRIVER_CONTROLLER_NAME, '.', SPARK_OPERATOR_NAME); pub const HISTORY_CONTROLLER_NAME: &str = "history"; pub const HISTORY_FULL_CONTROLLER_NAME: &str = - concatcp!(HISTORY_CONTROLLER_NAME, '.', OPERATOR_NAME); + concatcp!(HISTORY_CONTROLLER_NAME, '.', SPARK_OPERATOR_NAME); pub const HISTORY_APP_NAME: &str = "spark-history"; pub const HISTORY_ROLE_NAME: &str = "node"; @@ -97,7 +109,7 @@ pub const SPARK_CLUSTER_ROLE: &str = "spark-k8s-clusterrole"; pub const METRICS_PORT: Port = Port(18081); pub const HISTORY_UI_PORT: Port = Port(18080); -stackable_operator::constant!(pub LISTENER_VOLUME_NAME: VolumeName = "listener"); +constant!(pub LISTENER_VOLUME_NAME: VolumeName = "listener"); pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; pub const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; @@ -117,3 +129,24 @@ pub fn default_jvm_security_properties() -> BTreeMap { ] .into() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *CONTAINERDEBUG_LOG_DIRECTORY; + let _ = *LISTENER_VOLUME_NAME; + let _ = *SPARK_NO_DAEMONIZE; + let _ = *VOLUME_MOUNT_NAME_CONFIG; + let _ = *VOLUME_MOUNT_NAME_DRIVER_POD_TEMPLATES; + let _ = *VOLUME_MOUNT_NAME_EXECUTOR_POD_TEMPLATES; + let _ = *VOLUME_MOUNT_NAME_IVY2; + let _ = *VOLUME_MOUNT_NAME_JOB; + let _ = *VOLUME_MOUNT_NAME_LOG; + let _ = *VOLUME_MOUNT_NAME_LOG_CONFIG; + let _ = *VOLUME_MOUNT_NAME_REQ; + } +} diff --git a/rust/operator-binary/src/crd/history.rs b/rust/operator-binary/src/crd/history.rs index 0531c59c..470c51e4 100644 --- a/rust/operator-binary/src/crd/history.rs +++ b/rust/operator-binary/src/crd/history.rs @@ -17,12 +17,12 @@ use stackable_operator::{ k8s_openapi::apimachinery::pkg::api::resource::Quantity, kube::CustomResource, product_logging::{self, spec::Logging}, - role_utils::{GenericRoleConfig, Role}, + role_utils::GenericRoleConfig, schemars::{self, JsonSchema}, shared::time::Duration, v2::{ config_overrides::KeyValueConfigOverrides, - role_utils::JavaCommonConfig, + role_utils::{JavaCommonConfig, Role}, types::kubernetes::{ConfigMapName, ContainerName, ListenerClassName}, }, versioned::versioned, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index a456b1d9..1b9b824e 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -3,6 +3,7 @@ use std::{ cmp::max, collections::{BTreeMap, HashMap}, + str::FromStr, }; use constants::*; @@ -24,6 +25,7 @@ use stackable_operator::{ fragment::{self, ValidationError}, merge::Merge, }, + constant, crd::s3, k8s_openapi::{ api::core::v1::{EmptyDirVolumeSource, EnvVar, PodTemplateSpec, Volume, VolumeMount}, @@ -32,12 +34,13 @@ use stackable_operator::{ kube::{CustomResource, ResourceExt}, memory::{BinaryMultiple, MemoryQuantity}, product_logging, - role_utils::{CommonConfiguration, RoleGroup}, schemars::{self, JsonSchema}, shared::time::Duration, utils::crds::raw_object_list_schema, v2::{ - config_overrides::KeyValueConfigOverrides, role_utils::JavaCommonConfig, + builder::pod::container::{EnvVarName, EnvVarSet}, + config_overrides::KeyValueConfigOverrides, + role_utils::{CommonConfiguration, JavaCommonConfig, RoleGroup}, types::kubernetes::ConfigMapName, }, versioned::versioned, @@ -112,10 +115,24 @@ pub enum Error { source: s3::v1alpha1::ConnectionError, }, + #[snafu(display("invalid environment variable name in the `env` list"))] + ParseSparkEnvVarName { + source: stackable_operator::v2::builder::pod::container::Error, + }, + #[snafu(display("failed to configure log directory"))] ConfigureLogDir { source: logdir::Error }, } +// `_STACKABLE_PRE_HOOK` is evaluated by the entrypoint script (run-spark.sh) in the Spark images +// before the actual JVM process is started; the operator uses it to run `containerdebug` in the +// background of every `spark` container. +constant!(STACKABLE_PRE_HOOK: EnvVarName = "_STACKABLE_PRE_HOOK"); +constant!(PYTHONPATH: EnvVarName = "PYTHONPATH"); +// The environment variable holding the trust store password; its value is the +// `STACKABLE_TLS_STORE_PASSWORD` string constant. +constant!(STACKABLE_TLS_STORE_PASSWORD_ENV: EnvVarName = "STACKABLE_TLS_STORE_PASSWORD"); + pub type SparkApplicationJobRoleType = CommonConfiguration; @@ -281,7 +298,11 @@ impl v1alpha1::SparkApplication { } pub fn pod_template_config_map_name(&self, role: SparkApplicationRole) -> String { - format!("{app_name}-{role}-pod-template", app_name = self.name_any()) + format!( + "{app_name}-{role}-pod-template", + app_name = self.name_any(), + role = role.as_ref() + ) } pub fn application_artifact(&self) -> &str { @@ -756,12 +777,24 @@ impl v1alpha1::SparkApplication { Ok(vec![submit_cmd.join(" ")]) } + /// The base environment for the submit, driver and executor containers: the user-defined + /// `spec.env` entries plus the environment variables set by the operator. + /// + /// The role-specific `envOverrides` are merged on top in [`Self::merged_env`], so they take + /// precedence over everything set here. pub fn env( &self, s3conn: &Option, logdir: &Option, - ) -> Vec { - let mut e: Vec = self.spec.env.clone(); + ) -> Result { + // The CRD accepts raw `EnvVar` objects in `spec.env`, so the names are only validated + // here. + let mut env = EnvVarSet::new(); + for env_var in self.spec.env.clone() { + env = env + .with_env_var(env_var) + .context(ParseSparkEnvVarNameSnafu)?; + } // These env variables enable the `containerdebug` process in driver and executor pods. // More precisely, this process runs in the background of every `spark` container. @@ -770,36 +803,31 @@ impl v1alpha1::SparkApplication { // - `_STACKABLE_PRE_HOOK` - is evaluated by the entrypoint script (run-spark.sh) in the Spark images // before the actual JVM process is started. The result of this evaluation is that the // `containerdebug` process is executed in the background. - e.extend(vec![ - EnvVar { - name: "CONTAINERDEBUG_LOG_DIRECTORY".into(), - value: Some(format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug")), - value_from: None, - }, - EnvVar { - name: "_STACKABLE_PRE_HOOK".into(), - value: Some(format!( "containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &")), - value_from: None, - }, - ]); + env = env + .with_value( + &CONTAINERDEBUG_LOG_DIRECTORY, + format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug"), + ) + .with_value( + &STACKABLE_PRE_HOOK, + format!( + "containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &" + ), + ); if self.requirements().is_some() { - e.push(EnvVar { - name: "PYTHONPATH".to_string(), - value: Some(format!( - "$SPARK_HOME/python:{VOLUME_MOUNT_PATH_REQ}:$PYTHONPATH" - )), - value_from: None, - }); + env = env.with_value( + &PYTHONPATH, + format!("$SPARK_HOME/python:{VOLUME_MOUNT_PATH_REQ}:$PYTHONPATH"), + ); } if tlscerts::tls_secret_names(s3conn, logdir).is_some() { - e.push(EnvVar { - name: "STACKABLE_TLS_STORE_PASSWORD".to_string(), - value: Some(STACKABLE_TLS_STORE_PASSWORD.to_string()), - value_from: None, - }); + env = env.with_value( + &STACKABLE_TLS_STORE_PASSWORD_ENV, + STACKABLE_TLS_STORE_PASSWORD, + ); } - e + Ok(env) } pub fn submit_config(&self) -> Result { @@ -844,35 +872,24 @@ impl v1alpha1::SparkApplication { } } - pub fn merged_env(&self, role: SparkApplicationRole, env: &[EnvVar]) -> Vec { - // Use a BTreeMap internally to enable replacement of existing keys - let mut env: BTreeMap<&String, EnvVar> = env - .iter() - .map(|env_var| (&env_var.name, env_var.clone())) - .collect(); - - // Merge the role-specific envOverrides on top - let role_envs = match role { + /// The given base environment with the role-specific `envOverrides` merged on top. + /// + /// The overrides are merged in last so that they override any operator-set environment + /// variable. Callers must therefore add every operator-set environment variable to `env` + /// rather than appending it to the container afterwards. + pub fn merged_env(&self, role: SparkApplicationRole, env: EnvVarSet) -> EnvVarSet { + let role_env_overrides = match role { SparkApplicationRole::Submit => self.spec.job.as_ref().map(|j| &j.env_overrides), SparkApplicationRole::Driver => self.spec.driver.as_ref().map(|d| &d.env_overrides), SparkApplicationRole::Executor => { self.spec.executor.as_ref().map(|e| &e.config.env_overrides) } }; - if let Some(role_envs) = role_envs { - env.extend(role_envs.iter().map(|(k, v)| { - ( - k, - EnvVar { - name: k.clone(), - value: Some(v.clone()), - ..Default::default() - }, - ) - })) - } - env.into_values().collect() + match role_env_overrides { + Some(env_overrides) => env.merge(env_overrides.clone().into()), + None => env, + } } pub fn retry_on_failure_count(&self) -> i32 { @@ -1074,6 +1091,14 @@ mod tests { use super::*; use crate::crd::roles::SparkStorageConfig; + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *PYTHONPATH; + let _ = *STACKABLE_PRE_HOOK; + let _ = *STACKABLE_TLS_STORE_PASSWORD_ENV; + } + #[test] fn test_default_resource_limits() { let spark_application = serde_yaml::from_str::(indoc! {" diff --git a/rust/operator-binary/src/crd/roles.rs b/rust/operator-binary/src/crd/roles.rs index b2eb26f8..ed15befb 100644 --- a/rust/operator-binary/src/crd/roles.rs +++ b/rust/operator-binary/src/crd/roles.rs @@ -13,7 +13,7 @@ //! each role is named "default". These roles are transparent to the user. //! //! The history server has its own role completely unrelated to this module. -use std::{slice, str::FromStr}; +use std::{ops::Deref, slice, str::FromStr}; use serde::{Deserialize, Serialize}; use stackable_operator::{ @@ -28,26 +28,50 @@ use stackable_operator::{ fragment::Fragment, merge::{Atomic, Merge}, }, + constant, crd::s3, k8s_openapi::{api::core::v1::VolumeMount, apimachinery::pkg::api::resource::Quantity}, product_logging::{self, spec::Logging}, schemars::{self, JsonSchema}, shared::time::Duration, utils::crds::raw_object_list_schema, - v2::types::kubernetes::ContainerName, + v2::types::{kubernetes::ContainerName, operator::RoleName}, }; use strum::{Display, EnumIter}; use crate::crd::{ResolvedLogDir, constants::DEFAULT_SUBMIT_JOB_RETRY_ON_FAILURE_COUNT, v1alpha1}; -#[derive(Clone, Debug, Deserialize, Display, Eq, PartialEq, Serialize, JsonSchema)] -#[strum(serialize_all = "kebab-case")] +constant!(SUBMIT_ROLE_NAME: RoleName = "submit"); +constant!(DRIVER_ROLE_NAME: RoleName = "driver"); +constant!(EXECUTOR_ROLE_NAME: RoleName = "executor"); + +#[derive(Clone, Debug, Eq, PartialEq)] pub enum SparkApplicationRole { Submit, Driver, Executor, } +impl Deref for SparkApplicationRole { + type Target = RoleName; + + fn deref(&self) -> &Self::Target { + match self { + SparkApplicationRole::Submit => &SUBMIT_ROLE_NAME, + SparkApplicationRole::Driver => &DRIVER_ROLE_NAME, + SparkApplicationRole::Executor => &EXECUTOR_ROLE_NAME, + } + } +} + +impl SparkApplicationRole { + /// The type-safe name of this role, e.g. to build + /// [`ResourceNames`](stackable_operator::v2::role_group_utils::ResourceNames). + pub fn role_name(&self) -> RoleName { + RoleName::clone(self) + } +} + #[derive(Clone, Debug, Default, JsonSchema, PartialEq, Fragment)] #[allow(clippy::derive_partial_eq_without_eq)] #[fragment_attrs( @@ -257,3 +281,16 @@ impl From for Vec { value.volume_mounts } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *DRIVER_ROLE_NAME; + let _ = *EXECUTOR_ROLE_NAME; + let _ = *SUBMIT_ROLE_NAME; + } +} diff --git a/rust/operator-binary/src/crd/template_merger.rs b/rust/operator-binary/src/crd/template_merger.rs index 8f5e83f1..74cf0392 100644 --- a/rust/operator-binary/src/crd/template_merger.rs +++ b/rust/operator-binary/src/crd/template_merger.rs @@ -7,7 +7,10 @@ use stackable_operator::{ k8s_openapi::{ DeepMerge, api::core::v1::PodTemplateSpec, apimachinery::pkg::apis::meta::v1::ObjectMeta, }, - role_utils::{CommonConfiguration, RoleGroup}, + v2::{ + env_overrides::EnvOverrides, + role_utils::{CommonConfiguration, RoleGroup}, + }, }; use super::v1alpha1::SparkApplication; @@ -140,6 +143,13 @@ fn merge_option_hashmap( } } +/// Merge two [`EnvOverrides`], with overlay values taking precedence +fn merge_env_overrides(base: &EnvOverrides, overlay: &EnvOverrides) -> EnvOverrides { + let mut merged = base.clone(); + merged.extend(overlay.clone()); + merged +} + /// Merge two HashMaps, with overlay values taking precedence fn merge_hashmap(base: &HashMap, overlay: &HashMap) -> HashMap where @@ -223,7 +233,7 @@ where CommonConfiguration { config, config_overrides, - env_overrides: merge_hashmap(&base.env_overrides, &overlay.env_overrides), + env_overrides: merge_env_overrides(&base.env_overrides, &overlay.env_overrides), cli_overrides, pod_overrides: merge_pod_template_spec(&base.pod_overrides, &overlay.pod_overrides), product_specific_common_config, @@ -702,6 +712,14 @@ mod tests { ); } + /// The value of the env override with the given name, if present. + fn env_value(env_overrides: &EnvOverrides, name: &str) -> Option { + env_overrides + .iter() + .find(|(env_var_name, _)| env_var_name.as_ref() == name) + .map(|(_, value)| value.clone()) + } + #[test] fn test_deep_merge_env_overrides() { let base = serde_yaml::from_str::(indoc! {r#" @@ -761,39 +779,45 @@ mod tests { let merged = deep_merge(&base, &overlay); let submit_env = &merged.spec.job.as_ref().unwrap().env_overrides; - assert_eq!(submit_env.get("TEST_BASE_ONLY"), Some(&"base".to_string())); assert_eq!( - submit_env.get("TEST_OVERRIDDEN"), - Some(&"overlay".to_string()) + env_value(submit_env, "TEST_BASE_ONLY"), + Some("base".to_string()) ); assert_eq!( - submit_env.get("TEST_OVERLAY_ONLY"), - Some(&"overlay".to_string()) + env_value(submit_env, "TEST_OVERRIDDEN"), + Some("overlay".to_string()) + ); + assert_eq!( + env_value(submit_env, "TEST_OVERLAY_ONLY"), + Some("overlay".to_string()) ); let driver_env = &merged.spec.driver.as_ref().unwrap().env_overrides; - assert_eq!(driver_env.get("TEST_BASE_ONLY"), Some(&"base".to_string())); assert_eq!( - driver_env.get("TEST_OVERRIDDEN"), - Some(&"overlay".to_string()) + env_value(driver_env, "TEST_BASE_ONLY"), + Some("base".to_string()) ); assert_eq!( - driver_env.get("TEST_OVERLAY_ONLY"), - Some(&"overlay".to_string()) + env_value(driver_env, "TEST_OVERRIDDEN"), + Some("overlay".to_string()) + ); + assert_eq!( + env_value(driver_env, "TEST_OVERLAY_ONLY"), + Some("overlay".to_string()) ); let executor_env = &merged.spec.executor.as_ref().unwrap().config.env_overrides; assert_eq!( - executor_env.get("TEST_BASE_ONLY"), - Some(&"base".to_string()) + env_value(executor_env, "TEST_BASE_ONLY"), + Some("base".to_string()) ); assert_eq!( - executor_env.get("TEST_OVERRIDDEN"), - Some(&"overlay".to_string()) + env_value(executor_env, "TEST_OVERRIDDEN"), + Some("overlay".to_string()) ); assert_eq!( - executor_env.get("TEST_OVERLAY_ONLY"), - Some(&"overlay".to_string()) + env_value(executor_env, "TEST_OVERLAY_ONLY"), + Some("overlay".to_string()) ); } diff --git a/rust/operator-binary/src/history/controller/apply.rs b/rust/operator-binary/src/history/controller/apply.rs index 1ca31b74..00774e71 100644 --- a/rust/operator-binary/src/history/controller/apply.rs +++ b/rust/operator-binary/src/history/controller/apply.rs @@ -13,7 +13,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::history::controller::{ Applied, Prepared, SparkHistoryResources, - validate::{ValidatedSparkHistoryServer, controller_name, operator_name, product_name}, + validate::{CONTROLLER_NAME, OPERATOR_NAME, PRODUCT_NAME, ValidatedSparkHistoryServer}, }; #[derive(Snafu, Debug, EnumDiscriminants)] @@ -49,9 +49,9 @@ impl<'a> Applier<'a> { object_overrides: &'a ObjectOverrides, ) -> Applier<'a> { let cluster_resources = cluster_resources_new( - &product_name(), - &operator_name(), - &controller_name(), + &PRODUCT_NAME, + &OPERATOR_NAME, + &CONTROLLER_NAME, &cluster.name, &cluster.namespace, &cluster.uid, diff --git a/rust/operator-binary/src/history/controller/build/mod.rs b/rust/operator-binary/src/history/controller/build/mod.rs index 762afad2..6b61c974 100644 --- a/rust/operator-binary/src/history/controller/build/mod.rs +++ b/rust/operator-binary/src/history/controller/build/mod.rs @@ -5,22 +5,26 @@ use std::marker::PhantomData; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::meta::ObjectMetaBuilder, - v2::{builder::meta::ownerreference_from_resource, types::operator::RoleGroupName}, + kvp::Labels, + v2::{ + builder::meta::ownerreference_from_resource, + kvp::label, + types::operator::{RoleGroupName, RoleName}, + }, }; -use crate::{ - crd::constants::HISTORY_ROLE_NAME, - history::controller::{ - Prepared, SparkHistoryResources, - build::resource::{ - config_map::{self, build_config_map}, - listener::build_group_listener, - pdb::build_pdb, - rbac::{build_role_binding, build_service_account}, - service::build_rolegroup_metrics_service, - statefulset::{self, build_stateful_set}, - }, - validate::ValidatedSparkHistoryServer, +use crate::history::controller::{ + Prepared, SparkHistoryResources, + build::resource::{ + config_map::{self, build_config_map}, + listener::build_group_listener, + pdb::build_pdb, + rbac::{build_role_binding, build_service_account}, + service::build_rolegroup_metrics_service, + statefulset::{self, build_stateful_set}, + }, + validate::{ + CONTROLLER_NAME, NODE_ROLE_NAME, OPERATOR_NAME, PRODUCT_NAME, ValidatedSparkHistoryServer, }, }; @@ -56,7 +60,7 @@ pub fn build(validated: &ValidatedSparkHistoryServer) -> Result Labels { + label::recommended_labels_for_cluster_resources( + &server.name, + &PRODUCT_NAME, + &server.product_version, + &OPERATOR_NAME, + &CONTROLLER_NAME, + ) +} + +/// Recommended labels for resources shared by all role groups of the single `node` role, like +/// the group Listener. +pub(crate) fn recommended_labels_for_role_resources( + server: &ValidatedSparkHistoryServer, + role_name: &RoleName, +) -> Labels { + label::recommended_labels_for_role_resources( + &server.name, + &PRODUCT_NAME, + &server.product_version, + &OPERATOR_NAME, + &CONTROLLER_NAME, + role_name, + ) +} + +/// Recommended labels for resources of the given role group. +pub(crate) fn recommended_labels_for_role_group_resources( + server: &ValidatedSparkHistoryServer, + role_group_name: &RoleGroupName, +) -> Labels { + label::recommended_labels_for_role_group_resources( + &server.name, + &PRODUCT_NAME, + &server.product_version, + &OPERATOR_NAME, + &CONTROLLER_NAME, + &NODE_ROLE_NAME, + role_group_name, + ) +} + +/// Recommended labels for role group resources which cannot be mutated, like the listener PVC +/// template. The version label is omitted so the labels stay stable across version upgrades. +pub(crate) fn recommended_labels_for_unversioned_role_group_resources( + server: &ValidatedSparkHistoryServer, + role_group_name: &RoleGroupName, +) -> Labels { + label::recommended_labels_for_unversioned_role_group_resources( + &server.name, + &PRODUCT_NAME, + &OPERATOR_NAME, + &CONTROLLER_NAME, + &NODE_ROLE_NAME, + role_group_name, + ) +} + +/// Selector labels matching the pods of a role group. +pub(crate) fn role_group_selector( + server: &ValidatedSparkHistoryServer, + role_group_name: &RoleGroupName, +) -> Labels { + label::role_group_selector( + &server.name, + &PRODUCT_NAME, + &NODE_ROLE_NAME, + role_group_name, + ) +} + #[cfg(test)] pub(crate) mod test_support { use indoc::indoc; diff --git a/rust/operator-binary/src/history/controller/build/resource/config_map.rs b/rust/operator-binary/src/history/controller/build/resource/config_map.rs index cdfbb518..6daef8f7 100644 --- a/rust/operator-binary/src/history/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/history/controller/build/resource/config_map.rs @@ -20,7 +20,10 @@ use crate::{ history::SparkHistoryServerContainer, to_spark_env_sh_string, }, - history::controller::validate::{self, ValidatedHistoryRoleGroup}, + history::controller::{ + build::recommended_labels_for_role_group_resources, + validate::{self, ValidatedHistoryRoleGroup}, + }, product_logging::{self}, }; @@ -83,7 +86,10 @@ pub(crate) fn build_config_map( .namespace(validated.namespace.clone()) .name(&cm_name) .ownerreference(ownerreference_from_resource(validated, None, Some(true))) - .labels(validated.recommended_labels(role_group_name)) + .labels(recommended_labels_for_role_group_resources( + validated, + role_group_name, + )) .build(), ) .add_data(SPARK_DEFAULTS_FILE_NAME, spark_defaults) diff --git a/rust/operator-binary/src/history/controller/build/resource/listener.rs b/rust/operator-binary/src/history/controller/build/resource/listener.rs index 46908807..3db2fc47 100644 --- a/rust/operator-binary/src/history/controller/build/resource/listener.rs +++ b/rust/operator-binary/src/history/controller/build/resource/listener.rs @@ -2,27 +2,27 @@ use std::str::FromStr; use stackable_operator::{ crd::listener, - kube::ResourceExt, - v2::types::{kubernetes::ListenerClassName, operator::RoleGroupName}, + v2::types::{ + kubernetes::{ListenerClassName, ListenerName}, + operator::RoleName, + }, }; use crate::{ crd::{constants::HISTORY_UI_PORT, listener_ext}, - history::controller::validate, + history::controller::{build::recommended_labels_for_role_resources, validate}, }; pub(crate) fn build_group_listener( validated: &validate::ValidatedSparkHistoryServer, - role: &str, + role_name: &RoleName, listener_class: ListenerClassName, ) -> listener::v1alpha1::Listener { - let listener_name = group_listener_name(validated, role); + let listener_name = group_listener_name(validated, role_name); - // Group listeners are shared across role groups, so the role-group label is "none" (preserving - // the previous behaviour). - let recommended_object_labels = validated.recommended_labels( - &RoleGroupName::from_str("none").expect("\"none\" is a valid role group name"), - ); + // Group listeners are shared across all role groups of the role, so they carry role-level + // labels without a role group label. + let recommended_object_labels = recommended_labels_for_role_resources(validated, role_name); let listener_ports = [listener::v1alpha1::ListenerPort { name: "http".to_string(), @@ -32,7 +32,7 @@ pub(crate) fn build_group_listener( listener_ext::build_listener( validated, - &listener_name, + listener_name.as_ref(), &listener_class, recommended_object_labels, &listener_ports, @@ -41,7 +41,16 @@ pub(crate) fn build_group_listener( pub(crate) fn group_listener_name( validated: &validate::ValidatedSparkHistoryServer, - role: &str, -) -> String { - format!("{cluster}-{role}", cluster = validated.name_any()) + role_name: &RoleName, +) -> ListenerName { + ListenerName::from_str(&format!( + "{cluster}-{role}", + cluster = validated.name, + role = role_name + )) + .expect( + "the group listener name is a valid ListenerName, because a ClusterName is at most 40 \ + characters long and a RoleName is a RFC 1123 label of at most 63 characters, so the \ + joined name is a RFC 1123 DNS subdomain within the length limit", + ) } diff --git a/rust/operator-binary/src/history/controller/build/resource/pdb.rs b/rust/operator-binary/src/history/controller/build/resource/pdb.rs index 903d1538..3086c286 100644 --- a/rust/operator-binary/src/history/controller/build/resource/pdb.rs +++ b/rust/operator-binary/src/history/controller/build/resource/pdb.rs @@ -4,7 +4,7 @@ use stackable_operator::{ }; use crate::history::controller::validate::{ - ValidatedSparkHistoryServer, controller_name, operator_name, product_name, + CONTROLLER_NAME, NODE_ROLE_NAME, OPERATOR_NAME, PRODUCT_NAME, ValidatedSparkHistoryServer, }; /// Builds the [`PodDisruptionBudget`] for the history server role, or `None` if PDBs are disabled. @@ -20,10 +20,10 @@ pub fn build_pdb( .unwrap_or(max_unavailable_history_servers()); let pdb = pod_disruption_budget_builder_with_role( validated, - &product_name(), - &ValidatedSparkHistoryServer::role_name(), - &operator_name(), - &controller_name(), + &PRODUCT_NAME, + &NODE_ROLE_NAME, + &OPERATOR_NAME, + &CONTROLLER_NAME, ) .with_max_unavailable(max_unavailable) .build(); diff --git a/rust/operator-binary/src/history/controller/build/resource/rbac.rs b/rust/operator-binary/src/history/controller/build/resource/rbac.rs index c252bfa8..51fc82cc 100644 --- a/rust/operator-binary/src/history/controller/build/resource/rbac.rs +++ b/rust/operator-binary/src/history/controller/build/resource/rbac.rs @@ -1,26 +1,19 @@ //! Builds the RBAC resources (ServiceAccount + RoleBinding) shared by all role groups. -use std::str::FromStr; - use stackable_operator::{ k8s_openapi::api::{core::v1::ServiceAccount, rbac::v1::RoleBinding}, - kvp::Labels, - v2::{ - rbac, - types::operator::{RoleGroupName, RoleName}, - }, + v2::rbac, }; -use crate::history::controller::validate::ValidatedSparkHistoryServer; - -stackable_operator::constant!(NONE_ROLE_NAME: RoleName = "none"); -stackable_operator::constant!(NONE_ROLE_GROUP_NAME: RoleGroupName = "none"); +use crate::history::controller::{ + build::recommended_labels_for_cluster_resources, validate::ValidatedSparkHistoryServer, +}; pub fn build_service_account(server: &ValidatedSparkHistoryServer) -> ServiceAccount { rbac::build_service_account( server, &server.cluster_resource_names(), - rbac_labels(server), + recommended_labels_for_cluster_resources(server), ) } @@ -28,14 +21,10 @@ pub fn build_role_binding(server: &ValidatedSparkHistoryServer) -> RoleBinding { rbac::build_role_binding( server, &server.cluster_resource_names(), - rbac_labels(server), + recommended_labels_for_cluster_resources(server), ) } -fn rbac_labels(server: &ValidatedSparkHistoryServer) -> Labels { - server.recommended_labels_for(&NONE_ROLE_NAME, &NONE_ROLE_GROUP_NAME) -} - #[cfg(test)] mod tests { use serde_json::json; @@ -57,13 +46,10 @@ mod tests { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { - // The RBAC resources are cluster-shared, so role and role group are `none`. "labels": { - "app.kubernetes.io/component": "none", "app.kubernetes.io/instance": "my-history", "app.kubernetes.io/managed-by": "spark.stackable.tech_history", "app.kubernetes.io/name": "spark-history", - "app.kubernetes.io/role-group": "none", "app.kubernetes.io/version": app_version_label("3.5.8"), "stackable.tech/vendor": "Stackable" }, @@ -94,11 +80,9 @@ mod tests { "kind": "RoleBinding", "metadata": { "labels": { - "app.kubernetes.io/component": "none", "app.kubernetes.io/instance": "my-history", "app.kubernetes.io/managed-by": "spark.stackable.tech_history", "app.kubernetes.io/name": "spark-history", - "app.kubernetes.io/role-group": "none", "app.kubernetes.io/version": app_version_label("3.5.8"), "stackable.tech/vendor": "Stackable" }, diff --git a/rust/operator-binary/src/history/controller/build/resource/service.rs b/rust/operator-binary/src/history/controller/build/resource/service.rs index ddab39a9..a421949b 100644 --- a/rust/operator-binary/src/history/controller/build/resource/service.rs +++ b/rust/operator-binary/src/history/controller/build/resource/service.rs @@ -8,7 +8,10 @@ use stackable_operator::{ use crate::{ crd::constants::METRICS_PORT, - history::controller::{build::object_meta, validate::ValidatedSparkHistoryServer}, + history::controller::{ + build::{object_meta, role_group_selector}, + validate::ValidatedSparkHistoryServer, + }, }; /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label @@ -38,7 +41,7 @@ pub fn build_rolegroup_metrics_service( type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), ports: Some(metrics_ports()), - selector: Some(validated.role_group_selector(role_group_name).into()), + selector: Some(role_group_selector(validated, role_group_name).into()), publish_not_ready_addresses: Some(true), ..ServiceSpec::default() }), diff --git a/rust/operator-binary/src/history/controller/build/resource/statefulset.rs b/rust/operator-binary/src/history/controller/build/resource/statefulset.rs index 9c30c626..5cad8c51 100644 --- a/rust/operator-binary/src/history/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/history/controller/build/resource/statefulset.rs @@ -6,6 +6,7 @@ use stackable_operator::{ meta::ObjectMetaBuilder, pod::{PodBuilder, security::PodSecurityContextBuilder, volume::VolumeBuilder}, }, + constant, k8s_openapi::{ DeepMerge, api::apps::v1::{StatefulSet, StatefulSetSpec}, @@ -30,16 +31,21 @@ use stackable_operator::{ // PVC name for the listener volume, required by the v2 listener-volume builder. Its value matches // `LISTENER_VOLUME_NAME` in `crd::constants`. -stackable_operator::constant!(LISTENER_VOLUME_NAME_PVC: PersistentVolumeClaimName = "listener"); +constant!(LISTENER_VOLUME_NAME_PVC: PersistentVolumeClaimName = "listener"); + +// The classpath for extra JAR files of the history server. +constant!(SPARK_DAEMON_CLASSPATH: EnvVarName = "SPARK_DAEMON_CLASSPATH"); +// JVM arguments for the history server. +constant!(SPARK_HISTORY_OPTS: EnvVarName = "SPARK_HISTORY_OPTS"); use crate::{ crd::{ constants::{ - ACCESS_KEY_ID, HISTORY_ROLE_NAME, HISTORY_UI_PORT, LISTENER_VOLUME_DIR, + ACCESS_KEY_ID, CONTAINERDEBUG_LOG_DIRECTORY, HISTORY_UI_PORT, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, MAX_SPARK_LOG_FILES_SIZE, METRICS_PORT, SECRET_ACCESS_KEY, - SPARK_DEFAULTS_FILE_NAME, STACKABLE_TRUST_STORE, VOLUME_MOUNT_NAME_CONFIG, - VOLUME_MOUNT_NAME_LOG, VOLUME_MOUNT_NAME_LOG_CONFIG, VOLUME_MOUNT_PATH_CONFIG, - VOLUME_MOUNT_PATH_LOG, VOLUME_MOUNT_PATH_LOG_CONFIG, + SPARK_DEFAULTS_FILE_NAME, SPARK_NO_DAEMONIZE, STACKABLE_TRUST_STORE, + VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_NAME_LOG, VOLUME_MOUNT_NAME_LOG_CONFIG, + VOLUME_MOUNT_PATH_CONFIG, VOLUME_MOUNT_PATH_LOG, VOLUME_MOUNT_PATH_LOG_CONFIG, }, history::SparkHistoryServerContainer, logdir::ResolvedLogDir, @@ -48,8 +54,12 @@ use crate::{ history::{ config::jvm::construct_history_jvm_args, controller::{ - build::{object_meta, resource::listener::group_listener_name}, - validate::{self, ValidatedHistoryRoleGroup}, + build::{ + object_meta, recommended_labels_for_role_group_resources, + recommended_labels_for_unversioned_role_group_resources, + resource::listener::group_listener_name, role_group_selector, + }, + validate::{self, NODE_ROLE_NAME, ValidatedHistoryRoleGroup}, }, }, }; @@ -101,7 +111,8 @@ pub(crate) fn build_stateful_set( resource_names.role_group_config_map().to_string() }; - let recommended_labels = validated.recommended_labels(role_group_name); + let recommended_labels = + recommended_labels_for_role_group_resources(validated, role_group_name); let pb_metadata = ObjectMetaBuilder::new() .with_labels(recommended_labels.clone()) @@ -155,32 +166,19 @@ pub(crate) fn build_stateful_set( .build(), ); - // Base environment variables, with the already-merged (role + role group) env overrides - // layered on top (overrides win). The base names are static and known to be valid. - let known_env_var_name = |name: &str| { - EnvVarName::from_str(name).expect("the operator-generated env var name is valid") - }; + // Operator-set environment variables first; the already-merged (role + role group) env + // overrides are merged in last so that they override any operator-set environment variable. let merged_env = EnvVarSet::new() - .with_values([ - // Needed by the `containerdebug` running in the background of the history container - // to log it's tracing information to. - ( - known_env_var_name("CONTAINERDEBUG_LOG_DIRECTORY"), - format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug"), - ), - // This env var prevents the history server from detaching itself from the - // start script because this leads to the Pod terminating immediately. - (known_env_var_name("SPARK_NO_DAEMONIZE"), "true".to_owned()), - ( - known_env_var_name("SPARK_DAEMON_CLASSPATH"), - "/stackable/spark/extra-jars/*".to_owned(), - ), - // JVM arguments for the history server. - ( - known_env_var_name("SPARK_HISTORY_OPTS"), - construct_history_jvm_args(&rg.config, log_dir), - ), - ]) + .with_value( + &CONTAINERDEBUG_LOG_DIRECTORY, + format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug"), + ) + .with_value(&SPARK_NO_DAEMONIZE, "true") + .with_value(&SPARK_DAEMON_CLASSPATH, "/stackable/spark/extra-jars/*") + .with_value( + &SPARK_HISTORY_OPTS, + construct_history_jvm_args(&rg.config, log_dir), + ) .merge(rg.config.env_overrides.clone()); let container = @@ -218,13 +216,12 @@ pub(crate) fn build_stateful_set( // so that load balancers can hard-code the target addresses. This will // be the case even when no class is set (and the value defaults to // cluster-internal) as the address should still be consistent. + // + // PVC templates cannot be modified once they are deployed, so the version label is omitted + // from their labels to keep them stable across version upgrades. let volume_claim_templates = Some(vec![listener_operator_volume_source_builder_build_pvc( - &ListenerReference::Listener( - group_listener_name(validated, HISTORY_ROLE_NAME) - .parse() - .expect("the group listener name is a valid ListenerName"), - ), - &recommended_labels, + &ListenerReference::Listener(group_listener_name(validated, &NODE_ROLE_NAME)), + &recommended_labels_for_unversioned_role_group_resources(validated, role_group_name), &LISTENER_VOLUME_NAME_PVC, )]); @@ -259,7 +256,7 @@ pub(crate) fn build_stateful_set( volume_claim_templates, replicas: rg.config.replicas.map(i32::from), selector: LabelSelector { - match_labels: Some(validated.role_group_selector(role_group_name).into()), + match_labels: Some(role_group_selector(validated, role_group_name).into()), ..LabelSelector::default() }, ..StatefulSetSpec::default() @@ -290,3 +287,72 @@ fn command_args(logdir: &ResolvedLogDir) -> Vec { ]); vec![command.join("\n")] } + +#[cfg(test)] +mod tests { + use stackable_operator::k8s_openapi::api::core::v1::EnvVar; + + use super::*; + use crate::history::controller::build::test_support::minimal_validated_cluster; + + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *LISTENER_VOLUME_NAME_PVC; + let _ = *SPARK_DAEMON_CLASSPATH; + let _ = *SPARK_HISTORY_OPTS; + } + + /// `envOverrides` must be applied after all operator-set environment variables, so a user + /// override replaces the operator-set value instead of duplicating it or being ignored. + #[test] + fn env_overrides_override_operator_set_env_vars() { + let mut validated = minimal_validated_cluster(); + let role_group_name: RoleGroupName = "default".parse().expect("valid role group name"); + + validated + .role_groups + .get_mut(&role_group_name) + .expect("the default role group exists") + .config + .env_overrides = EnvVarSet::new().with_value( + &EnvVarName::from_str("SPARK_NO_DAEMONIZE").expect("valid env var name"), + "overridden", + ); + + let rg = validated + .role_groups + .get(&role_group_name) + .expect("the default role group exists"); + let stateful_set = build_stateful_set( + &validated, + &role_group_name, + rg, + &validated.cluster_config.log_dir, + ) + .expect("the StatefulSet can be built"); + + let env: Vec = stateful_set + .spec + .expect("the StatefulSet has a spec") + .template + .spec + .expect("the StatefulSet has a pod spec") + .containers + .iter() + .find(|container| container.name == "spark-history") + .expect("the spark-history container exists") + .env + .clone() + .expect("the spark-history container has env vars"); + + let matching: Vec<&EnvVar> = env + .iter() + .filter(|env_var| env_var.name == "SPARK_NO_DAEMONIZE") + .collect(); + + // The override must replace the operator-set value, not duplicate it. + assert_eq!(matching.len(), 1); + assert_eq!(matching[0].value.as_deref(), Some("overridden")); + } +} diff --git a/rust/operator-binary/src/history/controller/validate.rs b/rust/operator-binary/src/history/controller/validate.rs index b0aaaab9..3b5b39d1 100644 --- a/rust/operator-binary/src/history/controller/validate.rs +++ b/rust/operator-binary/src/history/controller/validate.rs @@ -13,15 +13,13 @@ use stackable_operator::{ product_image_selection::{self, ResolvedProductImage}, }, config::fragment, + constant, k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, kube::Resource, - kvp::Labels, product_logging::spec::Logging, v2::{ HasName, HasUid, NameIsValidLabelValue, - builder::pod::container::{EnvVarName, EnvVarSet}, controller_utils::{get_cluster_name, get_namespace, get_uid}, - kvp::label::{recommended_labels, role_group_selector}, product_logging::framework::{ VectorContainerLogConfig, validate_logging_configuration_for_container, }, @@ -41,7 +39,7 @@ use crate::{ crd::{ constants::{ CONTAINER_IMAGE_BASE_NAME, HISTORY_APP_NAME, HISTORY_CONTROLLER_NAME, - HISTORY_ROLE_NAME, OPERATOR_NAME, + HISTORY_ROLE_NAME, SPARK_OPERATOR_NAME, }, history::{HistoryConfig, HistoryConfigFragment, SparkHistoryServerContainer, v1alpha1}, logdir::ResolvedLogDir, @@ -89,12 +87,6 @@ pub enum Error { role_group: String, }, - #[snafu(display("invalid environment variable override name in role group {role_group}"))] - ParseEnvVarName { - source: stackable_operator::v2::macros::attributed_string_type::Error, - role_group: String, - }, - #[snafu(display("failed to validate the logging configuration"))] ValidateLoggingConfig { source: stackable_operator::v2::product_logging::framework::Error, @@ -138,6 +130,15 @@ fn validate_logging( type Result = std::result::Result; +// The product name (`spark-history`) as a type-safe label value. +constant!(pub(crate) PRODUCT_NAME: ProductName = HISTORY_APP_NAME); +// The operator name as a type-safe label value. +constant!(pub(crate) OPERATOR_NAME: OperatorName = SPARK_OPERATOR_NAME); +// The controller name as a type-safe label value. +constant!(pub(crate) CONTROLLER_NAME: ControllerName = HISTORY_CONTROLLER_NAME); +// The single history server role name (`node`). +constant!(pub(crate) NODE_ROLE_NAME: RoleName = HISTORY_ROLE_NAME); + /// A validated, merged history server role-group config. pub type HistoryRoleGroupConfig = RoleGroupConfig; @@ -193,17 +194,12 @@ pub struct ValidatedRoleConfig { } impl ValidatedSparkHistoryServer { - /// The single history server role name (`node`). - pub fn role_name() -> RoleName { - RoleName::from_str(HISTORY_ROLE_NAME).expect("HISTORY_ROLE_NAME is a valid role name") - } - /// Type-safe names for the per-cluster RBAC resources: the ServiceAccount, /// its (namespaced) RoleBinding, and the operator-deployed ClusterRole it binds. pub fn cluster_resource_names(&self) -> role_utils::ResourceNames { role_utils::ResourceNames { cluster_name: self.name.clone(), - product_name: product_name(), + product_name: PRODUCT_NAME.clone(), } } @@ -211,63 +207,10 @@ impl ValidatedSparkHistoryServer { pub fn role_group_resource_names(&self, role_group_name: &RoleGroupName) -> ResourceNames { ResourceNames { cluster_name: self.name.clone(), - role_name: Self::role_name(), + role_name: NODE_ROLE_NAME.clone(), role_group_name: role_group_name.clone(), } } - - /// Recommended labels for a resource of the given role. - pub fn recommended_labels(&self, role_group_name: &RoleGroupName) -> Labels { - self.recommended_labels_for(&Self::role_name(), role_group_name) - } - - /// Recommended labels for a resource that is not tied to a concrete role - /// (e.g. the cluster-shared RBAC resources), using a free-form role/role-group label value. - pub fn recommended_labels_for( - &self, - role_name: &RoleName, - role_group_name: &RoleGroupName, - ) -> Labels { - self.recommended_labels_with(&self.product_version, role_name, role_group_name) - } - - fn recommended_labels_with( - &self, - product_version: &ProductVersion, - role_name: &RoleName, - role_group_name: &RoleGroupName, - ) -> Labels { - recommended_labels( - self, - &product_name(), - product_version, - &operator_name(), - &controller_name(), - role_name, - role_group_name, - ) - } - - /// Selector labels matching the pods of a role group. - pub fn role_group_selector(&self, role_group_name: &RoleGroupName) -> Labels { - role_group_selector(self, &product_name(), &Self::role_name(), role_group_name) - } -} - -/// The product name (`spark-history`) as a type-safe label value. -pub fn product_name() -> ProductName { - ProductName::from_str(HISTORY_APP_NAME).expect("HISTORY_APP_NAME is a valid product name") -} - -/// The operator name as a type-safe label value. -pub fn operator_name() -> OperatorName { - OperatorName::from_str(OPERATOR_NAME).expect("the operator name is a valid label value") -} - -/// The controller name as a type-safe label value. -pub fn controller_name() -> ControllerName { - ControllerName::from_str(HISTORY_CONTROLLER_NAME) - .expect("the controller name is a valid label value") } impl NameIsValidLabelValue for ValidatedSparkHistoryServer { @@ -375,16 +318,6 @@ pub fn validate( role_group: rg_name.clone(), })?; - let mut env_overrides = EnvVarSet::new(); - for (env_var_name, env_var_value) in merged.config.env_overrides { - env_overrides = env_overrides.with_value( - &EnvVarName::from_str(&env_var_name).with_context(|_| ParseEnvVarNameSnafu { - role_group: rg_name.clone(), - })?, - env_var_value, - ); - } - let logging = validate_logging( &merged.config.config.logging, &vector_aggregator_config_map_name, @@ -394,7 +327,9 @@ pub fn validate( replicas: Some(merged.replicas.unwrap_or(1)), config: merged.config.config, config_overrides: merged.config.config_overrides, - env_overrides, + // The env override names were already validated when the custom resource was + // deserialized. + env_overrides: merged.config.env_overrides.into(), // The history server does not use CLI overrides; the field is carried (and merged // upstream) but unused. cli_overrides: merged.config.cli_overrides, @@ -443,6 +378,15 @@ mod tests { test_support::app_version_label, }; + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *CONTROLLER_NAME; + let _ = *NODE_ROLE_NAME; + let _ = *OPERATOR_NAME; + let _ = *PRODUCT_NAME; + } + /// Locks every value the validate step itself derives from the minimal fixture — so a /// validation regression fails here, with a validate-shaped message, instead of surfacing as /// a confusing build-test failure downstream. diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 31499410..5abdbe14 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -38,8 +38,8 @@ use crate::{ crd::{ SparkApplication, constants::{ - HISTORY_FULL_CONTROLLER_NAME, OPERATOR_NAME, POD_DRIVER_FULL_CONTROLLER_NAME, - SPARK_CONTROLLER_NAME, SPARK_FULL_CONTROLLER_NAME, + HISTORY_FULL_CONTROLLER_NAME, POD_DRIVER_FULL_CONTROLLER_NAME, SPARK_CONTROLLER_NAME, + SPARK_FULL_CONTROLLER_NAME, SPARK_OPERATOR_NAME, }, history::SparkHistoryServer, template_spec::{SparkApplicationTemplate, SparkApplicationTemplateVersion}, @@ -121,7 +121,7 @@ async fn main() -> anyhow::Result<()> { .map(anyhow::Ok); let client = stackable_operator::client::initialize_operator( - Some(OPERATOR_NAME.to_string()), + Some(SPARK_OPERATOR_NAME.to_string()), &common.cluster_info, ) .await?; @@ -194,7 +194,7 @@ async fn main() -> anyhow::Result<()> { let pod_driver_controller = Controller::new( watch_namespace.get_api::>(&client), watcher::Config::default() - .labels(&format!("app.kubernetes.io/managed-by={OPERATOR_NAME}_{SPARK_CONTROLLER_NAME},spark-role=driver")), + .labels(&format!("app.kubernetes.io/managed-by={SPARK_OPERATOR_NAME}_{SPARK_CONTROLLER_NAME},spark-role=driver")), ) .owns( watch_namespace.get_api::>(&client), diff --git a/rust/operator-binary/src/spark_k8s_controller/build/mod.rs b/rust/operator-binary/src/spark_k8s_controller/build/mod.rs index d1d05dee..68ceb2dc 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/mod.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/mod.rs @@ -1,24 +1,42 @@ pub mod pod; pub mod resource; -use std::marker::PhantomData; +use std::{marker::PhantomData, str::FromStr}; use resource::{config_map, job}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::meta::ObjectMetaBuilder, v2::builder::meta::ownerreference_from_resource, + builder::meta::ObjectMetaBuilder, + constant, + kvp::Labels, + v2::{builder::meta::ownerreference_from_resource, kvp::label, types::operator::RoleName}, }; use crate::{ crd::roles::SparkApplicationRole, - spark_k8s_controller::{Prepared, SparkResources, validate::ValidatedSparkApplication}, + spark_k8s_controller::{ + Prepared, SparkResources, + validate::{CONTROLLER_NAME, OPERATOR_NAME, PRODUCT_NAME, ValidatedSparkApplication}, + }, }; +// The `app.kubernetes.io/component` label values of the resources built by this controller. A +// SparkApplication has no Stackable roles, so these are free-form component names rather than +// role names. +constant!(pub(crate) SPARK_COMPONENT_NAME: RoleName = "spark"); +constant!(pub(crate) SPARK_JOB_COMPONENT_NAME: RoleName = "spark-job"); +constant!(pub(crate) SPARK_JOB_TEMPLATE_COMPONENT_NAME: RoleName = "spark-job-template"); +constant!(pub(crate) POD_TEMPLATES_COMPONENT_NAME: RoleName = "pod-templates"); +constant!(pub(crate) SPARK_SUBMIT_COMPONENT_NAME: RoleName = "spark-submit"); + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to resolve and merge config"))] FailedToResolveConfig { source: crate::crd::Error }, + #[snafu(display("failed to build the environment variables"))] + BuildEnvVars { source: crate::crd::Error }, + #[snafu(display("failed to build stark-submit command"))] BuildCommand { source: crate::crd::Error }, @@ -44,7 +62,9 @@ pub fn build(validated: &ValidatedSparkApplication) -> Result Result, - role: &str, + component_name: &RoleName, ) -> ObjectMetaBuilder { let mut builder = ObjectMetaBuilder::new(); builder .namespace(validated.namespace.clone()) .name(name) .ownerreference(ownerreference_from_resource(validated, None, Some(true))) - .with_labels(validated.recommended_labels(role)); + .with_labels(recommended_labels_for_component_resources( + validated, + component_name, + )); builder } + +/// Recommended labels for resources shared by the whole SparkApplication, like the RBAC +/// resources. +pub(crate) fn recommended_labels_for_cluster_resources( + validated: &ValidatedSparkApplication, +) -> Labels { + label::recommended_labels_for_cluster_resources( + &validated.name, + &PRODUCT_NAME, + &validated.product_version, + &OPERATOR_NAME, + &CONTROLLER_NAME, + ) +} + +/// Recommended labels for resources fulfilling the given component within the SparkApplication. +/// +/// A SparkApplication has no Stackable roles or role groups, so the `app.kubernetes.io/component` +/// label carries a free-form component name and there is no role group label. +pub(crate) fn recommended_labels_for_component_resources( + validated: &ValidatedSparkApplication, + component_name: &RoleName, +) -> Labels { + label::recommended_labels_for_role_resources( + &validated.name, + &PRODUCT_NAME, + &validated.product_version, + &OPERATOR_NAME, + &CONTROLLER_NAME, + component_name, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *POD_TEMPLATES_COMPONENT_NAME; + let _ = *SPARK_COMPONENT_NAME; + let _ = *SPARK_JOB_COMPONENT_NAME; + let _ = *SPARK_JOB_TEMPLATE_COMPONENT_NAME; + let _ = *SPARK_SUBMIT_COMPONENT_NAME; + } +} diff --git a/rust/operator-binary/src/spark_k8s_controller/build/pod.rs b/rust/operator-binary/src/spark_k8s_controller/build/pod.rs index 1c74f632..e7dccd26 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/pod.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/pod.rs @@ -8,11 +8,10 @@ use stackable_operator::{ PodBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, }, }, + constant, k8s_openapi::{ DeepMerge, - api::core::v1::{ - Container, EnvVar, PodSecurityContext, PodTemplateSpec, ServiceAccount, Volume, - }, + api::core::v1::{Container, PodSecurityContext, PodTemplateSpec, ServiceAccount, Volume}, }, kube::ResourceExt, product_logging::{ @@ -22,7 +21,7 @@ use stackable_operator::{ v2::{ builder::{ meta::ownerreference_from_resource, - pod::container::{EnvVarSet, new_container_builder}, + pod::container::{EnvVarName, EnvVarSet, new_container_builder}, service::{Scraping, prometheus_labels}, }, product_logging::framework::{ @@ -30,7 +29,7 @@ use stackable_operator::{ vector_container, }, role_group_utils::ResourceNames, - types::operator::{RoleGroupName, RoleName}, + types::operator::RoleGroupName, }, }; @@ -40,9 +39,17 @@ use crate::{ roles::{RoleConfig, SparkApplicationRole, SparkContainer}, tlscerts, }, - spark_k8s_controller::validate, + spark_k8s_controller::{ + build::{SPARK_COMPONENT_NAME, recommended_labels_for_component_resources}, + validate, + }, }; +// `_STACKABLE_POST_HOOK` is evaluated by the entrypoint script (run-spark.sh) in the Spark images +// after the actual JVM process has finished; the operator uses it to give Vector time to gather +// the logs and to shut it down afterwards. +constant!(STACKABLE_POST_HOOK: EnvVarName = "_STACKABLE_POST_HOOK"); + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to add needed volumeMount"))] @@ -223,7 +230,7 @@ pub(crate) fn pod_template( role: SparkApplicationRole, config: &RoleConfig, volumes: &[Volume], - env: &[EnvVar], + env: &EnvVarSet, service_account: &ServiceAccount, ) -> Result { let spark_application = &validated.spark_application; @@ -232,17 +239,11 @@ pub(crate) fn pod_template( let spark_image = &validated.resolved_product_image; let container_name = SparkContainer::Spark.to_string(); let mut cb = new_container_builder(&SparkContainer::Spark.to_container_name()); - let merged_env = spark_application.merged_env(role.clone(), env); - - cb.add_volume_mounts(config.volume_mounts(spark_application, s3conn, logdir)) - .context(AddVolumeMountSnafu)? - .add_env_vars(merged_env) - .resources(config.resources.clone().into()) - .image_from_product_image(spark_image); + let mut env = env.clone(); if config.logging.enable_vector_agent { - cb.add_env_var( - "_STACKABLE_POST_HOOK", + env = env.with_value( + &STACKABLE_POST_HOOK, [ // Wait for Vector to gather the logs. "sleep 10", @@ -251,13 +252,25 @@ pub(crate) fn pod_template( .join("; "), ); } + // The env overrides are merged in last so that they override any operator-set environment + // variable. + let merged_env = spark_application.merged_env(role.clone(), env); + + cb.add_volume_mounts(config.volume_mounts(spark_application, s3conn, logdir)) + .context(AddVolumeMountSnafu)? + .add_env_vars(merged_env) + .resources(config.resources.clone().into()) + .image_from_product_image(spark_image); let mut omb = ObjectMetaBuilder::new(); omb.name(&container_name) // this reference is not pointing to a controller but only provides a UID that can used to clean up resources // cleanly (specifically driver pods and related config maps) when the spark application is deleted. .ownerreference(ownerreference_from_resource(validated, None, None)) - .with_labels(validated.recommended_labels(&container_name)); + .with_labels(recommended_labels_for_component_resources( + validated, + &SPARK_COMPONENT_NAME, + )); // Only the driver pod should be scraped by Prometheus // because the executor metrics are also available via /metrics/executors/prometheus/ @@ -315,8 +328,7 @@ pub(crate) fn pod_template( // is a placeholder; the role name reflects the pod's Spark role (driver/executor). let vector_resource_names = ResourceNames { cluster_name: validated.name.clone(), - role_name: RoleName::from_str(&role.to_string()) - .expect("a SparkApplicationRole serializes to a valid role name"), + role_name: role.role_name(), role_group_name: RoleGroupName::from_str("default") .expect("\"default\" is a valid role group name"), }; @@ -343,3 +355,111 @@ pub(crate) fn security_context() -> PodSecurityContext { .fs_group(1000) .build() } + +#[cfg(test)] +mod tests { + use indoc::indoc; + use stackable_operator::{ + cli::OperatorEnvironmentOptions, + k8s_openapi::{api::core::v1::EnvVar, apimachinery::pkg::apis::meta::v1::ObjectMeta}, + }; + + use super::*; + use crate::{ + crd::v1alpha1, + spark_k8s_controller::{dereference::DereferencedSparkApplication, validate::validate}, + }; + + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *STACKABLE_POST_HOOK; + } + + /// `envOverrides` must be applied after all operator-set environment variables, so a user + /// override replaces the operator-set value instead of duplicating it or being ignored. + #[test] + fn env_overrides_override_operator_set_env_vars() { + let yaml = indoc! {r#" + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-example + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + driver: + envOverrides: + CONTAINERDEBUG_LOG_DIRECTORY: /custom/log/dir + "#}; + let deserializer = serde_yaml::Deserializer::from_str(yaml); + let spark_application: v1alpha1::SparkApplication = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer) + .expect("invalid test SparkApplication YAML"); + + let validated = validate( + DereferencedSparkApplication { + spark_application, + resolved_template_refs: Vec::new(), + s3_connection: None, + log_dir: None, + }, + &OperatorEnvironmentOptions { + operator_namespace: "stackable-operators".to_string(), + operator_service_name: "spark-k8s-operator".to_string(), + image_repository: "oci.example.org/sdp".to_string(), + }, + ) + .expect("the fixture validates"); + + let driver_config = validated + .spark_application + .driver_config() + .expect("the driver config resolves"); + let env = validated + .spark_application + .env(&None, &None) + .expect("the base environment can be built"); + let service_account = ServiceAccount { + metadata: ObjectMeta { + name: Some("spark-example".to_string()), + ..ObjectMeta::default() + }, + ..ServiceAccount::default() + }; + + let template = pod_template( + &validated, + SparkApplicationRole::Driver, + &driver_config, + &[], + &env, + &service_account, + ) + .expect("the driver pod template can be built"); + + let env: Vec = template + .spec + .expect("the pod template has a spec") + .containers + .iter() + .find(|container| container.name == "spark") + .expect("the spark container exists") + .env + .clone() + .expect("the spark container has env vars"); + + let matching: Vec<&EnvVar> = env + .iter() + .filter(|env_var| env_var.name == "CONTAINERDEBUG_LOG_DIRECTORY") + .collect(); + + // The override must replace the operator-set value, not duplicate it. + assert_eq!(matching.len(), 1); + assert_eq!(matching[0].value.as_deref(), Some("/custom/log/dir")); + } +} diff --git a/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs b/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs index 21c208b3..f38b1c33 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/resource/config_map.rs @@ -1,7 +1,7 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, pod::volume::VolumeBuilder}, - k8s_openapi::api::core::v1::{ConfigMap, EnvVar, ServiceAccount}, + k8s_openapi::api::core::v1::{ConfigMap, ServiceAccount}, product_logging::{ framework::VECTOR_CONFIG_FILE, spec::{ @@ -9,7 +9,7 @@ use stackable_operator::{ CustomContainerLogConfig, }, }, - v2::config_file_writer::to_java_properties_string, + v2::{builder::pod::container::EnvVarSet, config_file_writer::to_java_properties_string}, }; use crate::{ @@ -21,7 +21,7 @@ use crate::{ product_logging::{self}, spark_k8s_controller::{ build::{ - object_meta, + POD_TEMPLATES_COMPONENT_NAME, SPARK_SUBMIT_COMPONENT_NAME, object_meta, pod::{self, pod_template}, }, validate, @@ -36,7 +36,7 @@ pub enum Error { #[snafu(display("pod template serialization"))] PodTemplateSerde { source: serde_yaml::Error }, - #[snafu(display("failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", role))] + #[snafu(display("failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {role}", role = role.as_ref()))] JvmSecurityProperties { source: stackable_operator::v2::config_file_writer::PropertiesWriterError, role: SparkApplicationRole, @@ -61,7 +61,7 @@ pub(crate) fn pod_template_config_map( role: SparkApplicationRole, merged_config: &RoleConfig, config_overrides: &v1alpha1::ConfigOverrides, - env: &[EnvVar], + env: &EnvVarSet, service_account: &ServiceAccount, ) -> Result { let spark_application = &validated.spark_application; @@ -111,7 +111,7 @@ pub(crate) fn pod_template_config_map( let mut cm_builder = ConfigMapBuilder::new(); cm_builder - .metadata(object_meta(validated, &cm_name, "pod-templates").build()) + .metadata(object_meta(validated, &cm_name, &POD_TEMPLATES_COMPONENT_NAME).build()) .add_data( POD_TEMPLATE_FILE, serde_yaml::to_string(&template).context(PodTemplateSerdeSnafu)?, @@ -155,7 +155,7 @@ pub(crate) fn submit_job_config_map( let mut cm_builder = ConfigMapBuilder::new(); - cm_builder.metadata(object_meta(validated, &cm_name, "spark-submit").build()); + cm_builder.metadata(object_meta(validated, &cm_name, &SPARK_SUBMIT_COMPONENT_NAME).build()); cm_builder.add_data( SPARK_ENV_SH_FILE_NAME, diff --git a/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs b/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs index 6ff15f79..39820b20 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/resource/job.rs @@ -1,14 +1,17 @@ +use std::str::FromStr; + use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{meta::ObjectMetaBuilder, pod::volume::VolumeBuilder}, + constant, k8s_openapi::{ DeepMerge, api::{ batch::v1::{Job, JobSpec}, - core::v1::{Affinity, EnvVar, PodSpec, PodTemplateSpec, ServiceAccount}, + core::v1::{Affinity, PodSpec, PodTemplateSpec, ServiceAccount}, }, }, - v2::builder::pod::container::new_container_builder, + v2::builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, }; use crate::{ @@ -18,11 +21,19 @@ use crate::{ tlscerts, }, spark_k8s_controller::{ - build::{object_meta, pod::security_context}, + build::{ + SPARK_JOB_COMPONENT_NAME, SPARK_JOB_TEMPLATE_COMPONENT_NAME, object_meta, + pod::security_context, recommended_labels_for_component_resources, + }, validate, }, }; +// JVM settings of the spark-submit job. +constant!(SPARK_SUBMIT_OPTS: EnvVarName = "SPARK_SUBMIT_OPTS"); +// The Spark configuration directory of the spark-submit job. +constant!(SPARK_CONF_DIR: EnvVarName = "SPARK_CONF_DIR"); + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to add needed volumeMount"))] @@ -42,7 +53,7 @@ type Result = std::result::Result; pub(crate) fn spark_job( validated: &validate::ValidatedSparkApplication, serviceaccount: &ServiceAccount, - env: &[EnvVar], + env: &EnvVarSet, job_commands: &[String], job_config: &SubmitConfig, ) -> Result { @@ -52,8 +63,6 @@ pub(crate) fn spark_job( let logdir = &validated.cluster_config.log_dir; let mut cb = new_container_builder(&SparkContainer::SparkSubmit.to_container_name()); - let merged_env = spark_application.merged_env(SparkApplicationRole::Submit, env); - // The SPARK_SUBMIT_OPTS env var is used to configure the JVM settings of the spark-submit job. // Here we need to point the JVM to our logging configuration and if S3 is used for data or Spark History, // we also need to tell the JVM where the trust store is located. @@ -69,6 +78,17 @@ pub(crate) fn spark_job( "-Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD}" )); } + + // The env overrides are merged in last so that they override any operator-set environment + // variable. + let merged_env = spark_application.merged_env( + SparkApplicationRole::Submit, + env.clone() + .with_value(&SPARK_SUBMIT_OPTS, spark_submit_opts_env.join(" ")) + // TODO: move this to the image + .with_value(&SPARK_CONF_DIR, "/stackable/spark/conf"), + ); + cb.image_from_product_image(spark_image) .command(vec![ "/bin/bash".to_string(), @@ -81,10 +101,7 @@ pub(crate) fn spark_job( .resources(job_config.resources.clone().into()) .add_volume_mounts(spark_application.spark_job_volume_mounts(s3conn, logdir)) .context(AddVolumeMountSnafu)? - .add_env_vars(merged_env) - .add_env_var("SPARK_SUBMIT_OPTS", spark_submit_opts_env.join(" ")) - // TODO: move this to the image - .add_env_var("SPARK_CONF_DIR", "/stackable/spark/conf"); + .add_env_vars(merged_env); let mut volumes = vec![ VolumeBuilder::new(VOLUME_MOUNT_NAME_CONFIG.as_ref()) @@ -116,7 +133,10 @@ pub(crate) fn spark_job( metadata: Some( ObjectMetaBuilder::new() .name("spark-submit") - .with_labels(validated.recommended_labels("spark-job-template")) + .with_labels(recommended_labels_for_component_resources( + validated, + &SPARK_JOB_TEMPLATE_COMPONENT_NAME, + )) .build(), ), spec: Some(PodSpec { @@ -142,7 +162,12 @@ pub(crate) fn spark_job( } let job = Job { - metadata: object_meta(validated, validated.name.to_string(), "spark-job").build(), + metadata: object_meta( + validated, + validated.name.to_string(), + &SPARK_JOB_COMPONENT_NAME, + ) + .build(), spec: Some(JobSpec { template: pod, ttl_seconds_after_finished: Some(600), @@ -154,3 +179,114 @@ pub(crate) fn spark_job( Ok(job) } + +#[cfg(test)] +mod tests { + use indoc::indoc; + use stackable_operator::{ + cli::OperatorEnvironmentOptions, + k8s_openapi::{api::core::v1::EnvVar, apimachinery::pkg::apis::meta::v1::ObjectMeta}, + }; + + use super::*; + use crate::{ + crd::v1alpha1, + spark_k8s_controller::{dereference::DereferencedSparkApplication, validate::validate}, + }; + + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *SPARK_CONF_DIR; + let _ = *SPARK_SUBMIT_OPTS; + } + + /// `envOverrides` must be applied after all operator-set environment variables, so a user + /// override replaces the operator-set value instead of duplicating it or being ignored. + #[test] + fn env_overrides_override_operator_set_env_vars() { + let yaml = indoc! {r#" + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: spark-example + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + mode: cluster + mainApplicationFile: test.py + sparkImage: + productVersion: 1.2.3 + job: + envOverrides: + SPARK_CONF_DIR: /custom/conf + "#}; + let deserializer = serde_yaml::Deserializer::from_str(yaml); + let spark_application: v1alpha1::SparkApplication = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer) + .expect("invalid test SparkApplication YAML"); + + let validated = validate( + DereferencedSparkApplication { + spark_application, + resolved_template_refs: Vec::new(), + s3_connection: None, + log_dir: None, + }, + &OperatorEnvironmentOptions { + operator_namespace: "stackable-operators".to_string(), + operator_service_name: "spark-k8s-operator".to_string(), + image_repository: "oci.example.org/sdp".to_string(), + }, + ) + .expect("the fixture validates"); + + let submit_config = validated + .spark_application + .submit_config() + .expect("the submit config resolves"); + let env = validated + .spark_application + .env(&None, &None) + .expect("the base environment can be built"); + let service_account = ServiceAccount { + metadata: ObjectMeta { + name: Some("spark-example".to_string()), + ..ObjectMeta::default() + }, + ..ServiceAccount::default() + }; + + let job = spark_job( + &validated, + &service_account, + &env, + &["echo test".to_string()], + &submit_config, + ) + .expect("the spark-submit Job can be built"); + + let env: Vec = job + .spec + .expect("the Job has a spec") + .template + .spec + .expect("the Job has a pod spec") + .containers + .iter() + .find(|container| container.name == "spark-submit") + .expect("the spark-submit container exists") + .env + .clone() + .expect("the spark-submit container has env vars"); + + let matching: Vec<&EnvVar> = env + .iter() + .filter(|env_var| env_var.name == "SPARK_CONF_DIR") + .collect(); + + // The override must replace the operator-set value, not duplicate it. + assert_eq!(matching.len(), 1); + assert_eq!(matching[0].value.as_deref(), Some("/custom/conf")); + } +} diff --git a/rust/operator-binary/src/spark_k8s_controller/build/resource/serviceaccount.rs b/rust/operator-binary/src/spark_k8s_controller/build/resource/serviceaccount.rs index 2f88222c..7a4274dc 100644 --- a/rust/operator-binary/src/spark_k8s_controller/build/resource/serviceaccount.rs +++ b/rust/operator-binary/src/spark_k8s_controller/build/resource/serviceaccount.rs @@ -1,14 +1,18 @@ -use stackable_operator::k8s_openapi::{ - Resource, - api::{ - core::v1::ServiceAccount, - rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, +use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + k8s_openapi::{ + Resource, + api::{ + core::v1::ServiceAccount, + rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, + }, }, + v2::builder::meta::ownerreference_from_resource, }; use crate::{ crd::constants::*, - spark_k8s_controller::{build::object_meta, validate}, + spark_k8s_controller::{build::recommended_labels_for_cluster_resources, validate}, }; /// For a given SparkApplication, we create a ServiceAccount with a RoleBinding to the ClusterRole @@ -20,12 +24,12 @@ pub(crate) fn build_spark_role_serviceaccount( ) -> (ServiceAccount, RoleBinding) { let sa_name = validated.name.to_string(); let sa = ServiceAccount { - metadata: object_meta(validated, &sa_name, "service-account").build(), + metadata: cluster_resource_object_meta(validated, &sa_name), ..ServiceAccount::default() }; let binding_name = &sa_name; let binding = RoleBinding { - metadata: object_meta(validated, binding_name, "role-binding").build(), + metadata: cluster_resource_object_meta(validated, binding_name), role_ref: RoleRef { api_group: Some(ClusterRole::GROUP.to_string()), kind: ClusterRole::KIND.to_string(), @@ -40,3 +44,19 @@ pub(crate) fn build_spark_role_serviceaccount( }; (sa, binding) } + +/// Object metadata for a cluster-shared resource named `name`, owned by the SparkApplication. +/// +/// Unlike [`crate::spark_k8s_controller::build::object_meta`], the labels carry no +/// `app.kubernetes.io/component` label because the RBAC resources are not tied to a component. +fn cluster_resource_object_meta( + validated: &validate::ValidatedSparkApplication, + name: impl Into, +) -> stackable_operator::k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + ObjectMetaBuilder::new() + .namespace(validated.namespace.clone()) + .name(name) + .ownerreference(ownerreference_from_resource(validated, None, Some(true))) + .with_labels(recommended_labels_for_cluster_resources(validated)) + .build() +} diff --git a/rust/operator-binary/src/spark_k8s_controller/validate.rs b/rust/operator-binary/src/spark_k8s_controller/validate.rs index b48606eb..69623ed5 100644 --- a/rust/operator-binary/src/spark_k8s_controller/validate.rs +++ b/rust/operator-binary/src/spark_k8s_controller/validate.rs @@ -12,27 +12,25 @@ use stackable_operator::{ product_image_selection::{self, ResolvedProductImage}, tls_verification::TlsVerification, }, + constant, crd::s3, k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, kube::Resource, - kvp::Labels, v2::{ HasName, HasUid, NameIsValidLabelValue, controller_utils::{get_cluster_name, get_namespace, get_uid}, - kvp::label::recommended_labels, types::{ kubernetes::{NamespaceName, Uid}, - operator::{ - ClusterName, ControllerName, OperatorName, ProductName, ProductVersion, - RoleGroupName, RoleName, - }, + operator::{ClusterName, ControllerName, OperatorName, ProductName, ProductVersion}, }, }, }; use crate::{ crd::{ - constants::{APP_NAME, CONTAINER_IMAGE_BASE_NAME, OPERATOR_NAME, SPARK_CONTROLLER_NAME}, + constants::{ + APP_NAME, CONTAINER_IMAGE_BASE_NAME, SPARK_CONTROLLER_NAME, SPARK_OPERATOR_NAME, + }, logdir::ResolvedLogDir, v1alpha1, }, @@ -67,6 +65,13 @@ pub enum Error { type Result = std::result::Result; +// The product name (`spark-k8s`) as a type-safe label value. +constant!(pub(crate) PRODUCT_NAME: ProductName = APP_NAME); +// The operator name as a type-safe label value. +constant!(pub(crate) OPERATOR_NAME: OperatorName = SPARK_OPERATOR_NAME); +// The controller name as a type-safe label value. +constant!(pub(crate) CONTROLLER_NAME: ControllerName = SPARK_CONTROLLER_NAME); + /// Inputs the rest of `reconcile` needs after dereferencing. pub struct ValidatedSparkApplication { /// Metadata mirroring the source [`v1alpha1::SparkApplication`] (name, namespace and UID), so @@ -75,6 +80,10 @@ pub struct ValidatedSparkApplication { pub name: ClusterName, pub namespace: NamespaceName, pub uid: Uid, + /// The product version as a valid label value, used for the recommended + /// `app.kubernetes.io/version` label. Derived from the resolved image's app version label + /// value. + pub product_version: ProductVersion, /// The full source spec. /// /// Unlike the other operators' validated types, a `SparkApplication` cannot be reduced to @@ -143,49 +152,6 @@ impl Resource for ValidatedSparkApplication { } } -impl ValidatedSparkApplication { - /// Recommended labels for a resource fulfilling the given `role` within the SparkApplication. - /// - /// A SparkApplication has no Stackable role groups, so the role group label is fixed to the - /// controller name (preserving the previous `build_recommended_labels` behaviour). `role` is a - /// free-form component name such as "spark", "spark-submit" or "role-binding". - pub(crate) fn recommended_labels(&self, role: &str) -> Labels { - // `app_version_label_value` is constructed to be a valid label value, so it is also a - // valid `ProductVersion`. - let product_version = - ProductVersion::from_str(&self.resolved_product_image.app_version_label_value) - .expect("the app version label value is a valid product version"); - let role_name = RoleName::from_str(role).expect("the role is a valid role name"); - let role_group = RoleGroupName::from_str(SPARK_CONTROLLER_NAME) - .expect("SPARK_CONTROLLER_NAME is a valid role group name"); - recommended_labels( - self, - &product_name(), - &product_version, - &operator_name(), - &controller_name(), - &role_name, - &role_group, - ) - } -} - -/// The product name (`spark-k8s`) as a type-safe label value. -pub(crate) fn product_name() -> ProductName { - ProductName::from_str(APP_NAME).expect("APP_NAME is a valid product name") -} - -/// The operator name as a type-safe label value. -pub(crate) fn operator_name() -> OperatorName { - OperatorName::from_str(OPERATOR_NAME).expect("the operator name is a valid label value") -} - -/// The controller name as a type-safe label value. -pub(crate) fn controller_name() -> ControllerName { - ControllerName::from_str(SPARK_CONTROLLER_NAME) - .expect("the controller name is a valid label value") -} - pub fn validate( dereferenced: DereferencedSparkApplication, operator_environment: &OperatorEnvironmentOptions, @@ -208,6 +174,11 @@ pub fn validate( ) .context(ResolveProductImageSnafu)?; + // `app_version_label_value` is constructed to be a valid label value, so it is also a valid + // `ProductVersion`. + let product_version = ProductVersion::from_str(&resolved_product_image.app_version_label_value) + .expect("the app version label value is a valid product version"); + let name = get_cluster_name(&dereferenced.spark_application).context(ResolveClusterNameSnafu)?; let namespace = @@ -220,6 +191,7 @@ pub fn validate( name, namespace, uid, + product_version, spark_application: dereferenced.spark_application, resolved_product_image, cluster_config: ValidatedClusterConfig { @@ -250,6 +222,14 @@ mod tests { use super::*; use crate::test_support::app_version_label; + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *CONTROLLER_NAME; + let _ = *OPERATOR_NAME; + let _ = *PRODUCT_NAME; + } + /// Locks every value the validate step itself derives from the minimal fixture. The raw /// `SparkApplication` is deliberately retained on the validated type (see the field docs), /// so only the resolved identity, image and cluster config are derived here. diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json index 1e35fafb..5fb909b5 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json @@ -2,5 +2,5 @@ "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-env.sh": "", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/role-group: sparkapplication\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n prometheus.io/scrape: 'true'\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n prometheus.io/scrape: 'true'\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: config\n" } diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json index 096896c5..c82835cd 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json @@ -2,5 +2,5 @@ "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-env.sh": "", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/role-group: sparkapplication\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: config\n" } diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json index 22518c52..976109ef 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json @@ -18,16 +18,16 @@ "value": "/stackable/log/containerdebug" }, { - "name": "_STACKABLE_PRE_HOOK", - "value": "containerdebug --output=/stackable/log/containerdebug-state.json --loop &" + "name": "SPARK_CONF_DIR", + "value": "/stackable/spark/conf" }, { "name": "SPARK_SUBMIT_OPTS", "value": "-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties" }, { - "name": "SPARK_CONF_DIR", - "value": "/stackable/spark/conf" + "name": "_STACKABLE_PRE_HOOK", + "value": "containerdebug --output=/stackable/log/containerdebug-state.json --loop &" } ], "image": "oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev", diff --git a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json index 038159aa..ac0b2aef 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json @@ -2,5 +2,5 @@ "metrics.properties": "*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet\n*.sink.prometheusServlet.path=/metrics/prometheus\n", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-3.5.8.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=true\n", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/managed-by: spark.stackable.tech_connect\n app.kubernetes.io/name: spark-connect\n app.kubernetes.io/role-group: default\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\nspec:\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - podAffinityTerm:\n labelSelector:\n matchLabels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/name: spark-connect\n topologyKey: kubernetes.io/hostname\n weight: 70\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n name: spark\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log\n name: log\n - mountPath: /stackable/truststore\n name: stackable-truststore\n - mountPath: /stackable/log_config\n name: log-config\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n volumes:\n - emptyDir:\n sizeLimit: 30Mi\n name: log\n - configMap:\n name: spark-connect-executor\n name: config\n - emptyDir: {}\n name: stackable-truststore\n - configMap:\n name: spark-connect-log-config\n name: log-config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/managed-by: spark.stackable.tech_connect\n app.kubernetes.io/name: spark-connect\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\nspec:\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - podAffinityTerm:\n labelSelector:\n matchLabels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/name: spark-connect\n topologyKey: kubernetes.io/hostname\n weight: 70\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n name: spark\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log\n name: log\n - mountPath: /stackable/truststore\n name: stackable-truststore\n - mountPath: /stackable/log_config\n name: log-config\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n volumes:\n - emptyDir:\n sizeLimit: 30Mi\n name: log\n - configMap:\n name: spark-connect-executor\n name: config\n - emptyDir: {}\n name: stackable-truststore\n - configMap:\n name: spark-connect-log-config\n name: log-config\n" } diff --git a/tests/templates/kuttl/smoke/42-assert.yaml b/tests/templates/kuttl/smoke/42-assert.yaml index 707c699b..8e7b93d5 100644 --- a/tests/templates/kuttl/smoke/42-assert.yaml +++ b/tests/templates/kuttl/smoke/42-assert.yaml @@ -94,7 +94,6 @@ metadata: app.kubernetes.io/instance: spark-history app.kubernetes.io/managed-by: spark.stackable.tech_history app.kubernetes.io/name: spark-history - app.kubernetes.io/role-group: none stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 diff --git a/tests/templates/kuttl/smoke/52-assert.yaml b/tests/templates/kuttl/smoke/52-assert.yaml index 8404522b..2922b5cc 100644 --- a/tests/templates/kuttl/smoke/52-assert.yaml +++ b/tests/templates/kuttl/smoke/52-assert.yaml @@ -12,7 +12,6 @@ metadata: app.kubernetes.io/instance: spark-pi-s3-1 app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication app.kubernetes.io/name: spark-k8s - app.kubernetes.io/role-group: sparkapplication stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -29,7 +28,6 @@ metadata: app.kubernetes.io/instance: spark-pi-s3-1 app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication app.kubernetes.io/name: spark-k8s - app.kubernetes.io/role-group: sparkapplication stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -46,7 +44,6 @@ metadata: app.kubernetes.io/instance: spark-pi-s3-1 app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication app.kubernetes.io/name: spark-k8s - app.kubernetes.io/role-group: sparkapplication stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -59,11 +56,9 @@ kind: ServiceAccount metadata: name: spark-pi-s3-1 labels: - app.kubernetes.io/component: service-account app.kubernetes.io/instance: spark-pi-s3-1 app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication app.kubernetes.io/name: spark-k8s - app.kubernetes.io/role-group: sparkapplication stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -76,11 +71,9 @@ kind: RoleBinding metadata: name: spark-pi-s3-1 labels: - app.kubernetes.io/component: role-binding app.kubernetes.io/instance: spark-pi-s3-1 app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication app.kubernetes.io/name: spark-k8s - app.kubernetes.io/role-group: sparkapplication stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 diff --git a/tests/templates/kuttl/spark-connect-kerberos/12-assert.yaml b/tests/templates/kuttl/spark-connect-kerberos/12-assert.yaml index d25e9b1b..0e3a98f9 100644 --- a/tests/templates/kuttl/spark-connect-kerberos/12-assert.yaml +++ b/tests/templates/kuttl/spark-connect-kerberos/12-assert.yaml @@ -12,7 +12,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -26,7 +25,6 @@ spec: app.kubernetes.io/component: server app.kubernetes.io/instance: spark-connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default status: readyReplicas: 1 replicas: 1 @@ -40,7 +38,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -57,7 +54,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -74,7 +70,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -113,7 +108,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default prometheus.io/scrape: "true" stackable.tech/vendor: Stackable ownerReferences: @@ -144,7 +138,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 diff --git a/tests/templates/kuttl/spark-connect/12-assert.yaml b/tests/templates/kuttl/spark-connect/12-assert.yaml index d25e9b1b..0e3a98f9 100644 --- a/tests/templates/kuttl/spark-connect/12-assert.yaml +++ b/tests/templates/kuttl/spark-connect/12-assert.yaml @@ -12,7 +12,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -26,7 +25,6 @@ spec: app.kubernetes.io/component: server app.kubernetes.io/instance: spark-connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default status: readyReplicas: 1 replicas: 1 @@ -40,7 +38,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -57,7 +54,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -74,7 +70,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 @@ -113,7 +108,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default prometheus.io/scrape: "true" stackable.tech/vendor: Stackable ownerReferences: @@ -144,7 +138,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable ownerReferences: - apiVersion: spark.stackable.tech/v1alpha1 diff --git a/tests/templates/kuttl/spark-connect/14-assert.yaml.j2 b/tests/templates/kuttl/spark-connect/14-assert.yaml.j2 index 6fcac551..36a111c0 100644 --- a/tests/templates/kuttl/spark-connect/14-assert.yaml.j2 +++ b/tests/templates/kuttl/spark-connect/14-assert.yaml.j2 @@ -41,7 +41,6 @@ metadata: app.kubernetes.io/instance: spark-connect app.kubernetes.io/managed-by: spark.stackable.tech_connect app.kubernetes.io/name: spark-connect - app.kubernetes.io/role-group: default spark-role: executor stackable.tech/vendor: Stackable spec: From a787f819a7bffaac8c7db1f177e0b6fcab8779f6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 19 Aug 2026 15:16:58 +0200 Subject: [PATCH 2/2] changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fe0f3f4..577678ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,15 @@ All notable changes to this project will be documented in this file. - The reconcilers now apply resources in a discrete apply step; the connect server and application controllers additionally update the status in a discrete update_status step (the history server CRD has no status) ([#746]). +- Bump stackable-operator to 0.116.0 ([#753]). +- BREAKING: Remove the `app.kubernetes.io/component` and `app.kubernetes.io/role-group` labels + from the resources they don't apply to (previously set to `none` or a placeholder value such as + `sparkapplication` or `default`). StatefulSets created by older operator versions cannot be + updated in place: after the operator upgrade, delete each history server and Spark Connect + StatefulSet so that the operator immediately recreates it with the new labels ([#753]). +- Environment variable overrides (`envOverrides`) are now applied after all environment variables + set by the operator. In particular, `SPARK_CONF_DIR` of a SparkApplication can now be overridden, + whereas previously the operator's values always took precedence ([#753]). ### Fixed @@ -36,6 +45,7 @@ All notable changes to this project will be documented in this file. [#744]: https://github.com/stackabletech/spark-k8s-operator/pull/744 [#745]: https://github.com/stackabletech/spark-k8s-operator/pull/745 [#746]: https://github.com/stackabletech/spark-k8s-operator/pull/746 +[#753]: https://github.com/stackabletech/spark-k8s-operator/pull/753 ## [26.7.0] - 2026-07-21