From 2c00bf40a01476f6f0724bb231d777497db1fa5f Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:19:53 +0200 Subject: [PATCH 01/13] docs: Add design spec for Kerberized KRaft controllers Covers the rebase of #999 onto main, making the quorum-manager admin client Kerberos-aware so dynamic quorum scaling (#1010) keeps working with Kerberos enabled, and cleaning up the discovery ConfigMap client properties. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- ...-16-kerberized-kraft-controllers-design.md | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md diff --git a/docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md b/docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md new file mode 100644 index 00000000..330c79f9 --- /dev/null +++ b/docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md @@ -0,0 +1,202 @@ +# Kerberized KRaft controllers — design + +- Date: 2026-09-16 +- Tickets: stackabletech/issues#815, kafka-operator#899, kafka-operator#870 +- Existing work: kafka-operator#999 (draft, branch `feature/kraft-kerberos-support`) + +## Goal + +Let Apache Kafka KRaft controllers authenticate with Kerberos (GSSAPI), covering both +broker-to-controller and controller-to-controller (Raft) traffic, **without** losing the +dynamic quorum scaling added by kafka-operator#1010. + +## Background + +PR #999 implements most of the controller-side Kerberos support against base commit +`5211842`. Since then `main` has absorbed #1010 (dynamic KRaft quorum scaling) and a large +refactor, so the PR cannot be merged as-is. + +More importantly, #1010 and Kerberos are **mutually exclusive in `main` today**: + +- `build_quorum_manager_container` (`controller/build/resource/statefulset.rs:749`) returns + `None` when Kerberos is enabled. +- The controller `preStop` `remove-controller` hook is skipped for the same reason + (`statefulset.rs:506`). + +Both gates exist because `controller_admin_client_properties` +(`controller/build/security.rs:225`) ignores its `_security` argument and hardcodes +`security.protocol=SSL`. Merging #999 unchanged would therefore ship Kerberized controllers +that silently lose dynamic quorum scaling. + +### Why the admin client uses GSSAPI + +A Kafka listener has exactly one security protocol, so the mechanism used by +`kafka-metadata-quorum.sh` is decided by the listener it connects to. An alternative was +considered: define a second controller listener (`controller.listener.names` accepts a +comma-separated list) carrying plain `SSL` for admin traffic, leaving `CONTROLLER` on +`SASL_SSL`. + +Rejected, because: + +1. `add-controller` is not a plain admin call. The self-registering process reads `node.id` + and its own `listeners`/`controller.listener.names` from the **same** `--command-config` + file to build the voter-registration payload (see the comment at + `controller/build/command.rs:181-189`). With two controller listeners in that file, the + endpoint registered into the quorum becomes ambiguous — and registering the wrong endpoint + breaks the quorum, not just the admin call. +2. It creates a second identity to authorize: an X.509 principal (`CN=…`) alongside + `kafka/…@REALM`, so every controller-quorum ACL would need both. +3. The GSSAPI route is cheap. The sidecar runs *inside* the controller pod, which already + carries the correct pod-scoped keytab, and the controller's own principal is the right + identity for a voter registering itself. + +## Design + +### 1. Rebase of #999 + +`feature/kraft-kerberos-support` is 21 commits on `5211842` and contains a merge commit +(`a0adc2a`). Squash into a small set of logical commits first, then rebase onto `main`; a +plain `git rebase --onto` flattens the merge awkwardly. + +Hunks `main` has already obsoleted — **drop them, do not resolve the conflict**: + +- `kerberos.rs`: the `cb_kcat_prober: Option<&mut ContainerBuilder>` signature change. `main` + removed that parameter entirely. Keep only the core change: + `match role { Broker => listener volume scopes, Controller => with_pod_scope() }`. +- `kerberos.rs`: the `cb.add_env_var("KRB5_CONFIG", …)` loop. `main` extracted this into + `kerberos_env_vars() -> EnvVarSet` so that user `envOverrides` win on a name collision. +- String literals replaced by `constant!` newtypes throughout (`&*LISTENER_BROKER_VOLUME_NAME`, + `&*KERBEROS_VOLUME_NAME`, `EnvVarName`). Mechanical. + +Hunks needing genuine re-application: + +- `command.rs`: #1010 rewrote `controller_kafka_container_command` (`NODE_ID_OFFSET`, + `--no-initial-controllers`, `$FORMAT_QUORUM_FLAG`). The `set_realm_env` and `jaas_setup` + insertions must be re-placed into the new body. +- The PR's `controller_command_is_byte_identical_to_pre_kerberos_output_when_disabled` test + pins against a hand-copied *pre-#1010* function body. Re-baseline it against `main`'s + current body, otherwise it fails for the wrong reason and proves nothing. + +`controller/build/security.rs`, `crd/listener.rs` and `controller/build/properties/listener.rs` +hunks are expected to apply near-clean. + +Behaviour carried over unchanged from #999: + +- `CONTROLLER` listener becomes `SASL_SSL` when Kerberos is enabled, `SSL` otherwise. +- `sasl.mechanism.controller.protocol=GSSAPI` on both broker and controller properties. +- Controller keytabs are **pod-scoped** (`with_pod_scope()`); broker keytabs stay + listener-scoped. Controllers have no listener-operator `Listener` volume — they are only + reachable via their StatefulSet pod DNS name. +- A `controller.KafkaServer` JAAS section on both roles. It deliberately does **not** set + `isInitiator=false`: it is the only listener where the process must act as a GSSAPI + initiator as well as an acceptor, because controllers connect to each other for Raft. +- Kerberos-disabled output stays byte-identical to the pre-Kerberos implementation. + +### 2. Kerberos-aware admin client + +`controller_admin_client_properties` must branch on `has_kerberos_enabled()`: + +| Property | Value | +| --- | --- | +| `security.protocol` | `SASL_SSL` | +| `sasl.mechanism` | `GSSAPI` | +| `sasl.kerberos.service.name` | `kafka` | +| `sasl.jaas.config` | single-line `Krb5LoginModule`, `useKeyTab=true`, `storeKey=true`, `keyTab="/stackable/kerberos/keytab"`, `principal="kafka/${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}@${env:KERBEROS_REALM}"` | + +The existing internal keystore/truststore properties (`STACKABLE_TLS_KAFKA_INTERNAL_DIR`) are +kept in both branches. The non-Kerberos branch is unchanged. + +That principal contains `${env:…}` placeholders, so **`admin-client.properties` must pass +through `config-utils template` before use**. Today both consumers read it directly from +`/stackable/config`. Two changes: + +- **Sidecar** (`quorum_manager_container_command`): it already does + `cp controller.properties /tmp/ && config-utils template …` inside an `if … ; then` guard. + Extend that same `&&` chain to `admin-client.properties` and point `ADMIN_CLIENT_CONFIG` at + the `/tmp` copy. The existing degraded-mode `else` branch then covers a Kerberos render + failure with no new error handling. +- **Kafka container startup**: the same copy-and-template step, so the `preStop` hook's + `$ADMIN_CLIENT_CONFIG` resolves. + +The sidecar is a separate container and inherits nothing from the kafka container's startup, +so it additionally needs: + +- the `kerberos` volume mounted at `STACKABLE_KERBEROS_DIR`, +- `KRB5_CONFIG` set, +- its own `export KERBEROS_REALM=$(grep -oP 'default_realm = \K.*' …)`. + +Finally, remove both Kerberos gates — the `build_quorum_manager_container` early return and +the `preStop` skip — along with their now-false explanatory comments. + +### 3. Discovery ConfigMap client properties + +`client_properties` (`controller/build/security.rs:162`) emits +`principal="kafka/todo@$KERBEROS_REALM"`. This is not a missing value: the consumer is a +client running *outside* Kafka pods, with no `/stackable/kerberos/keytab` and no per-pod +principal. Supplying a real principal would produce a file that is confidently broken rather +than obviously broken. + +Resolution: + +- Delete the `sasl.jaas.config` entry from the discovery file. +- Delete `sasl.mechanism.inter.broker.protocol` from it — a broker-side property with no + meaning in a client config. +- Keep `security.protocol`, the `ssl.*` store properties and `sasl.kerberos.service.name`. +- Replace `sasl.enabled.mechanisms` with `sasl.mechanism=GSSAPI`. `sasl.enabled.mechanisms` is + the broker-side property (the list a broker accepts); the client-side equivalent — the one a + client actually reads — is `sasl.mechanism`. Same class of mistake as the two deletions + above, so it is fixed here rather than left behind. +- Document that clients supply their own principal and keytab (their own `jaas.conf`). + +The `TODO` comment above the block is discharged by the JAAS work in §1: the operator does +write real JAAS files, for the pods that actually hold keytabs. + +### 4. Tests + +Unit: + +- #999's `jaas_config_file`, `kerberos.rs` and `security.rs` tests, carried over. +- The re-baselined byte-identical command test (§1). +- New: Kerberized `controller_admin_client_properties` — asserts `SASL_SSL`, `GSSAPI`, the + service name, the pod-FQDN principal, and that the internal TLS stores are still present. +- New: non-Kerberos `controller_admin_client_properties` is unchanged. +- New: `quorum_manager_container_command` templates `admin-client.properties` and points + `ADMIN_CLIENT_CONFIG` at the `/tmp` copy. +- New: `build_quorum_manager_container` returns `Some` with Kerberos enabled, and the returned + container mounts the `kerberos` volume. + +Integration (kuttl): + +- #999's `kraft-kerberos` suite (MIT KDC, 3-controller quorum, produce/consume). +- **Extend it with controller scale-up and scale-down steps**, mirroring + `tests/templates/kuttl/operations-kraft/60-*` and `70-*`. This is the regression test for + the un-gating in §2 and is not optional — #999 predates the quorum manager and cannot have + covered it. +- Register the dimension in `tests/test-definition.yaml`. + +### 5. Documentation + +- `docs/modules/kafka/pages/usage-guide/kraft-controller.adoc`: Kerberos on the `CONTROLLER` + listener; pod-scoped controller keytabs vs listener-scoped broker keytabs; dynamic quorum + scaling is supported with Kerberos enabled. +- `docs/modules/kafka/pages/usage-guide/security.adoc`: clients must supply their own + principal and keytab when using the discovery ConfigMap (§3). +- `CHANGELOG.md` entry. + +## Risks + +- §2 puts GSSAPI on the `add-controller` self-registration path — the one call whose failure + corrupts quorum membership rather than merely erroring. The kuttl scale steps in §4 are what + make this safe to ship. +- The controller keytab needs secret-operator to support pod scope together with a Kerberos + service name. #999 reports this working on OKD, so it is assumed available; verify early in + implementation rather than at integration-test time. +- `sasl.jaas.config` must be a single logical line and correctly escaped for the Java + properties format. A malformed value fails at JAAS parse time inside the sidecar, which the + degraded-mode `else` branch will *not* catch (the render succeeds; the CLI call fails). + +## Out of scope + +- OPA/ACL authorization rules for the controller quorum principals. +- Kerberos support for the `kcat` readiness prober on brokers. +- KRaft migration from ZooKeeper with Kerberos enabled. From 3aadb259db1650c161b3c58032eeb8c4607a5db3 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:30:09 +0200 Subject: [PATCH 02/13] docs: Add implementation plan for Kerberized KRaft controllers Eight tasks reconstructing #999 against current main, adding the Kerberos-aware admin client so dynamic quorum scaling keeps working, and fixing the discovery ConfigMap client properties. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- ...2026-09-16-kerberized-kraft-controllers.md | 1417 +++++++++++++++++ 1 file changed, 1417 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-16-kerberized-kraft-controllers.md diff --git a/docs/superpowers/plans/2026-09-16-kerberized-kraft-controllers.md b/docs/superpowers/plans/2026-09-16-kerberized-kraft-controllers.md new file mode 100644 index 00000000..8f1bb372 --- /dev/null +++ b/docs/superpowers/plans/2026-09-16-kerberized-kraft-controllers.md @@ -0,0 +1,1417 @@ +# Kerberized KRaft Controllers Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let Apache Kafka KRaft controllers authenticate with Kerberos (GSSAPI) for both broker-to-controller and controller-to-controller (Raft) traffic, without losing the dynamic quorum scaling added in #1010. + +**Architecture:** The `CONTROLLER` listener switches from `SSL` to `SASL_SSL` when Kerberos is enabled. Controller pods get a *pod-scoped* keytab (they have no listener-operator `Listener` volume) and a `controller.KafkaServer` JAAS section. The `quorum-manager` sidecar and the `preStop` hook, which drive dynamic quorum membership, get a Kerberos-aware `admin-client.properties` so they keep working. + +**Tech Stack:** Rust, `stackable-operator` crate, `config-utils template` for runtime `${env:…}` placeholder resolution, kuttl + MIT KDC for integration tests. + +**Spec:** `docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md` + +## Relationship to PR #999 + +The spec frames this as "rebase #999". `main` has moved far enough that a literal `git rebase` produces more conflict resolution than reconstruction. This plan therefore **re-applies #999's changes task by task against current `main`**, using #999 as the reference for *what* to build. Task 0 sets up a read-only worktree of that branch so every later task can consult it. + +Three deviations from the spec, discovered while reading current `main`: + +1. **`add_kerberos_pod_config` is never called for controllers.** It is invoked only at `statefulset.rs:248`, inside `build_broker_rolegroup_statefulset`. Controller pods have no keytab volume at all today. This is prerequisite work the spec did not name; it is now Task 1. +2. **`kerberos_env_vars` must not go on the shared controller env.** It sets `KAFKA_OPTS=-Djava.security.auth.login.config=/tmp/jaas.properties`. `controller_pod_shared_env_vars` feeds both the `kafka` container and the `quorum-manager` sidecar, and the sidecar has no `/tmp/jaas.properties` — it uses an inline `sasl.jaas.config` instead. Kerberos env goes on the `kafka` container's `env` only; the sidecar gets `KRB5_CONFIG` alone. +3. **Drop the byte-identical command test rather than re-baselining it.** `broker_start_command` (`command.rs:86-88`) already copies and templates `jaas.properties` *unconditionally*, because the file is always present in the ConfigMap (empty string when Kerberos is off). Mirroring that for the controller is simpler than #999's conditional `jaas_setup`, and makes the byte-identical regression test pointless. + +## Global Constraints + +- Match the surrounding Rust style: `snafu` for errors, `constant!` newtypes for volume/env-var names, `expect` with a justifying message for statically-impossible failures. +- Kerberos-disabled behaviour must not change. Every task that touches a shared code path asserts the non-Kerberos branch is untouched. +- Product naming in docs: "Stackable Data Platform (SDP)" once, then SDP; "Apache Kafka" in formal prose. +- `sasl.jaas.config` must be a single logical line in a Java properties file. +- Kerberos principals are always `kafka/@`; the service name comes from `KafkaRole::kerberos_service_name()`, never a literal. +- Run `cargo test -p stackable-kafka-operator` for unit tests; `cargo clippy --all-targets -- -D warnings` before every commit. + +--- + +### Task 0: Reference worktree for PR #999 + +**Files:** + +- Create: none in the repo tree (worktree lives outside it) + +**Interfaces:** + +- Produces: a read-only checkout of `origin/feature/kraft-kerberos-support` that later tasks consult for reference implementations. + +- [ ] **Step 1: Fetch the branch and create the reference worktree** + +```bash +cd /home/razvan/repo/stackable/kafka-operator +git fetch origin feature/kraft-kerberos-support +git worktree add --detach /tmp/pr999 origin/feature/kraft-kerberos-support +``` + +- [ ] **Step 2: Confirm the reference files are readable** + +Run: + +```bash +ls /tmp/pr999/tests/templates/kuttl/kraft-kerberos/ +``` + +Expected: lists `01-install-krb5-kdc.yaml.j2`, `02-create-kerberos-secretclass.yaml.j2`, `20-install-kafka.yaml.j2`, `30-access-kafka.txt.j2` among others. + +- [ ] **Step 3: Confirm we are on the feature branch** + +Run: `git branch --show-current` +Expected: `feat/kerberized-kraft-controllers` + +No commit for this task — it creates no tracked files. + +--- + +### Task 1: Pod-scoped Kerberos keytab on controller pods + +Controllers are reachable only through their StatefulSet pod DNS name, so their keytab principal must be pod-scoped. Brokers keep listener-volume scoping. + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/kerberos.rs:52-82` (`add_kerberos_pod_config`) +- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs:413-460` (`build_controller_rolegroup_statefulset`) +- Test: `rust/operator-binary/src/controller/build/kerberos.rs` (`#[cfg(test)] mod tests`) + +**Interfaces:** + +- Consumes: `ValidatedKafkaSecurity::kerberos_secret_class()`, `KafkaRole`, `SecretOperatorVolumeSourceBuilder::with_pod_scope()`. +- Produces: `add_kerberos_pod_config` gains controller-aware behaviour; its signature is unchanged (`(&ValidatedKafkaSecurity, &KafkaRole, &mut ContainerBuilder, &mut PodBuilder) -> Result<(), Error>`). + +- [ ] **Step 1: Write the failing test** + +Add to the existing `mod tests` in `kerberos.rs`. (`security.rs`'s test module already exposes an identical `pub(crate) fn kerberos()`; importing it instead of redefining it locally is fine and preferable if it resolves cleanly.) + +```rust +use stackable_operator::{ + builder::{meta::ObjectMetaBuilder, pod::container::ContainerBuilder}, + crd::authentication::{core, kerberos}, +}; + +use crate::crd::authentication::ResolvedAuthenticationClasses; + +fn kerberos() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![core::v1alpha1::AuthenticationClass { + metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), + spec: core::v1alpha1::AuthenticationClassSpec { + provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( + kerberos::v1alpha1::AuthenticationProvider { + kerberos_secret_class: "kerberos-secret-class".to_string(), + }, + ), + }, + }]), + "tls".parse().expect("valid secret class name"), + Some("tls".parse().expect("valid secret class name")), + None, + ) +} + +/// Reads the `secrets.stackable.tech/*` annotations off the `kerberos` ephemeral volume. +fn kerberos_volume_annotations(pb: &mut PodBuilder) -> std::collections::BTreeMap { + let pod = pb.build_template(); + pod.spec + .as_ref() + .and_then(|spec| spec.volumes.as_ref()) + .and_then(|volumes| volumes.iter().find(|v| v.name == *KERBEROS_VOLUME_NAME)) + .expect("kerberos volume must be present") + .ephemeral + .as_ref() + .expect("kerberos volume must be an ephemeral secret-operator volume") + .volume_claim_template + .as_ref() + .and_then(|t| t.metadata.as_ref()) + .and_then(|m| m.annotations.clone()) + .expect("volume claim template must carry secrets.stackable.tech annotations") +} + +#[test] +fn controller_keytab_is_pod_scoped() { + let mut pb = PodBuilder::new(); + let mut cb_kafka = ContainerBuilder::new("kafka").expect("valid container name"); + + add_kerberos_pod_config( + &kerberos(), + &KafkaRole::Controller, + &mut cb_kafka, + &mut pb, + ) + .expect("kerberos pod config for the controller role"); + + let annotations = kerberos_volume_annotations(&mut pb); + // Controllers have no listener-operator Listener volume, so the keytab must be + // scoped to the pod's own DNS name, matching how their internal TLS cert is + // provisioned in `add_controller_volume_and_volume_mounts`. + assert_eq!( + annotations.get("secrets.stackable.tech/scope").map(String::as_str), + Some("pod"), + "controller keytab must be pod-scoped, got: {annotations:?}" + ); + assert_eq!( + annotations + .get("secrets.stackable.tech/kerberos.service.names") + .map(String::as_str), + Some("kafka") + ); +} + +#[test] +fn broker_keytab_stays_listener_scoped() { + let mut pb = PodBuilder::new(); + let mut cb_kafka = ContainerBuilder::new("kafka").expect("valid container name"); + + add_kerberos_pod_config( + &kerberos(), + &KafkaRole::Broker, + &mut cb_kafka, + &mut pb, + ) + .expect("kerberos pod config for the broker role"); + + let annotations = kerberos_volume_annotations(&mut pb); + let scope = annotations + .get("secrets.stackable.tech/scope") + .expect("scope annotation must be present"); + assert!( + scope.contains("listener-volume=listener-broker") + && scope.contains("listener-volume=listener-bootstrap"), + "broker keytab must stay listener-volume-scoped, got: {scope}" + ); + assert!( + !scope.split(',').any(|s| s == "pod"), + "broker keytab must not be pod-scoped, got: {scope}" + ); +} +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cargo test -p stackable-kafka-operator kerberos:: -- --nocapture` +Expected: `controller_keytab_is_pod_scoped` FAILS — the scope annotation is the broker's listener-volume scope, because the role is currently ignored. + +- [ ] **Step 3: Make the volume scope role-dependent** + +In `kerberos.rs`, replace the chained builder call inside `if let Some(kerberos_secret_class) = …` with: + +```rust + let mut volume_builder = SecretOperatorVolumeSourceBuilder::new( + kerberos_secret_class, + // We need both public (krb5.conf) and private (keytab) parts. + SecretClassVolumeProvisionParts::PublicPrivate, + ); + match role { + // Brokers are exposed through listener-operator `Listener` volumes (the broker + // and bootstrap listeners), so the keytab principal must cover both. + KafkaRole::Broker => { + volume_builder + .with_listener_volume_scope(&*LISTENER_BROKER_VOLUME_NAME) + .with_listener_volume_scope(&*LISTENER_BOOTSTRAP_VOLUME_NAME); + } + // KRaft controllers have no listener-operator `Listener` volume: they are only + // reachable through their own StatefulSet pod DNS name, so the keytab must be + // pod-scoped, matching how the controller's internal TLS cert is provisioned in + // `add_controller_volume_and_volume_mounts`. + KafkaRole::Controller => { + volume_builder.with_pod_scope(); + } + } + let kerberos_secret_operator_volume = volume_builder + .with_kerberos_service_name(role.kerberos_service_name()) + .build() + .context(KerberosSecretVolumeSnafu)?; +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `cargo test -p stackable-kafka-operator kerberos:: -- --nocapture` +Expected: PASS + +- [ ] **Step 5: Call it from the controller StatefulSet builder** + +In `statefulset.rs`, inside `build_controller_rolegroup_statefulset`, immediately after `let mut pod_builder = PodBuilder::new();`, add: + +```rust + if kafka_security.has_kerberos_enabled() { + add_kerberos_pod_config(kafka_security, kafka_role, &mut cb_kafka, &mut pod_builder) + .context(AddKerberosConfigSnafu)?; + } +``` + +Then, in the same function, add the Kerberos env vars to the **`kafka` container's** env only — `controller_shared_env` also feeds the `quorum-manager` sidecar, which has no `/tmp/jaas.properties` and must not receive `KAFKA_OPTS`. Change the `let env: Vec = …` chain to insert `.merge(kerberos_env_vars(kafka_security))` immediately before `.merge(validated_rg.env_overrides.clone())`: + +```rust + let env: Vec = controller_shared_env + .clone() + .merge(common_kafka_env( + merged_config, + &validated_rg + .product_specific_common_config + .jvm_argument_overrides, + resolved_product_image, + kafka_role, + role_group_name, + )?) + // Kerberos env goes on the `kafka` container only. `controller_shared_env` is also + // the sidecar's base, and `KAFKA_OPTS` points the JVM at `/tmp/jaas.properties`, + // which only the `kafka` container renders. + .merge(kerberos_env_vars(kafka_security)) + .merge(validated_rg.env_overrides.clone()) + .into(); +``` + +- [ ] **Step 6: Verify it compiles and the whole suite passes** + +Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` +Expected: no warnings, all tests pass. + +- [ ] **Step 7: Commit** + +```bash +git add rust/operator-binary/src/controller/build/kerberos.rs \ + rust/operator-binary/src/controller/build/resource/statefulset.rs +git commit -m "feat: mount a pod-scoped Kerberos keytab on KRaft controller pods" +``` + +--- + +### Task 2: `controller.KafkaServer` JAAS section and controller JAAS rendering + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/resource/config_map.rs:169-230` (`jaas_config_file` and its call site) +- Modify: `rust/operator-binary/src/controller/build/command.rs:145-176` (`controller_kafka_container_command`) +- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs:486-488` (call site) +- Test: the `#[cfg(test)] mod tests` blocks in `config_map.rs` and `command.rs` + +**Interfaces:** + +- Consumes: `KafkaRole` (Task 1's role plumbing), `node_address_cmd`, `ConfigFileName::Jaas`. +- Produces: + - `fn jaas_config_file(is_kerberos_enabled: bool, role: &KafkaRole) -> String` + - `pub fn controller_kafka_container_command(kafka_security: &ValidatedKafkaSecurity, controller_descriptors: Vec) -> String` + +- [ ] **Step 1: Write the failing tests** + +In `config_map.rs`, replace the existing `mod tests` contents with: + +```rust +#[cfg(test)] +mod tests { + use super::jaas_config_file; + use crate::crd::role::KafkaRole; + + const CONTROLLER_POD_FQDN: &str = "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}"; + + #[test] + fn jaas_config_file_empty_without_kerberos() { + assert_eq!(jaas_config_file(false, &KafkaRole::Broker), ""); + assert_eq!(jaas_config_file(false, &KafkaRole::Controller), ""); + } + + #[test] + fn jaas_config_file_renders_bootstrap_and_client_sections_with_kerberos() { + let jaas = jaas_config_file(true, &KafkaRole::Broker); + assert!(jaas.contains("bootstrap.KafkaServer")); + assert!(jaas.contains("client.KafkaServer")); + assert!(jaas.contains("Krb5LoginModule")); + assert!(jaas.contains("/stackable/kerberos/keytab")); + assert!(jaas.contains("/stackable/listener-bootstrap")); + assert!(jaas.contains("/stackable/listener-broker")); + } + + #[test] + fn broker_controller_section_uses_the_broker_listener_address() { + let jaas = jaas_config_file(true, &KafkaRole::Broker); + assert!(jaas.contains("controller.KafkaServer {")); + // Brokers connect *out* to controllers. The only principals in a broker's keytab are + // for its own listener addresses, so this section must reuse the broker address. + assert!(jaas.contains( + "kafka/${file:UTF-8:/stackable/listener-broker/default-address/address}@${env:KERBEROS_REALM}" + )); + } + + #[test] + fn controller_jaas_has_only_the_controller_section_with_a_pod_fqdn_principal() { + let jaas = jaas_config_file(true, &KafkaRole::Controller); + assert!(jaas.contains("controller.KafkaServer {")); + assert!(jaas.contains(&format!( + "kafka/{CONTROLLER_POD_FQDN}@${{env:KERBEROS_REALM}}" + ))); + // Controllers have no listener-operator Listener volume, so the broker-only + // sections must not appear in their JAAS file. + assert!(!jaas.contains("bootstrap.KafkaServer")); + assert!(!jaas.contains("client.KafkaServer")); + } + + #[test] + fn controller_section_allows_the_process_to_act_as_a_gssapi_initiator() { + for role in [KafkaRole::Broker, KafkaRole::Controller] { + let jaas = jaas_config_file(true, &role); + let start = jaas + .find("controller.KafkaServer {") + .expect("controller.KafkaServer section must be present"); + // Unlike the other sections, this context is used for BOTH sides of every + // CONTROLLER-listener connection: brokers connect out to controllers, and + // controllers connect to each other for Raft. So `isInitiator` must stay at its + // default (`true`). Scoped to this section so a broker-side `isInitiator=false` + // elsewhere stays fine. + assert!( + !jaas[start..].contains("isInitiator=false"), + "controller.KafkaServer for {role:?} must not disable GSSAPI initiation" + ); + } + } +} +``` + +In `command.rs`, add a `mod tests` block: + +```rust +#[cfg(test)] +mod tests { + use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + crd::authentication::{core, kerberos}, + }; + + use super::*; + use crate::crd::authentication::ResolvedAuthenticationClasses; + + fn kerberos() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![core::v1alpha1::AuthenticationClass { + metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), + spec: core::v1alpha1::AuthenticationClassSpec { + provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( + kerberos::v1alpha1::AuthenticationProvider { + kerberos_secret_class: "kerberos-secret-class".to_string(), + }, + ), + }, + }]), + "tls".parse().expect("valid secret class name"), + Some("tls".parse().expect("valid secret class name")), + None, + ) + } + + fn plaintext_security() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![]), + "tls".parse().expect("valid secret class name"), + None, + None, + ) + } + + #[test] + fn controller_command_exports_the_kerberos_realm_when_enabled() { + let command = controller_kafka_container_command(&kerberos(), vec![]); + assert!(command.contains("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*'")); + } + + #[test] + fn controller_command_does_not_export_a_realm_without_kerberos() { + let command = controller_kafka_container_command(&plaintext_security(), vec![]); + assert!(!command.contains("KERBEROS_REALM")); + } + + #[test] + fn controller_command_always_templates_the_jaas_file() { + // `jaas.properties` is always present in the ConfigMap (empty when Kerberos is off), + // so the copy is unconditional, matching `broker_start_command`. + for security in [kerberos(), plaintext_security()] { + let command = controller_kafka_container_command(&security, vec![]); + assert!(command.contains("cp /stackable/config/jaas.properties /tmp/jaas.properties")); + assert!(command.contains("config-utils template /tmp/jaas.properties")); + } + } +} +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cargo test -p stackable-kafka-operator` +Expected: compile errors — `jaas_config_file` takes one argument, `controller_kafka_container_command` takes one argument. + +- [ ] **Step 3: Give `jaas_config_file` a role and a controller section** + +In `config_map.rs`, add `KafkaRole` to the `crate::crd::role` import, change the call site to `jaas_config_file(kafka_security.has_kerberos_enabled(), &role)`, and replace the function with: + +```rust +// Generate JAAS configuration file for Kerberos authentication +// or an empty string if Kerberos is not enabled. +// See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html +fn jaas_config_file(is_kerberos_enabled: bool, role: &KafkaRole) -> String { + if !is_kerberos_enabled { + return String::new(); + } + + // Broker pods reach the CONTROLLER listener as SASL clients; the only principals in + // their keytab (see `add_kerberos_pod_config`) are for the broker and bootstrap listener + // addresses, so their CONTROLLER section must reuse the broker address. + // Controller pods have no listener-operator `Listener` volume; their keytab is + // pod-scoped, so their CONTROLLER section uses their own pod FQDN — the same expression + // already used for `KAFKA_LISTENERS` in `controller_properties.rs`. + let controller_principal_address = match role { + KafkaRole::Broker => node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), + KafkaRole::Controller => { + "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}" + .to_string() + } + }; + + // Unlike the bootstrap and client sections below, this context is used for BOTH sides of + // every CONTROLLER-listener connection: brokers connect out to controllers, and + // controllers connect to each other for Raft. This is the only listener in this operator + // where the process must act as a GSSAPI initiator as well as an acceptor, so + // `isInitiator` is intentionally left at its default (`true`). + let controller_section = formatdoc! {" + controller.KafkaServer {{ + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab=\"/stackable/kerberos/keytab\" + principal=\"kafka/{controller_principal_address}@${{env:KERBEROS_REALM}}\"; + }}; + "}; + + match role { + KafkaRole::Controller => controller_section, + KafkaRole::Broker => formatdoc! {" + bootstrap.KafkaServer {{ + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + isInitiator=false + keyTab=\"/stackable/kerberos/keytab\" + principal=\"kafka/{bootstrap_address}@${{env:KERBEROS_REALM}}\"; + }}; + + client.KafkaServer {{ + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + isInitiator=false + keyTab=\"/stackable/kerberos/keytab\" + principal=\"kafka/{broker_address}@${{env:KERBEROS_REALM}}\"; + }}; + + {controller_section} + ", + bootstrap_address = node_address_cmd(STACKABLE_LISTENER_BOOTSTRAP_DIR), + broker_address = node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), + }, + } +} +``` + +- [ ] **Step 4: Export the realm and template the JAAS file in the controller command** + +In `command.rs`, change the signature and body of `controller_kafka_container_command`: + +```rust +pub fn controller_kafka_container_command( + kafka_security: &ValidatedKafkaSecurity, + controller_descriptors: Vec, +) -> String { + formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + {remove_vector_shutdown_file_command} + prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + {set_realm_env} + + {derive_pod_index} + {export_replica_id} + + cp {config_dir}/{properties_file} /tmp/{properties_file} + + config-utils template /tmp/{properties_file} + + cp {config_dir}/{jaas_file} /tmp/{jaas_file} + config-utils template /tmp/{jaas_file} + + {quorum_format_flag} + bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted \"$FORMAT_QUORUM_FLAG\" + bin/kafka-server-start.sh /tmp/{properties_file} & + + wait_for_termination $! + {create_vector_shutdown_file_command} + ", + remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + // Mirrors `broker_kafka_container_commands`: empty when Kerberos is disabled. + set_realm_env = match kafka_security.has_kerberos_enabled() { + true => format!("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {STACKABLE_KERBEROS_KRB5_PATH})"), + false => "".to_string(), + }, + derive_pod_index = DERIVE_POD_INDEX, + export_replica_id = EXPORT_REPLICA_ID, + config_dir = STACKABLE_CONFIG_DIR, + properties_file = ConfigFileName::ControllerProperties, + jaas_file = ConfigFileName::Jaas, + quorum_format_flag = controller_quorum_format_flag(&controller_descriptors), + create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR) + } +} +``` + +- [ ] **Step 5: Update the call site** + +In `statefulset.rs`, change: + +```rust + .args(vec![controller_kafka_container_command( + kafka_security, + controller_pod_descriptors, + )]); +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` +Expected: no warnings, all tests pass. + +- [ ] **Step 7: Commit** + +```bash +git add rust/operator-binary/src/controller/build/resource/config_map.rs \ + rust/operator-binary/src/controller/build/command.rs \ + rust/operator-binary/src/controller/build/resource/statefulset.rs +git commit -m "feat: add a controller.KafkaServer JAAS section for KRaft controllers" +``` + +--- + +### Task 3: `SASL_SSL` on the CONTROLLER listener + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/properties/listener.rs:108-118` +- Modify: `rust/operator-binary/src/controller/build/security.rs:49` (new constant), and the Kerberos branches of `broker_config_settings` and `controller_config_settings` +- Modify: `rust/operator-binary/src/crd/listener.rs:55-67` (doc comment) +- Test: the `mod tests` blocks in `properties/listener.rs` and `security.rs` + +**Interfaces:** + +- Consumes: `KafkaListenerProtocol::SaslSsl`, `ValidatedKafkaSecurity::has_kerberos_enabled()`. +- Produces: no new public functions; `broker_config_settings` and `controller_config_settings` gain `sasl.mechanism.controller.protocol=GSSAPI` under Kerberos. + +- [ ] **Step 1: Write the failing tests** + +In `security.rs`, add to the existing Kerberos test for each role: + +```rust + #[test] + fn broker_config_sets_the_controller_sasl_mechanism_with_kerberos() { + let config = broker_config_settings(&kerberos()); + assert_eq!( + config.get("sasl.mechanism.controller.protocol"), + Some(&"GSSAPI".to_string()) + ); + } + + #[test] + fn controller_config_sets_the_controller_sasl_mechanism_with_kerberos() { + let config = controller_config_settings(&kerberos()); + assert_eq!( + config.get("sasl.mechanism.controller.protocol"), + Some(&"GSSAPI".to_string()) + ); + } + + #[test] + fn controller_sasl_mechanism_is_absent_without_kerberos() { + assert!( + !broker_config_settings(&internal_tls()).contains_key("sasl.mechanism.controller.protocol") + ); + assert!( + !controller_config_settings(&internal_tls()) + .contains_key("sasl.mechanism.controller.protocol") + ); + } +``` + +`kerberos()` and `internal_tls()` already exist in this module's `mod tests` (`kerberos()` is `pub(crate)`); do not add duplicates. + +In `properties/listener.rs`, update the existing `test_get_kafka_kerberos_listeners_config` expectation from `controller_protocol = KafkaListenerProtocol::Ssl` to `KafkaListenerProtocol::SaslSsl` (it is the last field of the `listener_security_protocol_map()` `format!` near the end of the module), and add this regression guard: + +```rust + #[test] + fn controller_listener_stays_ssl_without_kerberos() { + // Regression guard: only Kerberos may move CONTROLLER off plain SSL. + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + controllers: + roleGroups: + default: + replicas: 3 + brokers: + roleGroups: + default: + replicas: 1 + "#, + ); + let validated = validated_cluster(&kafka); + let kafka_security = ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![]), + "internal-tls".parse().expect("valid secret class name"), + Some("tls".parse().expect("valid secret class name")), + None, + ); + let role_group_name: RoleGroupName = "default".parse().expect("valid role group name"); + let config = get_kafka_listener_config( + &validated, + &kafka_security, + &KafkaRole::Controller, + &role_group_name, + ); + + assert!( + config.listener_security_protocol_map().contains(&format!( + "{name}:{protocol}", + name = KafkaListenerName::Controller, + protocol = KafkaListenerProtocol::Ssl + )), + "got: {}", + config.listener_security_protocol_map() + ); + } +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cargo test -p stackable-kafka-operator` +Expected: the three new `sasl.mechanism.controller.protocol` assertions FAIL (key absent); the updated listener assertion FAILS (`SSL` vs `SASL_SSL`). + +- [ ] **Step 3: Switch the CONTROLLER protocol** + +In `properties/listener.rs`, replace: + +```rust + listener_security_protocol_map.insert( + KafkaListenerName::Controller, + if kafka_security.has_kerberos_enabled() { + KafkaListenerProtocol::SaslSsl + } else { + KafkaListenerProtocol::Ssl + }, + ); +``` + +- [ ] **Step 4: Add the controller SASL mechanism property** + +In `security.rs`, next to the other property-name constants: + +```rust +const PROPERTY_SASL_CONTROLLER_MECHANISM: &str = "sasl.mechanism.controller.protocol"; +``` + +and inside the `has_kerberos_enabled()` branch of **both** `broker_config_settings` and `controller_config_settings`, next to the existing `PROPERTY_SASL_INTER_BROKER_MECHANISM` insert: + +```rust + config.insert( + PROPERTY_SASL_CONTROLLER_MECHANISM.to_string(), + SASL_MECHANISM_GSSAPI.to_string(), + ); +``` + +- [ ] **Step 5: Correct the CONTROLLER listener doc comment** + +In `crd/listener.rs`, replace the `Controller` variant's stale doc lines: + +```rust + /// This listener is defined when Kraft mode is enabled. + /// It is responsible for broker/controller as well as controller/controller communications + /// and therefore it is present on *both* brokers and controller properties files. + /// The protocol used is SSL, or SASL_SSL when Kerberos is enabled. + /// The advertised host names are FQDN pod names of the controllers. + /// + /// Note: there is no listener for client/controller communication. +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` +Expected: no warnings, all tests pass. + +- [ ] **Step 7: Commit** + +```bash +git add rust/operator-binary/src/controller/build/properties/listener.rs \ + rust/operator-binary/src/controller/build/security.rs \ + rust/operator-binary/src/crd/listener.rs +git commit -m "feat: use SASL_SSL on the CONTROLLER listener when Kerberos is enabled" +``` + +--- + +### Task 4: Kerberos-aware admin client properties + +The quorum-manager sidecar and the `preStop` hook both talk to the CONTROLLER listener, which Task 3 just moved to `SASL_SSL`. Without this task they can no longer connect. + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/security.rs:221-237` (`controller_admin_client_properties`) +- Test: the `mod tests` block in `security.rs` + +**Interfaces:** + +- Consumes: `ValidatedKafkaSecurity::has_kerberos_enabled()`, `push_client_ssl_stores`, `KafkaRole::kerberos_service_name()`. +- Produces: `pub fn controller_admin_client_properties(security: &ValidatedKafkaSecurity) -> Vec<(String, Option)>` — same signature, argument now used. + +- [ ] **Step 1: Write the failing tests** + +```rust + #[test] + fn admin_client_uses_gssapi_over_sasl_ssl_with_kerberos() { + let props = as_map(controller_admin_client_properties(&kerberos())); + assert_eq!(props.get("security.protocol"), Some(&"SASL_SSL".to_string())); + assert_eq!(props.get("sasl.mechanism"), Some(&"GSSAPI".to_string())); + assert_eq!( + props.get("sasl.kerberos.service.name"), + Some(&"kafka".to_string()) + ); + // The internal TLS stores stay: SASL_SSL is still SSL underneath. + assert_eq!( + props.get("ssl.truststore.location"), + Some(&"/stackable/tls-kafka-internal/truststore.p12".to_string()) + ); + } + + #[test] + fn admin_client_jaas_config_is_a_single_line_pod_principal() { + let props = as_map(controller_admin_client_properties(&kerberos())); + let jaas = props + .get("sasl.jaas.config") + .expect("sasl.jaas.config must be set when Kerberos is enabled"); + // Must be one logical line: a raw newline would truncate the value when the + // properties file is parsed. + assert!( + !jaas.contains('\n'), + "sasl.jaas.config must be a single line, got: {jaas}" + ); + assert!(jaas.contains("com.sun.security.auth.module.Krb5LoginModule required")); + assert!(jaas.contains("keyTab=\"/stackable/kerberos/keytab\"")); + // The controller's own pod-scoped principal (Task 1), resolved by + // `config-utils template` at container start. + assert!(jaas.contains( + "principal=\"kafka/${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}@${env:KERBEROS_REALM}\"" + )); + assert!(jaas.trim_end().ends_with(';')); + } + + #[test] + fn admin_client_is_unchanged_without_kerberos() { + let props = as_map(controller_admin_client_properties(&internal_tls())); + assert_eq!(props.get("security.protocol"), Some(&"SSL".to_string())); + assert!(!props.contains_key("sasl.mechanism")); + assert!(!props.contains_key("sasl.jaas.config")); + assert_eq!( + props.get("ssl.keystore.location"), + Some(&"/stackable/tls-kafka-internal/keystore.p12".to_string()) + ); + } +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cargo test -p stackable-kafka-operator admin_client` +Expected: the two Kerberos tests FAIL — `security.protocol` is `SSL` and `sasl.*` keys are absent. + +- [ ] **Step 3: Implement the Kerberos branch** + +Replace `controller_admin_client_properties`: + +```rust +/// Client-side (unprefixed `security.protocol`/`ssl.*`/`sasl.*`) properties for an admin CLI +/// tool (e.g. `kafka-metadata-quorum.sh`) talking to the CONTROLLER listener from *inside* a +/// controller pod, over the `tls-kafka-internal` volume mounted by +/// `add_controller_volume_and_volume_mounts`. +/// +/// When Kerberos is enabled the CONTROLLER listener is `SASL_SSL` (see +/// `get_kafka_listener_config`), so these calls must authenticate with GSSAPI. They do so as +/// the controller's *own* pod principal, from the pod-scoped keytab mounted by +/// `add_kerberos_pod_config` — which is the correct identity for a voter registering itself. +/// +/// The principal contains `${env:…}` placeholders, so the rendered file must be passed +/// through `config-utils template` before use; see `quorum_manager_container_command`. +pub fn controller_admin_client_properties( + security: &ValidatedKafkaSecurity, +) -> Vec<(String, Option)> { + let mut properties = vec![]; + + if security.has_kerberos_enabled() { + properties.push(( + PROPERTY_SECURITY_PROTOCOL.to_string(), + Some(KafkaListenerProtocol::SaslSsl.to_string()), + )); + // Client-side mechanism selection. `sasl.enabled.mechanisms` is the *broker-side* + // list and has no effect here. + properties.push(( + PROPERTY_SASL_MECHANISM.to_string(), + Some(SASL_MECHANISM_GSSAPI.to_string()), + )); + properties.push(( + PROPERTY_SASL_KERBEROS_SERVICE_NAME.to_string(), + Some(KafkaRole::Controller.kerberos_service_name().to_string()), + )); + properties.push(( + PROPERTY_SASL_JAAS_CONFIG.to_string(), + Some(format!( + "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true \ + storeKey=true keyTab=\"{keytab}\" \ + principal=\"{service}/{pod_fqdn}@${{env:KERBEROS_REALM}}\";", + keytab = STACKABLE_KERBEROS_KEYTAB_PATH, + service = KafkaRole::Controller.kerberos_service_name(), + pod_fqdn = CONTROLLER_POD_FQDN_TEMPLATE, + )), + )); + } else { + properties.push(( + PROPERTY_SECURITY_PROTOCOL.to_string(), + Some(KafkaListenerProtocol::Ssl.to_string()), + )); + } + + push_client_ssl_stores(&mut properties, STACKABLE_TLS_KAFKA_INTERNAL_DIR); + + properties +} +``` + +Add the supporting constants next to the other `PROPERTY_*` constants in `security.rs`: + +```rust +const PROPERTY_SASL_MECHANISM: &str = "sasl.mechanism"; +const PROPERTY_SASL_JAAS_CONFIG: &str = "sasl.jaas.config"; +const STACKABLE_KERBEROS_KEYTAB_PATH: &str = "/stackable/kerberos/keytab"; + +/// The controller pod's own FQDN, as `config-utils template` placeholders. Matches the +/// address used for `KAFKA_LISTENERS` in `controller_properties.rs` and for the +/// `controller.KafkaServer` JAAS principal in `jaas_config_file`. +const CONTROLLER_POD_FQDN_TEMPLATE: &str = + "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}"; +``` + +- [ ] **Step 4: Check the properties writer does not mangle the value** + +The `controller.properties` consumer strips escaped colons (`sed 's/\\:/:/g'` in `extract_bootstrap_servers_command`), which means the properties writer escapes `:` in values. The JAAS value above contains no colon, so no unescaping step is needed — but confirm by inspecting the rendered ConfigMap in Task 7's kuttl run before trusting it. + +Run: `cargo test -p stackable-kafka-operator admin_client` +Expected: PASS + +- [ ] **Step 5: Run the full suite** + +Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` +Expected: no warnings, all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add rust/operator-binary/src/controller/build/security.rs +git commit -m "feat: authenticate the controller admin client with GSSAPI under Kerberos" +``` + +--- + +### Task 5: Un-gate dynamic quorum scaling under Kerberos + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/command.rs:230-280` (`quorum_manager_container_command`) +- Modify: `rust/operator-binary/src/controller/build/command.rs` (`controller_kafka_container_command` — template `admin-client.properties` for the `preStop` hook) +- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs:504-521` (remove the `preStop` gate) +- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs:748-805` (`build_quorum_manager_container`) +- Test: the `mod tests` blocks in `command.rs` and `statefulset.rs` + +**Interfaces:** + +- Consumes: Task 4's Kerberos-aware `controller_admin_client_properties`, Task 1's `kerberos` pod volume. +- Produces: `fn build_quorum_manager_container(…) -> Container` (no longer `Option`); the `ADMIN_CLIENT_PROPERTIES_PATH` constant moves to `/tmp/admin-client.properties`. + +- [ ] **Step 1: Write the failing tests** + +In `command.rs`: + +```rust + #[test] + fn quorum_manager_templates_the_admin_client_config() { + let command = quorum_manager_container_command(); + assert!(command.contains("cp /stackable/config/admin-client.properties /tmp/admin-client.properties")); + assert!(command.contains("config-utils template /tmp/admin-client.properties")); + // It must connect with the *rendered* copy, not the raw ConfigMap file, or the + // `${env:…}` placeholders in `sasl.jaas.config` reach the JAAS parser verbatim. + assert!(command.contains("ADMIN_CLIENT_CONFIG=/tmp/admin-client.properties")); + assert!(!command.contains("ADMIN_CLIENT_CONFIG=/stackable/config/admin-client.properties")); + } + + #[test] + fn quorum_manager_exports_the_kerberos_realm() { + // The sidecar is a separate container: it inherits nothing from the kafka + // container's startup, so it must derive $KERBEROS_REALM itself for + // `config-utils template` to resolve the principal. + let command = quorum_manager_container_command(); + assert!(command.contains("KERBEROS_REALM")); + } + + #[test] + fn controller_command_templates_the_admin_client_config_for_pre_stop() { + let command = controller_kafka_container_command(&kerberos(), vec![]); + assert!(command.contains("cp /stackable/config/admin-client.properties /tmp/admin-client.properties")); + assert!(command.contains("config-utils template /tmp/admin-client.properties")); + } +``` + +In `statefulset.rs`, this module already has `kraft_mode_cluster()`, `controller_containers(&cluster)` and `controller_kafka_container(&cluster)`. Add a Kerberos variant of the cluster fixture next to `kraft_mode_cluster()`: + +```rust + /// Like [`kraft_mode_cluster`], but referencing a Kerberos `AuthenticationClass`. + fn kraft_mode_kerberos_cluster() -> crate::controller::ValidatedCluster { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + authentication: + - authenticationClass: kerberos-auth + controllers: + roleGroups: + default: + replicas: 3 + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + validated_cluster(&kafka) + } +``` + +> `validated_cluster` must be able to resolve the `kerberos-auth` AuthenticationClass. If `test_support` cannot dereference AuthenticationClasses, extend it to accept a pre-resolved one rather than weakening the test — check `rust/operator-binary/src/controller/test_support.rs` first and adapt. + +Then the assertions: + +```rust + #[test] + fn quorum_manager_sidecar_is_present_with_kerberos() { + let containers = controller_containers(&kraft_mode_kerberos_cluster()); + let sidecar = containers + .iter() + .find(|c| c.name == QUORUM_MANAGER_CONTAINER_NAME.to_string()) + .expect("the quorum-manager sidecar must exist when Kerberos is enabled"); + let mounts: Vec<&str> = sidecar + .volume_mounts + .as_ref() + .expect("sidecar must have volume mounts") + .iter() + .map(|m| m.mount_path.as_str()) + .collect(); + assert!( + mounts.contains(&"/stackable/kerberos"), + "sidecar needs the keytab and krb5.conf to authenticate, got: {mounts:?}" + ); + let env: Vec<&str> = sidecar + .env + .as_ref() + .expect("sidecar must have env vars") + .iter() + .map(|e| e.name.as_str()) + .collect(); + assert!(env.contains(&"KRB5_CONFIG")); + // `KAFKA_OPTS` points the JVM at `/tmp/jaas.properties`, which only the `kafka` + // container renders. The sidecar uses an inline `sasl.jaas.config` instead. + assert!( + !env.contains(&"KAFKA_OPTS"), + "sidecar must not inherit the kafka container's JAAS login config" + ); + } + + #[test] + fn controller_pre_stop_hook_is_present_with_kerberos() { + let pre_stop_command = controller_kafka_container(&kraft_mode_kerberos_cluster()) + .lifecycle + .as_ref() + .and_then(|l| l.pre_stop.as_ref()) + .and_then(|h| h.exec.as_ref()) + .and_then(|e| e.command.as_ref()) + .expect("voter removal on scale-down must run under Kerberos too") + .join(" "); + assert!(pre_stop_command.contains("remove-controller")); + } +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cargo test -p stackable-kafka-operator` +Expected: `quorum_manager_sidecar_is_present_with_kerberos` FAILS (no such container — `build_quorum_manager_container` returns `None`); `controller_pre_stop_hook_is_present_with_kerberos` FAILS (hook skipped); the `command.rs` templating tests FAIL. + +- [ ] **Step 3: Template `admin-client.properties` in the sidecar** + +In `command.rs`, change the constant and extend the render chain: + +```rust +/// The rendered admin-client config. The raw ConfigMap file is copied here and passed +/// through `config-utils template` first, because under Kerberos its `sasl.jaas.config` +/// carries `${env:…}` placeholders (see `controller_admin_client_properties`). +const ADMIN_CLIENT_PROPERTIES_PATH: &str = "/tmp/admin-client.properties"; +const ADMIN_CLIENT_PROPERTIES_SOURCE_PATH: &str = "/stackable/config/admin-client.properties"; +``` + +In `quorum_manager_container_command`, add the realm export after `{extract_bootstrap_servers}`: + +```rust + {set_realm_env} +``` + +with + +```rust + // The sidecar is a separate container and inherits nothing from the kafka + // container's startup, so it derives the realm itself. Harmless when the + // krb5.conf is absent: `config-utils template` only needs it under Kerberos. + set_realm_env = format!( + "KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {STACKABLE_KERBEROS_KRB5_PATH} 2>/dev/null) && export KERBEROS_REALM || true" + ), +``` + +and extend the existing `if cp … && … ; then` chain to render the admin client config: + +```rust + if cp {config_dir}/{controller_properties_file} /tmp/{controller_properties_file} \ + && config-utils template /tmp/{controller_properties_file} \ + && cp {admin_client_source} {admin_client_config} \ + && config-utils template {admin_client_config} \ + && cat /tmp/{controller_properties_file} {admin_client_config} > {add_controller_config}; then +``` + +adding `admin_client_source = ADMIN_CLIENT_PROPERTIES_SOURCE_PATH,` to the format arguments. The existing degraded-mode `else` branch now also covers a failed Kerberos render, with no new error handling. + +- [ ] **Step 4: Template it in the kafka container too, for the `preStop` hook** + +The `preStop` hook runs in the `kafka` container and reads `$ADMIN_CLIENT_CONFIG`. Add to `controller_kafka_container_command`, immediately after the `jaas.properties` copy from Task 2: + +```rust + cp {admin_client_source} {admin_client_config} + config-utils template {admin_client_config} +``` + +with `admin_client_source = ADMIN_CLIENT_PROPERTIES_SOURCE_PATH,` and `admin_client_config = ADMIN_CLIENT_PROPERTIES_PATH,` added to the format arguments. + +- [ ] **Step 5: Remove the two Kerberos gates** + +In `statefulset.rs`, delete the `if !kafka_security.has_kerberos_enabled() {` wrapper and its stale comment around the `cb_kafka.lifecycle_pre_stop(…)` call, leaving the call unconditional. + +Then in `build_quorum_manager_container`, delete the early return and its comment, and change the return type: + +```rust +/// Builds the `quorum-manager` sidecar for a controller pod. +fn build_quorum_manager_container( + resolved_product_image: &ResolvedProductImage, + kafka_security: &ValidatedKafkaSecurity, + env: Vec, +) -> stackable_operator::k8s_openapi::api::core::v1::Container { +``` + +Mount the Kerberos material and set `KRB5_CONFIG` when Kerberos is on, just before `Some(cb.build())` becomes `cb.build()`: + +```rust + if kafka_security.has_kerberos_enabled() { + // `controller_admin_client_properties` authenticates with the pod-scoped keytab + // mounted by `add_kerberos_pod_config`, so this container needs it too — the + // volume itself is already on the pod. + cb.add_volume_mount(&*KERBEROS_VOLUME_NAME, STACKABLE_KERBEROS_DIR) + .expect("The mount paths are statically defined and there should be no duplicates."); + cb.add_env_var(KRB5_CONFIG.to_string(), STACKABLE_KERBEROS_KRB5_PATH); + } + + cb.build() +``` + +This needs `KERBEROS_VOLUME_NAME` and `KRB5_CONFIG` to be `pub` in `kerberos.rs` (they are currently private) and `STACKABLE_KERBEROS_DIR`/`STACKABLE_KERBEROS_KRB5_PATH` imported from `crate::crd`. + +Update the call site to drop the `if let Some(…)`: + +```rust + pod_builder.add_container(build_quorum_manager_container( + resolved_product_image, + kafka_security, + quorum_manager_env, + )); +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` +Expected: no warnings, all tests pass. + +- [ ] **Step 7: Commit** + +```bash +git add rust/operator-binary/src/controller/build/command.rs \ + rust/operator-binary/src/controller/build/kerberos.rs \ + rust/operator-binary/src/controller/build/resource/statefulset.rs +git commit -m "feat: keep dynamic KRaft quorum scaling working with Kerberos enabled" +``` + +--- + +### Task 6: Fix the discovery ConfigMap client properties + +`client_properties` feeds the discovery ConfigMap, consumed by clients running *outside* Kafka pods. Those clients have no `/stackable/kerberos/keytab` and no per-pod principal, so three of its current entries are wrong for that consumer. + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/security.rs:161-218` (`client_properties`) +- Test: the `mod tests` block in `security.rs` + +**Interfaces:** + +- Consumes: nothing new. +- Produces: `client_properties` signature unchanged; output loses three keys and gains `sasl.mechanism`. + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn discovery_client_properties_carry_no_server_side_or_pod_local_settings() { + let props = as_map(client_properties(&kerberos())); + + // The consumer runs outside Kafka pods: it has no keytab and no pod principal, so a + // `sasl.jaas.config` here could only ever be wrong. Clients supply their own. + assert!(!props.contains_key("sasl.jaas.config")); + // Broker-side properties with no meaning in a client config. + assert!(!props.contains_key("sasl.mechanism.inter.broker.protocol")); + assert!(!props.contains_key("sasl.enabled.mechanisms")); + + // What a client actually needs. + assert_eq!(props.get("security.protocol"), Some(&"SASL_SSL".to_string())); + assert_eq!(props.get("sasl.mechanism"), Some(&"GSSAPI".to_string())); + assert_eq!( + props.get("sasl.kerberos.service.name"), + Some(&"kafka".to_string()) + ); + assert_eq!( + props.get("ssl.truststore.location"), + Some(&"/stackable/tls-kafka-server/truststore.p12".to_string()) + ); + } +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cargo test -p stackable-kafka-operator discovery_client_properties` +Expected: FAIL — `sasl.jaas.config` is present (with the `kafka/todo@…` placeholder principal). + +- [ ] **Step 3: Rewrite the Kerberos branch** + +Replace the `else if security.has_kerberos_enabled() {` arm of `client_properties` with: + +```rust + } else if security.has_kerberos_enabled() { + props.push(( + PROPERTY_SECURITY_PROTOCOL.to_string(), + Some(KafkaListenerProtocol::SaslSsl.to_string()), + )); + push_client_ssl_stores(&mut props, STACKABLE_TLS_KAFKA_SERVER_DIR); + // `sasl.mechanism` is the client-side selector. `sasl.enabled.mechanisms` is the + // broker-side list of accepted mechanisms and has no effect in a client config. + props.push(( + PROPERTY_SASL_MECHANISM.to_string(), + Some(SASL_MECHANISM_GSSAPI.to_string()), + )); + props.push(( + PROPERTY_SASL_KERBEROS_SERVICE_NAME.to_string(), + Some(KafkaRole::Broker.kerberos_service_name().to_string()), + )); + // Deliberately no `sasl.jaas.config`: this file is consumed by clients running + // outside Kafka pods, which have neither the keytab at /stackable/kerberos/keytab + // nor a per-pod principal. They supply their own login configuration; see + // docs/modules/kafka/pages/usage-guide/security.adoc. +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cargo test -p stackable-kafka-operator discovery_client_properties` +Expected: PASS + +- [ ] **Step 5: Run the full suite and fix any stale expectations** + +Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` +Expected: no warnings. Existing tests asserting the removed keys must be updated to assert their absence, not deleted silently. + +- [ ] **Step 6: Commit** + +```bash +git add rust/operator-binary/src/controller/build/security.rs +git commit -m "fix: remove pod-local and broker-side settings from the discovery client properties" +``` + +--- + +### Task 7: kuttl integration test + +This is the regression test for Task 5. PR #999 predates the quorum manager, so its suite must be extended with scale steps. + +**Files:** + +- Create: `tests/templates/kuttl/kraft-kerberos/` (copied from the reference worktree, then extended) +- Modify: `tests/test-definition.yaml` + +**Interfaces:** + +- Consumes: all preceding tasks. +- Produces: a `kraft-kerberos` kuttl suite registered as a test dimension. + +- [ ] **Step 1: Copy the reference suite** + +```bash +cp -r /tmp/pr999/tests/templates/kuttl/kraft-kerberos tests/templates/kuttl/kraft-kerberos +ls tests/templates/kuttl/kraft-kerberos +``` + +- [ ] **Step 2: Register the test dimension** + +In `tests/test-definition.yaml`, add a `kraft-kerberos` entry to `tests:`, mirroring the existing `kerberos` entry's dimensions (`kafka-latest`, `kerberos-realm`, `kerberos-backend`, `openshift`). Copy the shape from `/tmp/pr999/tests/test-definition.yaml`, adapting names to whatever `main` currently uses — `main` has since changed this file. + +- [ ] **Step 3: Run the suite as copied, to establish a baseline** + +Run: + +```bash +./scripts/run-tests --test-suite kraft-kerberos +``` + +Expected: the 3-controller quorum forms, produce/consume succeeds. If it fails, fix before extending — an already-red suite cannot validate Step 4. + +- [ ] **Step 4: Add controller scale-up and scale-down steps** + +Copy the scale steps from `tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2`, `60-assert.yaml.j2`, `70-scale-controller-down.yaml.j2` and `70-assert.yaml.j2` into the `kraft-kerberos` suite as steps `60-*` and `70-*`, adapting the KafkaCluster name and namespace to this suite's. + +The assertions must confirm the *quorum* changed, not just the StatefulSet replica count — a controller that starts but never joins the voter set is exactly the failure this guards against. Reuse `operations-kraft`'s existing `kafka-metadata-quorum describe` assertion verbatim. + +- [ ] **Step 5: Run the extended suite** + +Run: + +```bash +./scripts/run-tests --test-suite kraft-kerberos +``` + +Expected: PASS, including the scale steps. + +- [ ] **Step 6: Inspect the rendered admin client config (Task 4, Step 4 follow-up)** + +While the cluster is up: + +```bash +kubectl exec -n "$NAMESPACE" test-kafka-controller-default-0 -c quorum-manager -- cat /tmp/admin-client.properties +``` + +Expected: `sasl.jaas.config` is one line, with the placeholders resolved to a real pod FQDN and realm, and no stray backslash escapes. + +- [ ] **Step 7: Commit** + +```bash +git add tests/templates/kuttl/kraft-kerberos tests/test-definition.yaml +git commit -m "test: add a kraft-kerberos kuttl suite covering quorum scaling" +``` + +--- + +### Task 8: Documentation and changelog + +**Files:** + +- Modify: `docs/modules/kafka/pages/usage-guide/kraft-controller.adoc` +- Modify: `docs/modules/kafka/pages/usage-guide/security.adoc` +- Modify: `CHANGELOG.md` + +**Interfaces:** + +- Consumes: the behaviour built in Tasks 1-7. + +- [ ] **Step 1: Document Kerberized controllers** + +In `kraft-controller.adoc`, add a Kerberos section covering: the `CONTROLLER` listener uses `SASL_SSL` with GSSAPI when an `AuthenticationClass` with the Kerberos provider is referenced; controller keytabs are pod-scoped (controllers are reached by pod DNS name, not through a `Listener`) while broker keytabs are listener-scoped; dynamic quorum scaling is supported with Kerberos enabled. Use "Apache Kafka" in prose. Consult `/tmp/pr999/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc` for the reference wording, but do **not** carry over any statement that scaling is unsupported under Kerberos — Task 5 makes that false. + +- [ ] **Step 2: Document the client-side Kerberos requirement** + +In `security.adoc`, note that the discovery ConfigMap's `client.properties` carries `security.protocol`, `sasl.mechanism`, `sasl.kerberos.service.name` and the truststore settings, and that clients must supply their own principal and keytab (their own JAAS login configuration) — the operator cannot do so, as the file is consumed outside Kafka pods. + +- [ ] **Step 3: Add the changelog entry** + +Under `## [Unreleased]` → `### Added` in `CHANGELOG.md`: + +```markdown +- Support Kerberos authentication on KRaft controllers, covering both broker-to-controller + and controller-to-controller (Raft) traffic. Dynamic quorum scaling continues to work with + Kerberos enabled ([#999], [#815]). +``` + +and under `### Fixed`: + +```markdown +- Remove the pod-local `sasl.jaas.config` and the broker-side `sasl.enabled.mechanisms` and + `sasl.mechanism.inter.broker.protocol` settings from the discovery ConfigMap's client + properties; they were never usable by out-of-cluster clients ([#999]). +``` + +Add the link definitions at the bottom of the file in the existing style. + +- [ ] **Step 4: Verify the docs build** + +Run: `./scripts/docs_templating.sh && ./scripts/render_readme.sh` +Expected: no errors, no unexpected diff. + +- [ ] **Step 5: Commit** + +```bash +git add docs CHANGELOG.md +git commit -m "docs: document Kerberos support for KRaft controllers" +``` + +- [ ] **Step 6: Clean up the reference worktree** + +```bash +git worktree remove /tmp/pr999 +``` + +--- + +## Verification + +Before opening the PR: + +- [ ] `cargo clippy --all-targets -- -D warnings` — clean +- [ ] `cargo test -p stackable-kafka-operator` — all pass +- [ ] `./scripts/run-tests --test-suite kraft-kerberos` — passes including scale steps +- [ ] `./scripts/run-tests --test-suite smoke-kraft` — no regression with Kerberos off +- [ ] `./scripts/run-tests --test-suite operations-kraft` — no regression in quorum scaling with Kerberos off +- [ ] `./scripts/run-tests --test-suite kerberos` — no regression in broker Kerberos From 982be9962945320c75cf44d17d6565fe189c55ed Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:41:10 +0200 Subject: [PATCH 03/13] feat: mount a pod-scoped Kerberos keytab on KRaft controller pods Controllers have no listener-operator Listener volume: they are only reachable through their own StatefulSet pod DNS name, so their keytab must be pod-scoped. Brokers keep listener-volume scoping. `add_kerberos_pod_config` was previously only called from the broker StatefulSet builder, so controller pods had no keytab at all. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/controller/build/kerberos.rs | 106 ++++++++++++++++-- .../controller/build/resource/statefulset.rs | 9 ++ 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index cc2f1d91..1caca2d3 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -57,16 +57,31 @@ pub fn add_kerberos_pod_config( ) -> Result<(), Error> { if let Some(kerberos_secret_class) = kafka_security.kerberos_secret_class() { // Mount keytab - let kerberos_secret_operator_volume = SecretOperatorVolumeSourceBuilder::new( + let mut volume_builder = SecretOperatorVolumeSourceBuilder::new( kerberos_secret_class, // We need both public (krb5.conf) and private (keytab) parts. SecretClassVolumeProvisionParts::PublicPrivate, - ) - .with_listener_volume_scope(&*LISTENER_BROKER_VOLUME_NAME) - .with_listener_volume_scope(&*LISTENER_BOOTSTRAP_VOLUME_NAME) - .with_kerberos_service_name(role.kerberos_service_name()) - .build() - .context(KerberosSecretVolumeSnafu)?; + ); + match role { + // Brokers are exposed through listener-operator `Listener` volumes (the broker + // and bootstrap listeners), so the keytab principal must cover both. + KafkaRole::Broker => { + volume_builder + .with_listener_volume_scope(&*LISTENER_BROKER_VOLUME_NAME) + .with_listener_volume_scope(&*LISTENER_BOOTSTRAP_VOLUME_NAME); + } + // KRaft controllers have no listener-operator `Listener` volume: they are only + // reachable through their own StatefulSet pod DNS name, so the keytab must be + // pod-scoped, matching how the controller's internal TLS cert is provisioned in + // `add_controller_volume_and_volume_mounts`. + KafkaRole::Controller => { + volume_builder.with_pod_scope(); + } + } + let kerberos_secret_operator_volume = volume_builder + .with_kerberos_service_name(role.kerberos_service_name()) + .build() + .context(KerberosSecretVolumeSnafu)?; pb.add_volume( VolumeBuilder::new(&*KERBEROS_VOLUME_NAME) .ephemeral(kerberos_secret_operator_volume) @@ -106,7 +121,84 @@ pub fn kerberos_env_vars(kafka_security: &ValidatedKafkaSecurity) -> EnvVarSet { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + + use stackable_operator::builder::pod::container::ContainerBuilder; + use super::*; + use crate::controller::build::security::tests::kerberos; + + /// Reads the `secrets.stackable.tech/*` annotations off the `kerberos` ephemeral volume. + fn kerberos_volume_annotations(pb: &mut PodBuilder) -> BTreeMap { + pb.build_template() + .spec + .as_ref() + .and_then(|spec| spec.volumes.as_ref()) + .and_then(|volumes| { + volumes + .iter() + .find(|v| v.name == KERBEROS_VOLUME_NAME.to_string()) + }) + .expect("kerberos volume must be present") + .ephemeral + .as_ref() + .expect("kerberos volume must be an ephemeral secret-operator volume") + .volume_claim_template + .as_ref() + .and_then(|t| t.metadata.as_ref()) + .and_then(|m| m.annotations.clone()) + .expect("volume claim template must carry secrets.stackable.tech annotations") + } + + fn kerberos_volume_annotations_for(role: &KafkaRole) -> BTreeMap { + let mut pb = PodBuilder::new(); + let mut cb_kafka = ContainerBuilder::new("kafka").expect("valid container name"); + + add_kerberos_pod_config(&kerberos(), role, &mut cb_kafka, &mut pb) + .expect("kerberos pod config"); + + kerberos_volume_annotations(&mut pb) + } + + #[test] + fn controller_keytab_is_pod_scoped() { + let annotations = kerberos_volume_annotations_for(&KafkaRole::Controller); + + // Controllers have no listener-operator Listener volume, so the keytab must be + // scoped to the pod's own DNS name, matching how their internal TLS cert is + // provisioned in `add_controller_volume_and_volume_mounts`. + assert_eq!( + annotations + .get("secrets.stackable.tech/scope") + .map(String::as_str), + Some("pod"), + "controller keytab must be pod-scoped, got: {annotations:?}" + ); + assert_eq!( + annotations + .get("secrets.stackable.tech/kerberos.service.names") + .map(String::as_str), + Some("kafka") + ); + } + + #[test] + fn broker_keytab_stays_listener_scoped() { + let annotations = kerberos_volume_annotations_for(&KafkaRole::Broker); + + let scope = annotations + .get("secrets.stackable.tech/scope") + .expect("scope annotation must be present"); + assert!( + scope.contains("listener-volume=listener-broker") + && scope.contains("listener-volume=listener-bootstrap"), + "broker keytab must stay listener-volume-scoped, got: {scope}" + ); + assert!( + !scope.split(',').any(|s| s == "pod"), + "broker keytab must not be pod-scoped, got: {scope}" + ); + } #[test] fn test_constants() { diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 401cffdd..b0229ba4 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -427,6 +427,11 @@ pub fn build_controller_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); + if kafka_security.has_kerberos_enabled() { + add_kerberos_pod_config(kafka_security, kafka_role, &mut cb_kafka, &mut pod_builder) + .context(AddKerberosConfigSnafu)?; + } + let node_id_offset = node_id_hash32_offset(kafka_role, role_group_name.as_ref()).to_string(); // Operator-set env vars first (common + controller-specific); the user's `envOverrides` @@ -447,6 +452,10 @@ pub fn build_controller_rolegroup_statefulset( kafka_role, role_group_name, )?) + // Kerberos env goes on the `kafka` container only. `controller_shared_env` is also + // the `quorum-manager` sidecar's base, and `KAFKA_OPTS` points the JVM at + // `/tmp/jaas.properties`, which only the `kafka` container renders. + .merge(kerberos_env_vars(kafka_security)) .merge(validated_rg.env_overrides.clone()) .into(); From c456526b373c35baea9f4b1848a6bf66200e4d0d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:43:21 +0200 Subject: [PATCH 04/13] feat: add a controller.KafkaServer JAAS section for KRaft controllers Brokers reuse their broker-listener principal for the CONTROLLER section; controllers use their own pod FQDN, matching the pod-scoped keytab. The section deliberately leaves `isInitiator` at its default because controllers connect to each other for Raft. The controller startup command now exports $KERBEROS_REALM and templates jaas.properties, mirroring `broker_start_command`. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/controller/build/command.rs | 37 ++++++- .../controller/build/resource/config_map.rs | 97 +++++++++++++++++-- .../controller/build/resource/statefulset.rs | 1 + .../src/controller/build/security.rs | 2 +- 4 files changed, 127 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index dbf609b9..edc6ad80 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -142,6 +142,7 @@ fn controller_quorum_format_flag(controller_descriptors: &[KafkaPodDescriptor]) } pub fn controller_kafka_container_command( + kafka_security: &ValidatedKafkaSecurity, controller_descriptors: Vec, ) -> String { formatdoc! {" @@ -149,6 +150,7 @@ pub fn controller_kafka_container_command( {remove_vector_shutdown_file_command} prepare_signal_handlers containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + {set_realm_env} {derive_pod_index} {export_replica_id} @@ -157,6 +159,9 @@ pub fn controller_kafka_container_command( config-utils template /tmp/{properties_file} + cp {config_dir}/{jaas_file} /tmp/{jaas_file} + config-utils template /tmp/{jaas_file} + {quorum_format_flag} bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted \"$FORMAT_QUORUM_FLAG\" bin/kafka-server-start.sh /tmp/{properties_file} & @@ -165,10 +170,16 @@ pub fn controller_kafka_container_command( {create_vector_shutdown_file_command} ", remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + // Mirrors `broker_kafka_container_commands`: empty when Kerberos is disabled. + set_realm_env = match kafka_security.has_kerberos_enabled() { + true => format!("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {STACKABLE_KERBEROS_KRB5_PATH})"), + false => "".to_string(), + }, derive_pod_index = DERIVE_POD_INDEX, export_replica_id = EXPORT_REPLICA_ID, config_dir = STACKABLE_CONFIG_DIR, properties_file = ConfigFileName::ControllerProperties, + jaas_file = ConfigFileName::Jaas, quorum_format_flag = controller_quorum_format_flag(&controller_descriptors), create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR) } @@ -378,6 +389,30 @@ mod tests { use indoc::indoc; use super::*; + use crate::controller::build::security::tests::{kerberos, plaintext}; + + #[test] + fn controller_command_exports_the_kerberos_realm_when_enabled() { + let command = controller_kafka_container_command(&kerberos(), vec![]); + assert!(command.contains("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*'")); + } + + #[test] + fn controller_command_does_not_export_a_realm_without_kerberos() { + let command = controller_kafka_container_command(&plaintext(), vec![]); + assert!(!command.contains("KERBEROS_REALM")); + } + + #[test] + fn controller_command_always_templates_the_jaas_file() { + // `jaas.properties` is always present in the ConfigMap (empty when Kerberos is off), + // so the copy is unconditional, matching `broker_start_command`. + for security in [kerberos(), plaintext()] { + let command = controller_kafka_container_command(&security, vec![]); + assert!(command.contains("cp /stackable/config/jaas.properties /tmp/jaas.properties")); + assert!(command.contains("config-utils template /tmp/jaas.properties")); + } + } #[test] fn quorum_manager_container_command_targets_the_bootstrap_servers_not_localhost() { @@ -1119,7 +1154,7 @@ mod tests { pod_descriptor(KafkaRole::Controller, 1, 6), pod_descriptor(KafkaRole::Controller, 2, 7), ]; - let command = controller_kafka_container_command(descriptors); + let command = controller_kafka_container_command(&plaintext(), descriptors); assert!(command.contains(r#"if [ "$REPLICA_ID" = "5" ]; then"#)); assert!(command.contains("FORMAT_QUORUM_FLAG=--standalone")); diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 49c4e261..fd6ec48d 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -24,7 +24,7 @@ use crate::{ crd::{ STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, listener::{KafkaListenerConfig, node_address_cmd}, - role::AnyConfig, + role::{AnyConfig, KafkaRole}, }, }; @@ -177,7 +177,7 @@ pub fn build_rolegroup_config_map( // and this tool currently doesn't support the JAAS login configuration format. .add_data( ConfigFileName::Jaas.to_string(), - jaas_config_file(kafka_security.has_kerberos_enabled()), + jaas_config_file(kafka_security.has_kerberos_enabled(), &role), ); // `admin-client.properties` is only needed by the controller-side sidecar running @@ -223,10 +223,43 @@ pub fn build_rolegroup_config_map( // Generate JAAS configuration file for Kerberos authentication // or an empty string if Kerberos is not enabled. // See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html -fn jaas_config_file(is_kerberos_enabled: bool) -> String { - match is_kerberos_enabled { - false => String::new(), - true => formatdoc! {" +fn jaas_config_file(is_kerberos_enabled: bool, role: &KafkaRole) -> String { + if !is_kerberos_enabled { + return String::new(); + } + + // Broker pods reach the CONTROLLER listener as SASL clients; the only principals in + // their keytab (see `add_kerberos_pod_config`) are for the broker and bootstrap listener + // addresses, so their CONTROLLER section must reuse the broker address. + // Controller pods have no listener-operator `Listener` volume; their keytab is + // pod-scoped, so their CONTROLLER section uses their own pod FQDN — the same expression + // already used for `KAFKA_LISTENERS` in `controller_properties.rs`. + let controller_principal_address = match role { + KafkaRole::Broker => node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), + KafkaRole::Controller => { + "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}" + .to_string() + } + }; + + // Unlike the bootstrap and client sections below, this context is used for BOTH sides of + // every CONTROLLER-listener connection: brokers connect out to controllers, and + // controllers connect to each other for Raft. This is the only listener in this operator + // where the process must act as a GSSAPI initiator as well as an acceptor, so + // `isInitiator` is intentionally left at its default (`true`). + let controller_section = formatdoc! {" + controller.KafkaServer {{ + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + keyTab=\"/stackable/kerberos/keytab\" + principal=\"kafka/{controller_principal_address}@${{env:KERBEROS_REALM}}\"; + }}; + "}; + + match role { + KafkaRole::Controller => controller_section, + KafkaRole::Broker => formatdoc! {" bootstrap.KafkaServer {{ com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true @@ -245,6 +278,7 @@ fn jaas_config_file(is_kerberos_enabled: bool) -> String { principal=\"kafka/{broker_address}@${{env:KERBEROS_REALM}}\"; }}; + {controller_section} ", bootstrap_address = node_address_cmd(STACKABLE_LISTENER_BOOTSTRAP_DIR), broker_address = node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), @@ -255,15 +289,19 @@ fn jaas_config_file(is_kerberos_enabled: bool) -> String { #[cfg(test)] mod tests { use super::jaas_config_file; + use crate::crd::role::KafkaRole; + + const CONTROLLER_POD_FQDN: &str = "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}"; #[test] fn jaas_config_file_empty_without_kerberos() { - assert_eq!(jaas_config_file(false), ""); + assert_eq!(jaas_config_file(false, &KafkaRole::Broker), ""); + assert_eq!(jaas_config_file(false, &KafkaRole::Controller), ""); } #[test] fn jaas_config_file_renders_bootstrap_and_client_sections_with_kerberos() { - let jaas = jaas_config_file(true); + let jaas = jaas_config_file(true, &KafkaRole::Broker); assert!(jaas.contains("bootstrap.KafkaServer")); assert!(jaas.contains("client.KafkaServer")); assert!(jaas.contains("Krb5LoginModule")); @@ -272,4 +310,47 @@ mod tests { assert!(jaas.contains("/stackable/listener-bootstrap")); assert!(jaas.contains("/stackable/listener-broker")); } + + #[test] + fn broker_controller_section_uses_the_broker_listener_address() { + let jaas = jaas_config_file(true, &KafkaRole::Broker); + assert!(jaas.contains("controller.KafkaServer {")); + // Brokers connect *out* to controllers. The only principals in a broker's keytab are + // for its own listener addresses, so this section must reuse the broker address. + assert!(jaas.contains( + "kafka/${file:UTF-8:/stackable/listener-broker/default-address/address}@${env:KERBEROS_REALM}" + )); + } + + #[test] + fn controller_jaas_has_only_the_controller_section_with_a_pod_fqdn_principal() { + let jaas = jaas_config_file(true, &KafkaRole::Controller); + assert!(jaas.contains("controller.KafkaServer {")); + assert!(jaas.contains(&format!( + "kafka/{CONTROLLER_POD_FQDN}@${{env:KERBEROS_REALM}}" + ))); + // Controllers have no listener-operator Listener volume, so the broker-only + // sections must not appear in their JAAS file. + assert!(!jaas.contains("bootstrap.KafkaServer")); + assert!(!jaas.contains("client.KafkaServer")); + } + + #[test] + fn controller_section_allows_the_process_to_act_as_a_gssapi_initiator() { + for role in [KafkaRole::Broker, KafkaRole::Controller] { + let jaas = jaas_config_file(true, &role); + let start = jaas + .find("controller.KafkaServer {") + .expect("controller.KafkaServer section must be present"); + // Unlike the other sections, this context is used for BOTH sides of every + // CONTROLLER-listener connection: brokers connect out to controllers, and + // controllers connect to each other for Raft. So `isInitiator` must stay at its + // default (`true`). Scoped to this section so a broker-side `isInitiator=false` + // elsewhere stays fine. + assert!( + !jaas[start..].contains("isInitiator=false"), + "controller.KafkaServer for {role:?} must not disable GSSAPI initiation" + ); + } + } } diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index b0229ba4..fa9286cd 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -493,6 +493,7 @@ pub fn build_controller_rolegroup_statefulset( "-c".to_string(), ]) .args(vec![controller_kafka_container_command( + kafka_security, controller_pod_descriptors, )]); diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index b9eb650a..83cfc043 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -719,7 +719,7 @@ pub(crate) mod tests { } /// Plaintext: no TLS, no authentication, no OPA. - fn plaintext() -> ValidatedKafkaSecurity { + pub(crate) fn plaintext() -> ValidatedKafkaSecurity { ValidatedKafkaSecurity::new( no_auth(), SecretClassName::from_str("tls").expect("tls secret class name is valid"), From d083b170164a62784e1d3256c72f1202cb5bb357 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:44:53 +0200 Subject: [PATCH 05/13] feat: use SASL_SSL on the CONTROLLER listener when Kerberos is enabled Also sets sasl.mechanism.controller.protocol=GSSAPI on both broker and controller properties, and corrects the now-false CONTROLLER listener doc comment claiming SASL is unsupported. The controller admin client still assumes plain SSL and is repaired in the following commit; these two must land together. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- .../controller/build/properties/listener.rs | 63 ++++++++++++++++++- .../src/controller/build/security.rs | 39 ++++++++++++ rust/operator-binary/src/crd/listener.rs | 7 +-- 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/listener.rs b/rust/operator-binary/src/controller/build/properties/listener.rs index 3ba733bf..5bedcb68 100644 --- a/rust/operator-binary/src/controller/build/properties/listener.rs +++ b/rust/operator-binary/src/controller/build/properties/listener.rs @@ -108,8 +108,14 @@ pub fn get_kafka_listener_config( port: kafka_security.internal_port().to_string(), }); listener_security_protocol_map.insert(KafkaListenerName::Internal, KafkaListenerProtocol::Ssl); - listener_security_protocol_map - .insert(KafkaListenerName::Controller, KafkaListenerProtocol::Ssl); + listener_security_protocol_map.insert( + KafkaListenerName::Controller, + if kafka_security.has_kerberos_enabled() { + KafkaListenerProtocol::SaslSsl + } else { + KafkaListenerProtocol::Ssl + }, + ); // BOOTSTRAP if kafka_security.has_kerberos_enabled() { @@ -492,8 +498,59 @@ mod tests { bootstrap_name = KafkaListenerName::Bootstrap, bootstrap_protocol = KafkaListenerProtocol::SaslSsl, controller_name = KafkaListenerName::Controller, - controller_protocol = KafkaListenerProtocol::Ssl, + controller_protocol = KafkaListenerProtocol::SaslSsl, ) ); } + + #[test] + fn controller_listener_stays_ssl_without_kerberos() { + // Regression guard: only Kerberos may move CONTROLLER off plain SSL. + let kafka_cluster = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + controllers: + roleGroups: + default: + replicas: 3 + brokers: + roleGroups: + default: + replicas: 1 + "#; + let kafka = minimal_kafka(kafka_cluster); + let validated = validated_cluster(&kafka); + let kafka_security = ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![]), + "internal-tls".parse().unwrap(), + Some("tls".parse().unwrap()), + None, + ); + let role_group_name: RoleGroupName = "default".parse().unwrap(); + let config = get_kafka_listener_config( + &validated, + &kafka_security, + &KafkaRole::Controller, + &role_group_name, + ); + + assert!( + config.listener_security_protocol_map().contains(&format!( + "{name}:{protocol}", + name = KafkaListenerName::Controller, + protocol = KafkaListenerProtocol::Ssl + )), + "got: {}", + config.listener_security_protocol_map() + ); + } } diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index 83cfc043..33fc2030 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -51,6 +51,7 @@ const PROPERTY_SECURITY_PROTOCOL: &str = "security.protocol"; const PROPERTY_SASL_ENABLED_MECHANISMS: &str = "sasl.enabled.mechanisms"; const PROPERTY_SASL_KERBEROS_SERVICE_NAME: &str = "sasl.kerberos.service.name"; const PROPERTY_SASL_INTER_BROKER_MECHANISM: &str = "sasl.mechanism.inter.broker.protocol"; +const PROPERTY_SASL_CONTROLLER_MECHANISM: &str = "sasl.mechanism.controller.protocol"; pub(crate) const STACKABLE_TLS_KAFKA_INTERNAL_DIR: &str = "/stackable/tls-kafka-internal"; constant!(pub(crate) STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME: VolumeName = "tls-kafka-internal"); const STACKABLE_TLS_KAFKA_SERVER_DIR: &str = "/stackable/tls-kafka-server"; @@ -473,6 +474,10 @@ pub fn broker_config_settings(security: &ValidatedKafkaSecurity) -> BTreeMap BTreeMap PROPERTY_SASL_INTER_BROKER_MECHANISM.to_string(), SASL_MECHANISM_GSSAPI.to_string(), ); + config.insert( + PROPERTY_SASL_CONTROLLER_MECHANISM.to_string(), + SASL_MECHANISM_GSSAPI.to_string(), + ); tracing::debug!("Kerberos configs added: [{:#?}]", config); } @@ -1067,6 +1076,36 @@ pub(crate) mod tests { assert!(config.contains_key("listener.name.internal.ssl.keystore.location")); } + #[test] + fn broker_config_sets_the_controller_sasl_mechanism_with_kerberos() { + let config = broker_config_settings(&kerberos()); + assert_eq!( + config.get("sasl.mechanism.controller.protocol"), + Some(&"GSSAPI".to_string()) + ); + } + + #[test] + fn controller_config_sets_the_controller_sasl_mechanism_with_kerberos() { + let config = controller_config_settings(&kerberos()); + assert_eq!( + config.get("sasl.mechanism.controller.protocol"), + Some(&"GSSAPI".to_string()) + ); + } + + #[test] + fn controller_sasl_mechanism_is_absent_without_kerberos() { + assert!( + !broker_config_settings(&internal_tls()) + .contains_key("sasl.mechanism.controller.protocol") + ); + assert!( + !controller_config_settings(&internal_tls()) + .contains_key("sasl.mechanism.controller.protocol") + ); + } + #[test] fn controller_config_kerberos_adds_sasl() { let config = controller_config_settings(&kerberos()); diff --git a/rust/operator-binary/src/crd/listener.rs b/rust/operator-binary/src/crd/listener.rs index 7aabadad..8e014d14 100644 --- a/rust/operator-binary/src/crd/listener.rs +++ b/rust/operator-binary/src/crd/listener.rs @@ -58,13 +58,10 @@ pub enum KafkaListenerName { /// This listener is defined when Kraft mode is enabled. /// It is responsible for broker/controller as well as controller/controller communications /// and therefore it is present on *both* brokers and controller properties files. - /// The only protocol used is SSL. + /// The protocol used is SSL, or SASL_SSL when Kerberos is enabled. /// The advertised host names are FQDN pod names of the controllers. /// - /// Notes: - /// - /// - there is no listener for client/controller communication - /// - this listener does not support SSL_SASL. + /// Note: there is no listener for client/controller communication. #[strum(serialize = "CONTROLLER")] Controller, } From cd68319a877aac35a3af4460bc286c4560bfd9b5 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:46:13 +0200 Subject: [PATCH 06/13] feat: authenticate the controller admin client with GSSAPI under Kerberos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `controller_admin_client_properties` ignored its security argument and hardcoded SSL, so the quorum-manager sidecar and the preStop hook could not reach the now-SASL_SSL CONTROLLER listener. It authenticates as the controller's own pod principal, from the pod-scoped keytab — the correct identity for a voter registering itself. The principal uses config-utils placeholders, so the rendered file must be templated before use (wired up next). Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/controller/build/security.rs | 122 +++++++++++++++++- 1 file changed, 117 insertions(+), 5 deletions(-) diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index 33fc2030..745b91ae 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -52,6 +52,14 @@ const PROPERTY_SASL_ENABLED_MECHANISMS: &str = "sasl.enabled.mechanisms"; const PROPERTY_SASL_KERBEROS_SERVICE_NAME: &str = "sasl.kerberos.service.name"; const PROPERTY_SASL_INTER_BROKER_MECHANISM: &str = "sasl.mechanism.inter.broker.protocol"; const PROPERTY_SASL_CONTROLLER_MECHANISM: &str = "sasl.mechanism.controller.protocol"; +const PROPERTY_SASL_MECHANISM: &str = "sasl.mechanism"; +const PROPERTY_SASL_JAAS_CONFIG: &str = "sasl.jaas.config"; +const STACKABLE_KERBEROS_KEYTAB_PATH: &str = "/stackable/kerberos/keytab"; + +/// The controller pod's own FQDN, as `config-utils template` placeholders. Matches the address +/// used for `KAFKA_LISTENERS` in `controller_properties.rs` and for the `controller.KafkaServer` +/// JAAS principal in `jaas_config_file`. +const CONTROLLER_POD_FQDN_TEMPLATE: &str = "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}"; pub(crate) const STACKABLE_TLS_KAFKA_INTERNAL_DIR: &str = "/stackable/tls-kafka-internal"; constant!(pub(crate) STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME: VolumeName = "tls-kafka-internal"); const STACKABLE_TLS_KAFKA_SERVER_DIR: &str = "/stackable/tls-kafka-server"; @@ -223,15 +231,55 @@ pub fn client_properties(security: &ValidatedKafkaSecurity) -> Vec<(String, Opti /// (e.g. `kafka-metadata-quorum.sh`) talking to the CONTROLLER listener from *inside* a /// controller pod, over the `tls-kafka-internal` volume mounted by /// `add_controller_volume_and_volume_mounts`. +/// When Kerberos is enabled the CONTROLLER listener is `SASL_SSL` (see +/// [`get_kafka_listener_config`][glc]), so these calls must authenticate with GSSAPI. They do +/// so as the controller's *own* pod principal, from the pod-scoped keytab mounted by +/// [`add_kerberos_pod_config`][akpc] — the correct identity for a voter registering itself. +/// +/// The principal contains `${env:…}` placeholders, so the rendered file must be passed +/// through `config-utils template` before use; see [`quorum_manager_container_command`][qmcc]. +/// +/// [glc]: crate::controller::build::properties::listener::get_kafka_listener_config +/// [akpc]: crate::controller::build::kerberos::add_kerberos_pod_config +/// [qmcc]: crate::controller::build::command::quorum_manager_container_command pub fn controller_admin_client_properties( - _security: &ValidatedKafkaSecurity, + security: &ValidatedKafkaSecurity, ) -> Vec<(String, Option)> { let mut properties = vec![]; - properties.push(( - PROPERTY_SECURITY_PROTOCOL.to_string(), - Some(KafkaListenerProtocol::Ssl.to_string()), - )); + if security.has_kerberos_enabled() { + properties.push(( + PROPERTY_SECURITY_PROTOCOL.to_string(), + Some(KafkaListenerProtocol::SaslSsl.to_string()), + )); + // Client-side mechanism selection. `sasl.enabled.mechanisms` is the *broker-side* + // list of accepted mechanisms and has no effect here. + properties.push(( + PROPERTY_SASL_MECHANISM.to_string(), + Some(SASL_MECHANISM_GSSAPI.to_string()), + )); + properties.push(( + PROPERTY_SASL_KERBEROS_SERVICE_NAME.to_string(), + Some(KafkaRole::Controller.kerberos_service_name().to_string()), + )); + properties.push(( + PROPERTY_SASL_JAAS_CONFIG.to_string(), + Some(format!( + "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true \ + storeKey=true keyTab=\"{keytab}\" \ + principal=\"{service}/{pod_fqdn}@${{env:KERBEROS_REALM}}\";", + keytab = STACKABLE_KERBEROS_KEYTAB_PATH, + service = KafkaRole::Controller.kerberos_service_name(), + pod_fqdn = CONTROLLER_POD_FQDN_TEMPLATE, + )), + )); + } else { + properties.push(( + PROPERTY_SECURITY_PROTOCOL.to_string(), + Some(KafkaListenerProtocol::Ssl.to_string()), + )); + } + push_client_ssl_stores(&mut properties, STACKABLE_TLS_KAFKA_INTERNAL_DIR); properties @@ -933,6 +981,70 @@ pub(crate) mod tests { // ---- controller_admin_client_properties ---- + #[test] + fn admin_client_uses_gssapi_over_sasl_ssl_with_kerberos() { + let props = as_map(controller_admin_client_properties(&kerberos())); + assert_eq!( + props.get("security.protocol"), + Some(&Some("SASL_SSL".to_string())) + ); + assert_eq!( + props.get("sasl.mechanism"), + Some(&Some("GSSAPI".to_string())) + ); + assert_eq!( + props.get("sasl.kerberos.service.name"), + Some(&Some("kafka".to_string())) + ); + // The internal TLS stores stay: SASL_SSL is still SSL underneath. + assert_eq!( + props.get("ssl.truststore.location"), + Some(&Some( + "/stackable/tls-kafka-internal/truststore.p12".to_string() + )) + ); + } + + #[test] + fn admin_client_jaas_config_is_a_single_line_pod_principal() { + let props = as_map(controller_admin_client_properties(&kerberos())); + let jaas = props + .get("sasl.jaas.config") + .and_then(|v| v.as_ref()) + .expect("sasl.jaas.config must be set when Kerberos is enabled"); + // Must be one logical line: a raw newline would truncate the value when the + // properties file is parsed. + assert!( + !jaas.contains('\n'), + "sasl.jaas.config must be a single line, got: {jaas}" + ); + assert!(jaas.contains("com.sun.security.auth.module.Krb5LoginModule required")); + assert!(jaas.contains("keyTab=\"/stackable/kerberos/keytab\"")); + // The controller's own pod-scoped principal, resolved by `config-utils template` + // at container start. + assert!(jaas.contains( + "principal=\"kafka/${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}@${env:KERBEROS_REALM}\"" + )); + assert!(jaas.trim_end().ends_with(';')); + } + + #[test] + fn admin_client_is_unchanged_without_kerberos() { + let props = as_map(controller_admin_client_properties(&internal_tls())); + assert_eq!( + props.get("security.protocol"), + Some(&Some("SSL".to_string())) + ); + assert!(!props.contains_key("sasl.mechanism")); + assert!(!props.contains_key("sasl.jaas.config")); + assert_eq!( + props.get("ssl.keystore.location"), + Some(&Some( + "/stackable/tls-kafka-internal/keystore.p12".to_string() + )) + ); + } + #[test] fn controller_admin_client_properties_uses_the_internal_tls_directory() { let security = server_tls(); From 5c359de962fd681212883f5aad3be3383bbc0c0c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:50:10 +0200 Subject: [PATCH 07/13] feat: keep dynamic KRaft quorum scaling working with Kerberos enabled The quorum-manager sidecar and the controller preStop hook were both skipped whenever Kerberos was on, because admin-client.properties only covered the TLS/SSL case. Now that it carries GSSAPI settings, remove both gates. admin-client.properties is rendered through `config-utils template` before use in the sidecar and in the kafka container (for preStop), since its principal carries ${env:...} placeholders. The sidecar mounts the keytab and derives $KERBEROS_REALM itself; it deliberately does not get KAFKA_OPTS, which points at a jaas file only the kafka container renders. test_support gains `validated_cluster_with_auth` so fixtures can reference an AuthenticationClass. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- rust/operator-binary/src/controller.rs | 21 ++- .../src/controller/build/command.rs | 65 ++++++- .../src/controller/build/kerberos.rs | 4 +- .../controller/build/resource/statefulset.rs | 169 ++++++++++++++---- .../src/controller/build/security.rs | 34 ++++ 5 files changed, 252 insertions(+), 41 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 21e1d8de..4b7b0fbf 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -618,11 +618,30 @@ pub(crate) mod test_support { /// the result. Used for tests asserting on a specific validation failure. pub fn validate_err( kafka: &v1alpha1::KafkaCluster, + ) -> Result { + validate_with_auth_err(kafka, ResolvedAuthenticationClasses::new(Vec::new())) + } + + /// Like [`validated_cluster`], but with the given already-resolved `AuthenticationClass`es, + /// for fixtures whose `spec.clusterConfig.authentication` references one (e.g. Kerberos). + pub fn validated_cluster_with_auth( + kafka: &v1alpha1::KafkaCluster, + authentication_classes: ResolvedAuthenticationClasses, + ) -> ValidatedCluster { + validate_with_auth_err(kafka, authentication_classes) + .expect("validate should succeed for the test fixture") + } + + /// The shared body of [`validate_err`] and [`validated_cluster_with_auth`]: the real validate + /// step, parameterized on the resolved `AuthenticationClass`es. + pub fn validate_with_auth_err( + kafka: &v1alpha1::KafkaCluster, + authentication_classes: ResolvedAuthenticationClasses, ) -> Result { validate( kafka, DereferencedObjects { - authentication_classes: ResolvedAuthenticationClasses::new(Vec::new()), + authentication_classes, authorization_config: None, kubernetes_cluster_info: cluster_info(), bootstrap_listeners: Vec::new(), diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index edc6ad80..e8989d39 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -162,6 +162,9 @@ pub fn controller_kafka_container_command( cp {config_dir}/{jaas_file} /tmp/{jaas_file} config-utils template /tmp/{jaas_file} + cp {admin_client_source} {admin_client_config} + config-utils template {admin_client_config} + {quorum_format_flag} bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted \"$FORMAT_QUORUM_FLAG\" bin/kafka-server-start.sh /tmp/{properties_file} & @@ -180,6 +183,8 @@ pub fn controller_kafka_container_command( config_dir = STACKABLE_CONFIG_DIR, properties_file = ConfigFileName::ControllerProperties, jaas_file = ConfigFileName::Jaas, + admin_client_source = ADMIN_CLIENT_PROPERTIES_SOURCE_PATH, + admin_client_config = ADMIN_CLIENT_PROPERTIES_PATH, quorum_format_flag = controller_quorum_format_flag(&controller_descriptors), create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR) } @@ -187,7 +192,11 @@ pub fn controller_kafka_container_command( const KAFKA_METADATA_QUORUM_BINARY: &str = "/stackable/kafka/bin/kafka-metadata-quorum.sh"; -const ADMIN_CLIENT_PROPERTIES_PATH: &str = "/stackable/config/admin-client.properties"; +/// The rendered admin-client config. The raw ConfigMap file is copied here and passed through +/// `config-utils template` first, because under Kerberos its `sasl.jaas.config` carries +/// `${env:...}` placeholders (see `controller_admin_client_properties`). +const ADMIN_CLIENT_PROPERTIES_PATH: &str = "/tmp/admin-client.properties"; +const ADMIN_CLIENT_PROPERTIES_SOURCE_PATH: &str = "/stackable/config/admin-client.properties"; /// The merged config used only for `add-controller` (self-registration). /// @@ -241,6 +250,12 @@ const CONTROLLER_QUORUM_MANAGER_LOOP_SCRIPT: &str = /// The sidecar's main-loop command: while this controller's local Raft state is `observer`, /// admit it into the quorum's voter set once that is safe. pub fn quorum_manager_container_command() -> String { + // The sidecar is a separate container and inherits nothing from the kafka container's + // startup, so it derives the realm itself. Harmless when krb5.conf is absent: only the + // Kerberos case has a `${env:KERBEROS_REALM}` placeholder for `config-utils` to resolve. + let set_realm_env = format!( + "KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {STACKABLE_KERBEROS_KRB5_PATH} 2>/dev/null) && export KERBEROS_REALM || true" + ); format!( r#" set -uo pipefail @@ -249,9 +264,12 @@ pub fn quorum_manager_container_command() -> String { [ -n "$POD_INDEX" ] || exit 0 {export_replica_id} {extract_bootstrap_servers} + {set_realm_env} if cp {config_dir}/{controller_properties_file} /tmp/{controller_properties_file} \ && config-utils template /tmp/{controller_properties_file} \ + && cp {admin_client_source} {admin_client_config} \ + && config-utils template {admin_client_config} \ && cat /tmp/{controller_properties_file} {admin_client_config} > {add_controller_config}; then QUORUM_CLI={binary} ADMIN_CLIENT_CONFIG={admin_client_config} @@ -277,8 +295,10 @@ pub fn quorum_manager_container_command() -> String { derive_pod_index = DERIVE_POD_INDEX, export_replica_id = EXPORT_REPLICA_ID, extract_bootstrap_servers = extract_bootstrap_servers_command(), + set_realm_env = set_realm_env, config_dir = STACKABLE_CONFIG_DIR, controller_properties_file = ConfigFileName::ControllerProperties, + admin_client_source = ADMIN_CLIENT_PROPERTIES_SOURCE_PATH, admin_client_config = ADMIN_CLIENT_PROPERTIES_PATH, add_controller_config = ADD_CONTROLLER_PROPERTIES_PATH, cli_timeout = CLI_CALL_TIMEOUT_SECONDS, @@ -414,6 +434,41 @@ mod tests { } } + #[test] + fn quorum_manager_templates_the_admin_client_config() { + let command = quorum_manager_container_command(); + assert!( + command.contains( + "cp /stackable/config/admin-client.properties /tmp/admin-client.properties" + ) + ); + assert!(command.contains("config-utils template /tmp/admin-client.properties")); + // It must connect with the *rendered* copy, not the raw ConfigMap file, or the + // `${env:...}` placeholders in `sasl.jaas.config` reach the JAAS parser verbatim. + assert!(command.contains("ADMIN_CLIENT_CONFIG=/tmp/admin-client.properties")); + assert!(!command.contains("ADMIN_CLIENT_CONFIG=/stackable/config/admin-client.properties")); + } + + #[test] + fn quorum_manager_exports_the_kerberos_realm() { + // The sidecar is a separate container: it inherits nothing from the kafka container's + // startup, so it must derive $KERBEROS_REALM itself for `config-utils template` to + // resolve the principal. + let command = quorum_manager_container_command(); + assert!(command.contains("KERBEROS_REALM")); + } + + #[test] + fn controller_command_templates_the_admin_client_config_for_pre_stop() { + let command = controller_kafka_container_command(&kerberos(), vec![]); + assert!( + command.contains( + "cp /stackable/config/admin-client.properties /tmp/admin-client.properties" + ) + ); + assert!(command.contains("config-utils template /tmp/admin-client.properties")); + } + #[test] fn quorum_manager_container_command_targets_the_bootstrap_servers_not_localhost() { let command = quorum_manager_container_command(); @@ -481,11 +536,11 @@ mod tests { // `listeners`) via the same REPLICA_ID derivation used by the `kafka` container. assert!(command.contains("export REPLICA_ID=$((POD_INDEX + NODE_ID_OFFSET))")); assert!(command.contains("config-utils template /tmp/controller.properties")); - // Merges it with the plain admin-client config (carries `security.protocol`/`ssl.*`), - // controller.properties first so the client TLS config in admin-client.properties - // wins on any key collision (see `ADD_CONTROLLER_PROPERTIES_PATH`'s doc comment). + // Merges it with the *rendered* admin-client config (carries `security.protocol`, + // `ssl.*` and, under Kerberos, `sasl.jaas.config`), controller.properties first so + // the client config wins on any key collision (see `ADD_CONTROLLER_PROPERTIES_PATH`). assert!(command.contains( - "cat /tmp/controller.properties /stackable/config/admin-client.properties > /tmp/add-controller.properties" + "cat /tmp/controller.properties /tmp/admin-client.properties > /tmp/add-controller.properties" )); // The merged file is what `add-controller` — and only `add-controller` — connects // with; read-only `describe` calls keep using the plain admin-client config. diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index 1caca2d3..00bd290e 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -29,7 +29,7 @@ use crate::{ }, }; -constant!(KERBEROS_VOLUME_NAME: VolumeName = "kerberos"); +constant!(pub KERBEROS_VOLUME_NAME: VolumeName = "kerberos"); #[derive(Snafu, Debug)] pub enum Error { @@ -97,7 +97,7 @@ pub fn add_kerberos_pod_config( Ok(()) } -constant!(KRB5_CONFIG: EnvVarName = "KRB5_CONFIG"); +constant!(pub KRB5_CONFIG: EnvVarName = "KRB5_CONFIG"); constant!(KAFKA_OPTS: EnvVarName = "KAFKA_OPTS"); /// The environment variables the Kerberos configuration requires on the Kafka container, or an diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index fa9286cd..73f6d31d 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -50,7 +50,9 @@ use crate::{ kafka_log_opts, quorum_manager_container_command, }, graceful_shutdown::add_graceful_shutdown_config, - kerberos::{add_kerberos_pod_config, kerberos_env_vars}, + kerberos::{ + KERBEROS_VOLUME_NAME, KRB5_CONFIG, add_kerberos_pod_config, kerberos_env_vars, + }, properties::product_logging::MAX_KAFKA_LOG_FILES_SIZE, recommended_labels_for_role_group_resources, recommended_labels_for_unversioned_role_group_resources, role_group_selector, @@ -67,8 +69,9 @@ use crate::{ BROKER_ID_POD_MAP_DIR, BROKER_ID_POD_MAP_DIR_NAME, KAFKA_HEAP_OPTS, LISTENER_BOOTSTRAP_VOLUME_NAME, LISTENER_BROKER_VOLUME_NAME, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, - STACKABLE_DATA_DIR, STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, - STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_CONFIG_DIR_NAME, STACKABLE_LOG_DIR_NAME, + STACKABLE_DATA_DIR, STACKABLE_KERBEROS_DIR, STACKABLE_KERBEROS_KRB5_PATH, + STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, STACKABLE_LOG_CONFIG_DIR, + STACKABLE_LOG_CONFIG_DIR_NAME, STACKABLE_LOG_DIR_NAME, role::{ AnyConfig, KAFKA_NODE_ID_OFFSET, KafkaRole, broker::BrokerContainer, controller::ControllerContainer, @@ -512,23 +515,16 @@ pub fn build_controller_rolegroup_statefulset( .startup_probe(controller_startup_probe) .liveness_probe(controller_liveness_probe) .readiness_probe(controller_readiness_probe); - // Skipped when Kerberos is enabled, matching `build_quorum_manager_container`'s own - // gating — `admin-client.properties` (the file this removal call relies on) only covers - // the TLS/SSL case. - if !kafka_security.has_kerberos_enabled() { - cb_kafka.lifecycle_pre_stop(LifecycleHandler { - exec: Some(ExecAction { - command: Some(vec![ - "/bin/bash".to_string(), - "-c".to_string(), - controller_remove_self_pre_stop_command( - merged_config.graceful_shutdown_timeout, - ), - ]), - }), - ..LifecycleHandler::default() - }); - } + cb_kafka.lifecycle_pre_stop(LifecycleHandler { + exec: Some(ExecAction { + command: Some(vec![ + "/bin/bash".to_string(), + "-c".to_string(), + controller_remove_self_pre_stop_command(merged_config.graceful_shutdown_timeout), + ]), + }), + ..LifecycleHandler::default() + }); add_log_config_volume( &mut pod_builder, @@ -561,11 +557,11 @@ pub fn build_controller_rolegroup_statefulset( .add_container(kafka_container) .affinity(&merged_config.affinity); - if let Some(quorum_manager_container) = - build_quorum_manager_container(resolved_product_image, kafka_security, quorum_manager_env) - { - pod_builder.add_container(quorum_manager_container); - } + pod_builder.add_container(build_quorum_manager_container( + resolved_product_image, + kafka_security, + quorum_manager_env, + )); add_common_pod_config( &mut pod_builder, @@ -755,17 +751,12 @@ fn add_common_pod_config( // Name of the controller's `quorum-manager` sidecar container. stackable_operator::constant!(QUORUM_MANAGER_CONTAINER_NAME: ContainerName = "quorum-manager"); -/// Builds the `quorum-manager` sidecar for a controller pod. Returns `None` when Kerberos is -/// enabled (the sidecar's admin-client properties file only covers the TLS/SSL case). +/// Builds the `quorum-manager` sidecar for a controller pod. fn build_quorum_manager_container( resolved_product_image: &ResolvedProductImage, kafka_security: &ValidatedKafkaSecurity, env: Vec, -) -> Option { - if kafka_security.has_kerberos_enabled() { - return None; - } - +) -> stackable_operator::k8s_openapi::api::core::v1::Container { let mut cb = new_container_builder(&QUORUM_MANAGER_CONTAINER_NAME); cb.image_from_product_image(resolved_product_image) @@ -804,7 +795,18 @@ fn build_quorum_manager_container( .add_volume_mount(&*LOG_DIRS_VOLUME_NAME, STACKABLE_DATA_DIR) .expect("The mount paths are statically defined and there should be no duplicates."); - Some(cb.build()) + if kafka_security.has_kerberos_enabled() { + // `controller_admin_client_properties` authenticates with the pod-scoped keytab + // mounted by `add_kerberos_pod_config`; the volume is already on the pod, this + // container just needs its own mount and `KRB5_CONFIG`. It deliberately does *not* + // get `KAFKA_OPTS`: that points the JVM at `/tmp/jaas.properties`, which only the + // `kafka` container renders. + cb.add_volume_mount(&*KERBEROS_VOLUME_NAME, STACKABLE_KERBEROS_DIR) + .expect("The mount paths are statically defined and there should be no duplicates."); + cb.add_env_var(KRB5_CONFIG.to_string(), STACKABLE_KERBEROS_KRB5_PATH); + } + + cb.build() } /// Adds the Vector log-aggregation sidecar container, when the Vector agent is enabled. @@ -955,6 +957,107 @@ mod tests { validated_cluster(&kafka) } + /// Like [`kraft_mode_cluster`], but referencing a Kerberos `AuthenticationClass`. + fn kraft_mode_kerberos_cluster() -> crate::controller::ValidatedCluster { + use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + crd::authentication::{core, kerberos}, + }; + + use crate::{ + controller::test_support::validated_cluster_with_auth, + crd::authentication::ResolvedAuthenticationClasses, + }; + + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + authentication: + - authenticationClass: kerberos-auth + controllers: + roleGroups: + default: + replicas: 3 + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + validated_cluster_with_auth( + &kafka, + ResolvedAuthenticationClasses::new(vec![core::v1alpha1::AuthenticationClass { + metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), + spec: core::v1alpha1::AuthenticationClassSpec { + provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( + kerberos::v1alpha1::AuthenticationProvider { + kerberos_secret_class: "kerberos-secret-class".to_string(), + }, + ), + }, + }]), + ) + } + + #[test] + fn quorum_manager_sidecar_is_present_with_kerberos() { + let containers = controller_containers(&kraft_mode_kerberos_cluster()); + let sidecar = containers + .iter() + .find(|c| c.name == QUORUM_MANAGER_CONTAINER_NAME.to_string()) + .expect("the quorum-manager sidecar must exist when Kerberos is enabled"); + + let mounts: Vec<&str> = sidecar + .volume_mounts + .as_ref() + .expect("sidecar must have volume mounts") + .iter() + .map(|m| m.mount_path.as_str()) + .collect(); + assert!( + mounts.contains(&"/stackable/kerberos"), + "sidecar needs the keytab and krb5.conf to authenticate, got: {mounts:?}" + ); + + let env: Vec<&str> = sidecar + .env + .as_ref() + .expect("sidecar must have env vars") + .iter() + .map(|e| e.name.as_str()) + .collect(); + assert!(env.contains(&"KRB5_CONFIG")); + // `KAFKA_OPTS` points the JVM at `/tmp/jaas.properties`, which only the `kafka` + // container renders. The sidecar uses an inline `sasl.jaas.config` instead. + assert!( + !env.contains(&"KAFKA_OPTS"), + "sidecar must not inherit the kafka container's JAAS login config" + ); + } + + #[test] + fn controller_pre_stop_hook_is_present_with_kerberos() { + let pre_stop_command = controller_kafka_container(&kraft_mode_kerberos_cluster()) + .lifecycle + .as_ref() + .and_then(|l| l.pre_stop.as_ref()) + .and_then(|h| h.exec.as_ref()) + .and_then(|e| e.command.as_ref()) + .expect("voter removal on scale-down must run under Kerberos too") + .join(" "); + assert!(pre_stop_command.contains("remove-controller")); + } + #[test] fn statefulsets_use_ordered_ready_pod_management_for_controllers_only() { let cluster = kraft_mode_cluster(); diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index 745b91ae..cd81d7bf 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -981,6 +981,40 @@ pub(crate) mod tests { // ---- controller_admin_client_properties ---- + /// Renders the admin-client properties exactly as `build_rolegroup_config_map` does, so we + /// see what the Java properties writer actually puts on disk (it escapes `:` as `\:`, which + /// `config-utils` and the AdminClient must still be able to read back). + #[test] + fn admin_client_rendered_file_keeps_the_jaas_config_on_one_line() { + use stackable_operator::v2::config_file_writer::to_java_properties_string; + + let rendered = to_java_properties_string( + controller_admin_client_properties(&kerberos()) + .iter() + .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))), + ) + .expect("admin-client properties serialize"); + + let jaas_line = rendered + .lines() + .find(|l| l.starts_with("sasl.jaas.config")) + .expect("sasl.jaas.config must be present"); + assert!( + jaas_line.trim_end().ends_with(';'), + "the whole login module config must fit on one line, got: {jaas_line}" + ); + assert!(jaas_line.contains("Krb5LoginModule")); + // The writer escapes ` `, `=` and `:`, so the placeholders land as `${env\:NAME}`. + // Java's `Properties.load` unescapes all three on read, and `config-utils` already + // resolves this escaped form (`controller.properties` relies on it — see + // `extract_env_placeholders` in `statefulset.rs`), so the AdminClient ends up with + // the intended single-line value. + assert!( + jaas_line.contains("${env\\:POD_NAME}"), + "expected the escaped placeholder form, got: {jaas_line}" + ); + } + #[test] fn admin_client_uses_gssapi_over_sasl_ssl_with_kerberos() { let props = as_map(controller_admin_client_properties(&kerberos())); From c3ee82f0710e7b1574529b1ff1145ba7954829d1 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:51:44 +0200 Subject: [PATCH 08/13] fix: remove pod-local and broker-side settings from the discovery client properties The discovery ConfigMap's client.properties is consumed by clients running outside Kafka pods. It carried a sasl.jaas.config whose principal was the literal placeholder `kafka/todo@$KERBEROS_REALM`, which such a client could never use: it has neither the keytab at /stackable/kerberos/keytab nor a per-pod principal. Drop the entry rather than invent a principal, and document that clients bring their own login configuration. Also drops the broker-side sasl.mechanism.inter.broker.protocol and replaces sasl.enabled.mechanisms (broker-side) with the client-side sasl.mechanism. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/controller/build/security.rs | 64 +++++++++++++------ 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index cd81d7bf..eaa3fbaf 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -182,35 +182,25 @@ pub fn client_properties(security: &ValidatedKafkaSecurity) -> Vec<(String, Opti )); push_client_ssl_stores(&mut props, STACKABLE_TLS_KAFKA_SERVER_DIR); } else if security.has_kerberos_enabled() { - // TODO: to make this configuration file usable out of the box the operator needs to be - // refactored to write out Java jaas files instead of passing command line parameters - // to the Kafka daemon scripts. - // This will simplify the code and the command lines lot. - // It will also make the jaas files reusable by the Kafka shell scripts. props.push(( PROPERTY_SECURITY_PROTOCOL.to_string(), Some(KafkaListenerProtocol::SaslSsl.to_string()), )); push_client_ssl_stores(&mut props, STACKABLE_TLS_KAFKA_SERVER_DIR); + // `sasl.mechanism` is the client-side selector. `sasl.enabled.mechanisms` is the + // broker-side list of accepted mechanisms and has no effect in a client config. props.push(( - PROPERTY_SASL_ENABLED_MECHANISMS.to_string(), + PROPERTY_SASL_MECHANISM.to_string(), Some(SASL_MECHANISM_GSSAPI.to_string()), )); props.push(( PROPERTY_SASL_KERBEROS_SERVICE_NAME.to_string(), Some(KafkaRole::Broker.kerberos_service_name().to_string()), )); - props.push(( - PROPERTY_SASL_INTER_BROKER_MECHANISM.to_string(), - Some(SASL_MECHANISM_GSSAPI.to_string()), - )); - props.push(( - "sasl.jaas.config".to_string(), - Some(format!("com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab=\"{keytab}\" principal=\"{service}/{pod}@{realm}\"", - keytab="/stackable/kerberos/keytab", - service=KafkaRole::Broker.kerberos_service_name(), - pod="todo", - realm="$KERBEROS_REALM")))); + // Deliberately no `sasl.jaas.config`: this file is consumed by clients running + // outside Kafka pods, which have neither the keytab at /stackable/kerberos/keytab nor + // a per-pod principal, so any value here would be wrong. They supply their own login + // configuration; see docs/modules/kafka/pages/usage-guide/security.adoc. } else if security.tls_server_secret_class().is_some() { props.push(( PROPERTY_SECURITY_PROTOCOL.to_string(), @@ -968,15 +958,51 @@ pub(crate) mod tests { props.get("security.protocol"), Some(&Some("SASL_SSL".to_string())) ); + // `sasl.mechanism`, not the broker-side `sasl.enabled.mechanisms`; and no + // `sasl.jaas.config`, which this out-of-pod consumer cannot use. See + // `discovery_client_properties_carry_no_server_side_or_pod_local_settings`. assert_eq!( - props.get("sasl.enabled.mechanisms"), + props.get("sasl.mechanism"), Some(&Some("GSSAPI".to_string())) ); + assert!(!props.contains_key("sasl.enabled.mechanisms")); assert_eq!( props.get("sasl.kerberos.service.name"), Some(&Some("kafka".to_string())) ); - assert!(props.contains_key("sasl.jaas.config")); + assert!(!props.contains_key("sasl.jaas.config")); + } + + #[test] + fn discovery_client_properties_carry_no_server_side_or_pod_local_settings() { + let props = as_map(client_properties(&kerberos())); + + // The consumer runs outside Kafka pods: it has no keytab and no pod principal, so a + // `sasl.jaas.config` here could only ever be wrong. Clients supply their own. + assert!(!props.contains_key("sasl.jaas.config")); + // Broker-side properties with no meaning in a client config. + assert!(!props.contains_key("sasl.mechanism.inter.broker.protocol")); + assert!(!props.contains_key("sasl.enabled.mechanisms")); + + // What a client actually needs. + assert_eq!( + props.get("security.protocol"), + Some(&Some("SASL_SSL".to_string())) + ); + assert_eq!( + props.get("sasl.mechanism"), + Some(&Some("GSSAPI".to_string())) + ); + assert_eq!( + props.get("sasl.kerberos.service.name"), + Some(&Some("kafka".to_string())) + ); + assert_eq!( + props.get("ssl.truststore.location"), + Some(&Some( + "/stackable/tls-kafka-server/truststore.p12".to_string() + )) + ); } // ---- controller_admin_client_properties ---- From a16c651a4749f2a14d074b369059c11fdcadd4b0 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:53:34 +0200 Subject: [PATCH 09/13] test: add a kraft-kerberos kuttl suite covering quorum scaling Ports the suite from the draft PR (MIT KDC, 3-controller quorum, produce/consume) and extends it with controller scale-up and scale-down steps. Those steps are the regression test for un-gating the quorum manager under Kerberos, which the original suite predates. The scale asserts check the Raft voter set, not just the StatefulSet replica count, and point the admin client at the rendered /tmp/admin-client.properties. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- .../kuttl/kraft-kerberos/00-assert.yaml.j2 | 10 ++ ...tor-aggregator-discovery-configmap.yaml.j2 | 9 ++ .../kuttl/kraft-kerberos/00-patch-ns.yaml.j2 | 9 ++ .../kuttl/kraft-kerberos/00-rbac.yaml.j2 | 29 ++++ .../kuttl/kraft-kerberos/01-assert.yaml.j2 | 14 ++ .../01-install-krb5-kdc.yaml.j2 | 146 ++++++++++++++++++ .../02-create-kerberos-secretclass.yaml.j2 | 72 +++++++++ .../kuttl/kraft-kerberos/20-assert.yaml | 20 +++ .../kraft-kerberos/20-install-kafka.yaml.j2 | 65 ++++++++ .../kraft-kerberos/30-access-kafka.txt.j2 | 131 ++++++++++++++++ .../kuttl/kraft-kerberos/30-access-kafka.yaml | 6 + .../kuttl/kraft-kerberos/30-assert.yaml | 11 ++ .../kuttl/kraft-kerberos/60-assert.yaml.j2 | 39 +++++ .../60-scale-controller-up.yaml.j2 | 63 ++++++++ .../kuttl/kraft-kerberos/70-assert.yaml.j2 | 39 +++++ .../70-scale-controller-down.yaml.j2 | 63 ++++++++ .../templates/kuttl/kraft-kerberos/README.md | 9 ++ tests/test-definition.yaml | 8 + 18 files changed, 743 insertions(+) create mode 100644 tests/templates/kuttl/kraft-kerberos/00-assert.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/00-rbac.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/01-assert.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/01-install-krb5-kdc.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/02-create-kerberos-secretclass.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/20-assert.yaml create mode 100644 tests/templates/kuttl/kraft-kerberos/20-install-kafka.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/30-access-kafka.txt.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/30-access-kafka.yaml create mode 100644 tests/templates/kuttl/kraft-kerberos/30-assert.yaml create mode 100644 tests/templates/kuttl/kraft-kerberos/60-assert.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/60-scale-controller-up.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/70-assert.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/70-scale-controller-down.yaml.j2 create mode 100644 tests/templates/kuttl/kraft-kerberos/README.md diff --git a/tests/templates/kuttl/kraft-kerberos/00-assert.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/kraft-kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/kraft-kerberos/00-patch-ns.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/kraft-kerberos/00-rbac.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/00-rbac.yaml.j2 new file mode 100644 index 00000000..7ee61d23 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/00-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-rb +subjects: + - kind: ServiceAccount + name: test-sa +roleRef: + kind: Role + name: test-role + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/kraft-kerberos/01-assert.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/01-assert.yaml.j2 new file mode 100644 index 00000000..d34c1c63 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/01-assert.yaml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +{% if test_scenario['values']['kerberos-backend'] == 'mit' %} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +status: + readyReplicas: 1 + replicas: 1 +{% endif %} diff --git a/tests/templates/kuttl/kraft-kerberos/01-install-krb5-kdc.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/01-install-krb5-kdc.yaml.j2 new file mode 100644 index 00000000..69ceec81 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/01-install-krb5-kdc.yaml.j2 @@ -0,0 +1,146 @@ +{% if test_scenario['values']['kerberos-backend'] == 'mit' %} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +spec: + selector: + matchLabels: + app: krb5-kdc + template: + metadata: + labels: + app: krb5-kdc + spec: + serviceAccountName: test-sa + initContainers: + - name: init + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + args: + - sh + - -euo + - pipefail + - -c + - | + test -e /var/kerberos/krb5kdc/principal || kdb5_util create -s -P asdf + kadmin.local get_principal -terse root/admin || kadmin.local add_principal -pw asdf root/admin + # stackable-secret-operator principal must match the keytab specified in the SecretClass + kadmin.local get_principal -terse stackable-secret-operator || kadmin.local add_principal -e aes256-cts-hmac-sha384-192:normal -pw asdf stackable-secret-operator + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data + containers: + - name: kdc + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + args: + - krb5kdc + - -n + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data +# Root permissions required on Openshift to bind to privileged port numbers +{% if test_scenario['values']['openshift'] == "true" %} + securityContext: + runAsUser: 0 +{% endif %} + - name: kadmind + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + args: + - kadmind + - -nofork + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data +# Root permissions required on Openshift to bind to privileged port numbers +{% if test_scenario['values']['openshift'] == "true" %} + securityContext: + runAsUser: 0 +{% endif %} + - name: client + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + tty: true + stdin: true + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + volumes: + - name: config + configMap: + name: krb5-kdc + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: krb5-kdc +spec: + selector: + app: krb5-kdc + ports: + - name: kadmin + port: 749 + - name: kdc + port: 88 + - name: kdc-udp + port: 88 + protocol: UDP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: krb5-kdc +data: + krb5.conf: | + [logging] + default = STDERR + kdc = STDERR + admin_server = STDERR + # default = FILE:/var/log/krb5libs.log + # kdc = FILE:/var/log/krb5kdc.log + # admin_server = FILE:/vaggr/log/kadmind.log + [libdefaults] + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + default_realm = {{ test_scenario['values']['kerberos-realm'] }} + spake_preauth_groups = edwards25519 + [realms] + {{ test_scenario['values']['kerberos-realm'] }} = { + acl_file = /stackable/config/kadm5.acl + disable_encrypted_timestamp = false + } + [domain_realm] + .cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + kadm5.acl: | + root/admin *e + stackable-secret-operator *e +{% endif %} diff --git a/tests/templates/kuttl/kraft-kerberos/02-create-kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/02-create-kerberos-secretclass.yaml.j2 new file mode 100644 index 00000000..04ae9a63 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/02-create-kerberos-secretclass.yaml.j2 @@ -0,0 +1,72 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + # KRaft: metadata is managed by the controllers role, no ZooKeeper involved. + metadataManager: kraft + authentication: + - authenticationClass: kerberos-auth-$NAMESPACE + tls: + # Kerberos requires the use of server and internal TLS! + serverSecretClass: tls +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + controllers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + # 3 controller replicas so that this test actually exercises inter-controller + # (Raft) Kerberos-authenticated traffic on the CONTROLLER listener, not just + # broker-to-controller traffic. + replicas: 3 + brokers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + brokerListenerClass: {{ test_scenario['values']['broker-listener-class'] }} + # bootstrap-listener-class is orthogonal to this test's focus on Kerberos over the + # CONTROLLER listener (that axis is already covered by the plain `kerberos` test + # case), so it is pinned here rather than parameterized as a test dimension. + bootstrapListenerClass: cluster-internal + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 3 + EOF diff --git a/tests/templates/kuttl/kraft-kerberos/30-access-kafka.txt.j2 b/tests/templates/kuttl/kraft-kerberos/30-access-kafka.txt.j2 new file mode 100644 index 00000000..50a31864 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/30-access-kafka.txt.j2 @@ -0,0 +1,131 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: access-kafka +spec: + template: + spec: + serviceAccountName: test-sa + containers: + - name: access-kafka +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + image: {{ test_scenario['values']['kafka-kraft'].split(',')[1] }} +{% else %} + image: oci.stackable.tech/sdp/kafka:{{ test_scenario['values']['kafka-kraft'] }}-stackable0.0.0-dev +{% endif %} + command: + - /bin/bash + - /tmp/script/script.sh + env: + - name: KRB5_CONFIG + value: /stackable/kerberos/krb5.conf + - name: KAFKA_OPTS + value: -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf + - name: KAFKA + valueFrom: + configMapKeyRef: + name: test-kafka + key: KAFKA + volumeMounts: + - name: script + mountPath: /tmp/script + - mountPath: /stackable/tls-ca-cert-mount + name: tls-ca-cert-mount + - name: kerberos + mountPath: /stackable/kerberos + volumes: + - name: script + configMap: + name: access-kafka-script + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-$NAMESPACE + secrets.stackable.tech/scope: service=access-kafka + secrets.stackable.tech/kerberos.service.names: developer + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + - name: tls-ca-cert-mount + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMode: Filesystem + securityContext: + fsGroup: 1000 + restartPolicy: OnFailure +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: access-kafka-script +data: + script.sh: | + set -euxo pipefail + + export KCAT_CONFIG=/stackable/kcat.conf + TOPIC=test-topic + CONSUMER_GROUP=test-consumer-group + + echo -e -n "\ + metadata.broker.list=$KAFKA\n\ + auto.offset.reset=beginning\n\ + security.protocol=SASL_SSL\n\ + ssl.ca.location=/stackable/tls-ca-cert-mount/ca.crt\n\ + sasl.kerberos.keytab=/stackable/kerberos/keytab\n\ + sasl.kerberos.service.name=kafka\n\ + sasl.kerberos.principal=developer/access-kafka.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }}\n\ + sasl.mechanism=GSSAPI\n\ + " > $KCAT_CONFIG + + cat $KCAT_CONFIG + + sent_message="Hello Stackable!" + + echo $sent_message | kcat \ + -t $TOPIC \ + -P + + echo Sent message: \"$sent_message\" + + # Explicit numeric offset (not "-o stored"/"auto.offset.reset=beginning"): the bundled kcat's + # librdkafka (1.7.0) mis-detects broker feature support against Kafka >=4.0 brokers, which + # dropped old low-numbered API versions (KIP-896). It matches ApiVersions by exact version + # instead of range, so it wrongly reports the ListOffsets logical-offset query as unsupported + # ("Failed to query logical offset BEGINNING: Local: Required feature not supported by + # broker") even though the broker supports it fine -- see + # https://github.com/confluentinc/librdkafka/issues/4948. This is unrelated to SASL/Kerberos: + # authentication succeeds either way. The topic is freshly created and this is the only + # message ever produced to it, so offset 0 is always the message we just sent. + received_message=$(kcat \ + -G $CONSUMER_GROUP \ + -o 0 \ + -e \ + $TOPIC) + + echo Received message: \"$received_message\" + + if [ "$received_message" = "$sent_message" ]; then + echo "Test passed" + exit 0 + else + echo "Test failed" + exit 1 + fi diff --git a/tests/templates/kuttl/kraft-kerberos/30-access-kafka.yaml b/tests/templates/kuttl/kraft-kerberos/30-access-kafka.yaml new file mode 100644 index 00000000..eecc0f08 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/30-access-kafka.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 30-access-kafka.txt | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/kraft-kerberos/30-assert.yaml b/tests/templates/kuttl/kraft-kerberos/30-assert.yaml new file mode 100644 index 00000000..edc6c317 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/30-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: access-kafka +status: + succeeded: 1 diff --git a/tests/templates/kuttl/kraft-kerberos/60-assert.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/60-assert.yaml.j2 new file mode 100644 index 00000000..ef637733 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/60-assert.yaml.j2 @@ -0,0 +1,39 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available kafkaclusters.kafka.stackable.tech/test-kafka --timeout 301s + - script: | + # The voter set itself must have changed, not just the StatefulSet: a controller that + # starts but never joins the quorum is exactly what this test guards against. + # + # Kerberos-specific: the admin client must use the *rendered* /tmp copy. The raw + # ConfigMap file still has unresolved ${env:...} placeholders in sasl.jaas.config. + # + # :9093 is the TLS client port of this test fixture's security config, not a fixed + # Kafka port - if the fixture's TLS/port config changes, update this too. + kubectl exec -n $NAMESPACE test-kafka-controller-default-0 -c kafka -- \ + /stackable/kafka/bin/kafka-metadata-quorum.sh \ + --bootstrap-controller test-kafka-controller-default-0.test-kafka-controller-default-headless.$NAMESPACE.svc.cluster.local:9093 \ + --command-config /tmp/admin-client.properties \ + describe --replication | tail -n +2 | awk '$NF == "Leader" || $NF == "Follower"' | wc -l | grep -q '^5$' + # `timeout` is known-inert here: kuttl's TestAssert `commands` don't read this field (only + # TestStep commands do); left in place only as documentation of the intended budget. + timeout: 30 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + readyReplicas: 3 + replicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-controller-default +status: + readyReplicas: 5 + replicas: 5 diff --git a/tests/templates/kuttl/kraft-kerberos/60-scale-controller-up.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/60-scale-controller-up.yaml.j2 new file mode 100644 index 00000000..6854461e --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/60-scale-controller-up.yaml.j2 @@ -0,0 +1,63 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + # KRaft: metadata is managed by the controllers role, no ZooKeeper involved. + metadataManager: kraft + authentication: + - authenticationClass: kerberos-auth-$NAMESPACE + tls: + # Kerberos requires the use of server and internal TLS! + serverSecretClass: tls +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + controllers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 5 + brokers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + brokerListenerClass: {{ test_scenario['values']['broker-listener-class'] }} + # bootstrap-listener-class is orthogonal to this test's focus on Kerberos over the + # CONTROLLER listener (that axis is already covered by the plain `kerberos` test + # case), so it is pinned here rather than parameterized as a test dimension. + bootstrapListenerClass: cluster-internal + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 3 + EOF diff --git a/tests/templates/kuttl/kraft-kerberos/70-assert.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/70-assert.yaml.j2 new file mode 100644 index 00000000..21037372 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/70-assert.yaml.j2 @@ -0,0 +1,39 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available kafkaclusters.kafka.stackable.tech/test-kafka --timeout 301s + - script: | + # The voter set itself must have changed, not just the StatefulSet: a controller that + # starts but never joins the quorum is exactly what this test guards against. + # + # Kerberos-specific: the admin client must use the *rendered* /tmp copy. The raw + # ConfigMap file still has unresolved ${env:...} placeholders in sasl.jaas.config. + # + # :9093 is the TLS client port of this test fixture's security config, not a fixed + # Kafka port - if the fixture's TLS/port config changes, update this too. + kubectl exec -n $NAMESPACE test-kafka-controller-default-0 -c kafka -- \ + /stackable/kafka/bin/kafka-metadata-quorum.sh \ + --bootstrap-controller test-kafka-controller-default-0.test-kafka-controller-default-headless.$NAMESPACE.svc.cluster.local:9093 \ + --command-config /tmp/admin-client.properties \ + describe --replication | tail -n +2 | awk '$NF == "Leader" || $NF == "Follower"' | wc -l | grep -q '^3$' + # `timeout` is known-inert here: kuttl's TestAssert `commands` don't read this field (only + # TestStep commands do); left in place only as documentation of the intended budget. + timeout: 30 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + readyReplicas: 3 + replicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-controller-default +status: + readyReplicas: 3 + replicas: 3 diff --git a/tests/templates/kuttl/kraft-kerberos/70-scale-controller-down.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/70-scale-controller-down.yaml.j2 new file mode 100644 index 00000000..7a6bc90f --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/70-scale-controller-down.yaml.j2 @@ -0,0 +1,63 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + # KRaft: metadata is managed by the controllers role, no ZooKeeper involved. + metadataManager: kraft + authentication: + - authenticationClass: kerberos-auth-$NAMESPACE + tls: + # Kerberos requires the use of server and internal TLS! + serverSecretClass: tls +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + controllers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 3 + brokers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + brokerListenerClass: {{ test_scenario['values']['broker-listener-class'] }} + # bootstrap-listener-class is orthogonal to this test's focus on Kerberos over the + # CONTROLLER listener (that axis is already covered by the plain `kerberos` test + # case), so it is pinned here rather than parameterized as a test dimension. + bootstrapListenerClass: cluster-internal + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 3 + EOF diff --git a/tests/templates/kuttl/kraft-kerberos/README.md b/tests/templates/kuttl/kraft-kerberos/README.md new file mode 100644 index 00000000..a85e47b8 --- /dev/null +++ b/tests/templates/kuttl/kraft-kerberos/README.md @@ -0,0 +1,9 @@ +# Kraft + Kerberos test + +Proves that a KRaft-mode Kafka cluster (`spec.controllers` present, no ZooKeeper) can be +secured with Kerberos authentication (`spec.clusterConfig.authentication` referencing a +Kerberos `AuthenticationClass`) end to end: controllers form a quorum, brokers join, and a +client can authenticate via GSSAPI to produce/consume a message. + +This bundles the KRaft cluster setup from `smoke-kraft` with the KDC deployment, +`SecretClass`/`AuthenticationClass` and produce/consume job from `kerberos`. diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 9b47243a..5fba37ae 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -131,6 +131,14 @@ tests: - zookeeper-latest - kafka-latest - openshift + - name: kraft-kerberos + dimensions: + - kafka-kraft + - krb5 + - kerberos-realm + - kerberos-backend + - openshift + - broker-listener-class - name: kerberos dimensions: - kafka From 63116c0372e5088604a46303e10d220949f55d2b Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:54:52 +0200 Subject: [PATCH 10/13] docs: document Kerberos support for KRaft controllers Adds a Kerberos section to the KRaft controller guide (SASL_SSL on the CONTROLLER listener, pod-scoped controller keytabs, dynamic quorum scaling supported) and removes the now-false "Kerberos is currently not supported for KRaft" known issue. Documents in the security guide that clients using the discovery ConfigMap must supply their own principal and keytab. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 15 +++++++++++ .../pages/usage-guide/kraft-controller.adoc | 25 ++++++++++++++++++- .../kafka/pages/usage-guide/security.adoc | 10 ++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 454e23a6..d92f9d47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Support Kerberos (GSSAPI) authentication on KRaft controllers, covering both + broker-to-controller and controller-to-controller (Raft) traffic on the `CONTROLLER` + listener. Controller keytabs are pod-scoped, as controllers are reachable only under their + own StatefulSet pod DNS name. Dynamic quorum scaling continues to work with Kerberos + enabled ([#999]). + ### Changed - The dynamic KRaft quorum created by the operator is now scaled automatically. Previously, @@ -40,6 +48,12 @@ All notable changes to this project will be documented in this file. ### Fixed +- The discovery ConfigMap's `client.properties` no longer carries a `sasl.jaas.config` whose + principal was the placeholder `kafka/todo@$KERBEROS_REALM`. Its consumers run outside the + Kafka pods and have neither the pods' keytabs nor their per-pod principals, so they must + supply their own login configuration. The broker-side `sasl.enabled.mechanisms` and + `sasl.mechanism.inter.broker.protocol` entries were removed from that file for the same + reason, and the client-side `sasl.mechanism` added ([#999]). - Fix a longstanding problem of including empty `categories`, `shortNames` and `additionalPrinterColumns` in the CRDs, which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#998]). @@ -61,6 +75,7 @@ All notable changes to this project will be documented in this file. [#990]: https://github.com/stackabletech/kafka-operator/pull/990 [#994]: https://github.com/stackabletech/kafka-operator/pull/994 [#998]: https://github.com/stackabletech/kafka-operator/pull/998 +[#999]: https://github.com/stackabletech/kafka-operator/pull/999 [#1000]: https://github.com/stackabletech/kafka-operator/pull/1000 [#1010]: https://github.com/stackabletech/kafka-operator/pull/1010 [#1011]: https://github.com/stackabletech/kafka-operator/pull/1011 diff --git a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc index 76668635..238145f2 100644 --- a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc +++ b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc @@ -95,10 +95,33 @@ KRaft mode requires major configuration changes compared to ZooKeeper: * `controller.quorum.bootstrap.servers` points at each controller role group's own headless Service DNS name, not individual pod addresses. +== Kerberos + +Apache Kafka KRaft controllers support Kerberos (GSSAPI) authentication. +Reference an `AuthenticationClass` with the Kerberos provider as described in +xref:usage-guide/security.adoc#_kerberos[Security -- Kerberos]; it applies to the whole +`KafkaCluster`, controllers included. As always with Kerberos, server and internal TLS are +required. + +When Kerberos is enabled: + +* The `CONTROLLER` listener uses `SASL_SSL` instead of `SSL`, with `GSSAPI` as the mechanism + (`sasl.mechanism.controller.protocol`). This covers both broker-to-controller traffic and + the controller-to-controller (Raft) traffic between quorum members. +* Controller keytabs are *pod-scoped*, unlike broker keytabs, which are scoped to their + listener volumes. Controllers are not exposed through a listener-operator `Listener`; they + are reachable only under their own StatefulSet pod DNS name, which is therefore the + principal in their keytab. +* Each pod's `jaas.properties` gains a `controller.KafkaServer` login context. Unlike the + broker-side contexts it does not set `isInitiator=false`, because a controller must be able + to initiate GSSAPI connections to its peers, not only accept them. +* The `quorum-manager` sidecar and the `preStop` hook authenticate to the CONTROLLER listener + as the controller's own pod principal, so *dynamic quorum scaling works with Kerberos + enabled*. Scaling a controller role group up or down needs no manual intervention. + == Known Issues * Automatic migration from Apache ZooKeeper to KRaft is not supported. -* Kerberos is currently not supported for KRaft in all versions. * The quorum is created once by the controller with the lowest `node.id` using `--standalone`. If this controller loses it's PVC, a new conflicting quorum is created on restart. * A Controller that loses its persistent volume is not re-admitted to the voter set automatically, because it diff --git a/docs/modules/kafka/pages/usage-guide/security.adoc b/docs/modules/kafka/pages/usage-guide/security.adoc index 598bc96c..68736071 100644 --- a/docs/modules/kafka/pages/usage-guide/security.adoc +++ b/docs/modules/kafka/pages/usage-guide/security.adoc @@ -170,6 +170,16 @@ The bootstrap address is written to the discovery ConfigMap, using the Stackable NOTE: Port 9094 is reserved for non-secure kerberized connections which is not currently implemented. +The discovery ConfigMap's `client.properties` carries the settings a client needs to *reach* +a kerberized cluster -- `security.protocol`, `sasl.mechanism`, `sasl.kerberos.service.name` +and the truststore configuration -- but deliberately no login configuration. +Supplying credentials is the client's responsibility: the operator cannot do it, because +these clients run outside the Kafka pods and so have neither the pods' keytabs nor their +per-pod principals. +Add your own `sasl.jaas.config` (or a `java.security.auth.login.config` JAAS file) naming the +principal and keytab the client should authenticate with. + + == [[authorization]]Authorization If you wish to include integration with xref:opa:index.adoc[Open Policy Agent] and already have an OPA cluster, then you can include an `opa` field pointing to the OPA cluster discovery `ConfigMap` and the required package. From 7668da040db547af85a97d5de0674aba88d3ff25 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:29:08 +0200 Subject: [PATCH 11/13] fix: dial controller pod FQDNs, and give the quorum-manager krb5.conf Two defects found running the kraft-kerberos suite; both kept every peer from authenticating to the CONTROLLER listener. 1. `controller.quorum.bootstrap.servers` used the role group headless Service name (#1010), but a GSSAPI client derives its service principal from the hostname it dials, and the CONTROLLER acceptor can offer only one principal - the pod's own, which is also what the Raft voter endpoints advertise. Every broker and joining controller therefore asked for `kafka/` and was rejected with "invalid credentials". Verified on a live cluster: the same admin client against the same pod succeeds via the pod FQDN and fails via the Service name. The property now lists individual pod FQDNs. This gives up #1010's property that the list is stable across replica-count changes, so scaling a controller role group rolls the controller pods. How to handle that churn is deferred. 2. The quorum-manager sidecar had KRB5_CONFIG but not `-Djava.security.krb5.conf`. The JVM reads the system property; the env var only reaches native MIT tools, so the admin client failed with "Unable to locate KDC for realm". It gets KAFKA_OPTS with only that property - still not the kafka container's `java.security.auth.login.config`, which points at a jaas file only that container renders. Part of stackabletech/issues#815. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 9 ++- .../pages/usage-guide/kraft-controller.adoc | 6 +- .../src/controller/build/command.rs | 9 ++- .../src/controller/build/kerberos.rs | 2 +- .../src/controller/build/properties/mod.rs | 59 +++++++++++-------- .../controller/build/resource/statefulset.rs | 37 ++++++++++-- 6 files changed, 85 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d92f9d47..a29995ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,9 +20,12 @@ All notable changes to this project will be documented in this file. that adds the new controller to the voter list. On termination, a new `preStop` hook on the controller container (`kafka`) removes the pod from the voter list before shutdown. - The property `controller.quorum.bootstrap.servers` now contains the headless service names - of all controller role groups instead of individual peer host names. This prevents the - restart controller from restarting all pods in the quorum when a new one is added/deleted. + The property `controller.quorum.bootstrap.servers` lists the individual pod FQDNs of all + controllers. It briefly used the role group headless Service names instead, to stop the + restart controller rolling the quorum whenever a controller was added or removed, but that + is incompatible with Kerberos: a GSSAPI client derives the service principal from the + hostname it dials, and the CONTROLLER listener can only offer the pod's own principal + ([#999]). The controller `StatefulSet` is now scaled using `OrderedReady` instead of the `Parallel` strategy to ensure only one voter is added/removed at a time and thus keep the quorum healthy ([#1010]). - Internal operator refactoring: introduce a build() step in the reconciler that diff --git a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc index 238145f2..eea034e1 100644 --- a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc +++ b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc @@ -92,8 +92,10 @@ KRaft mode requires major configuration changes compared to ZooKeeper: with `kafka-storage.sh format --standalone`, bootstrapping a single-node quorum by itself. Every other controller formats with `--no-initial-controllers` and joins purely through the sidecar's `add-controller` call. Brokers always format with `--no-initial-controllers` too; they are never voters. -* `controller.quorum.bootstrap.servers` points at each controller role group's own headless Service DNS name, not - individual pod addresses. +* `controller.quorum.bootstrap.servers` lists the individual pod FQDNs of all controllers. + This is required by Kerberos -- a GSSAPI client derives the service principal from the hostname it dials, and the + CONTROLLER listener can only offer the pod's own principal -- but it means the property changes whenever a + controller role group is scaled, so the controller pods are rolled by the restart controller on scale. == Kerberos diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index e8989d39..210a9425 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -222,9 +222,12 @@ const CLI_CALL_KILL_AFTER_SECONDS: u32 = 5; /// ConfigMap file. /// /// Reading this at runtime, rather than baking the peer list into this script as a Rust -/// literal, keeps both sidecar scripts' content — and therefore the controller pod -/// template — identical across changes to an existing controller role group's *replica -/// count*. +/// literal, keeps both sidecar scripts' content identical across replica-count changes, so +/// the peer list lives in exactly one place (the ConfigMap). Note that since +/// [`kraft_controllers`][kc] lists individual pod FQDNs, that ConfigMap entry *does* change +/// with the replica count, and the pods roll with it. +/// +/// [kc]: super::properties::kraft_controllers fn extract_bootstrap_servers_command() -> String { format!( r#"BOOTSTRAP_SERVERS=$(grep '^controller.quorum.bootstrap.servers=' {config_dir}/{controller_properties_file} | cut -d= -f2- | sed 's/\\:/:/g')"#, diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index 00bd290e..1c4ec3fd 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -98,7 +98,7 @@ pub fn add_kerberos_pod_config( } constant!(pub KRB5_CONFIG: EnvVarName = "KRB5_CONFIG"); -constant!(KAFKA_OPTS: EnvVarName = "KAFKA_OPTS"); +constant!(pub KAFKA_OPTS: EnvVarName = "KAFKA_OPTS"); /// The environment variables the Kerberos configuration requires on the Kafka container, or an /// empty set when Kerberos is disabled. diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index d607d5f3..0d9d5285 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -63,18 +63,28 @@ pub fn uses_legacy_log4j(product_version: &str) -> bool { product_version.starts_with("3.") } -/// `controller.quorum.bootstrap.servers` addresses, one per distinct controller role group, -/// pointing at each role group's own headless Service DNS name rather than individual pod -/// FQDNs. +/// `controller.quorum.bootstrap.servers` addresses: one individual pod FQDN per controller, +/// across all controller role groups. /// -/// Only adding or removing a whole role group changes this list. +/// # Why pod FQDNs rather than the role group's headless Service +/// +/// Kerberos forces this. A GSSAPI client derives the service principal from the hostname it +/// dials, so dialling the headless Service asks for `kafka/`, while the CONTROLLER +/// listener's acceptor can offer only a single principal -- the pod's own +/// `kafka/`, which is also what the Raft voter endpoints advertise. Bootstrapping +/// through the Service therefore fails authentication for every peer. +/// +/// The trade-off is deliberate: unlike the headless-Service form, this list changes whenever +/// a controller role group's replica count changes, so scaling one rolls the controller pods. pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Vec { pod_descriptors .iter() .filter(|pd| pd.role == KafkaRole::Controller) .map(|desc| { format!( - "{service}.{namespace}.svc.{cluster_domain}:{client_port}", + "{sts}-{replica}.{service}.{namespace}.svc.{cluster_domain}:{client_port}", + sts = desc.role_group_statefulset_name, + replica = desc.replica, service = desc.role_group_service_name, namespace = desc.namespace, cluster_domain = desc.cluster_domain, @@ -132,7 +142,7 @@ mod tests { } #[test] - fn kraft_controllers_points_at_the_role_group_headless_service_not_individual_pods() { + fn kraft_controllers_lists_individual_pod_fqdns() { let pod_descriptors = vec![ pod_descriptor(KafkaRole::Controller, 0, 9093), pod_descriptor(KafkaRole::Controller, 1, 9093), @@ -143,14 +153,20 @@ mod tests { let quorum_bootstrap_servers = kraft_controllers(&pod_descriptors).join(","); + // Individual pod FQDNs, *not* the role group's headless Service. Under Kerberos the + // GSSAPI service principal is derived from the hostname the peer dials, and the + // CONTROLLER listener's acceptor can only offer one SPN -- the pod's own. Dialling + // the headless Service asks for `kafka/` instead and is rejected. assert_eq!( quorum_bootstrap_servers, - "kafka-controller-default-headless.default.svc.cluster.local:9093" + "kafka-controller-default-0.kafka-controller-default-headless.default.svc.cluster.local:9093,\ + kafka-controller-default-1.kafka-controller-default-headless.default.svc.cluster.local:9093,\ + kafka-controller-default-2.kafka-controller-default-headless.default.svc.cluster.local:9093" ); } #[test] - fn kraft_controllers_is_stable_across_replica_count_changes() { + fn kraft_controllers_grows_with_the_replica_count() { let three_replicas = vec![ pod_descriptor(KafkaRole::Controller, 0, 9093), pod_descriptor(KafkaRole::Controller, 1, 9093), @@ -164,15 +180,19 @@ mod tests { pod_descriptor(KafkaRole::Controller, 4, 9093), ]; - assert_eq!( + // Deliberate consequence of per-pod addressing: unlike the previous headless-Service + // form, this list changes with the replica count, so scaling a controller role group + // rolls the controller pods. + assert_eq!(kraft_controllers(&three_replicas).len(), 3); + assert_eq!(kraft_controllers(&five_replicas).len(), 5); + assert_ne!( kraft_controllers(&three_replicas), kraft_controllers(&five_replicas) ); } #[test] - fn kraft_controllers_lists_every_distinct_role_groups_service_once() { - let mut default_group_pod = pod_descriptor(KafkaRole::Controller, 0, 9093); + fn kraft_controllers_lists_pods_from_every_role_group() { let mut other_group_pod = pod_descriptor(KafkaRole::Controller, 0, 9093); other_group_pod.role_group_statefulset_name = "kafka-controller-other" .parse() @@ -180,18 +200,10 @@ mod tests { other_group_pod.role_group_service_name = "kafka-controller-other-headless" .parse() .expect("valid service name"); - // Second replica of the *same* role group as `default_group_pod` - must not produce - // a second entry for that Service. - let default_group_pod_replica_1 = { - let mut pod = pod_descriptor(KafkaRole::Controller, 1, 9093); - pod.node_id = 1; - pod - }; - default_group_pod.node_id = 0; let pod_descriptors = vec![ - default_group_pod, - default_group_pod_replica_1, + pod_descriptor(KafkaRole::Controller, 0, 9093), + pod_descriptor(KafkaRole::Controller, 1, 9093), other_group_pod, ]; @@ -200,8 +212,9 @@ mod tests { assert_eq!( quorum_bootstrap_servers, vec![ - "kafka-controller-default-headless.default.svc.cluster.local:9093".to_string(), - "kafka-controller-other-headless.default.svc.cluster.local:9093".to_string(), + "kafka-controller-default-0.kafka-controller-default-headless.default.svc.cluster.local:9093".to_string(), + "kafka-controller-default-1.kafka-controller-default-headless.default.svc.cluster.local:9093".to_string(), + "kafka-controller-other-0.kafka-controller-other-headless.default.svc.cluster.local:9093".to_string(), ] ); } diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 73f6d31d..3489625a 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -51,7 +51,8 @@ use crate::{ }, graceful_shutdown::add_graceful_shutdown_config, kerberos::{ - KERBEROS_VOLUME_NAME, KRB5_CONFIG, add_kerberos_pod_config, kerberos_env_vars, + KAFKA_OPTS, KERBEROS_VOLUME_NAME, KRB5_CONFIG, add_kerberos_pod_config, + kerberos_env_vars, }, properties::product_logging::MAX_KAFKA_LOG_FILES_SIZE, recommended_labels_for_role_group_resources, @@ -804,6 +805,16 @@ fn build_quorum_manager_container( cb.add_volume_mount(&*KERBEROS_VOLUME_NAME, STACKABLE_KERBEROS_DIR) .expect("The mount paths are statically defined and there should be no duplicates."); cb.add_env_var(KRB5_CONFIG.to_string(), STACKABLE_KERBEROS_KRB5_PATH); + // `KRB5_CONFIG` only reaches native MIT tools; the JVM reads the + // `java.security.krb5.conf` system property, without which the admin client fails + // with "Unable to locate KDC for realm". Unlike the `kafka` container's `KAFKA_OPTS` + // this deliberately omits `java.security.auth.login.config`: that points at + // `/tmp/jaas.properties`, which only the `kafka` container renders. This container + // authenticates with the inline `sasl.jaas.config` in `admin-client.properties`. + cb.add_env_var( + KAFKA_OPTS.to_string(), + format!("-Djava.security.krb5.conf={STACKABLE_KERBEROS_KRB5_PATH}"), + ); } cb.build() @@ -1037,11 +1048,27 @@ mod tests { .map(|e| e.name.as_str()) .collect(); assert!(env.contains(&"KRB5_CONFIG")); - // `KAFKA_OPTS` points the JVM at `/tmp/jaas.properties`, which only the `kafka` - // container renders. The sidecar uses an inline `sasl.jaas.config` instead. + + let kafka_opts = sidecar + .env + .as_ref() + .expect("sidecar must have env vars") + .iter() + .find(|e| e.name == "KAFKA_OPTS") + .and_then(|e| e.value.clone()) + .expect("sidecar needs KAFKA_OPTS to point the JVM at krb5.conf"); + // The JVM reads `java.security.krb5.conf`, *not* the `KRB5_CONFIG` env var (that only + // reaches native MIT tools), so without this the admin client cannot locate the KDC. + assert!( + kafka_opts.contains("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf"), + "got: {kafka_opts}" + ); + // But it must NOT inherit the kafka container's JAAS login config: that points at + // /tmp/jaas.properties, which only the `kafka` container renders. The sidecar + // authenticates with the inline `sasl.jaas.config` in admin-client.properties. assert!( - !env.contains(&"KAFKA_OPTS"), - "sidecar must not inherit the kafka container's JAAS login config" + !kafka_opts.contains("java.security.auth.login.config"), + "got: {kafka_opts}" ); } From f14175b49517049ebcf4a45c9eba8a70386b6a9c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:12:04 +0200 Subject: [PATCH 12/13] fix controller liveness probe This is a bugfix independent of Kerberos support. The controller's liveness probe tried to connect using `localhost`, but the controller doesn't bind to this address. This lead to the probe failing after every 10 minutes and causing the controller pods to be restarted. --- .../build/properties/controller_properties.rs | 18 +++++++++--------- .../controller/build/resource/config_map.rs | 8 +++----- .../src/controller/build/resource/probes.rs | 9 ++++++--- .../controller/build/resource/statefulset.rs | 6 ++++-- .../src/controller/build/security.rs | 8 ++------ rust/operator-binary/src/crd/mod.rs | 15 +++++++++++++++ 6 files changed, 39 insertions(+), 25 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/controller_properties.rs b/rust/operator-binary/src/controller/build/properties/controller_properties.rs index 4044e4f6..3849d253 100644 --- a/rust/operator-binary/src/controller/build/properties/controller_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/controller_properties.rs @@ -10,7 +10,7 @@ use crate::{ }, }, crd::{ - KafkaPodDescriptor, + CONTROLLER_POD_FQDN_TEMPLATE, KafkaPodDescriptor, listener::{KafkaListenerConfig, KafkaListenerName}, role::{ KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, @@ -32,27 +32,27 @@ pub fn build( KAFKA_LOG_DIRS.to_string(), "/stackable/data/kraft".to_string(), ), - (KAFKA_PROCESS_ROLES.to_string(), KafkaRole::Controller.to_string()), ( - "controller.listener.names".to_string(), - KafkaListenerName::Controller.to_string(), + KAFKA_PROCESS_ROLES.to_string(), + KafkaRole::Controller.to_string(), ), ( - KAFKA_NODE_ID.to_string(), - "${env:REPLICA_ID}".to_string(), + "controller.listener.names".to_string(), + KafkaListenerName::Controller.to_string(), ), + (KAFKA_NODE_ID.to_string(), "${env:REPLICA_ID}".to_string()), ( KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), kraft_controllers.clone(), ), ( KAFKA_LISTENERS.to_string(), - "CONTROLLER://${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}:${env:KAFKA_CLIENT_PORT}".to_string(), + format!("CONTROLLER://{CONTROLLER_POD_FQDN_TEMPLATE}:${{env:KAFKA_CLIENT_PORT}}"), ), ( KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), - listener_config - .listener_security_protocol_map_for_controller()), + listener_config.listener_security_protocol_map_for_controller(), + ), ]); result.insert( diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index fd6ec48d..5dafacc2 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -22,7 +22,8 @@ use crate::{ }, }, crd::{ - STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, + CONTROLLER_POD_FQDN_TEMPLATE, STACKABLE_LISTENER_BOOTSTRAP_DIR, + STACKABLE_LISTENER_BROKER_DIR, listener::{KafkaListenerConfig, node_address_cmd}, role::{AnyConfig, KafkaRole}, }, @@ -236,10 +237,7 @@ fn jaas_config_file(is_kerberos_enabled: bool, role: &KafkaRole) -> String { // already used for `KAFKA_LISTENERS` in `controller_properties.rs`. let controller_principal_address = match role { KafkaRole::Broker => node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), - KafkaRole::Controller => { - "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}" - .to_string() - } + KafkaRole::Controller => CONTROLLER_POD_FQDN_TEMPLATE.to_string(), }; // Unlike the bootstrap and client sections below, this context is used for BOTH sides of diff --git a/rust/operator-binary/src/controller/build/resource/probes.rs b/rust/operator-binary/src/controller/build/resource/probes.rs index 66426b60..468336ed 100644 --- a/rust/operator-binary/src/controller/build/resource/probes.rs +++ b/rust/operator-binary/src/controller/build/resource/probes.rs @@ -11,8 +11,11 @@ use stackable_operator::{ v2::types::common::Port, }; -use crate::controller::{ - build::security::kcat_prober_container_commands, security::ValidatedKafkaSecurity, +use crate::{ + controller::{ + build::security::kcat_prober_container_commands, security::ValidatedKafkaSecurity, + }, + crd::CONTROLLER_POD_FQDN_SHELL, }; #[derive(Snafu, Debug)] @@ -142,7 +145,7 @@ pub fn controller_stuck_unattached_liveness_probe( "bash".to_string(), "-c".to_string(), format!( - "timeout 2 bash -c 'cat < /dev/null > /dev/tcp/localhost/{client_port}' || exit 1\n\ + "timeout 2 bash -c 'cat < /dev/null > /dev/tcp/{CONTROLLER_POD_FQDN_SHELL}/{client_port}' || exit 1\n\ state=$(curl -s --max-time 2 localhost:{metrics_port}/metrics | grep -oE 'kafka_server_raft_metrics_current_state\\{{state=\"[a-z]+\",?\\}}' | grep -oE '\"[a-z]+\"' | tr -d '\"')\n\ [ \"$state\" != \"unattached\" ]" ), diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 3489625a..33d4254b 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -1357,8 +1357,10 @@ mod tests { let script = command.last().expect("the exec command has a script arg"); assert!( - script.contains(&format!("/dev/tcp/localhost/{client_port}")), - "expected a TCP reachability check against the controller's own port, script was: {script}" + script.contains(&format!( + "/dev/tcp/$POD_NAME.$ROLEGROUP_HEADLESS_SERVICE_NAME.$NAMESPACE.svc.$CLUSTER_DOMAIN/{client_port}" + )), + "expected the TCP check to dial the address the controller actually binds, script was: {script}" ); assert!( script.contains(r#"[ "$state" != "unattached" ]"#), diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index eaa3fbaf..055d60ce 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -26,8 +26,8 @@ use stackable_operator::{ use crate::{ controller::security::ValidatedKafkaSecurity, crd::{ - LISTENER_BOOTSTRAP_VOLUME_NAME, LISTENER_BROKER_VOLUME_NAME, STACKABLE_KERBEROS_KRB5_PATH, - STACKABLE_LISTENER_BROKER_DIR, + CONTROLLER_POD_FQDN_TEMPLATE, LISTENER_BOOTSTRAP_VOLUME_NAME, LISTENER_BROKER_VOLUME_NAME, + STACKABLE_KERBEROS_KRB5_PATH, STACKABLE_LISTENER_BROKER_DIR, listener::{ self, KafkaListenerName, KafkaListenerProtocol, node_address_cmd_env, node_port_cmd_env, }, @@ -56,10 +56,6 @@ const PROPERTY_SASL_MECHANISM: &str = "sasl.mechanism"; const PROPERTY_SASL_JAAS_CONFIG: &str = "sasl.jaas.config"; const STACKABLE_KERBEROS_KEYTAB_PATH: &str = "/stackable/kerberos/keytab"; -/// The controller pod's own FQDN, as `config-utils template` placeholders. Matches the address -/// used for `KAFKA_LISTENERS` in `controller_properties.rs` and for the `controller.KafkaServer` -/// JAAS principal in `jaas_config_file`. -const CONTROLLER_POD_FQDN_TEMPLATE: &str = "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}"; pub(crate) const STACKABLE_TLS_KAFKA_INTERNAL_DIR: &str = "/stackable/tls-kafka-internal"; constant!(pub(crate) STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME: VolumeName = "tls-kafka-internal"); const STACKABLE_TLS_KAFKA_SERVER_DIR: &str = "/stackable/tls-kafka-server"; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 3402151f..81a30d72 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -76,6 +76,21 @@ constant!(pub STACKABLE_LOG_DIR_NAME: VolumeName = "log"); pub const BROKER_ID_POD_MAP_DIR: &str = "/stackable/broker-id-pod-map"; constant!(pub BROKER_ID_POD_MAP_DIR_NAME: VolumeName = "broker-id-pod-map-dir"); +/// A KRaft controller pod's own fully-qualified domain name, as `config-utils` placeholders +/// resolved at container start. +/// +/// Used for: +/// - the address where the CONTROLLER listener is *bound*. +/// - the endpoint registered with the quorum's voter. +/// - when Kerberos is enabled, it is also the host in the controller's Kerberos service principal. +pub const CONTROLLER_POD_FQDN_TEMPLATE: &str = "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}"; + +/// Same as above ([`CONTROLLER_POD_FQDN_TEMPLATE`]) but for use in scripts +/// the operator generates (probes, startup commands) where the shell expands the value +/// rather than `config-utils`. +pub const CONTROLLER_POD_FQDN_SHELL: &str = + "$POD_NAME.$ROLEGROUP_HEADLESS_SERVICE_NAME.$NAMESPACE.svc.$CLUSTER_DOMAIN"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display( From 9f4a80365b868b2fd77be39bfa20380e054e4a8c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 17 Sep 2026 16:13:14 +0200 Subject: [PATCH 13/13] fix: cleanup LLM verbosity Update supported version text to be inline with the version bump for SDP 26.11 release. --- CHANGELOG.md | 21 +- .../pages/usage-guide/kraft-controller.adoc | 12 +- .../kafka/pages/usage-guide/security.adoc | 64 +- .../kafka/partials/supported-versions.adoc | 9 +- ...2026-09-16-kerberized-kraft-controllers.md | 1417 ----------------- ...-16-kerberized-kraft-controllers-design.md | 202 --- rust/operator-binary/src/controller.rs | 21 +- .../src/controller/build/command.rs | 8 - .../src/controller/build/kerberos.rs | 77 - .../controller/build/properties/listener.rs | 51 - .../src/controller/build/properties/mod.rs | 4 +- .../controller/build/resource/config_map.rs | 45 - .../controller/build/resource/statefulset.rs | 117 -- .../src/controller/build/security.rs | 61 - .../kuttl/kraft-kerberos/60-assert.yaml.j2 | 5 - .../60-scale-controller-up.yaml.j2 | 9 - .../kuttl/kraft-kerberos/70-assert.yaml.j2 | 5 - .../70-scale-controller-down.yaml.j2 | 9 - 18 files changed, 62 insertions(+), 2075 deletions(-) delete mode 100644 docs/superpowers/plans/2026-09-16-kerberized-kraft-controllers.md delete mode 100644 docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3de3641c..0e2e7bc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,10 @@ All notable changes to this project will be documented in this file. ### Added -- Support Kerberos (GSSAPI) authentication on KRaft controllers, covering both - broker-to-controller and controller-to-controller (Raft) traffic on the `CONTROLLER` - listener. Controller keytabs are pod-scoped, as controllers are reachable only under their - own StatefulSet pod DNS name. Dynamic quorum scaling continues to work with Kerberos - enabled ([#999]). - Support floating tags for product images via the new `spec.image.stackableVersionPolicy` field ([#1021]). +- Support Kerberos (GSSAPI) authentication on KRaft controllers, covering both + broker-to-controller and controller-to-controller (Raft) traffic ([#1024]). ### Changed @@ -26,12 +23,6 @@ All notable changes to this project will be documented in this file. that adds the new controller to the voter list. On termination, a new `preStop` hook on the controller container (`kafka`) removes the pod from the voter list before shutdown. - The property `controller.quorum.bootstrap.servers` lists the individual pod FQDNs of all - controllers. It briefly used the role group headless Service names instead, to stop the - restart controller rolling the quorum whenever a controller was added or removed, but that - is incompatible with Kerberos: a GSSAPI client derives the service principal from the - hostname it dials, and the CONTROLLER listener can only offer the pod's own principal - ([#999]). The controller `StatefulSet` is now scaled using `OrderedReady` instead of the `Parallel` strategy to ensure only one voter is added/removed at a time and thus keep the quorum healthy ([#1010]). - Internal operator refactoring: introduce a build() step in the reconciler that @@ -57,12 +48,6 @@ All notable changes to this project will be documented in this file. ### Fixed -- The discovery ConfigMap's `client.properties` no longer carries a `sasl.jaas.config` whose - principal was the placeholder `kafka/todo@$KERBEROS_REALM`. Its consumers run outside the - Kafka pods and have neither the pods' keytabs nor their per-pod principals, so they must - supply their own login configuration. The broker-side `sasl.enabled.mechanisms` and - `sasl.mechanism.inter.broker.protocol` entries were removed from that file for the same - reason, and the client-side `sasl.mechanism` added ([#999]). - Fix a longstanding problem of including empty `categories`, `shortNames` and `additionalPrinterColumns` in the CRDs, which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#998]). @@ -84,13 +69,13 @@ All notable changes to this project will be documented in this file. [#990]: https://github.com/stackabletech/kafka-operator/pull/990 [#994]: https://github.com/stackabletech/kafka-operator/pull/994 [#998]: https://github.com/stackabletech/kafka-operator/pull/998 -[#999]: https://github.com/stackabletech/kafka-operator/pull/999 [#1000]: https://github.com/stackabletech/kafka-operator/pull/1000 [#1010]: https://github.com/stackabletech/kafka-operator/pull/1010 [#1011]: https://github.com/stackabletech/kafka-operator/pull/1011 [#1014]: https://github.com/stackabletech/kafka-operator/pull/1014 [#1017]: https://github.com/stackabletech/kafka-operator/pull/1017 [#1021]: https://github.com/stackabletech/kafka-operator/pull/1021 +[#1024]: https://github.com/stackabletech/kafka-operator/pull/1024 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc index eea034e1..ce9f0f16 100644 --- a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc +++ b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc @@ -101,9 +101,7 @@ KRaft mode requires major configuration changes compared to ZooKeeper: Apache Kafka KRaft controllers support Kerberos (GSSAPI) authentication. Reference an `AuthenticationClass` with the Kerberos provider as described in -xref:usage-guide/security.adoc#_kerberos[Security -- Kerberos]; it applies to the whole -`KafkaCluster`, controllers included. As always with Kerberos, server and internal TLS are -required. +xref:usage-guide/security.adoc[Security]. When Kerberos is enabled: @@ -111,15 +109,13 @@ When Kerberos is enabled: (`sasl.mechanism.controller.protocol`). This covers both broker-to-controller traffic and the controller-to-controller (Raft) traffic between quorum members. * Controller keytabs are *pod-scoped*, unlike broker keytabs, which are scoped to their - listener volumes. Controllers are not exposed through a listener-operator `Listener`; they - are reachable only under their own StatefulSet pod DNS name, which is therefore the - principal in their keytab. + listener volumes. Controllers are reachable only under their own StatefulSet pod DNS name, + which is therefore the principal in their keytab. * Each pod's `jaas.properties` gains a `controller.KafkaServer` login context. Unlike the broker-side contexts it does not set `isInitiator=false`, because a controller must be able to initiate GSSAPI connections to its peers, not only accept them. * The `quorum-manager` sidecar and the `preStop` hook authenticate to the CONTROLLER listener - as the controller's own pod principal, so *dynamic quorum scaling works with Kerberos - enabled*. Scaling a controller role group up or down needs no manual intervention. + as the controller's own pod principal. == Known Issues diff --git a/docs/modules/kafka/pages/usage-guide/security.adoc b/docs/modules/kafka/pages/usage-guide/security.adoc index 68736071..5e5942c7 100644 --- a/docs/modules/kafka/pages/usage-guide/security.adoc +++ b/docs/modules/kafka/pages/usage-guide/security.adoc @@ -162,23 +162,55 @@ NOTE: When Kerberos is enabled it is also required to enable TLS for maximum sec ==== Clients -In order to keep client configuration as uncluttered as possible, each kerberized Kafka broker has two principals: one for the broker itself and one for the bootstrap service. -The client can connect to the bootstrap service, which returns the broker quorum for use in subsequent operations. -This is transparent as each connection dynamically uses the relevant principal (broker or bootstrap). -In order for this to work, it is necessary for kerberized clusters to define an extra Kafka listener for the bootstrap with a corresponding service (and port). -The bootstrap address is written to the discovery ConfigMap, using the Stackable bootstrap listener with the port being 9095 (secure) for kerberized clusters, and 9092 (non-secure) or 9093 (secure) for non-kerberized ones. - -NOTE: Port 9094 is reserved for non-secure kerberized connections which is not currently implemented. - -The discovery ConfigMap's `client.properties` carries the settings a client needs to *reach* -a kerberized cluster -- `security.protocol`, `sasl.mechanism`, `sasl.kerberos.service.name` -and the truststore configuration -- but deliberately no login configuration. -Supplying credentials is the client's responsibility: the operator cannot do it, because -these clients run outside the Kafka pods and so have neither the pods' keytabs nor their -per-pod principals. -Add your own `sasl.jaas.config` (or a `java.security.auth.login.config` JAAS file) naming the -principal and keytab the client should authenticate with. +===== Why each broker has two principals +A GSSAPI client derives the service principal it asks the KDC for from the *hostname it connects to*. +Connecting to a Kafka cluster takes two hops over two different addresses: a client first contacts a bootstrap address, receives the cluster metadata, and then connects to individual brokers directly. +Each of those addresses therefore needs its own principal. + +Every broker consequently gets two principals, `kafka/` and `kafka/`, both provisioned into its keytab by the Secret Operator. +The broker's JAAS configuration declares both, as the `bootstrap.KafkaServer` and `client.KafkaServer` login contexts. +No client-side configuration is needed to switch between them: whichever address a client dials, the broker already holds the matching principal. + +To make this work, a kerberized cluster exposes an additional Kafka listener, container port and `Listener` port for the bootstrap address. +These exist only when Kerberos is enabled; without it, clients reach brokers over the client listener alone. + +===== Bootstrap address and ports + +The bootstrap address is published in the xref:reference/discovery.adoc[discovery ConfigMap] under the `KAFKA` key, read from the ingress addresses of the Stackable bootstrap `Listener`. +The port depends on whether Kerberos and TLS are enabled: + +[cols="1,1,1"] +|=== +| Cluster | Port name | Port + +| Kerberos (always TLS) +| `bootstrap` +| 9095 + +| TLS, no Kerberos +| `kafka-tls` +| 9093 + +| No TLS, no Kerberos +| `kafka` +| 9092 +|=== + +NOTE: Kerberos requires TLS, so a kerberized cluster always uses port 9095 for bootstrapping. + +===== Client configuration + +The discovery ConfigMap's `client.properties` carries the properties needed to *reach* the cluster: + +- `security.protocol` +- `sasl.mechanism` +- `sasl.kerberos.service.name` +- and the truststore settings + +But the operator adds *no login configuration* to 'client.properties` because supplying credentials is the client's responsibility. +The operator cannot do it: these clients run outside the Kafka Pods, so they have neither the Pods' keytabs nor their principals. +Add your own `sasl.jaas.config`, or point the JVM at a JAAS file with `java.security.auth.login.config`, naming the principal and keytab the client should authenticate with. == [[authorization]]Authorization diff --git a/docs/modules/kafka/partials/supported-versions.adoc b/docs/modules/kafka/partials/supported-versions.adoc index 5a962cb4..0540998c 100644 --- a/docs/modules/kafka/partials/supported-versions.adoc +++ b/docs/modules/kafka/partials/supported-versions.adoc @@ -2,12 +2,11 @@ // This is a separate file, since it is used by both the direct Kafka documentation, and the overarching // Stackable Platform documentation. -* 4.2.1 (experimental, deprecated) - Requires KRaft, please read on the xref:kafka:usage-guide/kraft-controller.adoc[Kraft migration guide]. -* 4.1.1 (experimental, deprecated) - Requires KRaft, please read on the xref:kafka:usage-guide/kraft-controller.adoc[Kraft migration guide]. +* 4.2.1 +* 4.1.1 * 3.9.2 (LTS) * 3.9.1 (deprecated) -Support for clusters running in Kraft mode (which includes Apache Kafka >= 4.x) is experimental due to the following known issues: +Starting with Apache Kafka >=4.x, the quorum manager has been changed from Apache Zookeeper to the built in KRaft. -* Kerberos authentication is not tested yet. -* Service exposition is not definitive. +If you are running an older Kafka version and plan to upgrade, head on to the xref:kafka:usage-guide/kraft-controller.adoc[Kraft migration guide] for details on how to do so. diff --git a/docs/superpowers/plans/2026-09-16-kerberized-kraft-controllers.md b/docs/superpowers/plans/2026-09-16-kerberized-kraft-controllers.md deleted file mode 100644 index 8f1bb372..00000000 --- a/docs/superpowers/plans/2026-09-16-kerberized-kraft-controllers.md +++ /dev/null @@ -1,1417 +0,0 @@ -# Kerberized KRaft Controllers Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Let Apache Kafka KRaft controllers authenticate with Kerberos (GSSAPI) for both broker-to-controller and controller-to-controller (Raft) traffic, without losing the dynamic quorum scaling added in #1010. - -**Architecture:** The `CONTROLLER` listener switches from `SSL` to `SASL_SSL` when Kerberos is enabled. Controller pods get a *pod-scoped* keytab (they have no listener-operator `Listener` volume) and a `controller.KafkaServer` JAAS section. The `quorum-manager` sidecar and the `preStop` hook, which drive dynamic quorum membership, get a Kerberos-aware `admin-client.properties` so they keep working. - -**Tech Stack:** Rust, `stackable-operator` crate, `config-utils template` for runtime `${env:…}` placeholder resolution, kuttl + MIT KDC for integration tests. - -**Spec:** `docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md` - -## Relationship to PR #999 - -The spec frames this as "rebase #999". `main` has moved far enough that a literal `git rebase` produces more conflict resolution than reconstruction. This plan therefore **re-applies #999's changes task by task against current `main`**, using #999 as the reference for *what* to build. Task 0 sets up a read-only worktree of that branch so every later task can consult it. - -Three deviations from the spec, discovered while reading current `main`: - -1. **`add_kerberos_pod_config` is never called for controllers.** It is invoked only at `statefulset.rs:248`, inside `build_broker_rolegroup_statefulset`. Controller pods have no keytab volume at all today. This is prerequisite work the spec did not name; it is now Task 1. -2. **`kerberos_env_vars` must not go on the shared controller env.** It sets `KAFKA_OPTS=-Djava.security.auth.login.config=/tmp/jaas.properties`. `controller_pod_shared_env_vars` feeds both the `kafka` container and the `quorum-manager` sidecar, and the sidecar has no `/tmp/jaas.properties` — it uses an inline `sasl.jaas.config` instead. Kerberos env goes on the `kafka` container's `env` only; the sidecar gets `KRB5_CONFIG` alone. -3. **Drop the byte-identical command test rather than re-baselining it.** `broker_start_command` (`command.rs:86-88`) already copies and templates `jaas.properties` *unconditionally*, because the file is always present in the ConfigMap (empty string when Kerberos is off). Mirroring that for the controller is simpler than #999's conditional `jaas_setup`, and makes the byte-identical regression test pointless. - -## Global Constraints - -- Match the surrounding Rust style: `snafu` for errors, `constant!` newtypes for volume/env-var names, `expect` with a justifying message for statically-impossible failures. -- Kerberos-disabled behaviour must not change. Every task that touches a shared code path asserts the non-Kerberos branch is untouched. -- Product naming in docs: "Stackable Data Platform (SDP)" once, then SDP; "Apache Kafka" in formal prose. -- `sasl.jaas.config` must be a single logical line in a Java properties file. -- Kerberos principals are always `kafka/@`; the service name comes from `KafkaRole::kerberos_service_name()`, never a literal. -- Run `cargo test -p stackable-kafka-operator` for unit tests; `cargo clippy --all-targets -- -D warnings` before every commit. - ---- - -### Task 0: Reference worktree for PR #999 - -**Files:** - -- Create: none in the repo tree (worktree lives outside it) - -**Interfaces:** - -- Produces: a read-only checkout of `origin/feature/kraft-kerberos-support` that later tasks consult for reference implementations. - -- [ ] **Step 1: Fetch the branch and create the reference worktree** - -```bash -cd /home/razvan/repo/stackable/kafka-operator -git fetch origin feature/kraft-kerberos-support -git worktree add --detach /tmp/pr999 origin/feature/kraft-kerberos-support -``` - -- [ ] **Step 2: Confirm the reference files are readable** - -Run: - -```bash -ls /tmp/pr999/tests/templates/kuttl/kraft-kerberos/ -``` - -Expected: lists `01-install-krb5-kdc.yaml.j2`, `02-create-kerberos-secretclass.yaml.j2`, `20-install-kafka.yaml.j2`, `30-access-kafka.txt.j2` among others. - -- [ ] **Step 3: Confirm we are on the feature branch** - -Run: `git branch --show-current` -Expected: `feat/kerberized-kraft-controllers` - -No commit for this task — it creates no tracked files. - ---- - -### Task 1: Pod-scoped Kerberos keytab on controller pods - -Controllers are reachable only through their StatefulSet pod DNS name, so their keytab principal must be pod-scoped. Brokers keep listener-volume scoping. - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/kerberos.rs:52-82` (`add_kerberos_pod_config`) -- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs:413-460` (`build_controller_rolegroup_statefulset`) -- Test: `rust/operator-binary/src/controller/build/kerberos.rs` (`#[cfg(test)] mod tests`) - -**Interfaces:** - -- Consumes: `ValidatedKafkaSecurity::kerberos_secret_class()`, `KafkaRole`, `SecretOperatorVolumeSourceBuilder::with_pod_scope()`. -- Produces: `add_kerberos_pod_config` gains controller-aware behaviour; its signature is unchanged (`(&ValidatedKafkaSecurity, &KafkaRole, &mut ContainerBuilder, &mut PodBuilder) -> Result<(), Error>`). - -- [ ] **Step 1: Write the failing test** - -Add to the existing `mod tests` in `kerberos.rs`. (`security.rs`'s test module already exposes an identical `pub(crate) fn kerberos()`; importing it instead of redefining it locally is fine and preferable if it resolves cleanly.) - -```rust -use stackable_operator::{ - builder::{meta::ObjectMetaBuilder, pod::container::ContainerBuilder}, - crd::authentication::{core, kerberos}, -}; - -use crate::crd::authentication::ResolvedAuthenticationClasses; - -fn kerberos() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![core::v1alpha1::AuthenticationClass { - metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), - spec: core::v1alpha1::AuthenticationClassSpec { - provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( - kerberos::v1alpha1::AuthenticationProvider { - kerberos_secret_class: "kerberos-secret-class".to_string(), - }, - ), - }, - }]), - "tls".parse().expect("valid secret class name"), - Some("tls".parse().expect("valid secret class name")), - None, - ) -} - -/// Reads the `secrets.stackable.tech/*` annotations off the `kerberos` ephemeral volume. -fn kerberos_volume_annotations(pb: &mut PodBuilder) -> std::collections::BTreeMap { - let pod = pb.build_template(); - pod.spec - .as_ref() - .and_then(|spec| spec.volumes.as_ref()) - .and_then(|volumes| volumes.iter().find(|v| v.name == *KERBEROS_VOLUME_NAME)) - .expect("kerberos volume must be present") - .ephemeral - .as_ref() - .expect("kerberos volume must be an ephemeral secret-operator volume") - .volume_claim_template - .as_ref() - .and_then(|t| t.metadata.as_ref()) - .and_then(|m| m.annotations.clone()) - .expect("volume claim template must carry secrets.stackable.tech annotations") -} - -#[test] -fn controller_keytab_is_pod_scoped() { - let mut pb = PodBuilder::new(); - let mut cb_kafka = ContainerBuilder::new("kafka").expect("valid container name"); - - add_kerberos_pod_config( - &kerberos(), - &KafkaRole::Controller, - &mut cb_kafka, - &mut pb, - ) - .expect("kerberos pod config for the controller role"); - - let annotations = kerberos_volume_annotations(&mut pb); - // Controllers have no listener-operator Listener volume, so the keytab must be - // scoped to the pod's own DNS name, matching how their internal TLS cert is - // provisioned in `add_controller_volume_and_volume_mounts`. - assert_eq!( - annotations.get("secrets.stackable.tech/scope").map(String::as_str), - Some("pod"), - "controller keytab must be pod-scoped, got: {annotations:?}" - ); - assert_eq!( - annotations - .get("secrets.stackable.tech/kerberos.service.names") - .map(String::as_str), - Some("kafka") - ); -} - -#[test] -fn broker_keytab_stays_listener_scoped() { - let mut pb = PodBuilder::new(); - let mut cb_kafka = ContainerBuilder::new("kafka").expect("valid container name"); - - add_kerberos_pod_config( - &kerberos(), - &KafkaRole::Broker, - &mut cb_kafka, - &mut pb, - ) - .expect("kerberos pod config for the broker role"); - - let annotations = kerberos_volume_annotations(&mut pb); - let scope = annotations - .get("secrets.stackable.tech/scope") - .expect("scope annotation must be present"); - assert!( - scope.contains("listener-volume=listener-broker") - && scope.contains("listener-volume=listener-bootstrap"), - "broker keytab must stay listener-volume-scoped, got: {scope}" - ); - assert!( - !scope.split(',').any(|s| s == "pod"), - "broker keytab must not be pod-scoped, got: {scope}" - ); -} -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: `cargo test -p stackable-kafka-operator kerberos:: -- --nocapture` -Expected: `controller_keytab_is_pod_scoped` FAILS — the scope annotation is the broker's listener-volume scope, because the role is currently ignored. - -- [ ] **Step 3: Make the volume scope role-dependent** - -In `kerberos.rs`, replace the chained builder call inside `if let Some(kerberos_secret_class) = …` with: - -```rust - let mut volume_builder = SecretOperatorVolumeSourceBuilder::new( - kerberos_secret_class, - // We need both public (krb5.conf) and private (keytab) parts. - SecretClassVolumeProvisionParts::PublicPrivate, - ); - match role { - // Brokers are exposed through listener-operator `Listener` volumes (the broker - // and bootstrap listeners), so the keytab principal must cover both. - KafkaRole::Broker => { - volume_builder - .with_listener_volume_scope(&*LISTENER_BROKER_VOLUME_NAME) - .with_listener_volume_scope(&*LISTENER_BOOTSTRAP_VOLUME_NAME); - } - // KRaft controllers have no listener-operator `Listener` volume: they are only - // reachable through their own StatefulSet pod DNS name, so the keytab must be - // pod-scoped, matching how the controller's internal TLS cert is provisioned in - // `add_controller_volume_and_volume_mounts`. - KafkaRole::Controller => { - volume_builder.with_pod_scope(); - } - } - let kerberos_secret_operator_volume = volume_builder - .with_kerberos_service_name(role.kerberos_service_name()) - .build() - .context(KerberosSecretVolumeSnafu)?; -``` - -- [ ] **Step 4: Run the tests to verify they pass** - -Run: `cargo test -p stackable-kafka-operator kerberos:: -- --nocapture` -Expected: PASS - -- [ ] **Step 5: Call it from the controller StatefulSet builder** - -In `statefulset.rs`, inside `build_controller_rolegroup_statefulset`, immediately after `let mut pod_builder = PodBuilder::new();`, add: - -```rust - if kafka_security.has_kerberos_enabled() { - add_kerberos_pod_config(kafka_security, kafka_role, &mut cb_kafka, &mut pod_builder) - .context(AddKerberosConfigSnafu)?; - } -``` - -Then, in the same function, add the Kerberos env vars to the **`kafka` container's** env only — `controller_shared_env` also feeds the `quorum-manager` sidecar, which has no `/tmp/jaas.properties` and must not receive `KAFKA_OPTS`. Change the `let env: Vec = …` chain to insert `.merge(kerberos_env_vars(kafka_security))` immediately before `.merge(validated_rg.env_overrides.clone())`: - -```rust - let env: Vec = controller_shared_env - .clone() - .merge(common_kafka_env( - merged_config, - &validated_rg - .product_specific_common_config - .jvm_argument_overrides, - resolved_product_image, - kafka_role, - role_group_name, - )?) - // Kerberos env goes on the `kafka` container only. `controller_shared_env` is also - // the sidecar's base, and `KAFKA_OPTS` points the JVM at `/tmp/jaas.properties`, - // which only the `kafka` container renders. - .merge(kerberos_env_vars(kafka_security)) - .merge(validated_rg.env_overrides.clone()) - .into(); -``` - -- [ ] **Step 6: Verify it compiles and the whole suite passes** - -Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` -Expected: no warnings, all tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add rust/operator-binary/src/controller/build/kerberos.rs \ - rust/operator-binary/src/controller/build/resource/statefulset.rs -git commit -m "feat: mount a pod-scoped Kerberos keytab on KRaft controller pods" -``` - ---- - -### Task 2: `controller.KafkaServer` JAAS section and controller JAAS rendering - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/resource/config_map.rs:169-230` (`jaas_config_file` and its call site) -- Modify: `rust/operator-binary/src/controller/build/command.rs:145-176` (`controller_kafka_container_command`) -- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs:486-488` (call site) -- Test: the `#[cfg(test)] mod tests` blocks in `config_map.rs` and `command.rs` - -**Interfaces:** - -- Consumes: `KafkaRole` (Task 1's role plumbing), `node_address_cmd`, `ConfigFileName::Jaas`. -- Produces: - - `fn jaas_config_file(is_kerberos_enabled: bool, role: &KafkaRole) -> String` - - `pub fn controller_kafka_container_command(kafka_security: &ValidatedKafkaSecurity, controller_descriptors: Vec) -> String` - -- [ ] **Step 1: Write the failing tests** - -In `config_map.rs`, replace the existing `mod tests` contents with: - -```rust -#[cfg(test)] -mod tests { - use super::jaas_config_file; - use crate::crd::role::KafkaRole; - - const CONTROLLER_POD_FQDN: &str = "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}"; - - #[test] - fn jaas_config_file_empty_without_kerberos() { - assert_eq!(jaas_config_file(false, &KafkaRole::Broker), ""); - assert_eq!(jaas_config_file(false, &KafkaRole::Controller), ""); - } - - #[test] - fn jaas_config_file_renders_bootstrap_and_client_sections_with_kerberos() { - let jaas = jaas_config_file(true, &KafkaRole::Broker); - assert!(jaas.contains("bootstrap.KafkaServer")); - assert!(jaas.contains("client.KafkaServer")); - assert!(jaas.contains("Krb5LoginModule")); - assert!(jaas.contains("/stackable/kerberos/keytab")); - assert!(jaas.contains("/stackable/listener-bootstrap")); - assert!(jaas.contains("/stackable/listener-broker")); - } - - #[test] - fn broker_controller_section_uses_the_broker_listener_address() { - let jaas = jaas_config_file(true, &KafkaRole::Broker); - assert!(jaas.contains("controller.KafkaServer {")); - // Brokers connect *out* to controllers. The only principals in a broker's keytab are - // for its own listener addresses, so this section must reuse the broker address. - assert!(jaas.contains( - "kafka/${file:UTF-8:/stackable/listener-broker/default-address/address}@${env:KERBEROS_REALM}" - )); - } - - #[test] - fn controller_jaas_has_only_the_controller_section_with_a_pod_fqdn_principal() { - let jaas = jaas_config_file(true, &KafkaRole::Controller); - assert!(jaas.contains("controller.KafkaServer {")); - assert!(jaas.contains(&format!( - "kafka/{CONTROLLER_POD_FQDN}@${{env:KERBEROS_REALM}}" - ))); - // Controllers have no listener-operator Listener volume, so the broker-only - // sections must not appear in their JAAS file. - assert!(!jaas.contains("bootstrap.KafkaServer")); - assert!(!jaas.contains("client.KafkaServer")); - } - - #[test] - fn controller_section_allows_the_process_to_act_as_a_gssapi_initiator() { - for role in [KafkaRole::Broker, KafkaRole::Controller] { - let jaas = jaas_config_file(true, &role); - let start = jaas - .find("controller.KafkaServer {") - .expect("controller.KafkaServer section must be present"); - // Unlike the other sections, this context is used for BOTH sides of every - // CONTROLLER-listener connection: brokers connect out to controllers, and - // controllers connect to each other for Raft. So `isInitiator` must stay at its - // default (`true`). Scoped to this section so a broker-side `isInitiator=false` - // elsewhere stays fine. - assert!( - !jaas[start..].contains("isInitiator=false"), - "controller.KafkaServer for {role:?} must not disable GSSAPI initiation" - ); - } - } -} -``` - -In `command.rs`, add a `mod tests` block: - -```rust -#[cfg(test)] -mod tests { - use stackable_operator::{ - builder::meta::ObjectMetaBuilder, - crd::authentication::{core, kerberos}, - }; - - use super::*; - use crate::crd::authentication::ResolvedAuthenticationClasses; - - fn kerberos() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![core::v1alpha1::AuthenticationClass { - metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), - spec: core::v1alpha1::AuthenticationClassSpec { - provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( - kerberos::v1alpha1::AuthenticationProvider { - kerberos_secret_class: "kerberos-secret-class".to_string(), - }, - ), - }, - }]), - "tls".parse().expect("valid secret class name"), - Some("tls".parse().expect("valid secret class name")), - None, - ) - } - - fn plaintext_security() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![]), - "tls".parse().expect("valid secret class name"), - None, - None, - ) - } - - #[test] - fn controller_command_exports_the_kerberos_realm_when_enabled() { - let command = controller_kafka_container_command(&kerberos(), vec![]); - assert!(command.contains("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*'")); - } - - #[test] - fn controller_command_does_not_export_a_realm_without_kerberos() { - let command = controller_kafka_container_command(&plaintext_security(), vec![]); - assert!(!command.contains("KERBEROS_REALM")); - } - - #[test] - fn controller_command_always_templates_the_jaas_file() { - // `jaas.properties` is always present in the ConfigMap (empty when Kerberos is off), - // so the copy is unconditional, matching `broker_start_command`. - for security in [kerberos(), plaintext_security()] { - let command = controller_kafka_container_command(&security, vec![]); - assert!(command.contains("cp /stackable/config/jaas.properties /tmp/jaas.properties")); - assert!(command.contains("config-utils template /tmp/jaas.properties")); - } - } -} -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: `cargo test -p stackable-kafka-operator` -Expected: compile errors — `jaas_config_file` takes one argument, `controller_kafka_container_command` takes one argument. - -- [ ] **Step 3: Give `jaas_config_file` a role and a controller section** - -In `config_map.rs`, add `KafkaRole` to the `crate::crd::role` import, change the call site to `jaas_config_file(kafka_security.has_kerberos_enabled(), &role)`, and replace the function with: - -```rust -// Generate JAAS configuration file for Kerberos authentication -// or an empty string if Kerberos is not enabled. -// See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html -fn jaas_config_file(is_kerberos_enabled: bool, role: &KafkaRole) -> String { - if !is_kerberos_enabled { - return String::new(); - } - - // Broker pods reach the CONTROLLER listener as SASL clients; the only principals in - // their keytab (see `add_kerberos_pod_config`) are for the broker and bootstrap listener - // addresses, so their CONTROLLER section must reuse the broker address. - // Controller pods have no listener-operator `Listener` volume; their keytab is - // pod-scoped, so their CONTROLLER section uses their own pod FQDN — the same expression - // already used for `KAFKA_LISTENERS` in `controller_properties.rs`. - let controller_principal_address = match role { - KafkaRole::Broker => node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), - KafkaRole::Controller => { - "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}" - .to_string() - } - }; - - // Unlike the bootstrap and client sections below, this context is used for BOTH sides of - // every CONTROLLER-listener connection: brokers connect out to controllers, and - // controllers connect to each other for Raft. This is the only listener in this operator - // where the process must act as a GSSAPI initiator as well as an acceptor, so - // `isInitiator` is intentionally left at its default (`true`). - let controller_section = formatdoc! {" - controller.KafkaServer {{ - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - keyTab=\"/stackable/kerberos/keytab\" - principal=\"kafka/{controller_principal_address}@${{env:KERBEROS_REALM}}\"; - }}; - "}; - - match role { - KafkaRole::Controller => controller_section, - KafkaRole::Broker => formatdoc! {" - bootstrap.KafkaServer {{ - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - isInitiator=false - keyTab=\"/stackable/kerberos/keytab\" - principal=\"kafka/{bootstrap_address}@${{env:KERBEROS_REALM}}\"; - }}; - - client.KafkaServer {{ - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - isInitiator=false - keyTab=\"/stackable/kerberos/keytab\" - principal=\"kafka/{broker_address}@${{env:KERBEROS_REALM}}\"; - }}; - - {controller_section} - ", - bootstrap_address = node_address_cmd(STACKABLE_LISTENER_BOOTSTRAP_DIR), - broker_address = node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), - }, - } -} -``` - -- [ ] **Step 4: Export the realm and template the JAAS file in the controller command** - -In `command.rs`, change the signature and body of `controller_kafka_container_command`: - -```rust -pub fn controller_kafka_container_command( - kafka_security: &ValidatedKafkaSecurity, - controller_descriptors: Vec, -) -> String { - formatdoc! {" - {COMMON_BASH_TRAP_FUNCTIONS} - {remove_vector_shutdown_file_command} - prepare_signal_handlers - containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - {set_realm_env} - - {derive_pod_index} - {export_replica_id} - - cp {config_dir}/{properties_file} /tmp/{properties_file} - - config-utils template /tmp/{properties_file} - - cp {config_dir}/{jaas_file} /tmp/{jaas_file} - config-utils template /tmp/{jaas_file} - - {quorum_format_flag} - bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted \"$FORMAT_QUORUM_FLAG\" - bin/kafka-server-start.sh /tmp/{properties_file} & - - wait_for_termination $! - {create_vector_shutdown_file_command} - ", - remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), - // Mirrors `broker_kafka_container_commands`: empty when Kerberos is disabled. - set_realm_env = match kafka_security.has_kerberos_enabled() { - true => format!("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {STACKABLE_KERBEROS_KRB5_PATH})"), - false => "".to_string(), - }, - derive_pod_index = DERIVE_POD_INDEX, - export_replica_id = EXPORT_REPLICA_ID, - config_dir = STACKABLE_CONFIG_DIR, - properties_file = ConfigFileName::ControllerProperties, - jaas_file = ConfigFileName::Jaas, - quorum_format_flag = controller_quorum_format_flag(&controller_descriptors), - create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR) - } -} -``` - -- [ ] **Step 5: Update the call site** - -In `statefulset.rs`, change: - -```rust - .args(vec![controller_kafka_container_command( - kafka_security, - controller_pod_descriptors, - )]); -``` - -- [ ] **Step 6: Run the tests to verify they pass** - -Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` -Expected: no warnings, all tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add rust/operator-binary/src/controller/build/resource/config_map.rs \ - rust/operator-binary/src/controller/build/command.rs \ - rust/operator-binary/src/controller/build/resource/statefulset.rs -git commit -m "feat: add a controller.KafkaServer JAAS section for KRaft controllers" -``` - ---- - -### Task 3: `SASL_SSL` on the CONTROLLER listener - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/properties/listener.rs:108-118` -- Modify: `rust/operator-binary/src/controller/build/security.rs:49` (new constant), and the Kerberos branches of `broker_config_settings` and `controller_config_settings` -- Modify: `rust/operator-binary/src/crd/listener.rs:55-67` (doc comment) -- Test: the `mod tests` blocks in `properties/listener.rs` and `security.rs` - -**Interfaces:** - -- Consumes: `KafkaListenerProtocol::SaslSsl`, `ValidatedKafkaSecurity::has_kerberos_enabled()`. -- Produces: no new public functions; `broker_config_settings` and `controller_config_settings` gain `sasl.mechanism.controller.protocol=GSSAPI` under Kerberos. - -- [ ] **Step 1: Write the failing tests** - -In `security.rs`, add to the existing Kerberos test for each role: - -```rust - #[test] - fn broker_config_sets_the_controller_sasl_mechanism_with_kerberos() { - let config = broker_config_settings(&kerberos()); - assert_eq!( - config.get("sasl.mechanism.controller.protocol"), - Some(&"GSSAPI".to_string()) - ); - } - - #[test] - fn controller_config_sets_the_controller_sasl_mechanism_with_kerberos() { - let config = controller_config_settings(&kerberos()); - assert_eq!( - config.get("sasl.mechanism.controller.protocol"), - Some(&"GSSAPI".to_string()) - ); - } - - #[test] - fn controller_sasl_mechanism_is_absent_without_kerberos() { - assert!( - !broker_config_settings(&internal_tls()).contains_key("sasl.mechanism.controller.protocol") - ); - assert!( - !controller_config_settings(&internal_tls()) - .contains_key("sasl.mechanism.controller.protocol") - ); - } -``` - -`kerberos()` and `internal_tls()` already exist in this module's `mod tests` (`kerberos()` is `pub(crate)`); do not add duplicates. - -In `properties/listener.rs`, update the existing `test_get_kafka_kerberos_listeners_config` expectation from `controller_protocol = KafkaListenerProtocol::Ssl` to `KafkaListenerProtocol::SaslSsl` (it is the last field of the `listener_security_protocol_map()` `format!` near the end of the module), and add this regression guard: - -```rust - #[test] - fn controller_listener_stays_ssl_without_kerberos() { - // Regression guard: only Kerberos may move CONTROLLER off plain SSL. - let kafka = minimal_kafka( - r#" - apiVersion: kafka.stackable.tech/v1alpha1 - kind: KafkaCluster - metadata: - name: simple-kafka - namespace: default - uid: 12345678-1234-1234-1234-123456789012 - spec: - image: - productVersion: 3.9.2 - clusterConfig: - metadataManager: kraft - controllers: - roleGroups: - default: - replicas: 3 - brokers: - roleGroups: - default: - replicas: 1 - "#, - ); - let validated = validated_cluster(&kafka); - let kafka_security = ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![]), - "internal-tls".parse().expect("valid secret class name"), - Some("tls".parse().expect("valid secret class name")), - None, - ); - let role_group_name: RoleGroupName = "default".parse().expect("valid role group name"); - let config = get_kafka_listener_config( - &validated, - &kafka_security, - &KafkaRole::Controller, - &role_group_name, - ); - - assert!( - config.listener_security_protocol_map().contains(&format!( - "{name}:{protocol}", - name = KafkaListenerName::Controller, - protocol = KafkaListenerProtocol::Ssl - )), - "got: {}", - config.listener_security_protocol_map() - ); - } -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: `cargo test -p stackable-kafka-operator` -Expected: the three new `sasl.mechanism.controller.protocol` assertions FAIL (key absent); the updated listener assertion FAILS (`SSL` vs `SASL_SSL`). - -- [ ] **Step 3: Switch the CONTROLLER protocol** - -In `properties/listener.rs`, replace: - -```rust - listener_security_protocol_map.insert( - KafkaListenerName::Controller, - if kafka_security.has_kerberos_enabled() { - KafkaListenerProtocol::SaslSsl - } else { - KafkaListenerProtocol::Ssl - }, - ); -``` - -- [ ] **Step 4: Add the controller SASL mechanism property** - -In `security.rs`, next to the other property-name constants: - -```rust -const PROPERTY_SASL_CONTROLLER_MECHANISM: &str = "sasl.mechanism.controller.protocol"; -``` - -and inside the `has_kerberos_enabled()` branch of **both** `broker_config_settings` and `controller_config_settings`, next to the existing `PROPERTY_SASL_INTER_BROKER_MECHANISM` insert: - -```rust - config.insert( - PROPERTY_SASL_CONTROLLER_MECHANISM.to_string(), - SASL_MECHANISM_GSSAPI.to_string(), - ); -``` - -- [ ] **Step 5: Correct the CONTROLLER listener doc comment** - -In `crd/listener.rs`, replace the `Controller` variant's stale doc lines: - -```rust - /// This listener is defined when Kraft mode is enabled. - /// It is responsible for broker/controller as well as controller/controller communications - /// and therefore it is present on *both* brokers and controller properties files. - /// The protocol used is SSL, or SASL_SSL when Kerberos is enabled. - /// The advertised host names are FQDN pod names of the controllers. - /// - /// Note: there is no listener for client/controller communication. -``` - -- [ ] **Step 6: Run the tests to verify they pass** - -Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` -Expected: no warnings, all tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add rust/operator-binary/src/controller/build/properties/listener.rs \ - rust/operator-binary/src/controller/build/security.rs \ - rust/operator-binary/src/crd/listener.rs -git commit -m "feat: use SASL_SSL on the CONTROLLER listener when Kerberos is enabled" -``` - ---- - -### Task 4: Kerberos-aware admin client properties - -The quorum-manager sidecar and the `preStop` hook both talk to the CONTROLLER listener, which Task 3 just moved to `SASL_SSL`. Without this task they can no longer connect. - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/security.rs:221-237` (`controller_admin_client_properties`) -- Test: the `mod tests` block in `security.rs` - -**Interfaces:** - -- Consumes: `ValidatedKafkaSecurity::has_kerberos_enabled()`, `push_client_ssl_stores`, `KafkaRole::kerberos_service_name()`. -- Produces: `pub fn controller_admin_client_properties(security: &ValidatedKafkaSecurity) -> Vec<(String, Option)>` — same signature, argument now used. - -- [ ] **Step 1: Write the failing tests** - -```rust - #[test] - fn admin_client_uses_gssapi_over_sasl_ssl_with_kerberos() { - let props = as_map(controller_admin_client_properties(&kerberos())); - assert_eq!(props.get("security.protocol"), Some(&"SASL_SSL".to_string())); - assert_eq!(props.get("sasl.mechanism"), Some(&"GSSAPI".to_string())); - assert_eq!( - props.get("sasl.kerberos.service.name"), - Some(&"kafka".to_string()) - ); - // The internal TLS stores stay: SASL_SSL is still SSL underneath. - assert_eq!( - props.get("ssl.truststore.location"), - Some(&"/stackable/tls-kafka-internal/truststore.p12".to_string()) - ); - } - - #[test] - fn admin_client_jaas_config_is_a_single_line_pod_principal() { - let props = as_map(controller_admin_client_properties(&kerberos())); - let jaas = props - .get("sasl.jaas.config") - .expect("sasl.jaas.config must be set when Kerberos is enabled"); - // Must be one logical line: a raw newline would truncate the value when the - // properties file is parsed. - assert!( - !jaas.contains('\n'), - "sasl.jaas.config must be a single line, got: {jaas}" - ); - assert!(jaas.contains("com.sun.security.auth.module.Krb5LoginModule required")); - assert!(jaas.contains("keyTab=\"/stackable/kerberos/keytab\"")); - // The controller's own pod-scoped principal (Task 1), resolved by - // `config-utils template` at container start. - assert!(jaas.contains( - "principal=\"kafka/${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}@${env:KERBEROS_REALM}\"" - )); - assert!(jaas.trim_end().ends_with(';')); - } - - #[test] - fn admin_client_is_unchanged_without_kerberos() { - let props = as_map(controller_admin_client_properties(&internal_tls())); - assert_eq!(props.get("security.protocol"), Some(&"SSL".to_string())); - assert!(!props.contains_key("sasl.mechanism")); - assert!(!props.contains_key("sasl.jaas.config")); - assert_eq!( - props.get("ssl.keystore.location"), - Some(&"/stackable/tls-kafka-internal/keystore.p12".to_string()) - ); - } -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: `cargo test -p stackable-kafka-operator admin_client` -Expected: the two Kerberos tests FAIL — `security.protocol` is `SSL` and `sasl.*` keys are absent. - -- [ ] **Step 3: Implement the Kerberos branch** - -Replace `controller_admin_client_properties`: - -```rust -/// Client-side (unprefixed `security.protocol`/`ssl.*`/`sasl.*`) properties for an admin CLI -/// tool (e.g. `kafka-metadata-quorum.sh`) talking to the CONTROLLER listener from *inside* a -/// controller pod, over the `tls-kafka-internal` volume mounted by -/// `add_controller_volume_and_volume_mounts`. -/// -/// When Kerberos is enabled the CONTROLLER listener is `SASL_SSL` (see -/// `get_kafka_listener_config`), so these calls must authenticate with GSSAPI. They do so as -/// the controller's *own* pod principal, from the pod-scoped keytab mounted by -/// `add_kerberos_pod_config` — which is the correct identity for a voter registering itself. -/// -/// The principal contains `${env:…}` placeholders, so the rendered file must be passed -/// through `config-utils template` before use; see `quorum_manager_container_command`. -pub fn controller_admin_client_properties( - security: &ValidatedKafkaSecurity, -) -> Vec<(String, Option)> { - let mut properties = vec![]; - - if security.has_kerberos_enabled() { - properties.push(( - PROPERTY_SECURITY_PROTOCOL.to_string(), - Some(KafkaListenerProtocol::SaslSsl.to_string()), - )); - // Client-side mechanism selection. `sasl.enabled.mechanisms` is the *broker-side* - // list and has no effect here. - properties.push(( - PROPERTY_SASL_MECHANISM.to_string(), - Some(SASL_MECHANISM_GSSAPI.to_string()), - )); - properties.push(( - PROPERTY_SASL_KERBEROS_SERVICE_NAME.to_string(), - Some(KafkaRole::Controller.kerberos_service_name().to_string()), - )); - properties.push(( - PROPERTY_SASL_JAAS_CONFIG.to_string(), - Some(format!( - "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true \ - storeKey=true keyTab=\"{keytab}\" \ - principal=\"{service}/{pod_fqdn}@${{env:KERBEROS_REALM}}\";", - keytab = STACKABLE_KERBEROS_KEYTAB_PATH, - service = KafkaRole::Controller.kerberos_service_name(), - pod_fqdn = CONTROLLER_POD_FQDN_TEMPLATE, - )), - )); - } else { - properties.push(( - PROPERTY_SECURITY_PROTOCOL.to_string(), - Some(KafkaListenerProtocol::Ssl.to_string()), - )); - } - - push_client_ssl_stores(&mut properties, STACKABLE_TLS_KAFKA_INTERNAL_DIR); - - properties -} -``` - -Add the supporting constants next to the other `PROPERTY_*` constants in `security.rs`: - -```rust -const PROPERTY_SASL_MECHANISM: &str = "sasl.mechanism"; -const PROPERTY_SASL_JAAS_CONFIG: &str = "sasl.jaas.config"; -const STACKABLE_KERBEROS_KEYTAB_PATH: &str = "/stackable/kerberos/keytab"; - -/// The controller pod's own FQDN, as `config-utils template` placeholders. Matches the -/// address used for `KAFKA_LISTENERS` in `controller_properties.rs` and for the -/// `controller.KafkaServer` JAAS principal in `jaas_config_file`. -const CONTROLLER_POD_FQDN_TEMPLATE: &str = - "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}"; -``` - -- [ ] **Step 4: Check the properties writer does not mangle the value** - -The `controller.properties` consumer strips escaped colons (`sed 's/\\:/:/g'` in `extract_bootstrap_servers_command`), which means the properties writer escapes `:` in values. The JAAS value above contains no colon, so no unescaping step is needed — but confirm by inspecting the rendered ConfigMap in Task 7's kuttl run before trusting it. - -Run: `cargo test -p stackable-kafka-operator admin_client` -Expected: PASS - -- [ ] **Step 5: Run the full suite** - -Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` -Expected: no warnings, all tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add rust/operator-binary/src/controller/build/security.rs -git commit -m "feat: authenticate the controller admin client with GSSAPI under Kerberos" -``` - ---- - -### Task 5: Un-gate dynamic quorum scaling under Kerberos - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/command.rs:230-280` (`quorum_manager_container_command`) -- Modify: `rust/operator-binary/src/controller/build/command.rs` (`controller_kafka_container_command` — template `admin-client.properties` for the `preStop` hook) -- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs:504-521` (remove the `preStop` gate) -- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs:748-805` (`build_quorum_manager_container`) -- Test: the `mod tests` blocks in `command.rs` and `statefulset.rs` - -**Interfaces:** - -- Consumes: Task 4's Kerberos-aware `controller_admin_client_properties`, Task 1's `kerberos` pod volume. -- Produces: `fn build_quorum_manager_container(…) -> Container` (no longer `Option`); the `ADMIN_CLIENT_PROPERTIES_PATH` constant moves to `/tmp/admin-client.properties`. - -- [ ] **Step 1: Write the failing tests** - -In `command.rs`: - -```rust - #[test] - fn quorum_manager_templates_the_admin_client_config() { - let command = quorum_manager_container_command(); - assert!(command.contains("cp /stackable/config/admin-client.properties /tmp/admin-client.properties")); - assert!(command.contains("config-utils template /tmp/admin-client.properties")); - // It must connect with the *rendered* copy, not the raw ConfigMap file, or the - // `${env:…}` placeholders in `sasl.jaas.config` reach the JAAS parser verbatim. - assert!(command.contains("ADMIN_CLIENT_CONFIG=/tmp/admin-client.properties")); - assert!(!command.contains("ADMIN_CLIENT_CONFIG=/stackable/config/admin-client.properties")); - } - - #[test] - fn quorum_manager_exports_the_kerberos_realm() { - // The sidecar is a separate container: it inherits nothing from the kafka - // container's startup, so it must derive $KERBEROS_REALM itself for - // `config-utils template` to resolve the principal. - let command = quorum_manager_container_command(); - assert!(command.contains("KERBEROS_REALM")); - } - - #[test] - fn controller_command_templates_the_admin_client_config_for_pre_stop() { - let command = controller_kafka_container_command(&kerberos(), vec![]); - assert!(command.contains("cp /stackable/config/admin-client.properties /tmp/admin-client.properties")); - assert!(command.contains("config-utils template /tmp/admin-client.properties")); - } -``` - -In `statefulset.rs`, this module already has `kraft_mode_cluster()`, `controller_containers(&cluster)` and `controller_kafka_container(&cluster)`. Add a Kerberos variant of the cluster fixture next to `kraft_mode_cluster()`: - -```rust - /// Like [`kraft_mode_cluster`], but referencing a Kerberos `AuthenticationClass`. - fn kraft_mode_kerberos_cluster() -> crate::controller::ValidatedCluster { - let kafka = minimal_kafka( - r#" - apiVersion: kafka.stackable.tech/v1alpha1 - kind: KafkaCluster - metadata: - name: simple-kafka - namespace: default - uid: 12345678-1234-1234-1234-123456789012 - spec: - image: - productVersion: 3.9.2 - clusterConfig: - metadataManager: kraft - authentication: - - authenticationClass: kerberos-auth - controllers: - roleGroups: - default: - replicas: 3 - brokers: - roleGroups: - default: - replicas: 3 - "#, - ); - validated_cluster(&kafka) - } -``` - -> `validated_cluster` must be able to resolve the `kerberos-auth` AuthenticationClass. If `test_support` cannot dereference AuthenticationClasses, extend it to accept a pre-resolved one rather than weakening the test — check `rust/operator-binary/src/controller/test_support.rs` first and adapt. - -Then the assertions: - -```rust - #[test] - fn quorum_manager_sidecar_is_present_with_kerberos() { - let containers = controller_containers(&kraft_mode_kerberos_cluster()); - let sidecar = containers - .iter() - .find(|c| c.name == QUORUM_MANAGER_CONTAINER_NAME.to_string()) - .expect("the quorum-manager sidecar must exist when Kerberos is enabled"); - let mounts: Vec<&str> = sidecar - .volume_mounts - .as_ref() - .expect("sidecar must have volume mounts") - .iter() - .map(|m| m.mount_path.as_str()) - .collect(); - assert!( - mounts.contains(&"/stackable/kerberos"), - "sidecar needs the keytab and krb5.conf to authenticate, got: {mounts:?}" - ); - let env: Vec<&str> = sidecar - .env - .as_ref() - .expect("sidecar must have env vars") - .iter() - .map(|e| e.name.as_str()) - .collect(); - assert!(env.contains(&"KRB5_CONFIG")); - // `KAFKA_OPTS` points the JVM at `/tmp/jaas.properties`, which only the `kafka` - // container renders. The sidecar uses an inline `sasl.jaas.config` instead. - assert!( - !env.contains(&"KAFKA_OPTS"), - "sidecar must not inherit the kafka container's JAAS login config" - ); - } - - #[test] - fn controller_pre_stop_hook_is_present_with_kerberos() { - let pre_stop_command = controller_kafka_container(&kraft_mode_kerberos_cluster()) - .lifecycle - .as_ref() - .and_then(|l| l.pre_stop.as_ref()) - .and_then(|h| h.exec.as_ref()) - .and_then(|e| e.command.as_ref()) - .expect("voter removal on scale-down must run under Kerberos too") - .join(" "); - assert!(pre_stop_command.contains("remove-controller")); - } -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: `cargo test -p stackable-kafka-operator` -Expected: `quorum_manager_sidecar_is_present_with_kerberos` FAILS (no such container — `build_quorum_manager_container` returns `None`); `controller_pre_stop_hook_is_present_with_kerberos` FAILS (hook skipped); the `command.rs` templating tests FAIL. - -- [ ] **Step 3: Template `admin-client.properties` in the sidecar** - -In `command.rs`, change the constant and extend the render chain: - -```rust -/// The rendered admin-client config. The raw ConfigMap file is copied here and passed -/// through `config-utils template` first, because under Kerberos its `sasl.jaas.config` -/// carries `${env:…}` placeholders (see `controller_admin_client_properties`). -const ADMIN_CLIENT_PROPERTIES_PATH: &str = "/tmp/admin-client.properties"; -const ADMIN_CLIENT_PROPERTIES_SOURCE_PATH: &str = "/stackable/config/admin-client.properties"; -``` - -In `quorum_manager_container_command`, add the realm export after `{extract_bootstrap_servers}`: - -```rust - {set_realm_env} -``` - -with - -```rust - // The sidecar is a separate container and inherits nothing from the kafka - // container's startup, so it derives the realm itself. Harmless when the - // krb5.conf is absent: `config-utils template` only needs it under Kerberos. - set_realm_env = format!( - "KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {STACKABLE_KERBEROS_KRB5_PATH} 2>/dev/null) && export KERBEROS_REALM || true" - ), -``` - -and extend the existing `if cp … && … ; then` chain to render the admin client config: - -```rust - if cp {config_dir}/{controller_properties_file} /tmp/{controller_properties_file} \ - && config-utils template /tmp/{controller_properties_file} \ - && cp {admin_client_source} {admin_client_config} \ - && config-utils template {admin_client_config} \ - && cat /tmp/{controller_properties_file} {admin_client_config} > {add_controller_config}; then -``` - -adding `admin_client_source = ADMIN_CLIENT_PROPERTIES_SOURCE_PATH,` to the format arguments. The existing degraded-mode `else` branch now also covers a failed Kerberos render, with no new error handling. - -- [ ] **Step 4: Template it in the kafka container too, for the `preStop` hook** - -The `preStop` hook runs in the `kafka` container and reads `$ADMIN_CLIENT_CONFIG`. Add to `controller_kafka_container_command`, immediately after the `jaas.properties` copy from Task 2: - -```rust - cp {admin_client_source} {admin_client_config} - config-utils template {admin_client_config} -``` - -with `admin_client_source = ADMIN_CLIENT_PROPERTIES_SOURCE_PATH,` and `admin_client_config = ADMIN_CLIENT_PROPERTIES_PATH,` added to the format arguments. - -- [ ] **Step 5: Remove the two Kerberos gates** - -In `statefulset.rs`, delete the `if !kafka_security.has_kerberos_enabled() {` wrapper and its stale comment around the `cb_kafka.lifecycle_pre_stop(…)` call, leaving the call unconditional. - -Then in `build_quorum_manager_container`, delete the early return and its comment, and change the return type: - -```rust -/// Builds the `quorum-manager` sidecar for a controller pod. -fn build_quorum_manager_container( - resolved_product_image: &ResolvedProductImage, - kafka_security: &ValidatedKafkaSecurity, - env: Vec, -) -> stackable_operator::k8s_openapi::api::core::v1::Container { -``` - -Mount the Kerberos material and set `KRB5_CONFIG` when Kerberos is on, just before `Some(cb.build())` becomes `cb.build()`: - -```rust - if kafka_security.has_kerberos_enabled() { - // `controller_admin_client_properties` authenticates with the pod-scoped keytab - // mounted by `add_kerberos_pod_config`, so this container needs it too — the - // volume itself is already on the pod. - cb.add_volume_mount(&*KERBEROS_VOLUME_NAME, STACKABLE_KERBEROS_DIR) - .expect("The mount paths are statically defined and there should be no duplicates."); - cb.add_env_var(KRB5_CONFIG.to_string(), STACKABLE_KERBEROS_KRB5_PATH); - } - - cb.build() -``` - -This needs `KERBEROS_VOLUME_NAME` and `KRB5_CONFIG` to be `pub` in `kerberos.rs` (they are currently private) and `STACKABLE_KERBEROS_DIR`/`STACKABLE_KERBEROS_KRB5_PATH` imported from `crate::crd`. - -Update the call site to drop the `if let Some(…)`: - -```rust - pod_builder.add_container(build_quorum_manager_container( - resolved_product_image, - kafka_security, - quorum_manager_env, - )); -``` - -- [ ] **Step 6: Run the tests to verify they pass** - -Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` -Expected: no warnings, all tests pass. - -- [ ] **Step 7: Commit** - -```bash -git add rust/operator-binary/src/controller/build/command.rs \ - rust/operator-binary/src/controller/build/kerberos.rs \ - rust/operator-binary/src/controller/build/resource/statefulset.rs -git commit -m "feat: keep dynamic KRaft quorum scaling working with Kerberos enabled" -``` - ---- - -### Task 6: Fix the discovery ConfigMap client properties - -`client_properties` feeds the discovery ConfigMap, consumed by clients running *outside* Kafka pods. Those clients have no `/stackable/kerberos/keytab` and no per-pod principal, so three of its current entries are wrong for that consumer. - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/security.rs:161-218` (`client_properties`) -- Test: the `mod tests` block in `security.rs` - -**Interfaces:** - -- Consumes: nothing new. -- Produces: `client_properties` signature unchanged; output loses three keys and gains `sasl.mechanism`. - -- [ ] **Step 1: Write the failing test** - -```rust - #[test] - fn discovery_client_properties_carry_no_server_side_or_pod_local_settings() { - let props = as_map(client_properties(&kerberos())); - - // The consumer runs outside Kafka pods: it has no keytab and no pod principal, so a - // `sasl.jaas.config` here could only ever be wrong. Clients supply their own. - assert!(!props.contains_key("sasl.jaas.config")); - // Broker-side properties with no meaning in a client config. - assert!(!props.contains_key("sasl.mechanism.inter.broker.protocol")); - assert!(!props.contains_key("sasl.enabled.mechanisms")); - - // What a client actually needs. - assert_eq!(props.get("security.protocol"), Some(&"SASL_SSL".to_string())); - assert_eq!(props.get("sasl.mechanism"), Some(&"GSSAPI".to_string())); - assert_eq!( - props.get("sasl.kerberos.service.name"), - Some(&"kafka".to_string()) - ); - assert_eq!( - props.get("ssl.truststore.location"), - Some(&"/stackable/tls-kafka-server/truststore.p12".to_string()) - ); - } -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `cargo test -p stackable-kafka-operator discovery_client_properties` -Expected: FAIL — `sasl.jaas.config` is present (with the `kafka/todo@…` placeholder principal). - -- [ ] **Step 3: Rewrite the Kerberos branch** - -Replace the `else if security.has_kerberos_enabled() {` arm of `client_properties` with: - -```rust - } else if security.has_kerberos_enabled() { - props.push(( - PROPERTY_SECURITY_PROTOCOL.to_string(), - Some(KafkaListenerProtocol::SaslSsl.to_string()), - )); - push_client_ssl_stores(&mut props, STACKABLE_TLS_KAFKA_SERVER_DIR); - // `sasl.mechanism` is the client-side selector. `sasl.enabled.mechanisms` is the - // broker-side list of accepted mechanisms and has no effect in a client config. - props.push(( - PROPERTY_SASL_MECHANISM.to_string(), - Some(SASL_MECHANISM_GSSAPI.to_string()), - )); - props.push(( - PROPERTY_SASL_KERBEROS_SERVICE_NAME.to_string(), - Some(KafkaRole::Broker.kerberos_service_name().to_string()), - )); - // Deliberately no `sasl.jaas.config`: this file is consumed by clients running - // outside Kafka pods, which have neither the keytab at /stackable/kerberos/keytab - // nor a per-pod principal. They supply their own login configuration; see - // docs/modules/kafka/pages/usage-guide/security.adoc. -``` - -- [ ] **Step 4: Run the test to verify it passes** - -Run: `cargo test -p stackable-kafka-operator discovery_client_properties` -Expected: PASS - -- [ ] **Step 5: Run the full suite and fix any stale expectations** - -Run: `cargo clippy --all-targets -- -D warnings && cargo test -p stackable-kafka-operator` -Expected: no warnings. Existing tests asserting the removed keys must be updated to assert their absence, not deleted silently. - -- [ ] **Step 6: Commit** - -```bash -git add rust/operator-binary/src/controller/build/security.rs -git commit -m "fix: remove pod-local and broker-side settings from the discovery client properties" -``` - ---- - -### Task 7: kuttl integration test - -This is the regression test for Task 5. PR #999 predates the quorum manager, so its suite must be extended with scale steps. - -**Files:** - -- Create: `tests/templates/kuttl/kraft-kerberos/` (copied from the reference worktree, then extended) -- Modify: `tests/test-definition.yaml` - -**Interfaces:** - -- Consumes: all preceding tasks. -- Produces: a `kraft-kerberos` kuttl suite registered as a test dimension. - -- [ ] **Step 1: Copy the reference suite** - -```bash -cp -r /tmp/pr999/tests/templates/kuttl/kraft-kerberos tests/templates/kuttl/kraft-kerberos -ls tests/templates/kuttl/kraft-kerberos -``` - -- [ ] **Step 2: Register the test dimension** - -In `tests/test-definition.yaml`, add a `kraft-kerberos` entry to `tests:`, mirroring the existing `kerberos` entry's dimensions (`kafka-latest`, `kerberos-realm`, `kerberos-backend`, `openshift`). Copy the shape from `/tmp/pr999/tests/test-definition.yaml`, adapting names to whatever `main` currently uses — `main` has since changed this file. - -- [ ] **Step 3: Run the suite as copied, to establish a baseline** - -Run: - -```bash -./scripts/run-tests --test-suite kraft-kerberos -``` - -Expected: the 3-controller quorum forms, produce/consume succeeds. If it fails, fix before extending — an already-red suite cannot validate Step 4. - -- [ ] **Step 4: Add controller scale-up and scale-down steps** - -Copy the scale steps from `tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2`, `60-assert.yaml.j2`, `70-scale-controller-down.yaml.j2` and `70-assert.yaml.j2` into the `kraft-kerberos` suite as steps `60-*` and `70-*`, adapting the KafkaCluster name and namespace to this suite's. - -The assertions must confirm the *quorum* changed, not just the StatefulSet replica count — a controller that starts but never joins the voter set is exactly the failure this guards against. Reuse `operations-kraft`'s existing `kafka-metadata-quorum describe` assertion verbatim. - -- [ ] **Step 5: Run the extended suite** - -Run: - -```bash -./scripts/run-tests --test-suite kraft-kerberos -``` - -Expected: PASS, including the scale steps. - -- [ ] **Step 6: Inspect the rendered admin client config (Task 4, Step 4 follow-up)** - -While the cluster is up: - -```bash -kubectl exec -n "$NAMESPACE" test-kafka-controller-default-0 -c quorum-manager -- cat /tmp/admin-client.properties -``` - -Expected: `sasl.jaas.config` is one line, with the placeholders resolved to a real pod FQDN and realm, and no stray backslash escapes. - -- [ ] **Step 7: Commit** - -```bash -git add tests/templates/kuttl/kraft-kerberos tests/test-definition.yaml -git commit -m "test: add a kraft-kerberos kuttl suite covering quorum scaling" -``` - ---- - -### Task 8: Documentation and changelog - -**Files:** - -- Modify: `docs/modules/kafka/pages/usage-guide/kraft-controller.adoc` -- Modify: `docs/modules/kafka/pages/usage-guide/security.adoc` -- Modify: `CHANGELOG.md` - -**Interfaces:** - -- Consumes: the behaviour built in Tasks 1-7. - -- [ ] **Step 1: Document Kerberized controllers** - -In `kraft-controller.adoc`, add a Kerberos section covering: the `CONTROLLER` listener uses `SASL_SSL` with GSSAPI when an `AuthenticationClass` with the Kerberos provider is referenced; controller keytabs are pod-scoped (controllers are reached by pod DNS name, not through a `Listener`) while broker keytabs are listener-scoped; dynamic quorum scaling is supported with Kerberos enabled. Use "Apache Kafka" in prose. Consult `/tmp/pr999/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc` for the reference wording, but do **not** carry over any statement that scaling is unsupported under Kerberos — Task 5 makes that false. - -- [ ] **Step 2: Document the client-side Kerberos requirement** - -In `security.adoc`, note that the discovery ConfigMap's `client.properties` carries `security.protocol`, `sasl.mechanism`, `sasl.kerberos.service.name` and the truststore settings, and that clients must supply their own principal and keytab (their own JAAS login configuration) — the operator cannot do so, as the file is consumed outside Kafka pods. - -- [ ] **Step 3: Add the changelog entry** - -Under `## [Unreleased]` → `### Added` in `CHANGELOG.md`: - -```markdown -- Support Kerberos authentication on KRaft controllers, covering both broker-to-controller - and controller-to-controller (Raft) traffic. Dynamic quorum scaling continues to work with - Kerberos enabled ([#999], [#815]). -``` - -and under `### Fixed`: - -```markdown -- Remove the pod-local `sasl.jaas.config` and the broker-side `sasl.enabled.mechanisms` and - `sasl.mechanism.inter.broker.protocol` settings from the discovery ConfigMap's client - properties; they were never usable by out-of-cluster clients ([#999]). -``` - -Add the link definitions at the bottom of the file in the existing style. - -- [ ] **Step 4: Verify the docs build** - -Run: `./scripts/docs_templating.sh && ./scripts/render_readme.sh` -Expected: no errors, no unexpected diff. - -- [ ] **Step 5: Commit** - -```bash -git add docs CHANGELOG.md -git commit -m "docs: document Kerberos support for KRaft controllers" -``` - -- [ ] **Step 6: Clean up the reference worktree** - -```bash -git worktree remove /tmp/pr999 -``` - ---- - -## Verification - -Before opening the PR: - -- [ ] `cargo clippy --all-targets -- -D warnings` — clean -- [ ] `cargo test -p stackable-kafka-operator` — all pass -- [ ] `./scripts/run-tests --test-suite kraft-kerberos` — passes including scale steps -- [ ] `./scripts/run-tests --test-suite smoke-kraft` — no regression with Kerberos off -- [ ] `./scripts/run-tests --test-suite operations-kraft` — no regression in quorum scaling with Kerberos off -- [ ] `./scripts/run-tests --test-suite kerberos` — no regression in broker Kerberos diff --git a/docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md b/docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md deleted file mode 100644 index 330c79f9..00000000 --- a/docs/superpowers/specs/2026-09-16-kerberized-kraft-controllers-design.md +++ /dev/null @@ -1,202 +0,0 @@ -# Kerberized KRaft controllers — design - -- Date: 2026-09-16 -- Tickets: stackabletech/issues#815, kafka-operator#899, kafka-operator#870 -- Existing work: kafka-operator#999 (draft, branch `feature/kraft-kerberos-support`) - -## Goal - -Let Apache Kafka KRaft controllers authenticate with Kerberos (GSSAPI), covering both -broker-to-controller and controller-to-controller (Raft) traffic, **without** losing the -dynamic quorum scaling added by kafka-operator#1010. - -## Background - -PR #999 implements most of the controller-side Kerberos support against base commit -`5211842`. Since then `main` has absorbed #1010 (dynamic KRaft quorum scaling) and a large -refactor, so the PR cannot be merged as-is. - -More importantly, #1010 and Kerberos are **mutually exclusive in `main` today**: - -- `build_quorum_manager_container` (`controller/build/resource/statefulset.rs:749`) returns - `None` when Kerberos is enabled. -- The controller `preStop` `remove-controller` hook is skipped for the same reason - (`statefulset.rs:506`). - -Both gates exist because `controller_admin_client_properties` -(`controller/build/security.rs:225`) ignores its `_security` argument and hardcodes -`security.protocol=SSL`. Merging #999 unchanged would therefore ship Kerberized controllers -that silently lose dynamic quorum scaling. - -### Why the admin client uses GSSAPI - -A Kafka listener has exactly one security protocol, so the mechanism used by -`kafka-metadata-quorum.sh` is decided by the listener it connects to. An alternative was -considered: define a second controller listener (`controller.listener.names` accepts a -comma-separated list) carrying plain `SSL` for admin traffic, leaving `CONTROLLER` on -`SASL_SSL`. - -Rejected, because: - -1. `add-controller` is not a plain admin call. The self-registering process reads `node.id` - and its own `listeners`/`controller.listener.names` from the **same** `--command-config` - file to build the voter-registration payload (see the comment at - `controller/build/command.rs:181-189`). With two controller listeners in that file, the - endpoint registered into the quorum becomes ambiguous — and registering the wrong endpoint - breaks the quorum, not just the admin call. -2. It creates a second identity to authorize: an X.509 principal (`CN=…`) alongside - `kafka/…@REALM`, so every controller-quorum ACL would need both. -3. The GSSAPI route is cheap. The sidecar runs *inside* the controller pod, which already - carries the correct pod-scoped keytab, and the controller's own principal is the right - identity for a voter registering itself. - -## Design - -### 1. Rebase of #999 - -`feature/kraft-kerberos-support` is 21 commits on `5211842` and contains a merge commit -(`a0adc2a`). Squash into a small set of logical commits first, then rebase onto `main`; a -plain `git rebase --onto` flattens the merge awkwardly. - -Hunks `main` has already obsoleted — **drop them, do not resolve the conflict**: - -- `kerberos.rs`: the `cb_kcat_prober: Option<&mut ContainerBuilder>` signature change. `main` - removed that parameter entirely. Keep only the core change: - `match role { Broker => listener volume scopes, Controller => with_pod_scope() }`. -- `kerberos.rs`: the `cb.add_env_var("KRB5_CONFIG", …)` loop. `main` extracted this into - `kerberos_env_vars() -> EnvVarSet` so that user `envOverrides` win on a name collision. -- String literals replaced by `constant!` newtypes throughout (`&*LISTENER_BROKER_VOLUME_NAME`, - `&*KERBEROS_VOLUME_NAME`, `EnvVarName`). Mechanical. - -Hunks needing genuine re-application: - -- `command.rs`: #1010 rewrote `controller_kafka_container_command` (`NODE_ID_OFFSET`, - `--no-initial-controllers`, `$FORMAT_QUORUM_FLAG`). The `set_realm_env` and `jaas_setup` - insertions must be re-placed into the new body. -- The PR's `controller_command_is_byte_identical_to_pre_kerberos_output_when_disabled` test - pins against a hand-copied *pre-#1010* function body. Re-baseline it against `main`'s - current body, otherwise it fails for the wrong reason and proves nothing. - -`controller/build/security.rs`, `crd/listener.rs` and `controller/build/properties/listener.rs` -hunks are expected to apply near-clean. - -Behaviour carried over unchanged from #999: - -- `CONTROLLER` listener becomes `SASL_SSL` when Kerberos is enabled, `SSL` otherwise. -- `sasl.mechanism.controller.protocol=GSSAPI` on both broker and controller properties. -- Controller keytabs are **pod-scoped** (`with_pod_scope()`); broker keytabs stay - listener-scoped. Controllers have no listener-operator `Listener` volume — they are only - reachable via their StatefulSet pod DNS name. -- A `controller.KafkaServer` JAAS section on both roles. It deliberately does **not** set - `isInitiator=false`: it is the only listener where the process must act as a GSSAPI - initiator as well as an acceptor, because controllers connect to each other for Raft. -- Kerberos-disabled output stays byte-identical to the pre-Kerberos implementation. - -### 2. Kerberos-aware admin client - -`controller_admin_client_properties` must branch on `has_kerberos_enabled()`: - -| Property | Value | -| --- | --- | -| `security.protocol` | `SASL_SSL` | -| `sasl.mechanism` | `GSSAPI` | -| `sasl.kerberos.service.name` | `kafka` | -| `sasl.jaas.config` | single-line `Krb5LoginModule`, `useKeyTab=true`, `storeKey=true`, `keyTab="/stackable/kerberos/keytab"`, `principal="kafka/${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}@${env:KERBEROS_REALM}"` | - -The existing internal keystore/truststore properties (`STACKABLE_TLS_KAFKA_INTERNAL_DIR`) are -kept in both branches. The non-Kerberos branch is unchanged. - -That principal contains `${env:…}` placeholders, so **`admin-client.properties` must pass -through `config-utils template` before use**. Today both consumers read it directly from -`/stackable/config`. Two changes: - -- **Sidecar** (`quorum_manager_container_command`): it already does - `cp controller.properties /tmp/ && config-utils template …` inside an `if … ; then` guard. - Extend that same `&&` chain to `admin-client.properties` and point `ADMIN_CLIENT_CONFIG` at - the `/tmp` copy. The existing degraded-mode `else` branch then covers a Kerberos render - failure with no new error handling. -- **Kafka container startup**: the same copy-and-template step, so the `preStop` hook's - `$ADMIN_CLIENT_CONFIG` resolves. - -The sidecar is a separate container and inherits nothing from the kafka container's startup, -so it additionally needs: - -- the `kerberos` volume mounted at `STACKABLE_KERBEROS_DIR`, -- `KRB5_CONFIG` set, -- its own `export KERBEROS_REALM=$(grep -oP 'default_realm = \K.*' …)`. - -Finally, remove both Kerberos gates — the `build_quorum_manager_container` early return and -the `preStop` skip — along with their now-false explanatory comments. - -### 3. Discovery ConfigMap client properties - -`client_properties` (`controller/build/security.rs:162`) emits -`principal="kafka/todo@$KERBEROS_REALM"`. This is not a missing value: the consumer is a -client running *outside* Kafka pods, with no `/stackable/kerberos/keytab` and no per-pod -principal. Supplying a real principal would produce a file that is confidently broken rather -than obviously broken. - -Resolution: - -- Delete the `sasl.jaas.config` entry from the discovery file. -- Delete `sasl.mechanism.inter.broker.protocol` from it — a broker-side property with no - meaning in a client config. -- Keep `security.protocol`, the `ssl.*` store properties and `sasl.kerberos.service.name`. -- Replace `sasl.enabled.mechanisms` with `sasl.mechanism=GSSAPI`. `sasl.enabled.mechanisms` is - the broker-side property (the list a broker accepts); the client-side equivalent — the one a - client actually reads — is `sasl.mechanism`. Same class of mistake as the two deletions - above, so it is fixed here rather than left behind. -- Document that clients supply their own principal and keytab (their own `jaas.conf`). - -The `TODO` comment above the block is discharged by the JAAS work in §1: the operator does -write real JAAS files, for the pods that actually hold keytabs. - -### 4. Tests - -Unit: - -- #999's `jaas_config_file`, `kerberos.rs` and `security.rs` tests, carried over. -- The re-baselined byte-identical command test (§1). -- New: Kerberized `controller_admin_client_properties` — asserts `SASL_SSL`, `GSSAPI`, the - service name, the pod-FQDN principal, and that the internal TLS stores are still present. -- New: non-Kerberos `controller_admin_client_properties` is unchanged. -- New: `quorum_manager_container_command` templates `admin-client.properties` and points - `ADMIN_CLIENT_CONFIG` at the `/tmp` copy. -- New: `build_quorum_manager_container` returns `Some` with Kerberos enabled, and the returned - container mounts the `kerberos` volume. - -Integration (kuttl): - -- #999's `kraft-kerberos` suite (MIT KDC, 3-controller quorum, produce/consume). -- **Extend it with controller scale-up and scale-down steps**, mirroring - `tests/templates/kuttl/operations-kraft/60-*` and `70-*`. This is the regression test for - the un-gating in §2 and is not optional — #999 predates the quorum manager and cannot have - covered it. -- Register the dimension in `tests/test-definition.yaml`. - -### 5. Documentation - -- `docs/modules/kafka/pages/usage-guide/kraft-controller.adoc`: Kerberos on the `CONTROLLER` - listener; pod-scoped controller keytabs vs listener-scoped broker keytabs; dynamic quorum - scaling is supported with Kerberos enabled. -- `docs/modules/kafka/pages/usage-guide/security.adoc`: clients must supply their own - principal and keytab when using the discovery ConfigMap (§3). -- `CHANGELOG.md` entry. - -## Risks - -- §2 puts GSSAPI on the `add-controller` self-registration path — the one call whose failure - corrupts quorum membership rather than merely erroring. The kuttl scale steps in §4 are what - make this safe to ship. -- The controller keytab needs secret-operator to support pod scope together with a Kerberos - service name. #999 reports this working on OKD, so it is assumed available; verify early in - implementation rather than at integration-test time. -- `sasl.jaas.config` must be a single logical line and correctly escaped for the Java - properties format. A malformed value fails at JAAS parse time inside the sidecar, which the - degraded-mode `else` branch will *not* catch (the render succeeds; the CLI call fails). - -## Out of scope - -- OPA/ACL authorization rules for the controller quorum principals. -- Kerberos support for the `kcat` readiness prober on brokers. -- KRaft migration from ZooKeeper with Kerberos enabled. diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 4b7b0fbf..21e1d8de 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -618,30 +618,11 @@ pub(crate) mod test_support { /// the result. Used for tests asserting on a specific validation failure. pub fn validate_err( kafka: &v1alpha1::KafkaCluster, - ) -> Result { - validate_with_auth_err(kafka, ResolvedAuthenticationClasses::new(Vec::new())) - } - - /// Like [`validated_cluster`], but with the given already-resolved `AuthenticationClass`es, - /// for fixtures whose `spec.clusterConfig.authentication` references one (e.g. Kerberos). - pub fn validated_cluster_with_auth( - kafka: &v1alpha1::KafkaCluster, - authentication_classes: ResolvedAuthenticationClasses, - ) -> ValidatedCluster { - validate_with_auth_err(kafka, authentication_classes) - .expect("validate should succeed for the test fixture") - } - - /// The shared body of [`validate_err`] and [`validated_cluster_with_auth`]: the real validate - /// step, parameterized on the resolved `AuthenticationClass`es. - pub fn validate_with_auth_err( - kafka: &v1alpha1::KafkaCluster, - authentication_classes: ResolvedAuthenticationClasses, ) -> Result { validate( kafka, DereferencedObjects { - authentication_classes, + authentication_classes: ResolvedAuthenticationClasses::new(Vec::new()), authorization_config: None, kubernetes_cluster_info: cluster_info(), bootstrap_listeners: Vec::new(), diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index 210a9425..6b8c0d73 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -220,14 +220,6 @@ const CLI_CALL_KILL_AFTER_SECONDS: u32 = 5; /// Shell snippet setting `$BOOTSTRAP_SERVERS` by extracting /// `controller.quorum.bootstrap.servers` from the static, un-rendered `controller.properties` /// ConfigMap file. -/// -/// Reading this at runtime, rather than baking the peer list into this script as a Rust -/// literal, keeps both sidecar scripts' content identical across replica-count changes, so -/// the peer list lives in exactly one place (the ConfigMap). Note that since -/// [`kraft_controllers`][kc] lists individual pod FQDNs, that ConfigMap entry *does* change -/// with the replica count, and the pods roll with it. -/// -/// [kc]: super::properties::kraft_controllers fn extract_bootstrap_servers_command() -> String { format!( r#"BOOTSTRAP_SERVERS=$(grep '^controller.quorum.bootstrap.servers=' {config_dir}/{controller_properties_file} | cut -d= -f2- | sed 's/\\:/:/g')"#, diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index 1c4ec3fd..0335a4fd 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -121,84 +121,7 @@ pub fn kerberos_env_vars(kafka_security: &ValidatedKafkaSecurity) -> EnvVarSet { #[cfg(test)] mod tests { - use std::collections::BTreeMap; - - use stackable_operator::builder::pod::container::ContainerBuilder; - use super::*; - use crate::controller::build::security::tests::kerberos; - - /// Reads the `secrets.stackable.tech/*` annotations off the `kerberos` ephemeral volume. - fn kerberos_volume_annotations(pb: &mut PodBuilder) -> BTreeMap { - pb.build_template() - .spec - .as_ref() - .and_then(|spec| spec.volumes.as_ref()) - .and_then(|volumes| { - volumes - .iter() - .find(|v| v.name == KERBEROS_VOLUME_NAME.to_string()) - }) - .expect("kerberos volume must be present") - .ephemeral - .as_ref() - .expect("kerberos volume must be an ephemeral secret-operator volume") - .volume_claim_template - .as_ref() - .and_then(|t| t.metadata.as_ref()) - .and_then(|m| m.annotations.clone()) - .expect("volume claim template must carry secrets.stackable.tech annotations") - } - - fn kerberos_volume_annotations_for(role: &KafkaRole) -> BTreeMap { - let mut pb = PodBuilder::new(); - let mut cb_kafka = ContainerBuilder::new("kafka").expect("valid container name"); - - add_kerberos_pod_config(&kerberos(), role, &mut cb_kafka, &mut pb) - .expect("kerberos pod config"); - - kerberos_volume_annotations(&mut pb) - } - - #[test] - fn controller_keytab_is_pod_scoped() { - let annotations = kerberos_volume_annotations_for(&KafkaRole::Controller); - - // Controllers have no listener-operator Listener volume, so the keytab must be - // scoped to the pod's own DNS name, matching how their internal TLS cert is - // provisioned in `add_controller_volume_and_volume_mounts`. - assert_eq!( - annotations - .get("secrets.stackable.tech/scope") - .map(String::as_str), - Some("pod"), - "controller keytab must be pod-scoped, got: {annotations:?}" - ); - assert_eq!( - annotations - .get("secrets.stackable.tech/kerberos.service.names") - .map(String::as_str), - Some("kafka") - ); - } - - #[test] - fn broker_keytab_stays_listener_scoped() { - let annotations = kerberos_volume_annotations_for(&KafkaRole::Broker); - - let scope = annotations - .get("secrets.stackable.tech/scope") - .expect("scope annotation must be present"); - assert!( - scope.contains("listener-volume=listener-broker") - && scope.contains("listener-volume=listener-bootstrap"), - "broker keytab must stay listener-volume-scoped, got: {scope}" - ); - assert!( - !scope.split(',').any(|s| s == "pod"), - "broker keytab must not be pod-scoped, got: {scope}" - ); - } #[test] fn test_constants() { diff --git a/rust/operator-binary/src/controller/build/properties/listener.rs b/rust/operator-binary/src/controller/build/properties/listener.rs index 5bedcb68..acc63255 100644 --- a/rust/operator-binary/src/controller/build/properties/listener.rs +++ b/rust/operator-binary/src/controller/build/properties/listener.rs @@ -502,55 +502,4 @@ mod tests { ) ); } - - #[test] - fn controller_listener_stays_ssl_without_kerberos() { - // Regression guard: only Kerberos may move CONTROLLER off plain SSL. - let kafka_cluster = r#" - apiVersion: kafka.stackable.tech/v1alpha1 - kind: KafkaCluster - metadata: - name: simple-kafka - namespace: default - uid: 12345678-1234-1234-1234-123456789012 - spec: - image: - productVersion: 3.9.2 - clusterConfig: - metadataManager: kraft - controllers: - roleGroups: - default: - replicas: 3 - brokers: - roleGroups: - default: - replicas: 1 - "#; - let kafka = minimal_kafka(kafka_cluster); - let validated = validated_cluster(&kafka); - let kafka_security = ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![]), - "internal-tls".parse().unwrap(), - Some("tls".parse().unwrap()), - None, - ); - let role_group_name: RoleGroupName = "default".parse().unwrap(); - let config = get_kafka_listener_config( - &validated, - &kafka_security, - &KafkaRole::Controller, - &role_group_name, - ); - - assert!( - config.listener_security_protocol_map().contains(&format!( - "{name}:{protocol}", - name = KafkaListenerName::Controller, - protocol = KafkaListenerProtocol::Ssl - )), - "got: {}", - config.listener_security_protocol_map() - ); - } } diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 0d9d5285..0eed9d45 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -74,8 +74,8 @@ pub fn uses_legacy_log4j(product_version: &str) -> bool { /// `kafka/`, which is also what the Raft voter endpoints advertise. Bootstrapping /// through the Service therefore fails authentication for every peer. /// -/// The trade-off is deliberate: unlike the headless-Service form, this list changes whenever -/// a controller role group's replica count changes, so scaling one rolls the controller pods. +/// The side-effect of using pod FQDNs instead of service names is that this list changes on +/// on every scaling operation (replica count change), so scaling one rolls *all* controller pods. pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Vec { pod_descriptors .iter() diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 5dafacc2..9d21f7e3 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -289,8 +289,6 @@ mod tests { use super::jaas_config_file; use crate::crd::role::KafkaRole; - const CONTROLLER_POD_FQDN: &str = "${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}"; - #[test] fn jaas_config_file_empty_without_kerberos() { assert_eq!(jaas_config_file(false, &KafkaRole::Broker), ""); @@ -308,47 +306,4 @@ mod tests { assert!(jaas.contains("/stackable/listener-bootstrap")); assert!(jaas.contains("/stackable/listener-broker")); } - - #[test] - fn broker_controller_section_uses_the_broker_listener_address() { - let jaas = jaas_config_file(true, &KafkaRole::Broker); - assert!(jaas.contains("controller.KafkaServer {")); - // Brokers connect *out* to controllers. The only principals in a broker's keytab are - // for its own listener addresses, so this section must reuse the broker address. - assert!(jaas.contains( - "kafka/${file:UTF-8:/stackable/listener-broker/default-address/address}@${env:KERBEROS_REALM}" - )); - } - - #[test] - fn controller_jaas_has_only_the_controller_section_with_a_pod_fqdn_principal() { - let jaas = jaas_config_file(true, &KafkaRole::Controller); - assert!(jaas.contains("controller.KafkaServer {")); - assert!(jaas.contains(&format!( - "kafka/{CONTROLLER_POD_FQDN}@${{env:KERBEROS_REALM}}" - ))); - // Controllers have no listener-operator Listener volume, so the broker-only - // sections must not appear in their JAAS file. - assert!(!jaas.contains("bootstrap.KafkaServer")); - assert!(!jaas.contains("client.KafkaServer")); - } - - #[test] - fn controller_section_allows_the_process_to_act_as_a_gssapi_initiator() { - for role in [KafkaRole::Broker, KafkaRole::Controller] { - let jaas = jaas_config_file(true, &role); - let start = jaas - .find("controller.KafkaServer {") - .expect("controller.KafkaServer section must be present"); - // Unlike the other sections, this context is used for BOTH sides of every - // CONTROLLER-listener connection: brokers connect out to controllers, and - // controllers connect to each other for Raft. So `isInitiator` must stay at its - // default (`true`). Scoped to this section so a broker-side `isInitiator=false` - // elsewhere stays fine. - assert!( - !jaas[start..].contains("isInitiator=false"), - "controller.KafkaServer for {role:?} must not disable GSSAPI initiation" - ); - } - } } diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 33d4254b..d8c3ba42 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -968,123 +968,6 @@ mod tests { validated_cluster(&kafka) } - /// Like [`kraft_mode_cluster`], but referencing a Kerberos `AuthenticationClass`. - fn kraft_mode_kerberos_cluster() -> crate::controller::ValidatedCluster { - use stackable_operator::{ - builder::meta::ObjectMetaBuilder, - crd::authentication::{core, kerberos}, - }; - - use crate::{ - controller::test_support::validated_cluster_with_auth, - crd::authentication::ResolvedAuthenticationClasses, - }; - - let kafka = minimal_kafka( - r#" - apiVersion: kafka.stackable.tech/v1alpha1 - kind: KafkaCluster - metadata: - name: simple-kafka - namespace: default - uid: 12345678-1234-1234-1234-123456789012 - spec: - image: - productVersion: 3.9.2 - clusterConfig: - metadataManager: kraft - authentication: - - authenticationClass: kerberos-auth - controllers: - roleGroups: - default: - replicas: 3 - brokers: - roleGroups: - default: - replicas: 3 - "#, - ); - validated_cluster_with_auth( - &kafka, - ResolvedAuthenticationClasses::new(vec![core::v1alpha1::AuthenticationClass { - metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), - spec: core::v1alpha1::AuthenticationClassSpec { - provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( - kerberos::v1alpha1::AuthenticationProvider { - kerberos_secret_class: "kerberos-secret-class".to_string(), - }, - ), - }, - }]), - ) - } - - #[test] - fn quorum_manager_sidecar_is_present_with_kerberos() { - let containers = controller_containers(&kraft_mode_kerberos_cluster()); - let sidecar = containers - .iter() - .find(|c| c.name == QUORUM_MANAGER_CONTAINER_NAME.to_string()) - .expect("the quorum-manager sidecar must exist when Kerberos is enabled"); - - let mounts: Vec<&str> = sidecar - .volume_mounts - .as_ref() - .expect("sidecar must have volume mounts") - .iter() - .map(|m| m.mount_path.as_str()) - .collect(); - assert!( - mounts.contains(&"/stackable/kerberos"), - "sidecar needs the keytab and krb5.conf to authenticate, got: {mounts:?}" - ); - - let env: Vec<&str> = sidecar - .env - .as_ref() - .expect("sidecar must have env vars") - .iter() - .map(|e| e.name.as_str()) - .collect(); - assert!(env.contains(&"KRB5_CONFIG")); - - let kafka_opts = sidecar - .env - .as_ref() - .expect("sidecar must have env vars") - .iter() - .find(|e| e.name == "KAFKA_OPTS") - .and_then(|e| e.value.clone()) - .expect("sidecar needs KAFKA_OPTS to point the JVM at krb5.conf"); - // The JVM reads `java.security.krb5.conf`, *not* the `KRB5_CONFIG` env var (that only - // reaches native MIT tools), so without this the admin client cannot locate the KDC. - assert!( - kafka_opts.contains("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf"), - "got: {kafka_opts}" - ); - // But it must NOT inherit the kafka container's JAAS login config: that points at - // /tmp/jaas.properties, which only the `kafka` container renders. The sidecar - // authenticates with the inline `sasl.jaas.config` in admin-client.properties. - assert!( - !kafka_opts.contains("java.security.auth.login.config"), - "got: {kafka_opts}" - ); - } - - #[test] - fn controller_pre_stop_hook_is_present_with_kerberos() { - let pre_stop_command = controller_kafka_container(&kraft_mode_kerberos_cluster()) - .lifecycle - .as_ref() - .and_then(|l| l.pre_stop.as_ref()) - .and_then(|h| h.exec.as_ref()) - .and_then(|e| e.command.as_ref()) - .expect("voter removal on scale-down must run under Kerberos too") - .join(" "); - assert!(pre_stop_command.contains("remove-controller")); - } - #[test] fn statefulsets_use_ordered_ready_pod_management_for_controllers_only() { let cluster = kraft_mode_cluster(); diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index 055d60ce..6587faf4 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -193,10 +193,6 @@ pub fn client_properties(security: &ValidatedKafkaSecurity) -> Vec<(String, Opti PROPERTY_SASL_KERBEROS_SERVICE_NAME.to_string(), Some(KafkaRole::Broker.kerberos_service_name().to_string()), )); - // Deliberately no `sasl.jaas.config`: this file is consumed by clients running - // outside Kafka pods, which have neither the keytab at /stackable/kerberos/keytab nor - // a per-pod principal, so any value here would be wrong. They supply their own login - // configuration; see docs/modules/kafka/pages/usage-guide/security.adoc. } else if security.tls_server_secret_class().is_some() { props.push(( PROPERTY_SECURITY_PROTOCOL.to_string(), @@ -1003,40 +999,6 @@ pub(crate) mod tests { // ---- controller_admin_client_properties ---- - /// Renders the admin-client properties exactly as `build_rolegroup_config_map` does, so we - /// see what the Java properties writer actually puts on disk (it escapes `:` as `\:`, which - /// `config-utils` and the AdminClient must still be able to read back). - #[test] - fn admin_client_rendered_file_keeps_the_jaas_config_on_one_line() { - use stackable_operator::v2::config_file_writer::to_java_properties_string; - - let rendered = to_java_properties_string( - controller_admin_client_properties(&kerberos()) - .iter() - .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))), - ) - .expect("admin-client properties serialize"); - - let jaas_line = rendered - .lines() - .find(|l| l.starts_with("sasl.jaas.config")) - .expect("sasl.jaas.config must be present"); - assert!( - jaas_line.trim_end().ends_with(';'), - "the whole login module config must fit on one line, got: {jaas_line}" - ); - assert!(jaas_line.contains("Krb5LoginModule")); - // The writer escapes ` `, `=` and `:`, so the placeholders land as `${env\:NAME}`. - // Java's `Properties.load` unescapes all three on read, and `config-utils` already - // resolves this escaped form (`controller.properties` relies on it — see - // `extract_env_placeholders` in `statefulset.rs`), so the AdminClient ends up with - // the intended single-line value. - assert!( - jaas_line.contains("${env\\:POD_NAME}"), - "expected the escaped placeholder form, got: {jaas_line}" - ); - } - #[test] fn admin_client_uses_gssapi_over_sasl_ssl_with_kerberos() { let props = as_map(controller_admin_client_properties(&kerberos())); @@ -1061,29 +1023,6 @@ pub(crate) mod tests { ); } - #[test] - fn admin_client_jaas_config_is_a_single_line_pod_principal() { - let props = as_map(controller_admin_client_properties(&kerberos())); - let jaas = props - .get("sasl.jaas.config") - .and_then(|v| v.as_ref()) - .expect("sasl.jaas.config must be set when Kerberos is enabled"); - // Must be one logical line: a raw newline would truncate the value when the - // properties file is parsed. - assert!( - !jaas.contains('\n'), - "sasl.jaas.config must be a single line, got: {jaas}" - ); - assert!(jaas.contains("com.sun.security.auth.module.Krb5LoginModule required")); - assert!(jaas.contains("keyTab=\"/stackable/kerberos/keytab\"")); - // The controller's own pod-scoped principal, resolved by `config-utils template` - // at container start. - assert!(jaas.contains( - "principal=\"kafka/${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}@${env:KERBEROS_REALM}\"" - )); - assert!(jaas.trim_end().ends_with(';')); - } - #[test] fn admin_client_is_unchanged_without_kerberos() { let props = as_map(controller_admin_client_properties(&internal_tls())); diff --git a/tests/templates/kuttl/kraft-kerberos/60-assert.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/60-assert.yaml.j2 index ef637733..a8327181 100644 --- a/tests/templates/kuttl/kraft-kerberos/60-assert.yaml.j2 +++ b/tests/templates/kuttl/kraft-kerberos/60-assert.yaml.j2 @@ -11,16 +11,11 @@ commands: # Kerberos-specific: the admin client must use the *rendered* /tmp copy. The raw # ConfigMap file still has unresolved ${env:...} placeholders in sasl.jaas.config. # - # :9093 is the TLS client port of this test fixture's security config, not a fixed - # Kafka port - if the fixture's TLS/port config changes, update this too. kubectl exec -n $NAMESPACE test-kafka-controller-default-0 -c kafka -- \ /stackable/kafka/bin/kafka-metadata-quorum.sh \ --bootstrap-controller test-kafka-controller-default-0.test-kafka-controller-default-headless.$NAMESPACE.svc.cluster.local:9093 \ --command-config /tmp/admin-client.properties \ describe --replication | tail -n +2 | awk '$NF == "Leader" || $NF == "Follower"' | wc -l | grep -q '^5$' - # `timeout` is known-inert here: kuttl's TestAssert `commands` don't read this field (only - # TestStep commands do); left in place only as documentation of the intended budget. - timeout: 30 --- apiVersion: apps/v1 kind: StatefulSet diff --git a/tests/templates/kuttl/kraft-kerberos/60-scale-controller-up.yaml.j2 b/tests/templates/kuttl/kraft-kerberos/60-scale-controller-up.yaml.j2 index 6854461e..f1ce309b 100644 --- a/tests/templates/kuttl/kraft-kerberos/60-scale-controller-up.yaml.j2 +++ b/tests/templates/kuttl/kraft-kerberos/60-scale-controller-up.yaml.j2 @@ -6,15 +6,6 @@ commands: - script: | kubectl apply -n $NAMESPACE -f - <