From e1af94d7cb052707fcf74e6924ad9640731191d2 Mon Sep 17 00:00:00 2001 From: Stacky McStackface Date: Tue, 18 Aug 2026 21:27:15 +0000 Subject: [PATCH] chore: Generated commit to update templated files since the last template run up to stackabletech/operator-templating@86d5117ce9bab934886a413519682412ae90f92e Reference-to: stackabletech/operator-templating@86d5117 (Various fixes to Helm charts as well as support for ArtifactHub) --- .github/workflows/build.yaml | 206 ++++++++---------- .github/workflows/general_daily_security.yml | 6 +- .../workflows/integration-test-custom.yaml | 49 +++++ .../workflows/integration-test-profile.yaml | 59 +++++ .github/workflows/integration-test.yml | 63 ------ .github/workflows/pr_prek.yaml | 8 +- Makefile | 36 ++- deploy/helm/artifacthub-repo.yaml | 25 +++ deploy/helm/hive-operator/Chart.yaml | 10 + .../templates/serviceaccount.yaml | 5 +- nix/meta.json | 2 +- renovate.json | 2 +- scripts/run-tests | 61 +++++- 13 files changed, 337 insertions(+), 195 deletions(-) create mode 100644 .github/workflows/integration-test-custom.yaml create mode 100644 .github/workflows/integration-test-profile.yaml delete mode 100644 .github/workflows/integration-test.yml create mode 100644 deploy/helm/artifacthub-repo.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a5492298..c4a04c20 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,16 +39,18 @@ jobs: detect-changes: name: Detect relevant changed files runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false fetch-depth: 0 - name: Check for changed files id: check - uses: stackabletech/actions/detect-changes@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/detect-changes@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: patterns: | - '.github/workflows/build.yaml' @@ -57,8 +59,9 @@ jobs: - 'deploy/**' - '.cargo/**' - 'docker/**' - - 'Cargo.*' - - '*.rs' + - '**/Cargo.toml' + - 'Cargo.lock' + - '**/*.rs' outputs: detected: ${{ steps.check.outputs.detected }} @@ -74,11 +77,11 @@ jobs: with: persist-credentials: false - # Validates the templates and, since values.schema.json exists, the default - # values against it. + # Validates the chart templates and, where a values.schema.json exists, the + # default values against it. # # The chart cannot render without a registry overlay, because image.repository - # is only set there. Both published registries are linted. + # is only set there. - name: Lint Helm Chart env: CHART_DIRECTORY: deploy/helm/hive-operator @@ -92,17 +95,19 @@ jobs: if: needs.detect-changes.outputs.detected == 'true' needs: [detect-changes] runs-on: ubuntu-latest + permissions: + contents: read env: RUSTC_BOOTSTRAP: 1 steps: - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@2c09a5e66da6c8016428a2172bd76e5e4f14bb17 # v1.5.3 + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 with: packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https version: ubuntu-latest - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false submodules: recursive @@ -118,16 +123,22 @@ jobs: key: udeps - name: Install cargo-udeps - uses: stackabletech/cargo-install-action@8f7dbbcd2ebe22717efc132d0dd61e80841994b9 # cargo-udeps + uses: stackabletech/cargo-install-action@e3e2dcf8d0f0e5bdbc619bf6ee7560dd68152d3c + with: + crate: cargo-udeps - name: Run cargo-udeps run: cargo udeps --workspace --all-targets build-container-image: name: Build/Publish ${{ matrix.runner.arch }} Image - if: (github.event_name != 'merge_group') && needs.detect-changes.outputs.detected == 'true' + if: | + github.repository_owner == 'stackabletech' + && (github.event_name != 'merge_group') + && needs.detect-changes.outputs.detected == 'true' needs: [detect-changes] permissions: + contents: read id-token: write strategy: fail-fast: false @@ -140,13 +151,13 @@ jobs: operator-version: ${{ steps.version.outputs.OPERATOR_VERSION }} steps: - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@2c09a5e66da6c8016428a2172bd76e5e4f14bb17 # v1.5.3 + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 with: packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https version: ${{ matrix.runner.name }} - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false submodules: recursive @@ -165,12 +176,18 @@ jobs: CURRENT_VERSION=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[] | select(.name == "stackable-hive-operator") | .version') + # Include a PR suffix if this workflow is triggered by a PR if [ "$GITHUB_EVENT_NAME" == 'pull_request' ]; then - # Include a PR suffix if this workflow is triggered by a PR - if [ "$PR_BASE_REF" == 'main' ]; then - NEW_VERSION="0.0.0-pr$PR_NUMBER" - else + # If the PR is raised against a release branch, use the current operator version and + # append a suffix to it. If the PR is raised against ANY other base branch, use the + # established 0.0.0-prXXX tag. + if [[ "$PR_BASE_REF" =~ ^release-[0-9]{2}\.[0-9]{1,2}$ ]]; then NEW_VERSION="$CURRENT_VERSION-pr$PR_NUMBER" + else + # NOTE (@Techassi): One could argue that we should warn the developer when the current + # version is not 0.0.0-dev (and the PR potentially doesn't target main) that we will + # still rewrite the version to 0.0.0-prXXX. + NEW_VERSION="0.0.0-pr$PR_NUMBER" fi else # Just use the current version if this workflow is run on push, schedule, etc... @@ -181,7 +198,7 @@ jobs: echo "OPERATOR_VERSION=$NEW_VERSION" | tee -a "$GITHUB_OUTPUT" - name: Install Nix - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31.5.2 + uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31.11.0 - name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} Toolchain shell: bash @@ -189,7 +206,7 @@ jobs: - name: Build Container Image id: build - uses: stackabletech/actions/build-container-image@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/build-container-image@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: image-name: ${{ env.OPERATOR_NAME }} image-index-manifest-tag: ${{ steps.version.outputs.OPERATOR_VERSION }} @@ -198,7 +215,7 @@ jobs: - name: Publish Container Image to oci.stackable.tech if: ${{ !github.event.pull_request.head.repo.fork }} - uses: stackabletech/actions/publish-image@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/publish-image@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build @@ -209,7 +226,7 @@ jobs: - name: Publish Container Image to quay.io if: ${{ !github.event.pull_request.head.repo.fork }} - uses: stackabletech/actions/publish-image@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/publish-image@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: image-registry-uri: quay.io image-registry-username: stackable+robot_sdp_github_action_build @@ -221,27 +238,25 @@ jobs: publish-index-manifest: name: Publish/Sign ${{ needs.build-container-image.outputs.operator-version }} Index if: | - (github.event_name != 'merge_group') + github.repository_owner == 'stackabletech' + && (github.event_name != 'merge_group') && needs.detect-changes.outputs.detected == 'true' && !github.event.pull_request.head.repo.fork needs: - detect-changes - build-container-image permissions: + contents: read id-token: write runs-on: ubuntu-latest - outputs: - oci-index-digest: ${{ steps.publish-oci.outputs.image-index-manifest-digest }} - quay-index-digest: ${{ steps.publish-quay.outputs.image-index-manifest-digest }} steps: - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Publish and Sign Image Index to oci.stackable.tech - id: publish-oci - uses: stackabletech/actions/publish-image-index-manifest@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/publish-image-index-manifest@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build @@ -250,8 +265,7 @@ jobs: image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} - name: Publish and Sign Image Index to quay.io - id: publish-quay - uses: stackabletech/actions/publish-image-index-manifest@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/publish-image-index-manifest@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: image-registry-uri: quay.io image-registry-username: stackable+robot_sdp_github_action_build @@ -259,84 +273,29 @@ jobs: image-repository: stackable/sdp/${{ env.OPERATOR_NAME }} image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} - # Generate SLSA build provenance for the multi-arch image index and attach it - # to the published image in each registry. The reusable workflow signs the - # provenance with keyless signing (GitHub Actions as the OIDC identity) and - # pushes the attestation next to the image. - provenance-oci: - name: Generate Provenance for ${{ needs.build-container-image.outputs.operator-version }} (oci.stackable.tech) - if: | - (github.event_name != 'merge_group') - && needs.detect-changes.outputs.detected == 'true' - && !github.event.pull_request.head.repo.fork - needs: - - detect-changes - - build-container-image - - publish-index-manifest - permissions: - actions: read # detect the build workflow that generated the image - id-token: write # mint the OIDC token for keyless signing - packages: write # needed until https://github.com/slsa-framework/slsa-github-generator/issues/1257 is resolved - # MUST be referenced by a @vX.Y.Z tag (not a SHA), otherwise the reusable - # workflow cannot verify its own provenance. - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.1.0 - with: - # The 'env' context is not available in job-level 'with' inputs of - # reusable workflow calls (unlike step-level 'with'), so OPERATOR_NAME - # can't be used here and the operator name is templated in directly. - image: oci.stackable.tech/sdp/hive-operator - digest: ${{ needs.publish-index-manifest.outputs.oci-index-digest }} - registry-username: robot$sdp+github-action-build - secrets: - registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - - provenance-quay: - name: Generate Provenance for ${{ needs.build-container-image.outputs.operator-version }} (quay.io) - if: | - (github.event_name != 'merge_group') - && needs.detect-changes.outputs.detected == 'true' - && !github.event.pull_request.head.repo.fork - needs: - - detect-changes - - build-container-image - - publish-index-manifest - permissions: - actions: read # detect the build workflow that generated the image - id-token: write # mint the OIDC token for keyless signing - packages: write # needed until https://github.com/slsa-framework/slsa-github-generator/issues/1257 is resolved - # MUST be referenced by a @vX.Y.Z tag (not a SHA), otherwise the reusable - # workflow cannot verify its own provenance. - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.1.0 - with: - # The 'env' context is not available in job-level 'with' inputs of - # reusable workflow calls (unlike step-level 'with'), so OPERATOR_NAME - # can't be used here and the operator name is templated in directly. - image: quay.io/stackable/sdp/hive-operator - digest: ${{ needs.publish-index-manifest.outputs.quay-index-digest }} - registry-username: stackable+robot_sdp_github_action_build - secrets: - registry-password: ${{ secrets.QUAY_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - publish-helm-chart: name: Package/Publish ${{ needs.build-container-image.outputs.operator-version }} Helm Chart if: | - (github.event_name != 'merge_group') + github.repository_owner == 'stackabletech' + && (github.event_name != 'merge_group') && needs.detect-changes.outputs.detected == 'true' + && !github.event.pull_request.head.repo.fork needs: - detect-changes - build-container-image permissions: + contents: read id-token: write runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false submodules: recursive - name: Package, Publish, and Sign Helm Chart to oci.stackable.tech - uses: stackabletech/actions/publish-helm-chart@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/publish-helm-chart@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: chart-registry-uri: oci.stackable.tech chart-registry-username: robot$sdp-charts+github-action-build @@ -348,7 +307,7 @@ jobs: publish-and-sign: ${{ !github.event.pull_request.head.repo.fork }} - name: Package, Publish, and Sign Helm Chart to quay.io - uses: stackabletech/actions/publish-helm-chart@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/publish-helm-chart@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: chart-registry-uri: quay.io chart-registry-username: stackable+robot_sdp_charts_github_action_build @@ -363,7 +322,8 @@ jobs: openshift-preflight-check: name: Run OpenShift Preflight Check for ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }} if: | - (github.event_name != 'merge_group') + github.repository_owner == 'stackabletech' + && (github.event_name != 'merge_group') && needs.detect-changes.outputs.detected == 'true' && !github.event.pull_request.head.repo.fork needs: @@ -379,13 +339,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Run OpenShift Preflight Check for oci.stackable.tech - uses: stackabletech/actions/run-openshift-preflight@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/run-openshift-preflight@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: image-index-uri: oci.stackable.tech/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} image-architecture: ${{ matrix.arch }} - name: Run OpenShift Preflight Check for quay.io - uses: stackabletech/actions/run-openshift-preflight@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/run-openshift-preflight@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: image-index-uri: quay.io/stackable/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} image-architecture: ${{ matrix.arch }} @@ -397,50 +357,66 @@ jobs: # WARNING: Do not change the name unless you will also be changing the # Required Checks (in branch protections) in GitHub settings. name: Finished Build and Publish + # Run even when a dependency failed, was skipped or cancelled, so that this + # gate reflects the real outcome. Without `always()` a failed dependency + # would *skip* this job, and GitHub treats a skipped required check as + # passing - making a broken build mergeable. + if: always() + # List every leaf job directly. A transitive failure (e.g. a failed + # publish-index-manifest that skips openshift-preflight-check) does not + # surface as `failure` in `needs.*.result` unless the failing job is a + # direct dependency. needs: + - detect-changes - cargo-udeps - - openshift-preflight-check + - helm-lint + - build-container-image + - publish-index-manifest - publish-helm-chart + - openshift-preflight-check runs-on: ubuntu-latest steps: - - run: echo "We are done here" + # Skipped dependencies are fine (jobs skip legitimately on merge_group + # events, forks, or when detect-changes finds no relevant changes). Only + # a failure or cancellation must fail this gate. + - name: Fail on any failed or cancelled dependency + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + echo "Dependency results: $RESULTS" + for result in $RESULTS; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "::error::A required job did not succeed (result: $result)" + exit 1 + fi + done + echo "We are done here" notify: name: Failure Notification if: | - (failure() || github.run_attempt > 1) + !cancelled() + && github.repository_owner == 'stackabletech' + && (contains(needs.*.result, 'failure') || github.run_attempt > 1) && github.event_name != 'merge_group' && needs.detect-changes.outputs.detected == 'true' && !github.event.pull_request.head.repo.fork needs: - detect-changes + - cargo-udeps - build-container-image - publish-index-manifest - - provenance-oci - - provenance-quay - publish-helm-chart + - openshift-preflight-check + - finished runs-on: ubuntu-latest steps: - - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - - name: Send Notification - uses: stackabletech/actions/send-slack-notification@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + uses: stackabletech/actions/send-slack-notification@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: publish-helm-chart-result: ${{ needs.publish-helm-chart.result }} publish-manifests-result: ${{ needs.publish-index-manifest.result }} build-result: ${{ needs.build-container-image.result }} - # Provenance is generated per registry, but the notification only has a - # single field for it, so the two results are collapsed into the worst - # one. 'failure' must be reported verbatim, otherwise the notification - # is not marked as failed. - generate-provenance-result: >- - ${{ (needs.provenance-oci.result == 'failure' || needs.provenance-quay.result == 'failure') && 'failure' - || (needs.provenance-oci.result == 'cancelled' || needs.provenance-quay.result == 'cancelled') && 'cancelled' - || (needs.provenance-oci.result == 'skipped' || needs.provenance-quay.result == 'skipped') && 'skipped' - || 'success' }} slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }} channel-id: C07UG6JH44F # notifications-container-images type: container-image-build diff --git a/.github/workflows/general_daily_security.yml b/.github/workflows/general_daily_security.yml index 8b76446b..7a4c6fab 100644 --- a/.github/workflows/general_daily_security.yml +++ b/.github/workflows/general_daily_security.yml @@ -15,8 +15,12 @@ permissions: {} jobs: audit: runs-on: ubuntu-latest + permissions: + contents: read + # rustsec/audit-check publishes its results as a Check run, which requires this scope + checks: write steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 diff --git a/.github/workflows/integration-test-custom.yaml b/.github/workflows/integration-test-custom.yaml new file mode 100644 index 00000000..16e9839e --- /dev/null +++ b/.github/workflows/integration-test-custom.yaml @@ -0,0 +1,49 @@ +--- +name: Integration Test (Custom) + +on: + workflow_dispatch: + inputs: + test-runner: + description: Name of the test runner, eg. `amd64` (see test/interu.yaml) + required: true + test-suite: + description: Name of the test-suite, eg. `openshift` + test: + description: Name of the test, eg. `smoke` + +permissions: {} + +jobs: + test: + name: Run Integration Test + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + submodules: recursive + + - name: Run Integration Test + id: test + uses: stackabletech/actions/run-integration-test@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 + with: + replicated-api-token: ${{ secrets.REPLICATED_API_TOKEN }} + otlp-bearer-token: ${{ secrets.OTLP_BEARER_TOKEN }} + test-mode-input: ${{ inputs.test-runner }} + test-suite: ${{ inputs.test-suite }} + test-mode: custom + test: ${{ inputs.test }} + + - name: Send Notification + if: ${{ failure() || github.run_attempt > 1 }} + uses: stackabletech/actions/send-slack-notification@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 + with: + slack-token: ${{ secrets.SLACK_INTEGRATION_TEST_TOKEN }} + failed-tests: ${{ steps.test.outputs.failed-tests }} + test-health: ${{ steps.test.outputs.health }} + test-result: ${{ steps.test.conclusion }} + channel-id: C07UYJYSMSN # notifications-integration-tests + type: integration-test diff --git a/.github/workflows/integration-test-profile.yaml b/.github/workflows/integration-test-profile.yaml new file mode 100644 index 00000000..918d8090 --- /dev/null +++ b/.github/workflows/integration-test-profile.yaml @@ -0,0 +1,59 @@ +--- +name: Integration Test (Profile + Schedule) + +on: + schedule: + # At 04:00 on Sunday. See: https://crontab.guru/#0_4_*_*_0 + - cron: "0 4 * * 0" + workflow_dispatch: + inputs: + test-profile: + description: Name of the test runner, eg. `smoke-latest` or `schedule` (see test/interu.yaml) + required: true + +permissions: {} + +jobs: + test: + name: Run Integration Test + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + submodules: recursive + + - name: Determine Profile + id: profile + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + PROFILE_INPUT: ${{ inputs.test-profile }} + shell: bash + run: | + if [ "$EVENT_NAME" == "schedule" ]; then + echo "PROFILE=schedule" | tee -a "$GITHUB_OUTPUT" + else + echo "PROFILE=${PROFILE_INPUT}" | tee -a "$GITHUB_OUTPUT" + fi + + - name: Run Integration Test + id: test + uses: stackabletech/actions/run-integration-test@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 + with: + replicated-api-token: ${{ secrets.REPLICATED_API_TOKEN }} + otlp-bearer-token: ${{ secrets.OTLP_BEARER_TOKEN }} + test-mode-input: ${{ steps.profile.outputs.PROFILE }} + test-mode: profile + + - name: Send Notification + if: ${{ failure() || github.run_attempt > 1 }} + uses: stackabletech/actions/send-slack-notification@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 + with: + slack-token: ${{ secrets.SLACK_INTEGRATION_TEST_TOKEN }} + failed-tests: ${{ steps.test.outputs.failed-tests }} + test-health: ${{ steps.test.outputs.health }} + test-result: ${{ steps.test.conclusion }} + channel-id: C07UYJYSMSN # notifications-integration-tests + type: integration-test diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml deleted file mode 100644 index de17f182..00000000 --- a/.github/workflows/integration-test.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -name: Integration Test - -on: - # schedule: - # # At 00:00 on Sunday. See: https://crontab.guru/#0_0_*_*_0 - # - cron: "0 0 * * 0" - workflow_dispatch: - inputs: - test-mode: - description: Test mode - required: true - type: choice - options: - - profile - - custom - test-mode-input: - description: | - The profile or the runner used. Eg: `smoke-latest` or `amd64` (see test/interu.yaml) - required: true - test-suite: - description: Name of the test-suite. Only used if test-mode is `custom` - test: - description: Name of the test. Only used of test-mode is `custom` - -permissions: {} - -jobs: - test: - name: Run Integration Test - runs-on: ubuntu-latest - # services: - # otel-collector: - # image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s:0.131.1 - # volumes: - # - .:/mnt - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - submodules: recursive - - # TODO: Enable the scheduled runs which hard-code what profile to use - - name: Run Integration Test - id: test - uses: stackabletech/actions/run-integration-test@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 - with: - replicated-api-token: ${{ secrets.REPLICATED_API_TOKEN }} - test-mode-input: ${{ inputs.test-mode-input }} - test-suite: ${{ inputs.test-suite }} - test-mode: ${{ inputs.test-mode }} - test: ${{ inputs.test }} - - - name: Send Notification - if: ${{ failure() || github.run_attempt > 1 }} - uses: stackabletech/actions/send-slack-notification@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 - with: - slack-token: ${{ secrets.SLACK_INTEGRATION_TEST_TOKEN }} - failed-tests: ${{ steps.test.outputs.failed-tests }} - test-health: ${{ steps.test.outputs.health }} - test-result: ${{ steps.test.conclusion }} - channel-id: C07UYJYSMSN # notifications-integration-tests - type: integration-test diff --git a/.github/workflows/pr_prek.yaml b/.github/workflows/pr_prek.yaml index 5932e4bb..30cc7b93 100644 --- a/.github/workflows/pr_prek.yaml +++ b/.github/workflows/pr_prek.yaml @@ -17,18 +17,20 @@ env: jobs: prek: runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 + uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 with: packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https version: ubuntu-latest - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false submodules: recursive fetch-depth: 0 - - uses: stackabletech/actions/run-prek@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 + - uses: stackabletech/actions/run-prek@0fe048f952bee522474903a925ea48dc706a0bb4 # v0.17.5 with: rust: ${{ env.RUST_TOOLCHAIN_VERSION }} hadolint: ${{ env.HADOLINT_VERSION }} diff --git a/Makefile b/Makefile index 9d657f52..0c64fd4c 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ docker-build: docker build --force-rm --build-arg VERSION=${VERSION} -t "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}" -f docker/Dockerfile . ## Chart related targets -compile-chart: version crds +compile-chart: version crds crd-annotation version: cat "deploy/helm/${OPERATOR_NAME}/Chart.yaml" | yq ".version = \"${VERSION}\" | .appVersion = \"${VERSION}\"" > "deploy/helm/${OPERATOR_NAME}/Chart.yaml.new" @@ -40,6 +40,40 @@ crds: mkdir -p extra cargo run --bin stackable-"${OPERATOR_NAME}" -- crd > extra/crds.yaml +# This adds CRD metadata for artifacthub.io (AH) to Chart.yaml. +# We don't ship CRDs in our Helm charts (anymore), so we need these annotations to provide details. +# We only list the storage version as we're unsure if AH supports more than one (docs unclear). +# +# It appends a single annotation whose value is a list, one entry per CRD. +# Example: For the secret-operator that comes out as: +# +# artifacthub.io/crds: |- +# - kind: SecretClass +# version: v1alpha2 +# name: secretclasses.secrets.stackable.tech +# displayName: SecretClass +# description: A SecretClass is a cluster-global Kubernetes resource that defines ... +# - kind: TrustStore +# version: v1alpha1 +# name: truststores.secrets.stackable.tech +# displayName: TrustStore +# description: A TrustStore requests information about how to validate secrets ... +# +# The two yq expressions are split out below so it's easier to review and see what's going on. + +# READ step: turn every CRD document in extra/crds.yaml into one AH card entry. +# `[.] | map(...)` collects the documents into a list and maps each one to the format (see above) required by AH. +# The result of this is a list of these entries. +CRD_ANNOTATION_READ := [.] | map(.spec.versions |= map(select(.storage == true))) | map({"kind": .spec.names.kind, "version": .spec.versions[0].name, "name": .metadata.name, "displayName": .spec.names.kind, "description": (.spec.versions[0].schema.openAPIV3Schema.description // "")}) + +# WRITE step: Write that list to Chart.yaml as an annotation value. +# Literal block scalar so it stays readable in Chart.yaml. +CRD_ANNOTATION_WRITE := .annotations["artifacthub.io/crds"] = strenv(CRDS) | .annotations["artifacthub.io/crds"] style="literal" + +crd-annotation: crds + @CRDS="$$(yq ea -o=yaml '$(CRD_ANNOTATION_READ)' extra/crds.yaml)" \ + yq -i '$(CRD_ANNOTATION_WRITE)' "deploy/helm/${OPERATOR_NAME}/Chart.yaml" + chart-lint: compile-chart docker run -it -v $(shell pwd):/build/helm-charts -w /build/helm-charts quay.io/helmpack/chart-testing:v3.5.0 ct lint --config deploy/helm/ct.yaml diff --git a/deploy/helm/artifacthub-repo.yaml b/deploy/helm/artifacthub-repo.yaml new file mode 100644 index 00000000..770382e0 --- /dev/null +++ b/deploy/helm/artifacthub-repo.yaml @@ -0,0 +1,25 @@ +--- +# Artifact Hub (AH) repository metadata. +# +# This is NOT part of the chart. +# Artifact Hub reads it from a separate OCI artifact pushed to the chart repository under the special `artifacthub.io` tag: +# +# oras push oci.stackable.tech/sdp-charts/hive-operator:artifacthub.io \ +# --config /dev/null:application/vnd.cncf.artifacthub.config.v1+yaml \ +# artifacthub-repo.yaml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml +# +# Reference: https://github.com/artifacthub/hub/blob/master/docs/metadata/artifacthub-repo.yml and https://artifacthub.io/docs/topics/repositories/#verified-publisher + +# Enables the "Verified publisher" label in AH. Taken from the repository's card in +# the Artifact Hub control panel. Not a secret. +# +# Every operator that gets this file has an ID. An operator without an Artifact Hub +# repository excludes the file entirely via `ignored_files` in +# config/repositories.yaml, the way druid does while it is being removed from SDP. +repositoryID: "3dcde43a-1a00-44ba-a70c-837d37737140" + +# Versions Artifact Hub should not index. +# This excludes all dev/rc/pr charts. +ignore: + - name: "hive-operator" + version: "-(dev|rc|pr)" diff --git a/deploy/helm/hive-operator/Chart.yaml b/deploy/helm/hive-operator/Chart.yaml index a0430ded..db2a02fc 100644 --- a/deploy/helm/hive-operator/Chart.yaml +++ b/deploy/helm/hive-operator/Chart.yaml @@ -5,6 +5,10 @@ version: "0.0.0-dev" appVersion: "0.0.0-dev" description: "Kubernetes operator for Apache Hive. Deploy and run the Hive Metastore with the Stackable Data Platform (SDP)." home: https://github.com/stackabletech/hive-operator +# One known use of this is the Artifact Hub (AH) listing which shows the logo. +# AH downloads the logo and re-hosts it. +# We opted to not use the project logos (e.g. the Apache Spark logo) here due to trademark concerns. +icon: https://raw.githubusercontent.com/stackabletech/hive-operator/main/.readme/static/borrowed/Icon_Stackable.svg sources: - https://github.com/stackabletech/hive-operator keywords: ["apache-hive", "hive", "hive-metastore", "metastore", "kubernetes", "operator", "big-data"] @@ -21,3 +25,9 @@ annotations: url: https://docs.stackable.tech/home/stable/hive/ - name: support url: https://github.com/stackabletech/hive-operator/issues + artifacthub.io/crds: |- + - kind: HiveCluster + version: v1alpha1 + name: hiveclusters.hive.stackable.tech + displayName: HiveCluster + description: A Hive cluster stacklet. This resource is managed by the Stackable operator for Apache Hive. diff --git a/deploy/helm/hive-operator/templates/serviceaccount.yaml b/deploy/helm/hive-operator/templates/serviceaccount.yaml index 10633e5f..e0d959e2 100644 --- a/deploy/helm/hive-operator/templates/serviceaccount.yaml +++ b/deploy/helm/hive-operator/templates/serviceaccount.yaml @@ -10,9 +10,11 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 -# This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. +# Grants the operator ServiceAccount the ClusterRole from roles.yaml, which is what lets the +# operator watch and manage its custom resources across the cluster. kind: ClusterRoleBinding metadata: name: {{ include "operator.fullname" . }}-clusterrolebinding @@ -26,4 +28,3 @@ roleRef: kind: ClusterRole name: {{ include "operator.fullname" . }}-clusterrole apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/nix/meta.json b/nix/meta.json index 612b33ae..1a9eefca 100644 --- a/nix/meta.json +++ b/nix/meta.json @@ -1 +1 @@ -{"operator": {"name": "hive-operator", "pretty_string": "Apache Hive", "product_string": "hive", "url": "stackabletech/hive-operator.git", "chart_description": "Kubernetes operator for Apache Hive. Deploy and run the Hive Metastore with the Stackable Data Platform (SDP).", "keywords": ["apache-hive", "hive", "hive-metastore", "metastore", "kubernetes", "operator", "big-data"]}} +{"operator": {"name": "hive-operator", "pretty_string": "Apache Hive", "product_string": "hive", "url": "stackabletech/hive-operator.git", "artifacthub_repository_id": "3dcde43a-1a00-44ba-a70c-837d37737140", "chart_description": "Kubernetes operator for Apache Hive. Deploy and run the Hive Metastore with the Stackable Data Platform (SDP).", "keywords": ["apache-hive", "hive", "hive-metastore", "metastore", "kubernetes", "operator", "big-data"]}} diff --git a/renovate.json b/renovate.json index 486f30ab..049c0ee7 100644 --- a/renovate.json +++ b/renovate.json @@ -3,5 +3,5 @@ "extends": [ "local>stackabletech/.github:renovate-config" ], - "ignorePaths": [".github/workflows/build.yaml", ".github/workflows/general_daily_security.yml", ".github/workflows/integration-test.yml", ".github/workflows/pr_prek.yaml"] + "ignorePaths": [".github/workflows/build.yaml", ".github/workflows/general_daily_security.yml", ".github/workflows/integration-test-custom.yaml", ".github/workflows/integration-test-profile.yaml", ".github/workflows/pr_prek.yaml"] } diff --git a/scripts/run-tests b/scripts/run-tests index 00639f45..4e6496b5 100755 --- a/scripts/run-tests +++ b/scripts/run-tests @@ -144,6 +144,38 @@ def parse_args(argv: list[str]) -> argparse.Namespace: default=os.path.join("tests", "_work"), ) + parser.add_argument( + "--template-dir", + help="Directory containing the test templates (default tests/templates/kuttl)", + type=str, + required=False, + default=os.path.join("tests", "templates", "kuttl"), + ) + + parser.add_argument( + "--test-definition", + help="Path to the test definition file (default tests/test-definition.yaml)", + type=str, + required=False, + default=os.path.join("tests", "test-definition.yaml"), + ) + + parser.add_argument( + "--kuttl-test", + help="Path to the kuttl test definition file (default tests/kuttl-test.yaml.jinja2)", + type=str, + required=False, + default=os.path.join("tests", "kuttl-test.yaml.jinja2"), + ) + + parser.add_argument( + "--release-file", + help="Path to the test release file (default tests/release.yaml)", + type=str, + required=False, + default=os.path.join("tests", "release.yaml"), + ) + parser.add_argument( "--listener-class-preset", help="Choose the ListenerClass preset", @@ -201,7 +233,7 @@ def have_requirements() -> None: @contextlib.contextmanager def release_file( - operators: list[tuple[str, str]], skip_ops: list[str] + release_file: str, operators: list[tuple[str, str]], skip_ops: list[str] ) -> collections.abc.Generator[str, None, None]: """Generate a (possibly modified) copy of the release.yaml file. @@ -217,7 +249,6 @@ def release_file( """ def _patch(): - release_file = os.path.join("tests", "release.yaml") # A marker to validate that all ops were patched patched_release = [] with open(release_file, "r") as f: @@ -334,16 +365,23 @@ def maybe_install_release( raise TestRunnerException() -def gen_tests(test_suite: str, namespace: str, work_dir: str) -> None: +def gen_tests( + test_definition: str, + kuttl_test: str, + template_dir: str, + test_suite: str, + namespace: str, + work_dir: str, +) -> None: try: beku_cmd = [ "beku", "--test_definition", - os.path.join("tests", "test-definition.yaml"), + test_definition, "--kuttl_test", - os.path.join("tests", "kuttl-test.yaml.jinja2"), + kuttl_test, "--template_dir", - os.path.join("tests", "templates", "kuttl"), + template_dir, "--output_dir", work_dir, ] @@ -448,8 +486,15 @@ def main(argv) -> int: opts = parse_args(argv[1:]) logging.basicConfig(encoding="utf-8", level=opts.log_level) have_requirements() - gen_tests(opts.test_suite, opts.namespace, opts.work_dir) - with release_file(opts.operator, opts.skip_operator) as f: + gen_tests( + opts.test_definition, + opts.kuttl_test, + opts.template_dir, + opts.test_suite, + opts.namespace, + opts.work_dir, + ) + with release_file(opts.release_file, opts.operator, opts.skip_operator) as f: maybe_install_release(opts.skip_release, f, opts.listener_class_preset) if opts.skip_tests: logging.info("Skip running tests.")