Remove every backend that has never been run against real hardware

.github/workflows/real-world-tests.yml

@@ -105,11 +105,6 @@ jobs:
 
     - name: Run API tests (free tier)
       env:
-        LAMBDA_CLOUD_API_KEY: ${{ secrets.LAMBDA_CLOUD_API_KEY }}
-        GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-        GCP_JSON: ${{ secrets.GCP_JSON }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
         HF_USERNAME: ${{ secrets.HF_USERNAME }}
         HF_TOKEN: ${{ secrets.HF_TOKEN }}
       run: |
@@ -122,11 +117,6 @@ jobs:
     - name: Run expensive tests
       if: ${{ inputs.run_expensive }}
       env:
-        LAMBDA_CLOUD_API_KEY: ${{ secrets.LAMBDA_CLOUD_API_KEY }}
-        GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-        GCP_JSON: ${{ secrets.GCP_JSON }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
         HF_USERNAME: ${{ secrets.HF_USERNAME }}
         HF_TOKEN: ${{ secrets.HF_TOKEN }}
       run: |
@@ -199,11 +189,6 @@ jobs:
       env:
         CLUSTRIX_USERNAME: ${{ secrets.CLUSTRIX_USERNAME }}
         CLUSTRIX_PASSWORD: ${{ secrets.CLUSTRIX_PASSWORD }}
-        LAMBDA_CLOUD_API_KEY: ${{ secrets.LAMBDA_CLOUD_API_KEY }}
-        GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-        GCP_JSON: ${{ secrets.GCP_JSON }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
         HF_USERNAME: ${{ secrets.HF_USERNAME }}
         HF_TOKEN: ${{ secrets.HF_TOKEN }}
       run: |
@@ -213,11 +198,6 @@ jobs:
       env:
         CLUSTRIX_USERNAME: ${{ secrets.CLUSTRIX_USERNAME }}
         CLUSTRIX_PASSWORD: ${{ secrets.CLUSTRIX_PASSWORD }}
-        LAMBDA_CLOUD_API_KEY: ${{ secrets.LAMBDA_CLOUD_API_KEY }}
-        GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-        GCP_JSON: ${{ secrets.GCP_JSON }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
         HF_USERNAME: ${{ secrets.HF_USERNAME }}
         HF_TOKEN: ${{ secrets.HF_TOKEN }}
       run: |

.github/workflows/tests.yml

@@ -53,7 +53,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e ".[dev,test,kubernetes,widget]"
+        pip install -e ".[dev,test,widget]"
 
     # Linting is version-independent, so run it once on the primary target
     # rather than on every matrix combination.
@@ -103,6 +103,19 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+    # Must run BEFORE "Test installation": that step does a non-editable
+    # `pip install .`, after which `import clustrix` resolves to
+    # site-packages and this checker would be reading a different copy of
+    # the code than the docs in this checkout describe. Executes every code
+    # block the documentation publishes (#124) -- the script existed and
+    # passed, but nothing ran it, so the docs were correct only as long as
+    # someone remembered to check by hand. Run once on the primary target;
+    # the examples are not version-specific.
+    - name: Check documentation examples
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
+      run: |
+        python scripts/check_docs_examples.py
+
     - name: Test installation
       run: |
         pip install .
@@ -129,7 +142,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e ".[dev,test,kubernetes,widget]"
+        pip install -e ".[dev,test,widget]"
 
     - name: Run integration tests
       run: |
@@ -204,4 +217,43 @@ jobs:
       run: |
         pip install jupyter nbconvert
         jupyter nbconvert --to script docs/source/notebooks/basic_usage.ipynb
-        # Note: Full notebook execution would require cluster setup
+        # Note: Full notebook execution would require cluster setup
+  # A single stable name for branch protection to require. Without it,
+  # protection has to list all seven `test (os, version)` jobs individually
+  # and silently stops covering any combination added later -- and `master`
+  # had no protection at all, so a red run could always be merged past.
+  #
+  # `if: always()` so this job still runs when a dependency fails; without it
+  # the gate would be skipped rather than failing, and a skipped required
+  # check does not block a merge.
+  tests-status:
+    name: Tests Status
+    runs-on: ubuntu-latest
+    needs: [test, integration-test, docs-test]
+    if: always()
+    steps:
+    - name: Check status
+      run: |
+        # Every job in `needs` must be checked by name. An aggregator that
+        # forgets one reports success while that one burns -- the exact
+        # defect fixed in fast_ci.yml's status-check, which had
+        # security-scan in `needs` but not in its condition.
+        failed=0
+        for job in test integration-test docs-test; do
+          case "$job" in
+            test)             result="${{ needs.test.result }}" ;;
+            integration-test) result="${{ needs.integration-test.result }}" ;;
+            docs-test)        result="${{ needs.docs-test.result }}" ;;
+          esac
+          if [ "$result" != "success" ]; then
+            echo "::error::$job: $result"
+            failed=1
+          else
+            echo "$job: success"
+          fi
+        done
+        if [ "$failed" -ne 0 ]; then
+          echo "One or more test jobs did not succeed."
+          exit 1
+        fi
+        echo "All test jobs passed."

CHANGELOG.md

@@ -5,8 +5,8 @@ landed on `master`.
 
 The guiding rule for this file: a capability is only listed as working if it has
 been exercised against the real thing. Anything implemented but unproven is
-listed under **Implemented but unverified**, and stays there until someone runs
-it for real.
+labelled as such and stays labelled until someone runs it for real — and as of
+0.2.0, anything that stayed unproven was removed rather than shipped.
 
 ## [0.2.0] — unreleased
 
@@ -16,6 +16,11 @@ backend.
 
 ### Fixed — correctness
 
+Some entries below describe defects in backends that this same release then
+removed (see **Removed — unverified backends**). They are kept because the
+defects were real and the record matters; they are not claims that those
+backends now work.
+
 - **`@cluster` returned a fabricated GPU result instead of your answer.**
   `_attempt_client_side_gpu_parallelization` never called the decorated
   function. It ran a fixed `torch.randn(100, 100)` program on each GPU,
@@ -67,10 +72,6 @@ backend.
 - **Local auto-parallelization never parallelized anything.** It injected a
   `_parallel_<var>` keyword argument the callee could not accept, then swallowed
   the resulting `TypeError` and silently ran sequentially.
-- **Kubernetes reported failed jobs as successful.** `check_k8s_job_status`
-  returned `"completed"` from its exception paths, and results were decoded with
-  `ast.literal_eval` on the pod log, falling back to returning the raw log text
-  as the result.
 - **Environment replication silently dropped a third of the environment.**
   `get_environment_requirements` skipped every freeze line containing `@`. With
   `uv` on `PATH` — which is tried first — every conda-built package is rendered
@@ -79,12 +80,10 @@ backend.
   requirement that genuinely cannot be reproduced remotely (an editable install,
   a git checkout) is now refused at submit time, naming the package, instead of
   producing a job that fails on import.
-- **PBS never set up its remote environment**, unlike SLURM and SGE. All four
-  schedulers now share one staging and environment-setup path.
+- **The scheduler backends did not share a staging and environment-setup path**,
+  so they drifted. They now do.
 - **`cluster_type="local"` raised `ValueError: Unsupported cluster type`**, though
   it was offered in the widget and the CLI.
-- Cloud providers returned placeholder hostnames (`placeholder.example.com`) and
-  empty strings as if they were real, so failures surfaced far from their cause.
 - The by-value serialization walk missed instance attributes, local classes
   subclassing builtin containers, PEP-420 namespace packages, and
   `functools.partial`; and it silently degraded to by-reference at its 20,000-node
@@ -139,6 +138,17 @@ backend.
 - CI's flake8 step passed `--exit-zero` and could not fail; its mypy step carried
   `continue-on-error: true`; and it ran only `tests/unit/` — about 350 of the
   ~1,750 non-billable tests that exist.
+- **The pre-push hook could not block a push** (#147). `scripts/run_real_world_tests.py`
+  discarded every `runner.run_*_tests()` return value and never called `sys.exit`,
+  so it exited 0 whatever happened. All four categories printed `❌ ... failed`
+  and the hook then printed `✅ All real-world tests passed!` and allowed the
+  push. Its failure message also printed only `stdout`, which was empty in every
+  observed case because a pytest collection error goes to `stderr` — so an
+  operator was told something failed and not what.
+- **The host-key tests wrote into the developer's real `~/.ssh/known_hosts`**,
+  where 83 stale `[127.0.0.1]:<ephemeral port>` entries had accumulated. Port
+  reuse against a fresh server key then raised `BadHostKeyException` and failed
+  the reject test. The fixture now redirects `~`.
 
 ### Added
 
@@ -147,8 +157,8 @@ backend.
   default to a dry run and refuse to touch anything not tagged
   `clustrix:managed=true`. (The originals, recovered from git history, deleted
   every NAT gateway and VPC in the account with no ownership check at all.)
-- Kubernetes auto-provisioning documentation, and a usage-patterns tutorial. Every
-  example in the docs is executed by `scripts/check_docs_examples.py`.
+- A usage-patterns tutorial. Every example in the docs is executed by
+  `scripts/check_docs_examples.py`.
 - `clustrix.config.SUPPORTED_CLUSTER_TYPES` as the single source of truth. The
   CLI's own list had drifted and omitted `huggingface` entirely, so a working
   backend could not be selected from the command line.
@@ -162,15 +172,94 @@ backend.
 - Deleted 1,837 lines of genuinely orphaned modules. (The issue that requested
   this claimed ~5,100 lines and named five files that do not exist on `master`.)
 
-### Implemented but unverified
+### Removed — unverified backends (BREAKING)
+
+Seven execution backends were implemented in full and not one of them had ever
+been shown to run a job end to end against real hardware. Rather than keep
+publishing them as if they worked, they were removed. `SUPPORTED_CLUSTER_TYPES`
+is now exactly `local`, `ssh`, `slurm`, `huggingface` — the four backends that
+have each run a real job and returned the right answer.
+
+Asking for one of the removed backends is refused where you can act on it, and
+the message says which kind of wrong it is. A removed backend is named, with
+why it went and its tracking issue; an ordinary typo still gets the
+did-you-mean hint:
+
+```
+ClusterConfig(cluster_type="pbs")
+  ValueError: cluster_type='pbs' is no longer implemented. It was removed in
+  v0.2.0 because it had never been verified against real hardware. Its return
+  is tracked in issue #140. Supported types are: local, ssh, slurm, huggingface.
+
+load_config, file containing `k8s_namespace: compute`
+  ValueError: <path> contains unknown setting(s): k8s_namespace configured
+  Kubernetes, which has been removed (see issue #142)
+
+load_config, file containing a genuine typo `cluster_hostt`
+  ValueError: <path> contains unknown setting(s): cluster_hostt
+  (did you mean cluster_host?)
+```
+
+`validate_cluster_type()` is the single check, called from
+`ClusterConfig.__post_init__`, `configure()`, `load_config()` and the executor.
+Previously only `load_config` checked, so every other route carried a dead
+backend to `ClusterExecutor.submit_job` — which tested the type *after*
+`self.connect()`, spending an SSH round trip on a host that was never going to
+be used, and then saying only `Unsupported cluster type: pbs`.
+
+Each removed backend has a tracking issue and is planned for a future release.
+The gate for restoring one is the gate the surviving four already passed: a
+real job, on real hardware, whose result comes back and is checked in as
+evidence. No date is promised.
+
+| Removed | Issue | What it was |
+|-|-|-|
+| PBS | [#140](https://github.com/ContextLab/clustrix/issues/140) | `cluster_type="pbs"` — the PBS/Torque scheduler |
+| SGE | [#141](https://github.com/ContextLab/clustrix/issues/141) | `cluster_type="sge"` — Sun/Son of Grid Engine |
+| Kubernetes | [#142](https://github.com/ContextLab/clustrix/issues/142) | `cluster_type="kubernetes"`, the `k8s_*` settings, cluster auto-provisioning |
+| AWS | [#143](https://github.com/ContextLab/clustrix/issues/143) | `provider="aws"` — EC2 and EKS |
+| GCP | [#144](https://github.com/ContextLab/clustrix/issues/144) | `provider="gcp"` — Google Compute Engine |
+| Azure | [#145](https://github.com/ContextLab/clustrix/issues/145) | `provider="azure"` — Azure VMs |
+| Lambda Cloud | [#146](https://github.com/ContextLab/clustrix/issues/146) | `provider="lambda"` — Lambda Labs GPU cloud |
+
+The HuggingFace **Spaces** provider (`provider="huggingface"`) was removed with
+them. This is a different thing from `cluster_type="huggingface"`, which is
+HuggingFace **Jobs**: that backend is verified end to end and is fully
+supported. Note that `hf_hardware` and `hf_username` sat under a comment
+labelling them Spaces settings but are read by `hf_jobs.py`, so they stay;
+`hf_sdk` was genuinely Spaces-only and is gone.
+
+Removed with the backends:
+
+- **`configure(auto_install_deps=...)`** — it installed cloud provider
+  dependencies.
+- **Ten `@cluster` parameters**: `provider`, `instance_type`, `region`,
+  `platform`, `auto_provision`, `cluster_name`, `node_count`, `node_type`,
+  `kubernetes_version`, `from_scratch`.
+- **Every `k8s_*`, `aws_*`, `azure_*`, `gcp_*`, `lambda_*`, `cloud_*`,
+  `cost_monitoring` and `hf_sdk` field** on `ClusterConfig`.
+
+`configure()` also now validates every keyword before applying any of them. It
+used to `setattr` its way through and raise partway, so a call that *failed*
+had still changed the live configuration.
+
+### Removed — cost monitoring API (BREAKING)
+
+The cost monitoring and cloud pricing API is gone, along with all five of its
+public functions:
+
+- `cost_tracking_decorator`
+- `get_cost_monitor`
+- `start_cost_monitoring`
+- `generate_cost_report`
+- `get_pricing_info`
 
-These have code paths and error handling, but no one has run them against real
-hardware. They are not claimed to work.
+Importing any of them from `clustrix` now raises `ImportError`. They priced the
+cloud VM backends, so with those backends removed the API had nothing left to
+price. Use your provider's own pricing calculator instead.
 
-- PBS and SGE — never run against a real scheduler.
-- Kubernetes execution — never run against a real cluster.
-- AWS, GCP, Azure and Lambda VM backends — no cloud job has been shown to run end
-  to end.
+`scripts/aws/` is unaffected — it is operator cleanup tooling, not an execution
+backend, and it stays.
 
 ### Known limitations
 

CLAUDE.md

@@ -14,11 +14,22 @@ Backends, and how far each is actually proven — keep this honest, it is the fi
 | `ssh` | Verified end to end against a real GPU host |
 | `huggingface` | Verified end to end against real HF Jobs containers |
 | `local` | Runs in-process via `local_executor.py` |
-| `pbs`, `sge` | Implemented, **not** verified against real hardware |
-| `kubernetes` | Implemented, **not** verified against a real cluster |
-| AWS / GCP / Azure / Lambda VMs | **Unverified.** No cloud job has been shown to run end to end. |
 
-Evidence for the verified rows is regenerated by `scripts/verify_cluster_usecases.py` and committed under `docs/evidence/`.
+That is the whole list — `clustrix.config.SUPPORTED_CLUSTER_TYPES`. Evidence for the verified rows is regenerated by `scripts/verify_cluster_usecases.py` and committed under `docs/evidence/`.
+
+**Backends that are NOT currently supported.** `pbs`, `sge`, `kubernetes` and the AWS / GCP / Azure / Lambda Cloud VM providers were removed in v0.2.0 because none of them had ever been shown to run a job end to end. The cost monitoring and cloud pricing API (`cost_tracking_decorator`, `get_cost_monitor`, `start_cost_monitoring`, `generate_cost_report`, `get_pricing_info`) went with them, as did the HuggingFace **Spaces** provider — which is a different thing from `cluster_type="huggingface"` (HuggingFace **Jobs**), and that one stays. Each removed backend is planned for a future update and has a tracking issue; do not re-document any of them as working:
+
+| Removed | Issue |
+|-|-|
+| PBS | [#140](https://github.com/ContextLab/clustrix/issues/140) |
+| SGE | [#141](https://github.com/ContextLab/clustrix/issues/141) |
+| Kubernetes | [#142](https://github.com/ContextLab/clustrix/issues/142) |
+| AWS | [#143](https://github.com/ContextLab/clustrix/issues/143) |
+| GCP | [#144](https://github.com/ContextLab/clustrix/issues/144) |
+| Azure | [#145](https://github.com/ContextLab/clustrix/issues/145) |
+| Lambda Cloud | [#146](https://github.com/ContextLab/clustrix/issues/146) |
+
+`scripts/aws/` stays: it is operator cleanup tooling, not an execution backend.
 
 ## Development Commands
 
@@ -27,8 +38,8 @@ Evidence for the verified rows is regenerated by `scripts/verify_cluster_usecase
 # Install package with development dependencies
 pip install -e ".[dev]"
 
-# Install with Kubernetes support
-pip install -e ".[kubernetes,dev]"
+# Install with the Jupyter widget
+pip install -e ".[widget,dev]"
 ```
 
 ### Code Quality
@@ -61,10 +72,8 @@ pytest --cov=clustrix    # Run tests with coverage
 2. **ClusterExecutor** (`clustrix/executor_core.py`): Central execution engine. Note that `clustrix/executor.py` is a 39-line backward-compatibility shim that re-exports it; the implementation is split across:
    - `executor_core.py` — the `ClusterExecutor` class, dispatch, result retrieval and verification
    - `executor_connections.py` — SSH/SFTP connection management via Paramiko
-   - `executor_schedulers.py` — SLURM, PBS, SGE submission
+   - `executor_schedulers.py` — SLURM and SSH submission
    - `executor_scheduler_status.py` — scheduler status polling and error extraction
-   - `executor_kubernetes.py` — Kubernetes operations
-   - `executor_cloud.py` — cloud provider workflows
    - `hf_jobs.py` — the HuggingFace Jobs backend (`HFJobsManager`)
 
 3. **Configuration System** (`clustrix/config.py`): Singleton configuration management supporting:
@@ -128,12 +137,13 @@ VENV1 holds clustrix's own serialization dependencies; VENV2 holds the user's re
 
 ### Adding New Cluster Type Support
 
-There is **no `ClusterType` enum** — `ClusterConfig.cluster_type` is a plain `str`. The supported values are `local`, `ssh`, `slurm`, `pbs`, `sge`, `kubernetes`, `huggingface`.
+There is **no `ClusterType` enum** — `ClusterConfig.cluster_type` is a plain `str`. The supported values are `local`, `ssh`, `slurm`, `huggingface`, declared once in `clustrix.config.SUPPORTED_CLUSTER_TYPES`.
 
-1. Add the value wherever the valid set is declared (`clustrix/cli.py`'s `click.Choice`, the widget's dropdown, and the type comment in `config.py`) — these must not drift apart
+1. Add the value to `SUPPORTED_CLUSTER_TYPES`; the CLI's `click.Choice` and the widget's dropdown both read that tuple, so they cannot drift apart
 2. Implement submission in the appropriate `executor_*.py` module and dispatch from `ClusterExecutor` in `executor_core.py`
 3. Add status checking to `get_job_status` / `executor_scheduler_status.py`
-4. Update job script generation in `utils.py` if needed — reuse `job_execution_lines()` rather than writing a fourth variant
+4. Update job script generation in `utils.py` if needed — reuse `job_execution_lines()` rather than writing another variant
+5. **Do not mark it supported until a real job has run on real hardware and the evidence is committed.** That gate is why `pbs`, `sge`, `kubernetes` and the cloud VM providers were removed (#140–#146).
 
 ### Using Filesystem Utilities
 ```python