feat(self-host): align Docker Compose with Helm and overhaul self-hosting docs

.github/CONTRIBUTING.md

@@ -165,6 +165,15 @@ After running this command, open [http://localhost:3000/](http://localhost:3000/
 git clone https://github.com/<your-username>/sim.git
 cd sim
 
+# Generate the required secrets. The stack refuses to start without them
+# rather than booting with empty values.
+cat > .env << EOF
+BETTER_AUTH_SECRET=$(openssl rand -hex 32)
+ENCRYPTION_KEY=$(openssl rand -hex 32)
+INTERNAL_API_SECRET=$(openssl rand -hex 32)
+CRON_SECRET=$(openssl rand -hex 32)
+EOF
+
 # Start Sim
 docker compose -f docker-compose.prod.yml up -d
 ```

.github/workflows/ci.yml

@@ -294,6 +294,12 @@ jobs:
             ecr_repo_secret: ECR_PII
             gh_runner: ubuntu-latest
             bs_runner: blacksmith-4vcpu-ubuntu-2404
+          # No ECR repo is provisioned for cron, so it publishes to GHCR only.
+          # The tag step below omits the ECR tag when the repo name is empty.
+          - dockerfile: ./docker/cron.Dockerfile
+            ghcr_image: ghcr.io/simstudioai/cron
+            gh_runner: ubuntu-latest
+            bs_runner: blacksmith-2vcpu-ubuntu-2404
     steps:
       - name: Checkout code
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -338,15 +344,33 @@ jobs:
           ECR_REPO="${{ steps.ecr-repo.outputs.name }}"
           GHCR_IMAGE="${{ matrix.ghcr_image }}"
 
-          TAGS="${ECR_REGISTRY}/${ECR_REPO}:${{ github.sha }}"
+          TAGS=""
+          if [ -n "$ECR_REPO" ]; then
+            TAGS="${ECR_REGISTRY}/${ECR_REPO}:${{ github.sha }}"
+          fi
 
           if [ "${{ github.ref }}" = "refs/heads/main" ] && [ -n "$GHCR_IMAGE" ]; then
-            TAGS="${TAGS},${GHCR_IMAGE}:${{ github.sha }}-amd64"
+            if [ -n "$TAGS" ]; then
+              TAGS="${TAGS},${GHCR_IMAGE}:${{ github.sha }}-amd64"
+            else
+              TAGS="${GHCR_IMAGE}:${{ github.sha }}-amd64"
+            fi
+          fi
+
+          # An entry can legitimately resolve to no tags — e.g. the cron image has
+          # no ECR repo, so on staging/dev (where GHCR tags are not applied) there
+          # is nothing to push. Skip that build instead of failing the job.
+          if [ -z "$TAGS" ]; then
+            echo "No ECR repo and no GHCR tag for this entry on ${{ github.ref }} — skipping push."
+            echo "skip=true" >> $GITHUB_OUTPUT
+          else
+            echo "skip=false" >> $GITHUB_OUTPUT
           fi
 
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
 
       - name: Build and push images
+        if: steps.meta.outputs.skip != 'true'
         uses: ./.github/actions/docker-build
         with:
           provider: ${{ vars.CI_PROVIDER }}
@@ -470,6 +494,10 @@ jobs:
             image: ghcr.io/simstudioai/pii
             gh_runner: ubuntu-24.04-arm
             bs_runner: blacksmith-4vcpu-ubuntu-2404-arm
+          - dockerfile: ./docker/cron.Dockerfile
+            image: ghcr.io/simstudioai/cron
+            gh_runner: ubuntu-24.04-arm
+            bs_runner: blacksmith-4vcpu-ubuntu-2404-arm
 
     steps:
       - name: Checkout code
@@ -515,6 +543,7 @@ jobs:
           - image: ghcr.io/simstudioai/migrations
           - image: ghcr.io/simstudioai/realtime
           - image: ghcr.io/simstudioai/pii
+          - image: ghcr.io/simstudioai/cron
 
     steps:
       - name: Login to GHCR

.github/workflows/helm.yml

@@ -32,6 +32,17 @@ jobs:
         with:
           version: v3.16.4
 
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.13
+
+      # Docker Compose and Kubernetes must run the same background jobs on the
+      # same schedules; this fails the build if the two drift apart. The script
+      # imports only node builtins, so this job installs no dependencies.
+      - name: Scheduler parity (docker/crontab vs helm cronjobs)
+        run: bun run scripts/check-cron-parity.ts
+
       - name: Helm lint
         run: helm lint helm/sim --values helm/sim/ci/default-values.yaml
 

README.md

@@ -7,7 +7,7 @@
 
 <p align="center">
   <a href="https://deepwiki.com/simstudioai/sim" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/badge/Ask-DeepWiki-E6E6E6?labelColor=C3C3C3&color=E6E6E6" alt="Ask DeepWiki"></a>
-  <a href="https://cursor.com/link/prompt?text=Help%20me%20set%20up%20Sim%20locally.%20Follow%20these%20steps%3A%0A%0A1.%20First%2C%20verify%20Docker%20is%20installed%20and%20running%3A%0A%20%20%20docker%20--version%0A%20%20%20docker%20info%0A%0A2.%20Clone%20the%20repository%3A%0A%20%20%20git%20clone%20https%3A%2F%2Fgithub.com%2Fsimstudioai%2Fsim.git%0A%20%20%20cd%20sim%0A%0A3.%20Start%20the%20services%20with%20Docker%20Compose%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.prod.yml%20up%20-d%0A%0A4.%20Wait%20for%20all%20containers%20to%20be%20healthy%20(this%20may%20take%201-2%20minutes)%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.prod.yml%20ps%0A%0A5.%20Verify%20the%20app%20is%20accessible%20at%20http%3A%2F%2Flocalhost%3A3000%0A%0AIf%20there%20are%20any%20errors%2C%20help%20me%20troubleshoot%20them.%20Common%20issues%3A%0A-%20Port%203000%2C%203002%2C%20or%205432%20already%20in%20use%0A-%20Docker%20not%20running%0A-%20Insufficient%20memory%20(needs%2012GB%2B%20RAM)%0A%0AFor%20local%20AI%20models%20with%20Ollama%2C%20use%20this%20instead%20of%20step%203%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.ollama.yml%20--profile%20setup%20up%20-d"><img src="https://img.shields.io/badge/Set%20Up%20with-Cursor-E6E6E6?logo=cursor&logoColor=1A1A1A&labelColor=C3C3C3&color=E6E6E6" alt="Set Up with Cursor"></a>
+  <a href="https://cursor.com/link/prompt?text=Help%20me%20set%20up%20Sim%20locally.%20Follow%20these%20steps%3A%0A%0A1.%20First%2C%20verify%20Docker%20is%20installed%20and%20running%3A%0A%20%20%20docker%20--version%0A%20%20%20docker%20info%0A%0A2.%20Clone%20the%20repository%3A%0A%20%20%20git%20clone%20https%3A%2F%2Fgithub.com%2Fsimstudioai%2Fsim.git%0A%20%20%20cd%20sim%0A%0A3.%20Generate%20required%20secrets%20%28the%20stack%20will%20not%20start%20without%20them%29%3A%0A%20%20%20cat%20%3E%20.env%20%3C%3C%20EOF%0A%20%20%20BETTER_AUTH_SECRET%3D%24%28openssl%20rand%20-hex%2032%29%0A%20%20%20ENCRYPTION_KEY%3D%24%28openssl%20rand%20-hex%2032%29%0A%20%20%20INTERNAL_API_SECRET%3D%24%28openssl%20rand%20-hex%2032%29%0A%20%20%20CRON_SECRET%3D%24%28openssl%20rand%20-hex%2032%29%0A%20%20%20EOF%0A%0A4.%20Start%20the%20services%20with%20Docker%20Compose%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.prod.yml%20up%20-d%0A%0A4.%20Wait%20for%20all%20containers%20to%20be%20healthy%20(this%20may%20take%201-2%20minutes)%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.prod.yml%20ps%0A%0A5.%20Verify%20the%20app%20is%20accessible%20at%20http%3A%2F%2Flocalhost%3A3000%0A%0AIf%20there%20are%20any%20errors%2C%20help%20me%20troubleshoot%20them.%20Common%20issues%3A%0A-%20Port%203000%2C%203002%2C%20or%205432%20already%20in%20use%0A-%20Docker%20not%20running%0A-%20Insufficient%20memory%20(needs%2012GB%2B%20RAM)%0A%0AFor%20local%20AI%20models%20with%20Ollama%2C%20use%20this%20instead%20of%20step%203%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.ollama.yml%20--profile%20setup%20up%20-d"><img src="https://img.shields.io/badge/Set%20Up%20with-Cursor-E6E6E6?logo=cursor&logoColor=1A1A1A&labelColor=C3C3C3&color=E6E6E6" alt="Set Up with Cursor"></a>
 </p>
 
 <p align="center">

apps/docs/content/docs/en/platform/self-hosting/architecture.mdx

@@ -0,0 +1,112 @@
+---
+title: Architecture
+description: Every service Sim runs, what it depends on, and where state lives
+---
+
+import { Callout } from 'fumadocs-ui/components/callout'
+import { FAQ } from '@/components/ui/faq'
+
+Understanding what runs where makes every other operational decision — scaling, backup, network policy, upgrades — straightforward.
+
+## Services
+
+| Service | Image | Port | Stateless | Required |
+|---|---|---|---|---|
+| **app** | `ghcr.io/simstudioai/simstudio` | 3000 | Only with object storage configured | Yes |
+| **realtime** | `ghcr.io/simstudioai/realtime` | 3002 | Yes | Yes |
+| **migrations** | `ghcr.io/simstudioai/migrations` | — | Yes (runs once) | Yes |
+| **postgresql** | `pgvector/pgvector:pg17` | 5432 | **No** | Yes |
+| **redis** | `redis:7-alpine` | 6379 | Mostly | Bundled by both; swap for a managed instance in production |
+| **cron** | `ghcr.io/simstudioai/cron` (Compose) / `curlimages/curl` (CronJobs) | — | Yes | Yes |
+| **pii** | `ghcr.io/simstudioai/pii` | 5001 | Yes | Optional |
+| **ollama** | `ollama/ollama` | 11434 | **No** (model cache) | Optional |
+| **telemetry** | `otel/opentelemetry-collector-contrib` | 4317/4318 | Yes | Optional |
+
+### app
+
+The Next.js application: the editor UI, every API route, and the workflow execution engine. Workflow runs happen **inside the app process** by default, using an isolated-vm sandbox, which is why memory rather than CPU is the constraining resource. Both the chart and the compose file request 4 Gi and cap the app at 8 Gi. Configuring a remote sandbox provider (E2B or Daytona) moves code execution out of the process; see [Security](/platform/self-hosting/security).
+
+Any replica can serve any request **once object storage is configured**. Until then the app writes uploads to its own container filesystem, which makes it stateful — see [Where state lives](#where-state-lives). Scale it horizontally only after reading [Scaling & HA](/platform/self-hosting/scaling) for the Redis, storage, and connection-pool prerequisites.
+
+### realtime
+
+A Bun Socket.IO server handling collaborative editing, live execution updates, and collaborative documents. Clients connect at `/socket.io`.
+
+It shares the database and `BETTER_AUTH_SECRET` with the app (Better Auth's shared-database-session pattern), so it authenticates the same users without a separate login.
+
+<Callout type="warn">
+  Scaling realtime past one replica **requires** `REDIS_URL` — the Socket.IO Redis adapter is what carries events between pods. Without it, two users on different pods silently stop seeing each other's edits.
+</Callout>
+
+### migrations
+
+Applies Drizzle schema migrations, then exits. In Docker Compose it is a one-shot service; in Kubernetes it is an **init container on the app Deployment**, so migrations run before any app pod becomes ready and re-run (as a no-op) on every rollout.
+
+Migrations are forward-only. See [Upgrades](/platform/self-hosting/upgrades).
+
+### postgresql
+
+PostgreSQL 17 with the **pgvector** extension, which is required — knowledge base embeddings are stored and searched as vectors. The `pgvector/pgvector:pg17` image ships it; a managed instance needs the extension enabled (Sim's migrations issue `CREATE EXTENSION` automatically where permissions allow).
+
+This holds essentially all durable state: workflows, runs, logs, users, organizations, credentials, knowledge base chunks and embeddings, and table data.
+
+### redis
+
+Backs pub/sub, the Socket.IO adapter, the idempotency store, execution progress markers, distributed execution limits, and the CLI-auth approval store. The storage-like uses fall back to Postgres or in-process state. Pub/sub falls back to a **process-local** emitter, which is fine on one replica and drops every cross-pod event on more than one. See [Redis](/platform/self-hosting/redis).
+
+### cron
+
+Eighteen scheduled jobs that call internal endpoints — schedule execution, polling triggers, webhook-subscription renewal, connector syncs, outbox processing, data drains, and sandbox-image cleanup. Kubernetes runs them as CronJobs; Docker Compose runs them from a single supercronic service. Same paths, same schedules. See [Background Jobs](/platform/self-hosting/background-jobs).
+
+## Where state lives
+
+Three places once the deployment is configured for production. Everything else is disposable.
+
+| Store | Contents | Backup |
+|---|---|---|
+| **PostgreSQL** | All application data | `pg_dump` / managed snapshots + PITR |
+| **Object storage** | Uploaded files, KB documents, execution outputs, avatars, logos | Bucket versioning + lifecycle |
+| **Secrets** | `ENCRYPTION_KEY`, `API_ENCRYPTION_KEY`, `BETTER_AUTH_SECRET`, `INTERNAL_API_SECRET`, `CRON_SECRET` | Secret manager |
+
+<Callout type="error">
+  **Object storage is not configured by default, and the fallback is not durable.** Sim only uses S3, Azure Blob, or GCS when the corresponding variables are set (`S3_BUCKET_NAME` + `AWS_REGION`, `AZURE_STORAGE_CONTAINER_NAME` + credentials, or `GCS_BUCKET_NAME`). With none set it writes uploads to a directory inside the app container — and neither `docker-compose.prod.yml` nor the Helm chart mounts a volume there. Files are lost when the container is recreated and are invisible to other replicas. Configure [object storage](/platform/self-hosting/object-storage) before storing anything you care about, and before scaling past one replica.
+</Callout>
+
+<Callout type="error">
+  `ENCRYPTION_KEY` is not recoverable and not derivable. It encrypts workspace and personal environment variables, stored provider API keys, MCP OAuth credentials, and deployment/chat secrets at rest — a database restore paired with a *different* key yields a working app in which none of that can be decrypted. Back it up separately from the database, and never rotate it casually.
+</Callout>
+
+Redis is a cache and message bus. Losing it drops in-flight live updates; it does not lose committed data.
+
+## Request paths
+
+**Editor / API** — browser → ingress/reverse proxy → app:3000 → Postgres, Redis, object storage.
+
+**Collaboration** — browser → ingress → realtime:3002 (`/socket.io`, WebSocket upgrade) → Redis pub/sub → other realtime pods. The proxy must pass upgrade headers and allow long-lived idle connections; see [Networking](/platform/self-hosting/networking).
+
+**File upload (object storage configured)** — browser asks app for a presigned URL → browser `PUT`s **directly to object storage** → app records metadata. This is why buckets need a CORS policy naming your Sim origin. Downloads are proxied back through the app.
+
+**File upload (local disk)** — the presigned endpoint reports `directUploadSupported: false` and the browser uploads through the app instead. No CORS configuration is involved, and no bucket is used.
+
+**Workflow execution** — trigger (manual, API, webhook, or schedule) → app enqueues or runs inline → isolated-vm sandbox → results and logs to Postgres, progress markers to Redis.
+
+**Background work** — CronJob → `Authorization: Bearer $CRON_SECRET` → app endpoint → same execution path.
+
+## Network boundaries
+
+| From | To | Purpose |
+|---|---|---|
+| Internet | app:3000, realtime:3002 | Users |
+| app, realtime | postgresql:5432 | Data |
+| app, realtime | redis:6379 | Pub/sub, cache |
+| app | Object storage endpoint | Files (server side) |
+| **Browser** | **Object storage endpoint** | Presigned uploads — must be publicly reachable |
+| app | Model provider APIs, integration APIs, SMTP/email provider | Outbound |
+| cron | app:3000 (internal Service / compose network) | Scheduled triggers |
+
+The chart's optional NetworkPolicy blocks cloud metadata endpoints (`169.254.169.254`) by default but allows ingress from any pod in the cluster unless you scope `networkPolicy.ingressFrom`. See [Security](/platform/self-hosting/security).
+
+<FAQ items={[
+  { question: "Can I run the app without the realtime service?", answer: "A realtime server must exist — it carries collaborative editing and live execution updates, and the editor degrades badly without one. You can set realtime.enabled=false in the chart, but only when pointing app.env.SOCKET_SERVER_URL at a realtime instance you run yourself."},
+  { question: "Does workflow execution run in a separate worker process?", answer: "By default, no — executions run inside the app process using an isolated-vm sandbox, which is why the app has an 8 Gi memory limit. Setting E2B_ENABLED or SANDBOX_PROVIDER=daytona moves user code to a remote sandbox, and TRIGGER_DEV_ENABLED routes async jobs to Trigger.dev; otherwise the database-backed job queue is used."},
+]} />