diff --git a/.nextchanges/cli/ssh-connect-claude-launcher.md b/.nextchanges/cli/ssh-connect-claude-launcher.md new file mode 100644 index 00000000000..5d6b4091776 --- /dev/null +++ b/.nextchanges/cli/ssh-connect-claude-launcher.md @@ -0,0 +1 @@ +* `databricks ssh connect` serverless sessions now support Claude Code and Codex configured with Unity AI Gateway out of the box diff --git a/experimental/ssh/cmd/agent_shim.go b/experimental/ssh/cmd/agent_shim.go new file mode 100644 index 00000000000..fedd1589c3b --- /dev/null +++ b/experimental/ssh/cmd/agent_shim.go @@ -0,0 +1,43 @@ +package ssh + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/experimental/ssh/internal/client" + "github.com/databricks/cli/libs/cmdctx" + "github.com/spf13/cobra" +) + +func newAgentShimCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "agent-shim", + Short: "Launch a ucode-configured coding agent (invoked on the remote)", + Hidden: true, + } + for _, agent := range client.SupportedAgentNames() { + cmd.AddCommand(newAgentShimAgentCommand(agent)) + } + return cmd +} + +func newAgentShimAgentCommand(agent string) *cobra.Command { + cmd := &cobra.Command{ + Use: agent, + Short: "Launch the ucode-configured " + agent + " agent", + // Disable flag parsing: forward everything after the agent name to the agent verbatim. + DisableFlagParsing: true, + } + + cmd.PreRunE = func(cmd *cobra.Command, args []string) error { + // Runs on the driver with injected env auth; no bundle, no prompt. + cmd.SetContext(root.SkipLoadBundle(cmd.Context())) + cmd.SetContext(root.SkipPrompt(cmd.Context())) + return root.MustWorkspaceClient(cmd, args) + } + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + return client.RunAgentShim(ctx, cmdctx.WorkspaceClient(ctx), agent, args) + } + + return cmd +} diff --git a/experimental/ssh/cmd/ssh.go b/experimental/ssh/cmd/ssh.go index 9939d2f7050..19517cd6204 100644 --- a/experimental/ssh/cmd/ssh.go +++ b/experimental/ssh/cmd/ssh.go @@ -22,6 +22,7 @@ Use ` + "`databricks ssh connect --help`" + ` to see all available flags.`, cmd.AddCommand(newSetupCommand()) cmd.AddCommand(newConnectCommand()) cmd.AddCommand(newServerCommand()) + cmd.AddCommand(newAgentShimCommand()) return cmd } diff --git a/experimental/ssh/internal/client/agentshim.go b/experimental/ssh/internal/client/agentshim.go new file mode 100644 index 00000000000..36f5e72a660 --- /dev/null +++ b/experimental/ssh/internal/client/agentshim.go @@ -0,0 +1,776 @@ +package client + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" +) + +const ( + // agentDir (home-relative) is the shim's working area. + agentDir = ".agent-shim" + + // shimDir holds the per-agent wrappers; must match remoteShimDir in client.go. + shimDir = agentDir + "/bin" + + // ucodeRepo is the GitHub repo the shim installs ucode from (latest release). + ucodeRepo = "databricks/ucode" + + // workspaceHomeEnv passes the user's workspace home to the shim for the agent context. + workspaceHomeEnv = "DATABRICKS_WORKSPACE_HOME" + + // depsDir (home-relative) holds toolchain deps (Node/npm) fetched when the image ships none. + depsDir = agentDir + "/deps" + + // contextFile is the scratch file holding the Databricks session context. + contextFile = "agent-system-context.md" + + // setupLockName (under agentDir) serializes first-run toolchain setup across + // concurrent SSH clients sharing this driver's $HOME. + setupLockName = ".setup.lock" + + // setupLockStaleAfter reclaims a setup lock left behind by a process that died + // mid-install. It is deliberately generous: a cold first run downloads uv, + // ucode, and Node, which can legitimately take minutes. + setupLockStaleAfter = 10 * time.Minute +) + +type agentSpec struct { + name string + // contextFlag passes the context file via this CLI flag (Claude's --append-system-prompt-file). + contextFlag string + // contextHomeFile writes the context to this $HOME instructions file (codex's AGENTS.md). + contextHomeFile string +} + +// supportedAgents are the agents the shim launches — limited to those whose ucode +// command accepts --workspace (how we target the workspace headlessly). +var supportedAgents = []agentSpec{ + {name: "claude", contextFlag: "--append-system-prompt-file"}, + {name: "codex", contextHomeFile: ".codex/AGENTS.md"}, +} + +// SupportedAgentNames lists the agents the ssh command registers a subcommand for. +func SupportedAgentNames() []string { + names := make([]string, len(supportedAgents)) + for i, a := range supportedAgents { + names[i] = a.name + } + return names +} + +func agentByName(name string) (agentSpec, bool) { + for _, a := range supportedAgents { + if a.name == name { + return a, true + } + } + return agentSpec{}, false +} + +func RunAgentShim(ctx context.Context, client *databricks.WorkspaceClient, agentName string, agentArgs []string) error { + agent, ok := agentByName(agentName) + if !ok { + return fmt.Errorf("unsupported agent %q", agentName) + } + // Probe first: fail fast before the slow first-run bootstrap if the gateway is off. + if err := probeAIGateway(ctx, client); err != nil { + return err + } + // ucode targets this workspace via --workspace, so it configures without prompting. + workspace := strings.TrimRight(client.Config.Host, "/") + return bootstrapAndLaunchAgent(ctx, agent, workspace, agentArgs) +} + +// --- Unity AI Gateway preflight --- +// +// Kept in lockstep with ucode's probe_unity_gateway_capabilities +// (databricks/ucode, src/ucode/databricks.py): probe the Unity Catalog +// model-services API (v3) first, paging through it, then fall back to the legacy +// AI Gateway endpoints API (v2). A path counts only when its JSON body actually +// lists a usable resource — a 200 with an empty collection is "reachable but +// empty", not "enabled". We fail fast with the same actionable guidance ucode +// surfaces so the shim's preflight matches what ucode itself checks. + +const ( + modelServicesPath = "/api/2.1/unity-catalog/model-services" + legacyEndpointsPath = "/api/ai-gateway/v2/endpoints" + modelServiceProbePageSize = 50 + modelServiceProbeMaxPages = 20 + aiGatewayDocsURL = "https://docs.databricks.com/aws/en/ai-gateway/overview-beta" + + // modelServiceEmptyDetail matches ucode's wording for a reachable model-services + // API that lists nothing the caller can use — almost always a UC grant gap. + modelServiceEmptyDetail = "reachable, no accessible model services returned; " + + "check USE CATALOG on system, and USE SCHEMA and EXECUTE on system.ai" +) + +// gatewayProbe is the outcome of probing one AI Gateway API. resourceAvailable +// means the API returned at least one usable resource; conclusive is false when +// paging couldn't be completed, so "no resources" was never actually confirmed. +type gatewayProbe struct { + reachable bool + detail string + resourceAvailable bool + conclusive bool +} + +// probeAIGateway fails fast if the workspace's Unity AI Gateway can't run an +// agent. It returns nil when the gateway is usable — logging a warning when it's +// reachable but exposes no model services to this caller — and an actionable +// error otherwise. +func probeAIGateway(ctx context.Context, client *databricks.WorkspaceClient) error { + host := strings.TrimRight(client.Config.Host, "/") + + modelSvc := probeModelServices(ctx, client, host) + // A 401 (or a 400 "invalid token") can't be rescued by trying another API, + // so surface it before the fallback probe. + if !modelSvc.reachable && looksLikeDefinitiveAuthFailure(modelSvc.detail) { + return aiGatewayAuthError(host, modelSvc.detail) + } + if modelSvc.resourceAvailable { + return nil + } + + legacy := probeLegacyEndpoints(ctx, client, host) + switch { + case legacy.reachable: + // The legacy endpoints API answered, so the gateway is enabled even if no + // model services are visible to this caller. + log.Warnf(ctx, "Unity AI Gateway model service check: %s", modelSvc.detail) + return nil + case modelSvc.reachable && !modelSvc.conclusive: + // v3 answered but couldn't be paged to completion; give it the benefit of + // the doubt rather than blocking on an unconfirmed "empty". + return nil + case looksLikeDefinitiveAuthFailure(legacy.detail): + return aiGatewayAuthError(host, legacy.detail) + case looksLikeScopeFailure(modelSvc.detail): + return aiGatewayScopeError(host, modelSvc.detail) + case looksLikeScopeFailure(legacy.detail): + return aiGatewayScopeError(host, legacy.detail) + case looksLikeTransient(modelSvc.detail) || looksLikeTransient(legacy.detail): + // A rate-limit/5xx/network blip is not "disabled" — tell the user to retry + // rather than sending them to the enablement docs. + return fmt.Errorf("could not verify the Databricks Unity AI Gateway on %s: the probe hit a transient error (model services: %s; legacy endpoints: %s). Retry in a moment", host, modelSvc.detail, legacy.detail) + case looksLikePermissionFailure(modelSvc.detail): + return fmt.Errorf("model service access could not be verified on %s (%s). The legacy endpoint fallback also failed (%s). The model service probe requires permission to list Unity Catalog model services. Verify USE CATALOG on `system`, and USE SCHEMA and EXECUTE on `system.ai`", host, modelSvc.detail, legacy.detail) + case looksLikePermissionFailure(legacy.detail): + return fmt.Errorf("legacy endpoint access could not be verified on %s (%s). The model service probe also failed (%s). Verify the caller's workspace permissions for the legacy endpoints listing", host, legacy.detail, modelSvc.detail) + default: + return fmt.Errorf("the Databricks Unity AI Gateway is not enabled on this workspace (%s): neither model services (%s) nor legacy endpoints (%s) are available. See %s", host, modelSvc.detail, legacy.detail, aiGatewayDocsURL) + } +} + +// probeModelServices probes the UC model-services API (v3), paging until it finds +// an accessible model service, exhausts the cursor, or hits the page cap. +func probeModelServices(ctx context.Context, client *databricks.WorkspaceClient, host string) gatewayProbe { + pageToken := "" + for page := 0; page < modelServiceProbeMaxPages; page++ { + reqURL := fmt.Sprintf("%s%s?page_size=%d", host, modelServicesPath, modelServiceProbePageSize) + if pageToken != "" { + reqURL += "&page_token=" + url.QueryEscape(pageToken) + } + payload, reason := gatewayGetJSON(ctx, client, reqURL) + if payload == nil { + // A first-page failure is the real reachability signal; a later page + // failing still means the API answered at least once, so treat it as + // reachable-but-inconclusive rather than a confirmed "empty". + if page == 0 { + return gatewayProbe{detail: versionNeutralGatewayDetail(reason), conclusive: true} + } + return gatewayProbe{reachable: true, detail: "reachable"} + } + if hasNonEmptyCollection(payload, "model_services") { + return gatewayProbe{reachable: true, detail: "reachable, accessible model service returned", resourceAvailable: true, conclusive: true} + } + pageToken = stringField(payload, "next_page_token") + if pageToken == "" { + return gatewayProbe{reachable: true, detail: modelServiceEmptyDetail, conclusive: true} + } + } + return gatewayProbe{reachable: true, detail: "reachable"} +} + +// probeLegacyEndpoints probes the legacy AI Gateway endpoints API (v2). +func probeLegacyEndpoints(ctx context.Context, client *databricks.WorkspaceClient, host string) gatewayProbe { + payload, reason := gatewayGetJSON(ctx, client, host+legacyEndpointsPath+"?page_size=1") + if payload == nil { + return gatewayProbe{detail: versionNeutralGatewayDetail(reason), conclusive: true} + } + if hasNonEmptyCollection(payload, "endpoints") { + return gatewayProbe{reachable: true, detail: "reachable, accessible endpoint returned", resourceAvailable: true, conclusive: true} + } + return gatewayProbe{reachable: true, detail: "reachable, no accessible endpoints returned", conclusive: true} +} + +// gatewayGetJSON issues an authenticated GET expecting JSON. It returns the +// decoded payload on HTTP 200, or (nil, reason) on any failure — mirroring +// ucode's _http_get_json, including a short response-body excerpt so gateway auth +// errors (e.g. a 400 whose body is "Invalid Token") stay visible in the reason. +func gatewayGetJSON(ctx context.Context, client *databricks.WorkspaceClient, reqURL string) (any, string) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, fmt.Sprintf("network error: %v", err) + } + req.Header.Set("Accept", "application/json") + // Set X-Databricks-Workspace-Id (and suppress the legacy workspace_id=none + // sentinel) so these hand-written workspace-routed GETs reach the workspace + // plane on unified/SPOG hosts instead of the account plane. + for k, v := range auth.WorkspaceIDHeaders(client.Config) { + req.Header.Set(k, v) + } + if err := client.Config.Authenticate(req); err != nil { + return nil, fmt.Sprintf("network error: %v", err) + } + resp, err := (&http.Client{Transport: client.Config.HTTPTransport, Timeout: 10 * time.Second}).Do(req) + if err != nil { + return nil, fmt.Sprintf("network error: %v", err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode == http.StatusOK { + var payload any + if err := json.Unmarshal(body, &payload); err != nil { + return nil, fmt.Sprintf("response was not valid JSON (%v)", err) + } + return payload, "" + } + reason := fmt.Sprintf("HTTP %d %s", resp.StatusCode, http.StatusText(resp.StatusCode)) + if excerpt := strings.TrimSpace(string(body)); excerpt != "" { + if len(excerpt) > 200 { + excerpt = excerpt[:200] + } + reason += ": " + excerpt + } + return nil, reason +} + +var ( + gatewayDetailV3 = regexp.MustCompile(`(?i)\bv3\b`) + gatewayDetailV2 = regexp.MustCompile(`(?i)\bv2\b`) +) + +// versionNeutralGatewayDetail rewrites the internal v2/v3 API labels in a failure +// reason into user-facing terms, matching ucode so the error text stays in sync. +func versionNeutralGatewayDetail(detail string) string { + detail = gatewayDetailV3.ReplaceAllString(detail, "model service") + return gatewayDetailV2.ReplaceAllString(detail, "legacy endpoint") +} + +// looksLikeDefinitiveAuthFailure is true when retrying another workspace API +// can't rescue the token. A bare 403 is left out: it can be endpoint-specific +// authorization, so the preflight still tries the fallback before giving up. +func looksLikeDefinitiveAuthFailure(reason string) bool { + if strings.Contains(reason, "HTTP 401") { + return true + } + return strings.Contains(reason, "HTTP 400") && strings.Contains(strings.ToLower(reason), "invalid token") +} + +// looksLikeScopeFailure matches a 403 that reports the OAuth token is missing a +// required scope (which re-login can fix), as opposed to a plain permission 403. +func looksLikeScopeFailure(reason string) bool { + l := strings.ToLower(reason) + return strings.Contains(l, "http 403") && strings.Contains(l, "oauth token") && strings.Contains(l, "required scopes") +} + +func looksLikePermissionFailure(reason string) bool { + return strings.Contains(reason, "HTTP 403") +} + +// gatewayTransientStatus matches the HTTP statuses worth retrying (429 + any 5xx). +var gatewayTransientStatus = regexp.MustCompile(`HTTP (429|5\d\d)`) + +// looksLikeTransient is true for a probe failure that is likely temporary (rate +// limit, server error, or a network/transport error) rather than a stable +// "disabled"/"unauthorized" verdict. +func looksLikeTransient(reason string) bool { + return strings.HasPrefix(reason, "network error") || gatewayTransientStatus.MatchString(reason) +} + +func aiGatewayAuthError(host, reason string) error { + return fmt.Errorf("the Databricks workspace %s rejected the access token (%s). Try:\n databricks auth logout --host %s\n databricks auth login --host %s", host, reason, host, host) +} + +func aiGatewayScopeError(host, reason string) error { + return fmt.Errorf("the access token for %s is missing an OAuth scope required by the AI Gateway APIs (%s). Re-authenticate to mint a token with the needed scopes:\n databricks auth login --host %s", host, reason, host) +} + +// hasNonEmptyCollection reports whether payload is a JSON object with a non-empty +// array at key. +func hasNonEmptyCollection(payload any, key string) bool { + obj, ok := payload.(map[string]any) + if !ok { + return false + } + arr, ok := obj[key].([]any) + return ok && len(arr) > 0 +} + +// stringField returns payload[key] when payload is a JSON object holding a string +// there, else "". +func stringField(payload any, key string) string { + obj, ok := payload.(map[string]any) + if !ok { + return "" + } + s, _ := obj[key].(string) + return s +} + +func bootstrapAndLaunchAgent(ctx context.Context, agent agentSpec, workspace string, agentArgs []string) error { + home, err := env.UserHomeDir(ctx) + if err != nil { + return fmt.Errorf("failed to resolve home directory: %w", err) + } + + // Reconstruct the PATH tooling runs under on every launch rather than caching + // it: every entry is a known location, so deriving it can't restore a stale + // path (e.g. a versioned CLI dir from an older session). Drop the shim dir so + // ucode execs the real agent rather than this wrapper, then prepend uv/ucode's + // bin, this databricks CLI's own dir (so ucode finds it and skips its own + // install), and the Node bin ensureNode installs into. + removePath(ctx, filepath.Join(home, shimDir)) + prependPath(ctx, filepath.Join(home, ".local", "bin")) + if self, err := os.Executable(); err == nil { + prependPath(ctx, filepath.Dir(self)) + } + prependPath(ctx, filepath.Join(home, depsDir, "node", "bin")) + + if err := ensureToolchain(ctx, home); err != nil { + return err + } + + // npm is on PATH now (shipped or just installed); silence its update-notifier + // box so it doesn't clutter the agent session. + disableNpmUpdateNotifier(ctx) + + // Put npm's global bin on PATH so an npm-installed agent binary resolves after + // ucode installs it. + if prefix := npmGlobalPrefix(ctx); prefix != "" { + prependPath(ctx, filepath.Join(prefix, "bin")) + } + + return launchUcodeAgent(ctx, home, agent, workspace, agentArgs) +} + +// toolchainReady reports whether the agent toolchain is already installed, derived +// purely from PATH (which the caller has already set up). ucode launches the agent +// and npm provides the Node runtime agents like Claude Code need. +func toolchainReady() bool { + _, ucodeErr := exec.LookPath("ucode") + _, npmErr := exec.LookPath("npm") + return ucodeErr == nil && npmErr == nil +} + +// ensureToolchain installs uv, ucode, and Node/npm when missing. It is a no-op +// once the toolchain is present, and holds a machine-local lock while installing +// so two first-run SSH clients sharing this $HOME don't concurrently install into +// (and corrupt) the same paths. +func ensureToolchain(ctx context.Context, home string) error { + if toolchainReady() { + return nil + } + unlock, err := acquireSetupLock(ctx, home) + if err != nil { + return err + } + defer unlock() + // Another client may have finished the install while we waited for the lock. + if toolchainReady() { + return nil + } + return installToolchain(ctx, home) +} + +// installToolchain performs the actual installs. The caller holds the setup lock. +func installToolchain(ctx context.Context, home string) error { + ui := newProgressUI(ctx) + + // 1. uv (installs into ~/.local/bin). + if _, err := exec.LookPath("uv"); err != nil { + if err := ui.runStep(ctx, "Installing dependencies", func(out io.Writer) error { + if err := runShell(ctx, out, "curl -LsSf https://astral.sh/uv/install.sh | sh"); err != nil { + return fmt.Errorf("failed to install uv: %w", err) + } + return nil + }); err != nil { + return err + } + } + + // 2. ucode (latest stock upstream release). + if _, err := exec.LookPath("ucode"); err != nil { + if err := ui.runStep(ctx, "Installing ucode", func(out io.Writer) error { + ref, err := latestUcodeRef(ctx, "https://api.github.com/repos/"+ucodeRepo+"/releases/latest") + if err != nil { + return err + } + if err := runCommand(ctx, out, "uv", "tool", "install", "git+https://github.com/"+ucodeRepo+"@"+ref); err != nil { + return fmt.Errorf("failed to install ucode: %w", err) + } + return nil + }); err != nil { + return err + } + } + + // 3. Node/npm. Some agents (e.g. Claude Code) need it and serverless images + // don't ship it. It installs into depsDir/node/bin, already prepended to PATH. + if _, err := exec.LookPath("npm"); err != nil { + if err := ui.runStep(ctx, "Installing Node.js", func(out io.Writer) error { + _, err := ensureNode(ctx, home, out) + return err + }); err != nil { + return err + } + } + + return nil +} + +// acquireSetupLock takes a machine-local lock (an O_EXCL sentinel file, matching +// the pattern in libs/versioncheck) serializing first-run setup across SSH +// clients. It blocks until the lock is free, reclaiming one left behind by a dead +// process after setupLockStaleAfter. The returned func releases the lock. +func acquireSetupLock(ctx context.Context, home string) (func(), error) { + lockPath := filepath.Join(home, agentDir, setupLockName) + if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil { + return nil, fmt.Errorf("failed to create %s: %w", filepath.Dir(lockPath), err) + } + for { + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err == nil { + _ = f.Close() + return func() { _ = os.Remove(lockPath) }, nil + } + if !errors.Is(err, os.ErrExist) { + return nil, fmt.Errorf("failed to acquire setup lock: %w", err) + } + // Held by another process: reclaim it if stale, otherwise wait and retry. + if info, statErr := os.Stat(lockPath); statErr == nil && time.Since(info.ModTime()) > setupLockStaleAfter { + log.Warnf(ctx, "reclaiming stale agent-shim setup lock at %s", lockPath) + _ = os.Remove(lockPath) + continue + } + log.Infof(ctx, "waiting for a concurrent agent-shim setup to finish") + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Second): + } + } +} + +// launchUcodeAgent launches the ucode-configured agent, replacing this process. +// home is the already-resolved user home directory. +func launchUcodeAgent(ctx context.Context, home string, agent agentSpec, workspace string, agentArgs []string) error { + ucodePath, err := exec.LookPath("ucode") + if err != nil { + return fmt.Errorf("ucode not found on PATH after setup: %w", err) + } + contextArgs, err := injectAgentContext(ctx, home, agent) + if err != nil { + return err + } + argv := []string{"ucode", agent.name} + if workspace != "" { + // Every supported agent accepts --workspace (the gate for supportedAgents). + argv = append(argv, "--workspace", workspace) + } + argv = append(argv, contextArgs...) + argv = append(argv, agentArgs...) + // Pass the session token as DATABRICKS_BEARER so stock ucode authenticates headlessly. + if env.Get(ctx, "DATABRICKS_BEARER") == "" { + if token := env.Get(ctx, "DATABRICKS_TOKEN"); token != "" { + _ = os.Setenv("DATABRICKS_BEARER", token) + } + } + return execProcess(ucodePath, argv, os.Environ()) +} + +// injectAgentContext puts the Databricks context where the agent reads it, returning any extra argv. +func injectAgentContext(ctx context.Context, home string, agent agentSpec) ([]string, error) { + systemContext := agentSystemContext(env.Get(ctx, workspaceHomeEnv)) + switch { + case agent.contextFlag != "": + // Keep the scratch context file under the shim's own directory so it never + // clobbers an unrelated file the user happens to have in their home. + dir := filepath.Join(home, agentDir) + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("failed to create %s: %w", dir, err) + } + f := filepath.Join(dir, contextFile) + if err := os.WriteFile(f, []byte(systemContext), 0o644); err != nil { + return nil, fmt.Errorf("failed to write agent context file: %w", err) + } + return []string{agent.contextFlag, f}, nil + case agent.contextHomeFile != "": + f := filepath.Join(home, agent.contextHomeFile) + if _, err := os.Stat(f); err == nil { + return nil, nil // already present — don't clobber + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("failed to stat %s: %w", f, err) + } + if err := os.MkdirAll(filepath.Dir(f), 0o755); err != nil { + return nil, fmt.Errorf("failed to create %s: %w", filepath.Dir(f), err) + } + if err := os.WriteFile(f, []byte(systemContext), 0o644); err != nil { + return nil, fmt.Errorf("failed to write %s: %w", f, err) + } + return nil, nil + default: + return nil, nil + } +} + +// ensureNode downloads the latest Krypton LTS Node into deps/node once, returning +// its bin; extraction output is written to out. Linux-only by design: the shim runs +// on the serverless driver, so the tarball name is hardcoded to linux while +// nodeDownloadArch guards the arch. +func ensureNode(ctx context.Context, home string, out io.Writer) (string, error) { + depsRoot := filepath.Join(home, depsDir) + nodeDir := filepath.Join(depsRoot, "node") + nodeBin := filepath.Join(nodeDir, "bin") + if _, err := os.Stat(filepath.Join(nodeBin, "npm")); err == nil { + return nodeBin, nil + } + + arch := nodeDownloadArch(runtime.GOARCH) + if arch == "" { + return "", fmt.Errorf("unsupported architecture for Node download: %s", runtime.GOARCH) + } + const base = "https://nodejs.org/dist/latest-krypton" + tarName, wantSum, err := latestNodeTarball(ctx, base+"/SHASUMS256.txt", arch) + if err != nil { + return "", err + } + if err := os.MkdirAll(depsRoot, 0o755); err != nil { + return "", fmt.Errorf("failed to create %s: %w", depsRoot, err) + } + // Download to a temp file, verifying its SHA256 from SHASUMS256.txt before use. + tarball, err := downloadVerified(ctx, base+"/"+tarName, wantSum) + if err != nil { + return "", err + } + defer os.Remove(tarball) + // Extract into a sibling temp dir and atomically rename it into place, so an + // interrupted extraction never leaves a half-populated node dir that the + // os.Stat(npm) check above would then wrongly accept as a finished install. + tmpDir, err := os.MkdirTemp(depsRoot, "node-*") + if err != nil { + return "", fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) // no-op once renamed; cleans up a failed extraction + // Node's .tar.xz is the smallest download; extract it with the system tar. + if err := runCommand(ctx, out, "tar", "-xJf", tarball, "--strip-components=1", "-C", tmpDir); err != nil { + return "", fmt.Errorf("failed to extract Node.js: %w", err) + } + // Clear any partial leftover from a previously-interrupted run, then publish + // atomically (same-filesystem rename, since tmpDir is a sibling of nodeDir). + if err := os.RemoveAll(nodeDir); err != nil { + return "", fmt.Errorf("failed to remove %s: %w", nodeDir, err) + } + if err := os.Rename(tmpDir, nodeDir); err != nil { + return "", fmt.Errorf("failed to install Node.js: %w", err) + } + return nodeBin, nil +} + +// downloadVerified fetches url to a temp file, failing unless its SHA256 matches +// wantSum. The caller is responsible for removing the returned file. +func downloadVerified(ctx context.Context, url, wantSum string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("failed to download %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to download %s: HTTP %d", url, resp.StatusCode) + } + + f, err := os.CreateTemp("", "node-*.tar.xz") + if err != nil { + return "", err + } + sum := sha256.New() + if _, err := io.Copy(io.MultiWriter(f, sum), resp.Body); err != nil { + f.Close() + os.Remove(f.Name()) + return "", fmt.Errorf("failed to download %s: %w", url, err) + } + if err := f.Close(); err != nil { + os.Remove(f.Name()) + return "", err + } + if got := hex.EncodeToString(sum.Sum(nil)); !strings.EqualFold(got, wantSum) { + os.Remove(f.Name()) + return "", fmt.Errorf("checksum mismatch for %s: got %s, want %s", url, got, wantSum) + } + return f.Name(), nil +} + +// latestUcodeRef returns the tag of ucode's latest GitHub release. +func latestUcodeRef(ctx context.Context, apiURL string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return "", err + } + req.Header.Set("Accept", "application/vnd.github+json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("failed to fetch latest ucode release: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to fetch latest ucode release: HTTP %d", resp.StatusCode) + } + var payload struct { + TagName string `json:"tag_name"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&payload); err != nil { + return "", fmt.Errorf("failed to parse latest ucode release: %w", err) + } + if tag := strings.TrimSpace(payload.TagName); tag != "" { + return tag, nil + } + return "", errors.New("latest ucode release has empty tag_name") +} + +// nodeDownloadArch maps a Go arch to Node's release arch token, or "" if unsupported. +func nodeDownloadArch(goarch string) string { + switch goarch { + case "amd64": + return "x64" + case "arm64": + return "arm64" + default: + return "" + } +} + +// latestNodeTarball returns the linux tarball name and its SHA256 for arch from +// the SHASUMS256.txt listing (lines of " "). +func latestNodeTarball(ctx context.Context, shasumsURL, arch string) (name, sum string, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, shasumsURL, nil) + if err != nil { + return "", "", err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", "", fmt.Errorf("failed to fetch Node checksums: %w", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", "", fmt.Errorf("failed to read Node checksums: %w", err) + } + re := regexp.MustCompile(`^([0-9a-f]{64})\s+(node-v[0-9.]+-linux-` + regexp.QuoteMeta(arch) + `\.tar\.xz)$`) + for _, line := range strings.Split(string(body), "\n") { + if m := re.FindStringSubmatch(strings.TrimSpace(line)); m != nil { + return m[2], m[1], nil + } + } + return "", "", fmt.Errorf("no linux-%s Node tarball found in %s", arch, shasumsURL) +} + +// disableNpmUpdateNotifier turns off npm's "new version available" box so it +// doesn't clutter the agent session. Best-effort: it's cosmetic, so a failure +// (e.g. npm not runnable) is logged and ignored rather than blocking the launch. +func disableNpmUpdateNotifier(ctx context.Context) { + if err := exec.CommandContext(ctx, "npm", "config", "set", "update-notifier", "false").Run(); err != nil { + log.Debugf(ctx, "failed to disable npm update-notifier: %v", err) + } +} + +// npmGlobalPrefix returns `npm prefix -g`, or "" if npm isn't runnable. +func npmGlobalPrefix(ctx context.Context) string { + out, err := exec.CommandContext(ctx, "npm", "prefix", "-g").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} + +// runCommand runs name with the given args, writing combined stdout+stderr to out +// (captured so it surfaces only on failure) and inheriting the process env. Stdin +// is left closed: the shim's install steps are non-interactive. +func runCommand(ctx context.Context, out io.Writer, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = out + cmd.Stderr = out + return cmd.Run() +} + +// runShell runs a shell snippet — for the curl|sh / curl|tar pipelines — writing +// combined stdout+stderr to out. +func runShell(ctx context.Context, out io.Writer, script string) error { + cmd := exec.CommandContext(ctx, "sh", "-c", script) + cmd.Stdout = out + cmd.Stderr = out + return cmd.Run() +} + +// prependPath puts dir at the front of $PATH, dropping any later duplicate. +func prependPath(ctx context.Context, dir string) { + dirs := []string{dir} + for _, d := range filepath.SplitList(env.Get(ctx, "PATH")) { + if d != dir { + dirs = append(dirs, d) + } + } + _ = os.Setenv("PATH", strings.Join(dirs, string(os.PathListSeparator))) +} + +// removePath drops every occurrence of dir from $PATH. +func removePath(ctx context.Context, dir string) { + var dirs []string + for _, d := range filepath.SplitList(env.Get(ctx, "PATH")) { + if d != dir { + dirs = append(dirs, d) + } + } + _ = os.Setenv("PATH", strings.Join(dirs, string(os.PathListSeparator))) +} + +// agentSystemContext returns the Databricks session context; wsHome names the working directory. +func agentSystemContext(wsHome string) string { + cwd := "the user's Databricks workspace home directory" + if wsHome != "" { + cwd = wsHome + } + return fmt.Sprintf(`You are running inside a "databricks ssh connect" session on the driver node of a +Databricks serverless cluster. +- The "databricks" CLI is installed and already authenticated: DATABRICKS_HOST and + DATABRICKS_TOKEN are set in the environment, so "databricks ..." commands work with no + "databricks auth login". The same token governs Unity Catalog and serving-endpoint access. +- This container is ephemeral; only paths under /Workspace persist across sessions. Your + working directory is %s. +- DATABRICKS_TOKEN is a static session token that may expire during a long session; if + "databricks" calls start failing with auth errors, the session likely needs reconnecting. +- You can run shell commands and use the "databricks" CLI to explore the workspace + (clusters, jobs, Unity Catalog, DBFS, etc.).`, cwd) +} diff --git a/experimental/ssh/internal/client/agentshim_test.go b/experimental/ssh/internal/client/agentshim_test.go new file mode 100644 index 00000000000..19396689341 --- /dev/null +++ b/experimental/ssh/internal/client/agentshim_test.go @@ -0,0 +1,372 @@ +package client + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrependPath(t *testing.T) { + t.Setenv("PATH", strings.Join([]string{"/a", "/b"}, string(os.PathListSeparator))) + prependPath(t.Context(), "/b") // existing entry moves to the front (deduped) + assert.Equal(t, []string{"/b", "/a"}, filepath.SplitList(os.Getenv("PATH"))) + prependPath(t.Context(), "/new") + assert.Equal(t, []string{"/new", "/b", "/a"}, filepath.SplitList(os.Getenv("PATH"))) +} + +func TestRemovePath(t *testing.T) { + t.Setenv("PATH", strings.Join([]string{"/a", "/b", "/a"}, string(os.PathListSeparator))) + removePath(t.Context(), "/a") + assert.Equal(t, []string{"/b"}, filepath.SplitList(os.Getenv("PATH"))) +} + +func TestAcquireSetupLock(t *testing.T) { + home := t.TempDir() + lockPath := filepath.Join(home, agentDir, setupLockName) + + // Acquire creates the sentinel; release removes it. + unlock, err := acquireSetupLock(t.Context(), home) + require.NoError(t, err) + _, statErr := os.Stat(lockPath) + require.NoError(t, statErr, "lock sentinel should exist while held") + unlock() + _, statErr = os.Stat(lockPath) + assert.True(t, os.IsNotExist(statErr), "lock sentinel should be gone after release") + + // A lock left behind by a dead process is reclaimed once stale, not waited on + // forever. + require.NoError(t, os.WriteFile(lockPath, nil, 0o644)) + stale := time.Now().Add(-2 * setupLockStaleAfter) + require.NoError(t, os.Chtimes(lockPath, stale, stale)) + unlock2, err := acquireSetupLock(t.Context(), home) + require.NoError(t, err) + unlock2() +} + +func TestNodeDownloadArch(t *testing.T) { + assert.Equal(t, "x64", nodeDownloadArch("amd64")) + assert.Equal(t, "arm64", nodeDownloadArch("arm64")) + assert.Empty(t, nodeDownloadArch("mips")) +} + +func TestLatestNodeTarball(t *testing.T) { + gzSum, x64Sum, armSum := strings.Repeat("0", 64), strings.Repeat("a", 64), strings.Repeat("b", 64) + body := gzSum + " node-v24.1.0-linux-x64.tar.gz\n" + // .gz is skipped (want .xz) + x64Sum + " node-v24.1.0-linux-x64.tar.xz\n" + + armSum + " node-v24.1.0-linux-arm64.tar.xz\n" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(body)) + })) + defer srv.Close() + + name, sum, err := latestNodeTarball(t.Context(), srv.URL, "x64") + require.NoError(t, err) + assert.Equal(t, "node-v24.1.0-linux-x64.tar.xz", name) + assert.Equal(t, x64Sum, sum) + + _, _, err = latestNodeTarball(t.Context(), srv.URL, "ppc64le") + assert.Error(t, err) +} + +func TestDownloadVerified(t *testing.T) { + payload := []byte("fake node tarball") + sum := sha256.Sum256(payload) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(payload) + })) + defer srv.Close() + + t.Run("matching checksum writes the file", func(t *testing.T) { + path, err := downloadVerified(t.Context(), srv.URL, hex.EncodeToString(sum[:])) + require.NoError(t, err) + defer os.Remove(path) + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, payload, got) + }) + + t.Run("mismatched checksum errors and leaves no file", func(t *testing.T) { + _, err := downloadVerified(t.Context(), srv.URL, strings.Repeat("0", 64)) + require.Error(t, err) + assert.Contains(t, err.Error(), "checksum mismatch") + }) +} + +func TestLatestUcodeRef(t *testing.T) { + t.Run("returns the release tag", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"tag_name":"v1.2.3"}`)) + })) + defer srv.Close() + ref, err := latestUcodeRef(t.Context(), srv.URL) + require.NoError(t, err) + assert.Equal(t, "v1.2.3", ref) + }) + + t.Run("errors on non-200", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + _, err := latestUcodeRef(t.Context(), srv.URL) + assert.Error(t, err) + }) + + t.Run("errors on empty tag", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{}`)) + })) + defer srv.Close() + _, err := latestUcodeRef(t.Context(), srv.URL) + assert.Error(t, err) + }) +} + +func TestSupportedAgents(t *testing.T) { + names := SupportedAgentNames() + // Limited to the agents whose ucode command accepts --workspace. + assert.Equal(t, []string{"claude", "codex"}, names) + + // Each agent injects context by exactly one mechanism (flag OR home file); + // the two are never both set. + expect := map[string]struct{ flag, home string }{ + "claude": {flag: "--append-system-prompt-file"}, + "codex": {home: ".codex/AGENTS.md"}, + } + for name, want := range expect { + a, ok := agentByName(name) + require.True(t, ok) + assert.Equal(t, want.flag, a.contextFlag, "%s contextFlag", name) + assert.Equal(t, want.home, a.contextHomeFile, "%s contextHomeFile", name) + } + + _, ok := agentByName("not-an-agent") + assert.False(t, ok) +} + +func TestInjectAgentContext(t *testing.T) { + claude, _ := agentByName("claude") + codex, _ := agentByName("codex") + noContext := agentSpec{name: "none"} // an agent with neither mechanism + + t.Run("flag agent writes a scratch file and returns the flag", func(t *testing.T) { + home := t.TempDir() + args, err := injectAgentContext(t.Context(), home, claude) + require.NoError(t, err) + require.Len(t, args, 2) + assert.Equal(t, "--append-system-prompt-file", args[0]) + data, err := os.ReadFile(args[1]) + require.NoError(t, err) + assert.Contains(t, string(data), "databricks ssh connect") + }) + + t.Run("home-file agent writes the global instructions file, no args", func(t *testing.T) { + home := t.TempDir() + args, err := injectAgentContext(t.Context(), home, codex) + require.NoError(t, err) + assert.Nil(t, args) + data, err := os.ReadFile(filepath.Join(home, ".codex", "AGENTS.md")) + require.NoError(t, err) + assert.Contains(t, string(data), "databricks ssh connect") + }) + + t.Run("home-file agent does not clobber an existing file", func(t *testing.T) { + home := t.TempDir() + target := filepath.Join(home, ".codex", "AGENTS.md") + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755)) + require.NoError(t, os.WriteFile(target, []byte("user's own instructions"), 0o644)) + args, err := injectAgentContext(t.Context(), home, codex) + require.NoError(t, err) + assert.Nil(t, args) + data, err := os.ReadFile(target) + require.NoError(t, err) + assert.Equal(t, "user's own instructions", string(data)) + }) + + t.Run("agent without a mechanism writes nothing", func(t *testing.T) { + home := t.TempDir() + args, err := injectAgentContext(t.Context(), home, noContext) + require.NoError(t, err) + assert.Nil(t, args) + entries, err := os.ReadDir(home) + require.NoError(t, err) + assert.Empty(t, entries) + }) +} + +func TestAgentSystemContext(t *testing.T) { + assert.Contains(t, agentSystemContext("/Workspace/Users/me@example.com"), "working directory is /Workspace/Users/me@example.com") + // Falls back to a generic phrase when the workspace home is unknown. + assert.Contains(t, agentSystemContext(""), "working directory is the user's Databricks workspace home directory") +} + +func newProbeClient(t *testing.T, host string) *databricks.WorkspaceClient { + t.Helper() + w, err := databricks.NewWorkspaceClient((*databricks.Config)(&config.Config{Host: host, Token: "test-token"})) + require.NoError(t, err) + return w +} + +// newGatewayServer routes the model-services (v3) and legacy endpoints (v2) probe +// requests to per-API handlers, 404ing anything else (e.g. the SDK's config +// discovery). A nil handler means that API is not served (404). +func newGatewayServer(t *testing.T, modelServices, endpoints http.HandlerFunc) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/api/2.1/unity-catalog/model-services") && modelServices != nil: + assert.Equal(t, "Bearer test-token", r.Header.Get("Authorization")) + modelServices(w, r) + case strings.HasPrefix(r.URL.Path, "/api/ai-gateway/v2/endpoints") && endpoints != nil: + assert.Equal(t, "Bearer test-token", r.Header.Get("Authorization")) + endpoints(w, r) + default: + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("{}")) + } + })) +} + +// jsonHandler replies with status and body for every request it receives. +func jsonHandler(status int, body string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + } +} + +func TestProbeAIGateway(t *testing.T) { + const modelServicesBody = `{"model_services":[{"name":"model-services/system.ai.gpt-5"}]}` + const endpointsBody = `{"endpoints":[{"name":"databricks-gpt-5"}]}` + const emptyBody = `{}` + + t.Run("model service available: connected, legacy not probed", func(t *testing.T) { + var legacyCalls int + srv := newGatewayServer(t, + jsonHandler(http.StatusOK, modelServicesBody), + func(w http.ResponseWriter, r *http.Request) { + legacyCalls++ + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(emptyBody)) + }, + ) + defer srv.Close() + require.NoError(t, probeAIGateway(t.Context(), newProbeClient(t, srv.URL))) + assert.Zero(t, legacyCalls, "legacy endpoints must not be probed once a model service is found") + }) + + t.Run("model service found across pages", func(t *testing.T) { + srv := newGatewayServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if r.URL.Query().Get("page_token") == "" { + _, _ = w.Write([]byte(`{"next_page_token":"cursor-1"}`)) + return + } + _, _ = w.Write([]byte(modelServicesBody)) + }, nil) + defer srv.Close() + require.NoError(t, probeAIGateway(t.Context(), newProbeClient(t, srv.URL))) + }) + + t.Run("empty model service but legacy reachable: proceeds", func(t *testing.T) { + // v3 200-empty and v2 200-empty: both reachable, no resources → proceed (warn). + srv := newGatewayServer(t, jsonHandler(http.StatusOK, emptyBody), jsonHandler(http.StatusOK, emptyBody)) + defer srv.Close() + assert.NoError(t, probeAIGateway(t.Context(), newProbeClient(t, srv.URL))) + }) + + t.Run("legacy-only workspace (v3 404, v2 reachable): proceeds", func(t *testing.T) { + srv := newGatewayServer(t, + jsonHandler(http.StatusNotFound, `{"message":"not found"}`), + jsonHandler(http.StatusOK, endpointsBody), + ) + defer srv.Close() + assert.NoError(t, probeAIGateway(t.Context(), newProbeClient(t, srv.URL))) + }) + + t.Run("auth failure (401) fails fast without probing legacy", func(t *testing.T) { + var legacyCalls int + srv := newGatewayServer(t, + jsonHandler(http.StatusUnauthorized, `{"message":"unauthorized"}`), + func(w http.ResponseWriter, r *http.Request) { + legacyCalls++ + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(endpointsBody)) + }, + ) + defer srv.Close() + err := probeAIGateway(t.Context(), newProbeClient(t, srv.URL)) + require.Error(t, err) + assert.Contains(t, err.Error(), "rejected the access token") + assert.Zero(t, legacyCalls, "a definitive auth failure must not fall back to the legacy probe") + }) + + t.Run("missing OAuth scope on both paths routes to re-auth", func(t *testing.T) { + scope := jsonHandler(http.StatusForbidden, "Provided OAuth token does not have required scopes: unity-catalog") + srv := newGatewayServer(t, scope, scope) + defer srv.Close() + err := probeAIGateway(t.Context(), newProbeClient(t, srv.URL)) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing an OAuth scope") + assert.Contains(t, err.Error(), "databricks auth login") + }) + + t.Run("plain 403 on both paths is a permission failure", func(t *testing.T) { + forbidden := jsonHandler(http.StatusForbidden, `{"message":"forbidden"}`) + srv := newGatewayServer(t, forbidden, forbidden) + defer srv.Close() + err := probeAIGateway(t.Context(), newProbeClient(t, srv.URL)) + require.Error(t, err) + assert.Contains(t, err.Error(), "model service access could not be verified") + }) + + t.Run("neither gateway available: not enabled", func(t *testing.T) { + notFound := jsonHandler(http.StatusNotFound, `{"message":"not found"}`) + srv := newGatewayServer(t, notFound, notFound) + defer srv.Close() + err := probeAIGateway(t.Context(), newProbeClient(t, srv.URL)) + require.Error(t, err) + assert.Contains(t, err.Error(), "not enabled on this workspace") + }) + + t.Run("transient failure (5xx both) is retryable, not disabled", func(t *testing.T) { + unavailable := jsonHandler(http.StatusServiceUnavailable, `{"message":"try later"}`) + srv := newGatewayServer(t, unavailable, unavailable) + defer srv.Close() + err := probeAIGateway(t.Context(), newProbeClient(t, srv.URL)) + require.Error(t, err) + assert.Contains(t, err.Error(), "transient error") + assert.NotContains(t, err.Error(), "not enabled") + }) +} + +func TestProbeModelServicesInconclusive(t *testing.T) { + // A later page failing (after the first page succeeded) is reachable-but- + // inconclusive, never a confirmed "empty". + srv := newGatewayServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page_token") == "" { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"next_page_token":"cursor-1"}`)) + return + } + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("boom")) + }, nil) + defer srv.Close() + + probe := probeModelServices(t.Context(), newProbeClient(t, srv.URL), strings.TrimRight(srv.URL, "/")) + assert.True(t, probe.reachable) + assert.False(t, probe.conclusive) + assert.False(t, probe.resourceAvailable) +} diff --git a/experimental/ssh/internal/client/client.go b/experimental/ssh/internal/client/client.go index 97cc5334126..8d848f76295 100644 --- a/experimental/ssh/internal/client/client.go +++ b/experimental/ssh/internal/client/client.go @@ -454,8 +454,9 @@ func Run(ctx context.Context, client *databricks.WorkspaceClient, opts ClientOpt } else if opts.IDE != "" { return runIDE(ctx, client, userName, keyPath, serverPort, clusterID, opts) } else { + // Default shell session: install the agent launchers, then open bash (see buildRemoteShellArgs). log.Infof(ctx, "Additional SSH arguments: %v", opts.AdditionalArgs) - return spawnSSHClient(ctx, client, userName, keyPath, serverPort, clusterID, opts) + return spawnSSHClient(ctx, client, userName, keyPath, serverPort, clusterID, version, opts) } } @@ -782,6 +783,38 @@ func shellSingleQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } +// remoteShimDir is where the interactive session installs the per-agent wrappers; +// must expand to the same path the shim computes from shimDir. +const remoteShimDir = "$HOME/.agent-shim/bin" + +// agentWrapperScript returns one agent's on-PATH launcher: it resolves the uploaded +// databricks CLI for the driver's arch, then delegates to `ssh agent-shim `. +// wsHome anchors that binary path and is forwarded (via env) for the agent context. +func agentWrapperScript(wsHome, version, agent string) string { + versionedDir := wsHome + "/.databricks/ssh-tunnel/" + version + // Mirror getReleaseName (minus .zip); ${_arch} is expanded on the remote. + subdir := strings.TrimSuffix(getReleaseName("${_arch}", version), ".zip") + // Single-quote the workspace/version-derived path so a username or version with + // shell metacharacters can't break (or inject into) the exec line, while leaving + // ${_arch} unquoted so the remote shell still expands it (getReleaseName always + // embeds it, so Cut always splits here). + before, after, _ := strings.Cut(subdir, "${_arch}") + binary := shellSingleQuote(versionedDir+"/"+before) + "${_arch}" + shellSingleQuote(after+"/databricks") + + // buildRemoteShellArgs only installs launchers when wsHome is set, so it is + // always non-empty here. + exports := "export " + workspaceHomeEnv + "=" + shellSingleQuote(wsHome) + "\n" + + return fmt.Sprintf(`#!/usr/bin/env bash +# "%[3]s" launcher from "databricks ssh connect"; delegates to "ssh agent-shim %[3]s". +case "$(uname -m)" in + x86_64|amd64) _arch=amd64 ;; + aarch64|arm64) _arch=arm64 ;; + *) echo "%[3]s: unsupported architecture $(uname -m)" >&2; exit 1 ;; +esac +%[1]sexec %[2]s ssh agent-shim %[3]s "$@"`, exports, binary, agent) +} + // buildRemoteShellArgs returns the ssh arguments that follow the hostname. // // For the interactive case (no remote command given), it forces PTY allocation @@ -793,9 +826,8 @@ func shellSingleQuote(s string) string { // would resolve to the system interpreter instead of $DATABRICKS_VIRTUAL_ENV. Using // -i avoids that reset; the server's ~/.bashrc snippet (see seedEnvActivation) then // re-prepends the environment bin after /etc/bash.bashrc runs, so bare `python`/`pip` -// resolve to the environment interpreter. When wsHome is set, the shell first changes -// into the user's workspace home folder; if that directory is missing the cd is -// ignored and the shell still launches from $HOME. +// resolve to the environment interpreter. It first installs a per-agent launcher +// (see agentWrapperScript) on PATH, then cds into wsHome when set (else $HOME). // // For the non-interactive case (e.g. `databricks ssh connect ... -- ls -la`), // the user's command is returned verbatim so behavior is unchanged. @@ -803,15 +835,38 @@ func shellSingleQuote(s string) string { // Note: this returns the remote command only. PTY allocation (-t) is added to // the ssh options *before* the destination by the caller; -t placed after the // host would be parsed as part of the remote command, not as ssh's flag. -func buildRemoteShellArgs(opts ClientOptions, wsHome string) []string { +func buildRemoteShellArgs(opts ClientOptions, wsHome, version string) []string { if len(opts.AdditionalArgs) > 0 { return opts.AdditionalArgs } - cmd := `command -v bash >/dev/null 2>&1 && exec bash -i || exec "${SHELL:-/bin/sh}" -i` + shell := `command -v bash >/dev/null 2>&1 && exec bash -i || exec "${SHELL:-/bin/sh}" -i` + // For any interactive session, land in the user's workspace home when known; + // if that directory is missing the cd is ignored and the shell still launches + // from $HOME. This is independent of the launcher install below, so dedicated + // clusters keep the workspace cwd they had before agent launchers existed. if wsHome != "" { - cmd = "cd " + shellSingleQuote(wsHome) + " 2>/dev/null; " + cmd - } - return []string{cmd} + shell = "cd " + shellSingleQuote(wsHome) + " 2>/dev/null; " + shell + } + // The agent launchers are serverless-only for now: dedicated clusters have no + // story for persisting the shims or respecting existing installs. They also + // need the workspace path to locate the uploaded CLI. Otherwise just open the shell. + if !opts.IsServerlessMode() || wsHome == "" { + return []string{shell} + } + // Write each agent's wrapper (quoted heredoc keeps its $VARS literal), then open the shell. + var cmd strings.Builder + fmt.Fprintf(&cmd, "mkdir -p \"%s\"\n", remoteShimDir) + for _, agent := range SupportedAgentNames() { + fmt.Fprintf(&cmd, `cat > "%[1]s/%[2]s" <<'AGENT_SHIM_WRAPPER_EOF' +%[3]s +AGENT_SHIM_WRAPPER_EOF +chmod +x "%[1]s/%[2]s" +`, remoteShimDir, agent, agentWrapperScript(wsHome, version, agent)) + } + // Append the shim dir (don't prepend) so the wrappers never shadow a real + // binary of the same name earlier on PATH. + fmt.Fprintf(&cmd, "export PATH=\"$PATH:%s\"\n", remoteShimDir) + return []string{cmd.String() + shell} } // buildSSHArgs assembles the argument list for the ssh client. Options come @@ -819,7 +874,7 @@ func buildRemoteShellArgs(opts ClientOptions, wsHome string) []string { // allocation (-t) for the interactive case is added before the host: ssh stops // parsing options at the destination, so a -t placed after the host would be // treated as part of the remote command rather than as ssh's force-PTY flag. -func buildSSHArgs(userName, privateKeyPath, proxyCommand, hostName, wsHome string, opts ClientOptions) []string { +func buildSSHArgs(userName, privateKeyPath, proxyCommand, hostName, wsHome, version string, opts ClientOptions) []string { sshArgs := []string{ "-l", userName, "-i", privateKeyPath, @@ -836,11 +891,11 @@ func buildSSHArgs(userName, privateKeyPath, proxyCommand, hostName, wsHome strin sshArgs = append(sshArgs, "-t") } sshArgs = append(sshArgs, hostName) - sshArgs = append(sshArgs, buildRemoteShellArgs(opts, wsHome)...) + sshArgs = append(sshArgs, buildRemoteShellArgs(opts, wsHome, version)...) return sshArgs } -func spawnSSHClient(ctx context.Context, client *databricks.WorkspaceClient, userName, privateKeyPath string, serverPort int, clusterID string, opts ClientOptions) error { +func spawnSSHClient(ctx context.Context, client *databricks.WorkspaceClient, userName, privateKeyPath string, serverPort int, clusterID, version string, opts ClientOptions) error { // Create a copy with metadata for the ProxyCommand optsWithMetadata := opts optsWithMetadata.ServerMetadata = FormatMetadata(userName, serverPort, clusterID) @@ -852,9 +907,8 @@ func spawnSSHClient(ctx context.Context, client *databricks.WorkspaceClient, use hostName := opts.SessionIdentifier() - // For an interactive session (no remote command supplied), land the shell in - // the user's workspace home folder (/Workspace/Users/) instead of the - // OS home. Only needed for an interactive session; skip the lookup otherwise. + // For an interactive session, land the shell in the user's workspace home + // (/Workspace/Users/) instead of the OS home. var wsHome string if len(opts.AdditionalArgs) == 0 { if currentUser, err := client.CurrentUser.Me(ctx, iam.MeRequest{}); err != nil { @@ -864,7 +918,7 @@ func spawnSSHClient(ctx context.Context, client *databricks.WorkspaceClient, use } } - sshArgs := buildSSHArgs(userName, privateKeyPath, proxyCommand, hostName, wsHome, opts) + sshArgs := buildSSHArgs(userName, privateKeyPath, proxyCommand, hostName, wsHome, version, opts) log.Debugf(ctx, "Launching SSH client: ssh %s", strings.Join(sshArgs, " ")) sshCmd := exec.CommandContext(ctx, "ssh", sshArgs...) diff --git a/experimental/ssh/internal/client/client_internal_test.go b/experimental/ssh/internal/client/client_internal_test.go index ba15384bb46..89cf081dc7d 100644 --- a/experimental/ssh/internal/client/client_internal_test.go +++ b/experimental/ssh/internal/client/client_internal_test.go @@ -357,28 +357,51 @@ func TestHostKeyChangedHint(t *testing.T) { func TestBuildRemoteShellArgs(t *testing.T) { const bashCmd = `command -v bash >/dev/null 2>&1 && exec bash -i || exec "${SHELL:-/bin/sh}" -i` + const version = "1.2.3" - t.Run("interactive returns non-login bash command", func(t *testing.T) { - args := buildRemoteShellArgs(ClientOptions{}, "") + // A serverless session has a connection name and no cluster ID. + serverless := ClientOptions{ConnectionName: "my-conn"} + + t.Run("serverless installs a launcher per agent, cds home, then opens bash", func(t *testing.T) { + const wsHome = "/Workspace/Users/me@example.com" + args := buildRemoteShellArgs(serverless, wsHome, version) require.Len(t, args, 1) - assert.Equal(t, bashCmd, args[0]) + // A wrapper is written for every supported agent before the shell opens. + for _, agent := range SupportedAgentNames() { + assert.Contains(t, args[0], `cat > "$HOME/.agent-shim/bin/`+agent+`"`) + assert.Contains(t, args[0], agentWrapperScript(wsHome, version, agent)) + } + assert.Contains(t, args[0], `export PATH="$PATH:$HOME/.agent-shim/bin"`) + // The cd into the workspace home precedes the shell launch. + assert.True(t, strings.HasSuffix(args[0], `cd '`+wsHome+`' 2>/dev/null; `+bashCmd)) + }) + + t.Run("dedicated cluster skips the launchers but keeps the workspace cwd", func(t *testing.T) { + const wsHome = "/Workspace/Users/me@example.com" + args := buildRemoteShellArgs(ClientOptions{ClusterID: "abc-123"}, wsHome, version) + require.Len(t, args, 1) + // No launchers on a dedicated cluster... + assert.NotContains(t, args[0], "agent-shim/bin") + // ...but the cd into the workspace home is preserved (regression guard). + assert.Equal(t, `cd '`+wsHome+`' 2>/dev/null; `+bashCmd, args[0]) }) - t.Run("interactive cds into workspace home when set", func(t *testing.T) { - args := buildRemoteShellArgs(ClientOptions{}, "/Workspace/Users/me@example.com") + t.Run("serverless without a workspace home skips the launchers", func(t *testing.T) { + // Without wsHome the wrappers can't locate the CLI, so only the shell opens. + args := buildRemoteShellArgs(serverless, "", version) require.Len(t, args, 1) - assert.Equal(t, `cd '/Workspace/Users/me@example.com' 2>/dev/null; `+bashCmd, args[0]) + assert.Equal(t, bashCmd, args[0]) }) t.Run("non-interactive passes additional args verbatim", func(t *testing.T) { additional := []string{"ls", "-la"} - args := buildRemoteShellArgs(ClientOptions{AdditionalArgs: additional}, "/Workspace/Users/me@example.com") + args := buildRemoteShellArgs(ClientOptions{ConnectionName: "my-conn", AdditionalArgs: additional}, "/Workspace/Users/me@example.com", version) assert.Equal(t, additional, args) }) } func TestBuildSSHArgsSetsServerAliveInterval(t *testing.T) { - args := buildSSHArgs("user", "/key", "proxy command", "myhost", "", ClientOptions{}) + args := buildSSHArgs("user", "/key", "proxy command", "myhost", "", "1.2.3", ClientOptions{}) // ssh stops parsing options at the destination, so an option placed after the host would be // treated as part of the remote command rather than as an ssh option. @@ -388,6 +411,27 @@ func TestBuildSSHArgsSetsServerAliveInterval(t *testing.T) { assert.Less(t, optIdx, slices.Index(args, "myhost"), "the option must precede the destination host") } +func TestAgentWrapperScript(t *testing.T) { + const wsHome = "/Workspace/Users/me@example.com" + + t.Run("resolves the uploaded binary and delegates to agent-shim ", func(t *testing.T) { + wrapper := agentWrapperScript(wsHome, "1.2.3", "codex") + assert.True(t, strings.HasPrefix(wrapper, "#!/usr/bin/env bash")) + // Architecture is resolved on the remote via uname -m. + assert.Contains(t, wrapper, "case \"$(uname -m)\" in") + // Binary path uses the version dir + arch subdir; the workspace/version-derived + // portion is single-quoted while ${_arch} stays expandable on the remote. + assert.Contains(t, wrapper, `exec '`+wsHome+`/.databricks/ssh-tunnel/1.2.3/databricks_cli_1.2.3_linux_'${_arch}'/databricks' ssh agent-shim codex "$@"`) + // The workspace home is forwarded so the shim can name the working directory. + assert.Contains(t, wrapper, "export "+workspaceHomeEnv+"='"+wsHome+"'") + }) + + t.Run("dev builds use the version-less release subdir", func(t *testing.T) { + wrapper := agentWrapperScript(wsHome, "1.2.3-dev+abc", "claude") + assert.Contains(t, wrapper, `/ssh-tunnel/1.2.3-dev+abc/databricks_cli_linux_'${_arch}'/databricks`) + }) +} + func TestBuildSSHArgsPTYPlacement(t *testing.T) { indexOf := func(args []string, want string) int { for i, a := range args { @@ -399,7 +443,7 @@ func TestBuildSSHArgsPTYPlacement(t *testing.T) { } t.Run("interactive forces a PTY before the destination", func(t *testing.T) { - args := buildSSHArgs("user", "/key", "proxy command", "myhost", "/Workspace/Users/me@example.com", ClientOptions{}) + args := buildSSHArgs("user", "/key", "proxy command", "myhost", "/Workspace/Users/me@example.com", "1.2.3", ClientOptions{}) ptyIdx := indexOf(args, "-t") hostIdx := indexOf(args, "myhost") require.NotEqual(t, -1, ptyIdx, "-t must be present for interactive sessions") @@ -411,7 +455,7 @@ func TestBuildSSHArgsPTYPlacement(t *testing.T) { }) t.Run("non-interactive does not force a PTY", func(t *testing.T) { - args := buildSSHArgs("user", "/key", "proxy command", "myhost", "", ClientOptions{AdditionalArgs: []string{"ls", "-la"}}) + args := buildSSHArgs("user", "/key", "proxy command", "myhost", "", "1.2.3", ClientOptions{AdditionalArgs: []string{"ls", "-la"}}) assert.Equal(t, -1, indexOf(args, "-t"), "no PTY for non-interactive passthrough") hostIdx := indexOf(args, "myhost") require.NotEqual(t, -1, hostIdx) diff --git a/experimental/ssh/internal/client/exec_unix.go b/experimental/ssh/internal/client/exec_unix.go new file mode 100644 index 00000000000..8c024b980bf --- /dev/null +++ b/experimental/ssh/internal/client/exec_unix.go @@ -0,0 +1,11 @@ +//go:build !windows + +package client + +import "syscall" + +// execProcess replaces the current process (execve(2)) so the ssh PTY connects +// straight to the agent and its exit code propagates. +func execProcess(argv0 string, argv, env []string) error { + return syscall.Exec(argv0, argv, env) +} diff --git a/experimental/ssh/internal/client/exec_windows.go b/experimental/ssh/internal/client/exec_windows.go new file mode 100644 index 00000000000..3cfc5b2e1ed --- /dev/null +++ b/experimental/ssh/internal/client/exec_windows.go @@ -0,0 +1,10 @@ +//go:build windows + +package client + +import "errors" + +// execProcess is unsupported on Windows (no syscall.Exec); the shim only runs on the Linux driver. +func execProcess(argv0 string, argv, env []string) error { + return errors.New("process exec is not supported on Windows") +} diff --git a/experimental/ssh/internal/client/progress.go b/experimental/ssh/internal/client/progress.go new file mode 100644 index 00000000000..d850ba3714b --- /dev/null +++ b/experimental/ssh/internal/client/progress.go @@ -0,0 +1,68 @@ +package client + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "strings" + + "github.com/charmbracelet/lipgloss" + "github.com/databricks/cli/libs/cmdio" +) + +// progressUI renders the bootstrap steps: each step spins while it runs and +// leaves a checkmark line when it finishes. Subprocess output is captured per +// step and printed only when that step fails. +type progressUI struct { + w io.Writer + check string // styled "✓" prefix for a finished step + cross string // styled "✗" prefix for a failed step +} + +// newProgressUI builds a progress renderer writing its checkmark lines to stderr. +func newProgressUI(ctx context.Context) *progressUI { + // Mint the checkmark styles from a renderer targeting stderr so color handling + // stays centralized in cmdio.NewRenderer, matching cmdio's own spinner. + r, _ := cmdio.NewRenderer(ctx, os.Stderr) + return &progressUI{ + w: os.Stderr, + check: r.NewStyle().Foreground(lipgloss.Color("10")).Render("✓"), // green + cross: r.NewStyle().Foreground(lipgloss.Color("9")).Render("✗"), // red + } +} + +// runStep shows a cmdio spinner labelled desc while fn runs, giving fn a writer +// that captures the step's subprocess output. On success it leaves a checkmark +// line; on failure it prints the captured output (stdout+stderr, in order) before +// returning fn's error. The shared spinner shows elapsed time and degrades to no +// output in a non-interactive terminal, so the checkmark line is what the reader +// sees either way. +func (ui *progressUI) runStep(ctx context.Context, desc string, fn func(out io.Writer) error) error { + sp := cmdio.NewSpinner(ctx, cmdio.WithElapsedTime()) + // Close is idempotent; defer it so a panic in fn can't leave the spinner (and + // its tea-program slot) running, while the explicit Close below still controls + // output ordering on the normal path. + defer sp.Close() + sp.Update(desc) + + var buf bytes.Buffer + err := fn(&buf) + // Stop the spinner (clearing its line) before printing the step's outcome. + sp.Close() + + if err != nil { + fmt.Fprintln(ui.w, ui.cross+" "+desc) + if out := buf.String(); out != "" { + fmt.Fprint(ui.w, out) + if !strings.HasSuffix(out, "\n") { + fmt.Fprintln(ui.w) + } + } + return err + } + + fmt.Fprintln(ui.w, ui.check+" "+desc) + return nil +} diff --git a/experimental/ssh/internal/client/progress_test.go b/experimental/ssh/internal/client/progress_test.go new file mode 100644 index 00000000000..8da3ad3c0d3 --- /dev/null +++ b/experimental/ssh/internal/client/progress_test.go @@ -0,0 +1,53 @@ +package client + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/databricks/cli/libs/cmdio" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestProgressUI returns a progressUI writing to w with sentinel check/cross +// markers so the emitted outcome line is assertable. runStep is driven with a +// non-interactive cmdio context (the mode tests run in), where the spinner +// degrades to no output, exercising the capture-and-dump logic directly. +func newTestProgressUI(w io.Writer) *progressUI { + return &progressUI{w: w, check: "OK", cross: "FAIL"} +} + +func TestRunStepHidesOutputOnSuccess(t *testing.T) { + var w bytes.Buffer + ui := newTestProgressUI(&w) + + err := ui.runStep(cmdio.MockDiscard(t.Context()), "Installing dependencies", func(out io.Writer) error { + _, _ = io.WriteString(out, "verbose installer chatter\n") + return nil + }) + require.NoError(t, err) + + // The step's subprocess output must not surface on success, but the checkmark + // line for the step is still emitted. + assert.NotContains(t, w.String(), "verbose installer chatter") + assert.Contains(t, w.String(), "OK Installing dependencies") +} + +func TestRunStepShowsOutputOnFailure(t *testing.T) { + var w bytes.Buffer + ui := newTestProgressUI(&w) + + sentinel := errors.New("install failed") + err := ui.runStep(cmdio.MockDiscard(t.Context()), "Installing ucode", func(out io.Writer) error { + _, _ = io.WriteString(out, "line to stdout\nline to stderr") + return sentinel + }) + require.ErrorIs(t, err, sentinel) + + // On failure the cross line and the full captured output are printed, with a + // trailing newline added. + assert.Contains(t, w.String(), "FAIL Installing ucode") + assert.Contains(t, w.String(), "line to stdout\nline to stderr\n") +} diff --git a/internal/bugbash/exec.sh b/internal/bugbash/exec.sh index 07833b42614..7275b78328b 100755 --- a/internal/bugbash/exec.sh +++ b/internal/bugbash/exec.sh @@ -76,28 +76,33 @@ if [ -z "$last_successful_run_id" ]; then exit 1 fi -# Create a temporary directory to download and extract the artifact. -dir=$(mktemp -d) +# Download the release archives into ./bugbash relative to where the script is +# run. This is a stable location (not a temp dir) so `databricks ssh connect` can +# point its --releases-dir at the downloaded CLI archives. Use an absolute path +# so the reference stays valid if the user cd's elsewhere in the bugbash shell. +releases_dir="$PWD/bugbash" +rm -rf "$releases_dir" +mkdir -p "$releases_dir" # Download the artifact. echo "Downloading the snapshot build..." -gh run --repo databricks/cli download "$last_successful_run_id" --name cli --dir "$dir/.download" +gh run --repo databricks/cli download "$last_successful_run_id" --name cli --dir "$releases_dir" -# Extract the archive for this platform. +# Extract the archive for this platform into a temporary directory. archive=$(cli_snapshot_archive) -if [ ! -f "$dir/.download/$archive" ]; then +if [ ! -f "$releases_dir/$archive" ]; then echo "Archive not found: $archive" echo "Available archives:" - ls "$dir/.download/" + ls "$releases_dir/" exit 1 fi -mkdir -p "$dir/.bin" -tar -xzf "$dir/.download/$archive" -C "$dir/.bin" +bin_dir=$(mktemp -d) +tar -xzf "$releases_dir/$archive" -C "$bin_dir" # Make CLI available on $PATH. -chmod +x "$dir/.bin/databricks" -export PATH="$dir/.bin:$PATH" +chmod +x "$bin_dir/databricks" +export PATH="$bin_dir:$PATH" # Set the prompt to indicate the bugbash environment and exec. export PS1="(bugbash $BRANCH) \[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ " @@ -126,6 +131,13 @@ echo " source <(databricks completion bash)" echo "" echo "==================================================================" echo "" +echo "To test 'databricks ssh connect', point --releases-dir at the downloaded" +echo "archives:" +echo "" +echo " databricks ssh connect ... --releases-dir $releases_dir" +echo "" +echo "==================================================================" +echo "" # Exec into a new shell. # Note: don't use zsh because on macOS it _always_ overwrites PS1.