feat(feedback): add feedback send/dismiss, the survey constants, and the cadence store

.changeset/qualitative-survey-feedback.md

@@ -3,3 +3,5 @@
 ---
 
 `.taskless/.gitignore` now ignores `/.tmp-*`, the scratch request files the agent recipes write (`.tmp-rule-request.json`, `.tmp-improve-request.json`), so a file an agent forgot to clean up is a stray rather than a commit. This is scaffold migration 7; the scaffold's own `version` field carries the compatibility signal, and a project at 6 gains one ignore line the next time it is bootstrapped.
+
+A `feedback` subcommand joins the CLI, reached only through the survey invite a served recipe carries and so absent from the `taskless agent` index: `feedback send --from <file>` validates a human-keyed payload (`verbatim`, `goal`, `completed` as `Yes`/`No`/`Unknown`, optional `workedWell` and `needsImprovement`), maps it to the PostHog survey's question ids, and captures `survey sent`; `feedback dismiss` captures `survey dismissed`. Both hold the next invite off for 20 days and, under the telemetry opt-out, say nothing was sent and exit 0.

packages/cli/src/commands/agent.ts

@@ -45,8 +45,8 @@ async function resolveDescription(
 /**
  * Commands this index deliberately does not advertise.
  *
- * A declared list rather than a condition, because the two entries are absent
- * for unrelated reasons and a bare `name === "agent"` recorded neither.
+ * A declared list rather than a condition, because the entries are absent for
+ * unrelated reasons and a bare `name === "agent"` recorded none of them.
  *
  * - `agent` is the index itself, so listing it would be circular.
  * - `demo` writes a fixed example rule for someone learning what a rule is. It
@@ -59,10 +59,16 @@ async function resolveDescription(
  *   verb gave it discoverability it does not want yet, and this is the cost of
  *   that choice, paid here.
  *
+ * - `feedback` is reached only through the survey invite a served recipe
+ *   carries. Listed, it would invite an agent to run it unprompted, and a
+ *   `survey sent` with no invite behind it is noise in the funnel. When a
+ *   general feedback channel exists this surface folds into it, and that is
+ *   the point to reconsider listing.
+ *
  * Absence from this list is what puts a command in the index, so adding one is
  * a decision someone made rather than a step they forgot.
  */
-const UNLISTED_COMMANDS = new Set(["agent", "demo"]);
+const UNLISTED_COMMANDS = new Set(["agent", "demo", "feedback"]);
 
 export function createAgentCommand(subCommands: SubCommandsDef) {
   return defineCommand({

packages/cli/src/commands/feedback.ts

@@ -0,0 +1,183 @@
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+import { defineCommand } from "citty";
+import { ZodError } from "zod";
+
+import { inputSchema, type FeedbackInput } from "../schemas/feedback";
+import { writeNextAsk } from "../survey/cadence";
+import {
+  ANSWERED_INTERVAL_MS,
+  SURVEY_ID,
+  SURVEY_QUESTIONS,
+} from "../survey/constants";
+import { getTelemetry, isTelemetryEnabled } from "../telemetry";
+import { type CLIErrorCode, writeJsonError } from "../types/errors";
+import { CLIError } from "../util/cli-error";
+
+/**
+ * What both verbs say under the telemetry opt-out. An agent should never
+ * reach them in that state, because the invite is not served in it, so this
+ * is a defensive line rather than a path the recipe describes. Exit zero: the
+ * user asked for nothing to be sent, and nothing was.
+ */
+const NOTHING_SENT =
+  "Telemetry is disabled, so no feedback was sent. Nothing else to do.";
+
+/**
+ * The `survey sent` properties for a validated payload.
+ *
+ * Exactly PostHog's contract: `$survey_id` and one `$survey_response_<id>`
+ * per answered question. An optional question left blank is absent rather
+ * than sent as an empty string, so the responses view shows a gap and not an
+ * empty answer.
+ */
+export function buildSurveyResponse(
+  input: FeedbackInput
+): Record<string, string> {
+  const properties: Record<string, string> = { $survey_id: SURVEY_ID };
+  for (const { key, id } of SURVEY_QUESTIONS) {
+    const answer = input[key];
+    if (answer !== undefined) properties[`$survey_response_${id}`] = answer;
+  }
+  return properties;
+}
+
+const dismissCommand = defineCommand({
+  meta: {
+    name: "dismiss",
+    description: "Record that the user declined the feedback survey",
+  },
+  args: {
+    dir: {
+      type: "string",
+      alias: "d",
+      description: "Working directory",
+    },
+  },
+  async run({ args }) {
+    const cwd = resolve(args.dir ?? process.cwd());
+    if (!isTelemetryEnabled()) {
+      console.log(NOTHING_SENT);
+      return;
+    }
+    const telemetry = await getTelemetry(cwd);
+    telemetry.capture("survey dismissed", { $survey_id: SURVEY_ID });
+    await writeNextAsk(SURVEY_ID, Date.now() + ANSWERED_INTERVAL_MS);
+    console.log("Thanks. Taskless will not ask again for a while.");
+  },
+});
+
+const sendCommand = defineCommand({
+  meta: {
+    name: "send",
+    description:
+      "Send a completed feedback survey (use --from to specify the input file)",
+  },
+  args: {
+    dir: {
+      type: "string",
+      alias: "d",
+      description: "Working directory",
+    },
+    from: {
+      type: "string",
+      description:
+        "Path to a JSON file containing the feedback payload (required). Example: --from .taskless/.tmp-feedback.json",
+    },
+    json: {
+      type: "boolean",
+      description:
+        "On error, write the standardized { ok:false, code, message } envelope to stdout instead of human text on stderr",
+      default: false,
+    },
+  },
+  async run({ args }) {
+    const cwd = resolve(args.dir ?? process.cwd());
+
+    /** Emit an error and exit, respecting --json mode */
+    function fail(message: string, code: CLIErrorCode): never {
+      if (args.json) {
+        writeJsonError(code, message);
+      } else {
+        console.error(`Error: ${message}`);
+      }
+      process.exitCode = 1;
+      throw new CLIError(message, code, { reported: true });
+    }
+
+    // Validate before checking the opt-out: a malformed payload is wrong
+    // whether or not anything would be sent, and the agent should hear so.
+    if (!args.from) {
+      fail(
+        "--from is required. Provide a path to a JSON file.\n  Example: taskless feedback send --from .taskless/.tmp-feedback.json",
+        "INVALID_INPUT"
+      );
+    }
+
+    const filePath = resolve(cwd, args.from);
+    let fileContent: string;
+    try {
+      fileContent = await readFile(filePath, "utf8");
+    } catch {
+      fail(`Could not read file "${args.from}".`, "INVALID_INPUT");
+    }
+
+    let rawJson: unknown;
+    try {
+      rawJson = JSON.parse(fileContent) as unknown;
+    } catch {
+      fail(`"${args.from}" is not valid JSON.`, "INVALID_INPUT");
+    }
+
+    let input: FeedbackInput;
+    try {
+      input = inputSchema.parse(rawJson);
+    } catch (error) {
+      if (error instanceof ZodError) {
+        fail(
+          `Invalid input: ${error.issues
+            .map(
+              (issue) =>
+                `${issue.path.join(".") || "payload"}: ${issue.message}`
+            )
+            .join(", ")}`,
+          "INVALID_INPUT"
+        );
+      }
+      fail(
+        error instanceof Error ? error.message : String(error),
+        "INVALID_INPUT"
+      );
+    }
+
+    if (!isTelemetryEnabled()) {
+      console.log(NOTHING_SENT);
+      return;
+    }
+
+    const telemetry = await getTelemetry(cwd);
+    telemetry.capture("survey sent", buildSurveyResponse(input));
+    await writeNextAsk(SURVEY_ID, Date.now() + ANSWERED_INTERVAL_MS);
+    // The input file is left where it is, like `rule create --from`; the
+    // recipe's clean-up step deletes it, and `/.tmp-*` is ignored regardless.
+    console.log("Feedback sent. Thank you.");
+  },
+});
+
+/**
+ * Reached only through the survey invite. Deliberately absent from the
+ * `taskless agent` index (see `UNLISTED_COMMANDS` there): listing it would
+ * invite an agent to run it unprompted. When a general feedback channel
+ * exists, this surface folds into it.
+ */
+export const feedbackCommand = defineCommand({
+  meta: {
+    name: "feedback",
+    description: "Send or dismiss the Taskless feedback survey",
+  },
+  subCommands: {
+    dismiss: dismissCommand,
+    send: sendCommand,
+  },
+});

packages/cli/src/commands/names.ts

@@ -15,6 +15,7 @@ export const SUBCOMMAND_NAMES = [
   "check",
   "demo",
   "detect",
+  "feedback",
   "info",
   "init",
   "onboard",

packages/cli/src/index.ts

@@ -5,6 +5,7 @@ import { authCommand } from "./commands/auth";
 import { checkCommand } from "./commands/check";
 import { demoCommand } from "./commands/demo";
 import { detectCommand } from "./commands/detect";
+import { feedbackCommand } from "./commands/feedback";
 import { initCommand, updateCommand } from "./commands/init";
 import { testCommand, verifyCommand } from "./commands/verify";
 import { infoCommand } from "./commands/info";
@@ -35,6 +36,7 @@ const subCommands = {
   detect: detectCommand,
   check: checkCommand,
   demo: demoCommand,
+  feedback: feedbackCommand,
   auth: authCommand,
   onboard: onboardCommand,
   rule: ruleCommand,

packages/cli/src/schemas/feedback.ts

@@ -0,0 +1,45 @@
+import { z } from "zod";
+
+import { COMPLETED_CHOICES } from "../survey/constants";
+
+/**
+ * Input schema for `taskless feedback send --from` JSON file.
+ *
+ * Human keys only. The map to the survey's question identifiers lives in
+ * `src/survey/constants.ts`, and a payload never carries a `$survey_*` key.
+ */
+export const inputSchema = z.object({
+  verbatim: z
+    .string()
+    .trim()
+    .min(1, "verbatim must be the user's own words, non-empty")
+    .describe("The user's reply, in their own words, unedited"),
+  goal: z
+    .string()
+    .trim()
+    .min(1, "goal must be a non-empty string")
+    .describe("What the user was trying to accomplish, in your words"),
+  completed: z
+    .enum(COMPLETED_CHOICES, {
+      error: `completed must be one of ${COMPLETED_CHOICES.map((choice) => `"${choice}"`).join(", ")}`,
+    })
+    .describe(
+      "Whether the user completed the task, in your opinion. Success is binary; use Unknown when you cannot tell"
+    ),
+  workedWell: z
+    .string()
+    .trim()
+    .min(1, "workedWell, when present, must be non-empty")
+    .optional()
+    .describe("Steps of the interaction with Taskless that worked well"),
+  needsImprovement: z
+    .string()
+    .trim()
+    .min(1, "needsImprovement, when present, must be non-empty")
+    .optional()
+    .describe(
+      "Steps of the interaction with Taskless that could use improvement"
+    ),
+});
+
+export type FeedbackInput = z.infer<typeof inputSchema>;

packages/cli/src/survey/cadence.ts

@@ -0,0 +1,53 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { getConfigDirectory } from "../auth/token";
+
+const NEXT_ASK_FILE = "next_ask";
+
+/**
+ * Where a survey's cadence lives: one file per survey under the same XDG
+ * config directory that holds the anonymous telemetry id.
+ *
+ * Keyed by survey rather than by CLI version. A directory per release would
+ * grow without bound, and an upgrade should not reset the cadence: a newer
+ * CLI reads the same file and may not ask right away, which is fine. A new
+ * survey is a new id and therefore a new ask. Per-release segmentation still
+ * works because `cliVersion` rides on every capture.
+ */
+export function nextAskPath(surveyId: string): string {
+  return join(getConfigDirectory(), "surveys", surveyId, NEXT_ASK_FILE);
+}
+
+/**
+ * The earliest time the next invite may be served, as epoch milliseconds, or
+ * `undefined` when the file is absent or not a number. Both read as "ask
+ * now"; the write that follows repairs a corrupt file.
+ */
+export async function readNextAsk(
+  surveyId: string
+): Promise<number | undefined> {
+  let content: string;
+  try {
+    content = await readFile(nextAskPath(surveyId), "utf8");
+  } catch {
+    return undefined;
+  }
+  const value = Number(content.trim());
+  return Number.isFinite(value) ? value : undefined;
+}
+
+/** Record when the next invite may be served. Best-effort, like the anonymous id. */
+export async function writeNextAsk(
+  surveyId: string,
+  at: number
+): Promise<void> {
+  const path = nextAskPath(surveyId);
+  try {
+    await mkdir(join(path, ".."), { recursive: true });
+    await writeFile(path, String(Math.trunc(at)), "utf8");
+  } catch {
+    // A cadence that could not be written means the invite may be served
+    // again sooner than intended, which is the cheaper failure.
+  }
+}