From da87a51fb7c0267b096c9deac3d12d2ee01e1e05 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 17 Sep 2026 18:50:50 -0700 Subject: [PATCH] docs(agent): add the feedback and feedback-invite recipes feedback tells the agent it is the survey's respondent: the user's reply is recorded verbatim, the other four answers come from the agent's own account of the session, and the survey's questions are never put to the user one by one. The payload schema is embedded through the existing INPUT_SCHEMA mechanism. feedback-invite is the fragment the gate will append to a served recipe, kept under src/agent/ so Vale and the cross-reference tests cover its prose; the agent command renders it header-less. Both are internal topics. --- .changeset/qualitative-survey-feedback.md | 2 + packages/cli/src/agent/feedback-invite.md | 26 +++++ packages/cli/src/agent/feedback.md | 116 +++++++++++++++++++++ packages/cli/src/prompts/index.ts | 8 ++ packages/cli/src/prompts/recipes.ts | 2 + packages/cli/test/feedback-recipes.test.ts | 102 ++++++++++++++++++ packages/cli/test/prompts.test.ts | 6 +- 7 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/agent/feedback-invite.md create mode 100644 packages/cli/src/agent/feedback.md create mode 100644 packages/cli/test/feedback-recipes.test.ts diff --git a/.changeset/qualitative-survey-feedback.md b/.changeset/qualitative-survey-feedback.md index d16368f9..b6bbb8d4 100644 --- a/.changeset/qualitative-survey-feedback.md +++ b/.changeset/qualitative-survey-feedback.md @@ -5,3 +5,5 @@ `.taskless/.gitignore` now ignores `/.tmp-*`, the scratch request files the agent recipes write (`.tmp-rule-request.json`, `.tmp-improve-request.json`), so a file an agent forgot to clean up is a stray rather than a commit. This is scaffold migration 7; the scaffold's own `version` field carries the compatibility signal, and a project at 6 gains one ignore line the next time it is bootstrapped. A `feedback` subcommand joins the CLI, reached only through the survey invite a served recipe carries and so absent from the `taskless agent` index: `feedback send --from ` validates a human-keyed payload (`verbatim`, `goal`, `completed` as `Yes`/`No`/`Unknown`, optional `workedWell` and `needsImprovement`), maps it to the PostHog survey's question ids, and captures `survey sent`; `feedback dismiss` captures `survey dismissed`. Both hold the next invite off for 20 days and, under the telemetry opt-out, say nothing was sent and exit 0. + +Two recipes back it: `taskless agent feedback` tells the agent it is the survey's respondent (the user's reply verbatim, the rest from its own account of the session) and embeds the payload schema; `feedback-invite` is the fragment a served recipe carries, rendered header-less. diff --git a/packages/cli/src/agent/feedback-invite.md b/packages/cli/src/agent/feedback-invite.md new file mode 100644 index 00000000..4b6a9a1f --- /dev/null +++ b/packages/cli/src/agent/feedback-invite.md @@ -0,0 +1,26 @@ +# Topic: feedback-invite (CLI v%(CLI_VERSION)s / topic v1) + +## Before you finish: one question for the user + +Taskless would like to hear how this went. When the task above is done, +and only then, put this to the user exactly once, in these words: + +> Taskless would like to know how the CLI is doing. Would you be okay +> sharing a few sentences about your experience? Or just skip it with +> `skip`. + +Then act on the reply: + +- **They gave feedback.** Fetch `%(TASKLESS_CLI)s agent feedback` and follow it. + It takes their words as they are and asks you, not them, for the rest. +- **They said `skip`, said nothing, or replied about something else.** + That is a decline. Run: + ``` + %(TASKLESS_CLI)s feedback dismiss + ``` + and carry on with whatever they asked for. An unrelated reply is not + feedback, and it is not a reason to ask again. + +Ask once. Do not repeat the question later in this session, do not ask +before the task is finished, and do not run `feedback send` without +having asked. diff --git a/packages/cli/src/agent/feedback.md b/packages/cli/src/agent/feedback.md new file mode 100644 index 00000000..1a172140 --- /dev/null +++ b/packages/cli/src/agent/feedback.md @@ -0,0 +1,116 @@ +# Topic: feedback (CLI v%(CLI_VERSION)s / topic v1) + +## You are here +This is `feedback`. It helps you turn what a user just said about +Taskless into a survey response the CLI can send, and send it. +You reach it from the invite at the end of an authoring or onboarding +recipe, after the user replied with something other than `skip`. +If that is not why you are reading this, re-run `%(TASKLESS_CLI)s agent` and +find the topic you meant. + +## Goal +Produce one JSON payload that answers the survey, write it to +`.taskless/.tmp-feedback.json`, send it with `feedback send`, and delete +the file. The whole thing is one short exchange with the user and a few +sentences from you; it is not an interview. + +## Preconditions +- The user replied to the invite with feedback rather than `skip`, + silence, or something unrelated. If they did any of those, this is the + wrong recipe: run `%(TASKLESS_CLI)s feedback dismiss` and continue with what + they asked for. +- The agent can write a file and run a shell command. +- No auth required. + +## You are the respondent + +The survey is addressed to you, the agent, not to the user. One answer is +the user's words and you record them verbatim. The other four are your +own account of the session you just ran: what they were trying to do, +whether they got it, what went well, and what did not. You already know +all of that. Do not put the survey's questions to the user one by one. + +## Steps + +1. **Take the user's reply as it is.** Whatever they wrote after the + invite is `verbatim`. Do not paraphrase, shorten, or tidy it. If they + wrote several messages, join them in order with a blank line between. + +2. **Ask one follow-up at most, and only if needed.** If the reply is + feedback but leaves you unable to fill `completed`, ask whether they + got what they came for. Otherwise ask nothing further; the invite + already asked for their time once. + +3. **Fill the rest from the session.** + - `goal`: what the user was trying to accomplish, in one or two of + your own sentences. Name the recipe you were following and the + rule or task it was for. + - `completed`: `Yes`, `No`, or `Unknown`. Success is binary here. A + rule that verifies and the user accepted is `Yes`; a rule the user + abandoned or that never verified is `No`; if the session ended + before you could tell, `Unknown`. There is no partial. + - `workedWell`: the steps of the interaction with Taskless that went + smoothly. Omit the key if nothing stands out. + - `needsImprovement`: the steps that cost time, needed a retry, or + that you had to work around. Be specific: name the command, the + field, or the message. Omit the key if nothing stands out. + + Keep your own answers to a few sentences each. The people reading + them want the shape of the friction, not a transcript. + +4. **Write the payload** to `.taskless/.tmp-feedback.json`, matching the + input schema below. Use the human keys exactly as given; the CLI maps + them to the survey's own question identifiers, and a payload carrying + a `$survey_` key is not what it expects. + +5. **Send.** Run: + ``` + %(TASKLESS_CLI)s feedback send --from .taskless/.tmp-feedback.json --json + ``` + Under `--json`, a failure is `{ ok: false, code, message }`; see the + table below. On success the command prints a thank-you. + +6. **Clean up.** Delete `.taskless/.tmp-feedback.json` whether the call + succeeded or failed. `.taskless/.gitignore` already ignores it, so a + forgotten file is a stray rather than a commit, but leave nothing + behind. + +7. **Return to the user's task.** Thank them in one line and carry on. + Do not ask for more, and do not run this recipe a second time in the + same session. + +## Input schema + +The `--from` JSON file conforms to: + +```json +%(INPUT_SCHEMA)s +``` + +`verbatim`, `goal`, and `completed` are required. `workedWell` and +`needsImprovement` are optional, and an optional answer you have nothing +for is an omitted key rather than an empty string. + +## Important Notes + +- Do NOT edit the user's words. `verbatim` is the one answer that is + theirs, and its value to the people reading it is that it is theirs. +- Do NOT invent a follow-up interview. The invite asked once; step 2 is + the only question this recipe allows, and only when `completed` would + otherwise be a guess. +- If telemetry is disabled in this environment the command says so and + exits 0 with nothing sent. That is the expected outcome there, not an + error to retry. + +## Errors + +With `--json`, failures emit `{ ok: false, code, message }`: + +| code | meaning | fix | +|-----------------|---------------------------------------------|-------------------------------------------------------| +| `INVALID_INPUT` | `--from` missing, unreadable, or failed validation | the message names the field; fix the payload and retry | + +## See Also + +- `%(TASKLESS_CLI)s feedback dismiss`: what to run when the user declined +- `%(TASKLESS_CLI)s agent`: the topic index, if you arrived here by mistake diff --git a/packages/cli/src/prompts/index.ts b/packages/cli/src/prompts/index.ts index 1a054053..e72c05bc 100644 --- a/packages/cli/src/prompts/index.ts +++ b/packages/cli/src/prompts/index.ts @@ -77,6 +77,12 @@ export const TOPICS = [ * the boundary from the client's side, `detect` documents a CLI subprocess a * Worker cannot spawn, `create-legacy-rule` targets a local toolchain, and * `rule-meta` describes a local sidecar file the CLI never writes. + * - `feedback` and `feedback-invite` belong to the survey the CLI appends to + * a served recipe. The invite is a fragment the `agent` command renders + * header-less and attaches after a recipe's last section; it lives here as + * a recipe so Vale and the cross-reference tests cover its prose, and it is + * servable by name only as a side effect of that. Neither has a reader + * outside the CLI that sends the response. */ export const INTERNAL_TOPICS = [ "auth", @@ -86,6 +92,8 @@ export const INTERNAL_TOPICS = [ "create-remote-rule", "delete-rule", "detect", + "feedback", + "feedback-invite", "improve-rule", "info", "init", diff --git a/packages/cli/src/prompts/recipes.ts b/packages/cli/src/prompts/recipes.ts index b26d9bc0..bf8fe5e0 100644 --- a/packages/cli/src/prompts/recipes.ts +++ b/packages/cli/src/prompts/recipes.ts @@ -8,6 +8,7 @@ import { } from "../util/invocation"; import { inputSchema as ruleCreateInputSchema } from "../schemas/rules-create"; import { inputSchema as ruleImproveInputSchema } from "../schemas/rules-improve"; +import { inputSchema as feedbackInputSchema } from "../schemas/feedback"; import { AST_GREP_VERSION, VALE_VERSION, @@ -79,6 +80,7 @@ export function canonicalRecipeTopics(): string[] { const TOPIC_INPUT_SCHEMAS: Record = { "create-remote-rule": ruleCreateInputSchema, "improve-rule": ruleImproveInputSchema, + feedback: feedbackInputSchema, }; /** Agent-fill marker used when the caller does not supply a real value. */ diff --git a/packages/cli/test/feedback-recipes.test.ts b/packages/cli/test/feedback-recipes.test.ts new file mode 100644 index 00000000..1c5dd606 --- /dev/null +++ b/packages/cli/test/feedback-recipes.test.ts @@ -0,0 +1,102 @@ +import { execFile } from "node:child_process"; +import { resolve } from "node:path"; +import { promisify } from "node:util"; + +import { describe, expect, it } from "vitest"; + +import { getRecipe } from "../src/prompts/recipes"; +import { inputSchema } from "../src/schemas/feedback"; +import { COMPLETED_CHOICES } from "../src/survey/constants"; + +const execFileAsync = promisify(execFile); +const binPath = resolve(import.meta.dirname, "../dist/index.js"); +const invocation = "npx @taskless/cli"; + +/** The sentence the invite puts to the user, as the design fixed it. */ +const ASK = + "Taskless would like to know how the CLI is doing. Would you be okay sharing a few sentences about your experience? Or just skip it with `skip`."; + +/** Blockquote prose, unwrapped: the recipe hard-wraps and prefixes `> `. */ +function unwrapQuote(text: string): string { + return text + .split("\n") + .filter((line) => line.startsWith("> ")) + .map((line) => line.slice(2).trim()) + .join(" "); +} + +describe("the feedback recipe", () => { + it("opens with its header and embeds the payload schema", async () => { + const { stdout } = await execFileAsync("node", [ + binPath, + "agent", + "feedback", + ]); + expect(stdout.startsWith("# Topic: feedback ")).toBe(true); + // The schema is rendered from the Zod source, so the choices the agent + // reads are the ones `feedback send` accepts. + for (const choice of COMPLETED_CHOICES) { + expect(stdout).toContain(`"${choice}"`); + } + for (const key of Object.keys(inputSchema.shape)) { + expect(stdout).toContain(`"${key}"`); + } + }); + + it("names the send and dismiss commands by the rendered invocation", () => { + const rendered = getRecipe("feedback", { invocation }); + expect(rendered).toContain( + `${invocation} feedback send --from .taskless/.tmp-feedback.json --json` + ); + expect(rendered).toContain(`${invocation} feedback dismiss`); + }); + + it("tells the agent it is the respondent and hands the user's words through verbatim", () => { + const rendered = getRecipe("feedback", { invocation }) ?? ""; + expect(rendered).toContain("You are the respondent"); + expect(rendered).toContain( + "Do not put the survey's questions to the user one by one" + ); + expect(rendered).toContain("Do NOT edit the user's words"); + }); +}); + +describe("the feedback invite", () => { + it("renders header-less as the fragment the gate appends", () => { + const fragment = getRecipe("feedback-invite", { + invocation, + header: false, + }); + expect(fragment).toBeDefined(); + expect(fragment).not.toContain("# Topic:"); + expect(fragment?.startsWith("## Before you finish")).toBe(true); + }); + + it("puts the fixed sentence to the user, once", () => { + const fragment = + getRecipe("feedback-invite", { + invocation, + header: false, + }) ?? ""; + expect(unwrapQuote(fragment)).toBe(ASK); + expect(fragment).toContain("Ask once"); + }); + + it("names both doors by the rendered invocation", () => { + const fragment = + getRecipe("feedback-invite", { + invocation, + header: false, + }) ?? ""; + expect(fragment).toContain(`${invocation} agent feedback`); + expect(fragment).toContain(`${invocation} feedback dismiss`); + }); + + it("treats skip, silence, and an unrelated reply as a decline", () => { + const fragment = getRecipe("feedback-invite", { header: false }) ?? ""; + expect(fragment).toMatch( + /`skip`, said nothing, or replied about something else/ + ); + expect(fragment).toContain("That is a decline"); + }); +}); diff --git a/packages/cli/test/prompts.test.ts b/packages/cli/test/prompts.test.ts index 00f57615..725142ae 100644 --- a/packages/cli/test/prompts.test.ts +++ b/packages/cli/test/prompts.test.ts @@ -580,7 +580,7 @@ function importSpecifiers(source: string): string[] { } describe("prompts entry carries no CLI runtime", () => { - // Everything the render path is allowed to reach: embedded text, the two leaf + // Everything the render path is allowed to reach: embedded text, the leaf // Zod schemas, the invocation rewrite, the engine capability constants, and // the templating library. // @@ -597,6 +597,10 @@ describe("prompts entry carries no CLI runtime", () => { "../rules/capabilities", "../schemas/rules-create", "../schemas/rules-improve", + // A third leaf schema, embedded into the `feedback` recipe. It imports + // `zod` and `src/survey/constants.ts`, which is pure data like + // `../rules/capabilities` and imports nothing at all. + "../schemas/feedback", "./recipes.js", ]);