diff --git a/.changeset/qualitative-survey-feedback.md b/.changeset/qualitative-survey-feedback.md index d16368f9..b6bbb8d4 100644 --- a/.changeset/qualitative-survey-feedback.md +++ b/.changeset/qualitative-survey-feedback.md @@ -5,3 +5,5 @@ `.taskless/.gitignore` now ignores `/.tmp-*`, the scratch request files the agent recipes write (`.tmp-rule-request.json`, `.tmp-improve-request.json`), so a file an agent forgot to clean up is a stray rather than a commit. This is scaffold migration 7; the scaffold's own `version` field carries the compatibility signal, and a project at 6 gains one ignore line the next time it is bootstrapped. A `feedback` subcommand joins the CLI, reached only through the survey invite a served recipe carries and so absent from the `taskless agent` index: `feedback send --from ` validates a human-keyed payload (`verbatim`, `goal`, `completed` as `Yes`/`No`/`Unknown`, optional `workedWell` and `needsImprovement`), maps it to the PostHog survey's question ids, and captures `survey sent`; `feedback dismiss` captures `survey dismissed`. Both hold the next invite off for 20 days and, under the telemetry opt-out, say nothing was sent and exit 0. + +Two recipes back it: `taskless agent feedback` tells the agent it is the survey's respondent (the user's reply verbatim, the rest from its own account of the session) and embeds the payload schema; `feedback-invite` is the fragment a served recipe carries, rendered header-less. diff --git a/packages/cli/src/agent/feedback-invite.md b/packages/cli/src/agent/feedback-invite.md new file mode 100644 index 00000000..4b6a9a1f --- /dev/null +++ b/packages/cli/src/agent/feedback-invite.md @@ -0,0 +1,26 @@ +# Topic: feedback-invite (CLI v%(CLI_VERSION)s / topic v1) + +## Before you finish: one question for the user + +Taskless would like to hear how this went. When the task above is done, +and only then, put this to the user exactly once, in these words: + +> Taskless would like to know how the CLI is doing. Would you be okay +> sharing a few sentences about your experience? Or just skip it with +> `skip`. + +Then act on the reply: + +- **They gave feedback.** Fetch `%(TASKLESS_CLI)s agent feedback` and follow it. + It takes their words as they are and asks you, not them, for the rest. +- **They said `skip`, said nothing, or replied about something else.** + That is a decline. Run: + ``` + %(TASKLESS_CLI)s feedback dismiss + ``` + and carry on with whatever they asked for. An unrelated reply is not + feedback, and it is not a reason to ask again. + +Ask once. Do not repeat the question later in this session, do not ask +before the task is finished, and do not run `feedback send` without +having asked. diff --git a/packages/cli/src/agent/feedback.md b/packages/cli/src/agent/feedback.md new file mode 100644 index 00000000..1a172140 --- /dev/null +++ b/packages/cli/src/agent/feedback.md @@ -0,0 +1,116 @@ +# Topic: feedback (CLI v%(CLI_VERSION)s / topic v1) + +## You are here +This is `feedback`. It helps you turn what a user just said about +Taskless into a survey response the CLI can send, and send it. +You reach it from the invite at the end of an authoring or onboarding +recipe, after the user replied with something other than `skip`. +If that is not why you are reading this, re-run `%(TASKLESS_CLI)s agent` and +find the topic you meant. + +## Goal +Produce one JSON payload that answers the survey, write it to +`.taskless/.tmp-feedback.json`, send it with `feedback send`, and delete +the file. The whole thing is one short exchange with the user and a few +sentences from you; it is not an interview. + +## Preconditions +- The user replied to the invite with feedback rather than `skip`, + silence, or something unrelated. If they did any of those, this is the + wrong recipe: run `%(TASKLESS_CLI)s feedback dismiss` and continue with what + they asked for. +- The agent can write a file and run a shell command. +- No auth required. + +## You are the respondent + +The survey is addressed to you, the agent, not to the user. One answer is +the user's words and you record them verbatim. The other four are your +own account of the session you just ran: what they were trying to do, +whether they got it, what went well, and what did not. You already know +all of that. Do not put the survey's questions to the user one by one. + +## Steps + +1. **Take the user's reply as it is.** Whatever they wrote after the + invite is `verbatim`. Do not paraphrase, shorten, or tidy it. If they + wrote several messages, join them in order with a blank line between. + +2. **Ask one follow-up at most, and only if needed.** If the reply is + feedback but leaves you unable to fill `completed`, ask whether they + got what they came for. Otherwise ask nothing further; the invite + already asked for their time once. + +3. **Fill the rest from the session.** + - `goal`: what the user was trying to accomplish, in one or two of + your own sentences. Name the recipe you were following and the + rule or task it was for. + - `completed`: `Yes`, `No`, or `Unknown`. Success is binary here. A + rule that verifies and the user accepted is `Yes`; a rule the user + abandoned or that never verified is `No`; if the session ended + before you could tell, `Unknown`. There is no partial. + - `workedWell`: the steps of the interaction with Taskless that went + smoothly. Omit the key if nothing stands out. + - `needsImprovement`: the steps that cost time, needed a retry, or + that you had to work around. Be specific: name the command, the + field, or the message. Omit the key if nothing stands out. + + Keep your own answers to a few sentences each. The people reading + them want the shape of the friction, not a transcript. + +4. **Write the payload** to `.taskless/.tmp-feedback.json`, matching the + input schema below. Use the human keys exactly as given; the CLI maps + them to the survey's own question identifiers, and a payload carrying + a `$survey_` key is not what it expects. + +5. **Send.** Run: + ``` + %(TASKLESS_CLI)s feedback send --from .taskless/.tmp-feedback.json --json + ``` + Under `--json`, a failure is `{ ok: false, code, message }`; see the + table below. On success the command prints a thank-you. + +6. **Clean up.** Delete `.taskless/.tmp-feedback.json` whether the call + succeeded or failed. `.taskless/.gitignore` already ignores it, so a + forgotten file is a stray rather than a commit, but leave nothing + behind. + +7. **Return to the user's task.** Thank them in one line and carry on. + Do not ask for more, and do not run this recipe a second time in the + same session. + +## Input schema + +The `--from` JSON file conforms to: + +```json +%(INPUT_SCHEMA)s +``` + +`verbatim`, `goal`, and `completed` are required. `workedWell` and +`needsImprovement` are optional, and an optional answer you have nothing +for is an omitted key rather than an empty string. + +## Important Notes + +- Do NOT edit the user's words. `verbatim` is the one answer that is + theirs, and its value to the people reading it is that it is theirs. +- Do NOT invent a follow-up interview. The invite asked once; step 2 is + the only question this recipe allows, and only when `completed` would + otherwise be a guess. +- If telemetry is disabled in this environment the command says so and + exits 0 with nothing sent. That is the expected outcome there, not an + error to retry. + +## Errors + +With `--json`, failures emit `{ ok: false, code, message }`: + +| code | meaning | fix | +|-----------------|---------------------------------------------|-------------------------------------------------------| +| `INVALID_INPUT` | `--from` missing, unreadable, or failed validation | the message names the field; fix the payload and retry | + +## See Also + +- `%(TASKLESS_CLI)s feedback dismiss`: what to run when the user declined +- `%(TASKLESS_CLI)s agent`: the topic index, if you arrived here by mistake diff --git a/packages/cli/src/prompts/index.ts b/packages/cli/src/prompts/index.ts index 1a054053..e72c05bc 100644 --- a/packages/cli/src/prompts/index.ts +++ b/packages/cli/src/prompts/index.ts @@ -77,6 +77,12 @@ export const TOPICS = [ * the boundary from the client's side, `detect` documents a CLI subprocess a * Worker cannot spawn, `create-legacy-rule` targets a local toolchain, and * `rule-meta` describes a local sidecar file the CLI never writes. + * - `feedback` and `feedback-invite` belong to the survey the CLI appends to + * a served recipe. The invite is a fragment the `agent` command renders + * header-less and attaches after a recipe's last section; it lives here as + * a recipe so Vale and the cross-reference tests cover its prose, and it is + * servable by name only as a side effect of that. Neither has a reader + * outside the CLI that sends the response. */ export const INTERNAL_TOPICS = [ "auth", @@ -86,6 +92,8 @@ export const INTERNAL_TOPICS = [ "create-remote-rule", "delete-rule", "detect", + "feedback", + "feedback-invite", "improve-rule", "info", "init", diff --git a/packages/cli/src/prompts/recipes.ts b/packages/cli/src/prompts/recipes.ts index b26d9bc0..bf8fe5e0 100644 --- a/packages/cli/src/prompts/recipes.ts +++ b/packages/cli/src/prompts/recipes.ts @@ -8,6 +8,7 @@ import { } from "../util/invocation"; import { inputSchema as ruleCreateInputSchema } from "../schemas/rules-create"; import { inputSchema as ruleImproveInputSchema } from "../schemas/rules-improve"; +import { inputSchema as feedbackInputSchema } from "../schemas/feedback"; import { AST_GREP_VERSION, VALE_VERSION, @@ -79,6 +80,7 @@ export function canonicalRecipeTopics(): string[] { const TOPIC_INPUT_SCHEMAS: Record = { "create-remote-rule": ruleCreateInputSchema, "improve-rule": ruleImproveInputSchema, + feedback: feedbackInputSchema, }; /** Agent-fill marker used when the caller does not supply a real value. */ diff --git a/packages/cli/test/feedback-recipes.test.ts b/packages/cli/test/feedback-recipes.test.ts new file mode 100644 index 00000000..1c5dd606 --- /dev/null +++ b/packages/cli/test/feedback-recipes.test.ts @@ -0,0 +1,102 @@ +import { execFile } from "node:child_process"; +import { resolve } from "node:path"; +import { promisify } from "node:util"; + +import { describe, expect, it } from "vitest"; + +import { getRecipe } from "../src/prompts/recipes"; +import { inputSchema } from "../src/schemas/feedback"; +import { COMPLETED_CHOICES } from "../src/survey/constants"; + +const execFileAsync = promisify(execFile); +const binPath = resolve(import.meta.dirname, "../dist/index.js"); +const invocation = "npx @taskless/cli"; + +/** The sentence the invite puts to the user, as the design fixed it. */ +const ASK = + "Taskless would like to know how the CLI is doing. Would you be okay sharing a few sentences about your experience? Or just skip it with `skip`."; + +/** Blockquote prose, unwrapped: the recipe hard-wraps and prefixes `> `. */ +function unwrapQuote(text: string): string { + return text + .split("\n") + .filter((line) => line.startsWith("> ")) + .map((line) => line.slice(2).trim()) + .join(" "); +} + +describe("the feedback recipe", () => { + it("opens with its header and embeds the payload schema", async () => { + const { stdout } = await execFileAsync("node", [ + binPath, + "agent", + "feedback", + ]); + expect(stdout.startsWith("# Topic: feedback ")).toBe(true); + // The schema is rendered from the Zod source, so the choices the agent + // reads are the ones `feedback send` accepts. + for (const choice of COMPLETED_CHOICES) { + expect(stdout).toContain(`"${choice}"`); + } + for (const key of Object.keys(inputSchema.shape)) { + expect(stdout).toContain(`"${key}"`); + } + }); + + it("names the send and dismiss commands by the rendered invocation", () => { + const rendered = getRecipe("feedback", { invocation }); + expect(rendered).toContain( + `${invocation} feedback send --from .taskless/.tmp-feedback.json --json` + ); + expect(rendered).toContain(`${invocation} feedback dismiss`); + }); + + it("tells the agent it is the respondent and hands the user's words through verbatim", () => { + const rendered = getRecipe("feedback", { invocation }) ?? ""; + expect(rendered).toContain("You are the respondent"); + expect(rendered).toContain( + "Do not put the survey's questions to the user one by one" + ); + expect(rendered).toContain("Do NOT edit the user's words"); + }); +}); + +describe("the feedback invite", () => { + it("renders header-less as the fragment the gate appends", () => { + const fragment = getRecipe("feedback-invite", { + invocation, + header: false, + }); + expect(fragment).toBeDefined(); + expect(fragment).not.toContain("# Topic:"); + expect(fragment?.startsWith("## Before you finish")).toBe(true); + }); + + it("puts the fixed sentence to the user, once", () => { + const fragment = + getRecipe("feedback-invite", { + invocation, + header: false, + }) ?? ""; + expect(unwrapQuote(fragment)).toBe(ASK); + expect(fragment).toContain("Ask once"); + }); + + it("names both doors by the rendered invocation", () => { + const fragment = + getRecipe("feedback-invite", { + invocation, + header: false, + }) ?? ""; + expect(fragment).toContain(`${invocation} agent feedback`); + expect(fragment).toContain(`${invocation} feedback dismiss`); + }); + + it("treats skip, silence, and an unrelated reply as a decline", () => { + const fragment = getRecipe("feedback-invite", { header: false }) ?? ""; + expect(fragment).toMatch( + /`skip`, said nothing, or replied about something else/ + ); + expect(fragment).toContain("That is a decline"); + }); +}); diff --git a/packages/cli/test/prompts.test.ts b/packages/cli/test/prompts.test.ts index 00f57615..725142ae 100644 --- a/packages/cli/test/prompts.test.ts +++ b/packages/cli/test/prompts.test.ts @@ -580,7 +580,7 @@ function importSpecifiers(source: string): string[] { } describe("prompts entry carries no CLI runtime", () => { - // Everything the render path is allowed to reach: embedded text, the two leaf + // Everything the render path is allowed to reach: embedded text, the leaf // Zod schemas, the invocation rewrite, the engine capability constants, and // the templating library. // @@ -597,6 +597,10 @@ describe("prompts entry carries no CLI runtime", () => { "../rules/capabilities", "../schemas/rules-create", "../schemas/rules-improve", + // A third leaf schema, embedded into the `feedback` recipe. It imports + // `zod` and `src/survey/constants.ts`, which is pure data like + // `../rules/capabilities` and imports nothing at all. + "../schemas/feedback", "./recipes.js", ]);