diff --git a/src/assets/evaluators/autoevals-lambda/README.md b/src/assets/evaluators/autoevals-lambda/README.md deleted file mode 100644 index 5caac21ff..000000000 --- a/src/assets/evaluators/autoevals-lambda/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# {{ Name }} - -An AgentCore **code-based evaluator** backed by -[autoevals](https://github.com/braintrustdata/autoevals) — scores each session -with autoevals' `{{ EvaluatorClass }}` scorer. - -## What's here - -- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter` - behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock - judge model set, autoevals grades via a LiteLLM client → Bedrock. -- `pyproject.toml` — autoevals + judge dependencies, managed with - [uv](https://docs.astral.sh/uv/). -- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the - judge model. - -## Customize - -- Change the scorer or its arguments in `lambda_function.py`. -- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference - output — provide it when you invoke the evaluator. - -`agentcore project deploy` packages this directory into the evaluator Lambda. diff --git a/src/assets/evaluators/autoevals-lambda/execution-role-policy.json b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json deleted file mode 100644 index 6b47af830..000000000 --- a/src/assets/evaluators/autoevals-lambda/execution-role-policy.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], - "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" - }, - { - "Effect": "Allow", - "Action": ["bedrock:InvokeModel"], - "Resource": "*" - } - ] -} diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py deleted file mode 100644 index 4161dbbf7..000000000 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ /dev/null @@ -1,37 +0,0 @@ -{{#if ModelProviderBedrock}} -import os - -# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION. -os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2")) - -from autoevals import {{ EvaluatorClass }}, init -from autoevals.litellm import LiteLLMClient - -from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( - EvaluatorInput, - EvaluatorOutput, - custom_code_based_evaluator, -) -from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter - -client = LiteLLMClient() -init(client=client, default_model="bedrock/{{ Model }}") - -adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) -{{else}} -from autoevals import {{ EvaluatorClass }} - -from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( - EvaluatorInput, - EvaluatorOutput, - custom_code_based_evaluator, -) -from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter - -adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) -{{/if}} - - -@custom_code_based_evaluator() -def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: - return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml deleted file mode 100644 index 8f34b0890..000000000 --- a/src/assets/evaluators/autoevals-lambda/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "{{ Name }}" -version = "0.1.0" -description = "AgentCore Code-Based Evaluator (Autoevals)" -requires-python = ">=3.10" -dependencies = [ - # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter; - # the extra owns the autoevals version, so pinning it here only conflicts. - "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0", -{{#if ModelProviderBedrock}} - # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge - "litellm>=1.60,<1.85", -{{else}} - "openai>=1.0.0,<2.0.0", -{{/if}} -] - -[tool.hatch.build.targets.wheel] -packages = ["."] diff --git a/src/assets/evaluators/deepeval-lambda/README.md b/src/assets/evaluators/deepeval-lambda/README.md deleted file mode 100644 index 5e66a8316..000000000 --- a/src/assets/evaluators/deepeval-lambda/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# {{ Name }} - -An AgentCore **code-based evaluator** backed by -[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's -`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock. - -## What's here - -- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter` - behind the standard `@custom_code_based_evaluator()` handler. -- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with - [uv](https://docs.astral.sh/uv/). -- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the - judge model; add more if your metric needs it. - -## Customize - -- Swap the metric or tune its threshold in `lambda_function.py`. -- Some DeepEval metrics need retrieval context or a reference/expected output — - supply those when you invoke the evaluator, or the metric returns - `MISSING_REQUIRED_FIELD`. - -`agentcore project deploy` packages this directory into the evaluator Lambda. diff --git a/src/assets/evaluators/deepeval-lambda/execution-role-policy.json b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json deleted file mode 100644 index 6b47af830..000000000 --- a/src/assets/evaluators/deepeval-lambda/execution-role-policy.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], - "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" - }, - { - "Effect": "Allow", - "Action": ["bedrock:InvokeModel"], - "Resource": "*" - } - ] -} diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py deleted file mode 100644 index a89b22051..000000000 --- a/src/assets/evaluators/deepeval-lambda/lambda_function.py +++ /dev/null @@ -1,29 +0,0 @@ -import os - -os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval") -os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES") -os.chdir("/tmp") - -{{#if ModelProviderBedrock}} -from deepeval.models import AmazonBedrockModel -{{/if}} -from deepeval.metrics import {{ EvaluatorClass }} - -from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( - EvaluatorInput, - EvaluatorOutput, - custom_code_based_evaluator, -) -from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter - -{{#if ModelProviderBedrock}} -model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2")) -adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})) -{{else}} -adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) -{{/if}} - - -@custom_code_based_evaluator() -def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: - return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml deleted file mode 100644 index c718ebf5a..000000000 --- a/src/assets/evaluators/deepeval-lambda/pyproject.toml +++ /dev/null @@ -1,22 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "{{ Name }}" -version = "0.1.0" -description = "AgentCore Code-Based Evaluator (DeepEval)" -requires-python = ">=3.10" -dependencies = [ - # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter; - # the extra owns the deepeval version, so pinning it here only conflicts. - "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0", -{{#if ModelProviderBedrock}} - # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x - # allows the botocore that bedrock-agentcore requires. - "aiobotocore>=3.0.0,<4.0.0", -{{/if}} -] - -[tool.hatch.build.targets.wheel] -packages = ["."] diff --git a/src/assets/evaluators/python-lambda/README.md b/src/assets/evaluators/python-lambda/README.md index f0d81bc2b..7c4732b98 100644 --- a/src/assets/evaluators/python-lambda/README.md +++ b/src/assets/evaluators/python-lambda/README.md @@ -13,7 +13,11 @@ with your own logic. - `pyproject.toml` — Python dependencies, managed with [uv](https://docs.astral.sh/uv/). - `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime. - Add statements here for anything your logic calls (DynamoDB, S3, …). + Add statements here for anything your logic calls (Bedrock, DynamoDB, S3, …). + +Third-party evaluation libraries are ordinary application dependencies. Add +them to `pyproject.toml`, configure them in `lambda_function.py`, and grant any +required permissions in `execution-role-policy.json`. ## Write your evaluator diff --git a/src/core/project/templates/evaluator.ts b/src/core/project/templates/evaluator.ts index 498a30318..d57ccd252 100644 --- a/src/core/project/templates/evaluator.ts +++ b/src/core/project/templates/evaluator.ts @@ -3,27 +3,12 @@ import type { AssetSource } from "../source"; import type { Evaluator } from "../../../projectSchemas/evaluator"; import type { TemplateRenderer, TemplateResolver } from "./types"; import { toPythonPackageName } from "../fsUtils"; -import type { - EvaluatorLibrary, - ManagedEvaluatorScaffoldInput, -} from "../../../handlers/project/types"; +import type { ManagedEvaluatorScaffoldInput } from "../../../handlers/project/types"; const DEFAULT_TIMEOUT = 60; - -const EVALUATOR_ASSETS: Record< - EvaluatorLibrary, - { assetDir: string; defaultTimeoutSeconds: number } -> = { - deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 }, - autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT }, -}; - -const EMPTY_ASSET_DIR = "evaluators/python-lambda"; +const ASSET_DIR = "evaluators/python-lambda"; function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator { - const timeoutSeconds = - input.timeoutSeconds ?? - (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT); return { name: input.name, level: input.level, @@ -33,7 +18,7 @@ function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evalua managed: { codeLocation: `app/${input.name}`, entrypoint: "lambda_function.handler", - timeoutSeconds, + timeoutSeconds: input.timeoutSeconds ?? DEFAULT_TIMEOUT, additionalPolicies: ["execution-role-policy.json"], }, }, @@ -43,17 +28,6 @@ function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evalua }; } -function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record { - const context: Record = { Name: toPythonPackageName(input.name) }; - if (input.metric) { - context["EvaluatorClass"] = input.metric.metricClass; - context["Model"] = input.model ?? ""; - context["ModelProviderBedrock"] = input.model !== undefined; - context["EvaluatorParams"] = ""; - } - return context; -} - type GetEvaluatorTemplateResolverConfig = { assetSource: AssetSource; templateRenderer: TemplateRenderer; @@ -64,15 +38,13 @@ export function getEvaluatorTemplateResolver( ): TemplateResolver { return { async resolve(input) { - const assetDir = input.metric - ? EVALUATOR_ASSETS[input.metric.library].assetDir - : EMPTY_ASSET_DIR; const tree = await FsTreeNode.fromAssetSource( { assetSource: config.assetSource }, - { assetDir }, + { assetDir: ASSET_DIR }, { rootDirName: input.name, - transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)), + transformContent: (raw) => + config.templateRenderer.render(raw, { Name: toPythonPackageName(input.name) }), }, ); return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } }; diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index 99d8b4509..b21b1130d 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -53,7 +53,7 @@ const evaluator = async (projectRoot: string, name: string) => ((await spec(projectRoot)).evaluators ?? []).find((e: { name: string }) => e.name === name); describe("project add evaluator code-based", () => { - test("3P metric → managed config + scaffolded, rendered Lambda source", async () => { + test("scaffolds managed evaluator code with an explicit timeout", async () => { const projectRoot = await inProject(); await run([ "add", @@ -63,10 +63,8 @@ describe("project add evaluator code-based", () => { "answer_faithfulness", "--level", "SESSION", - "--metric", - "deepeval.FaithfulnessMetric", - "--model", - "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "--timeout-seconds", + "120", ]); expect(await evaluator(projectRoot, "answer_faithfulness")).toMatchObject({ @@ -77,7 +75,7 @@ describe("project add evaluator code-based", () => { managed: { codeLocation: "app/answer_faithfulness", entrypoint: "lambda_function.handler", - timeoutSeconds: 300, + timeoutSeconds: 120, additionalPolicies: ["execution-role-policy.json"], }, }, @@ -86,38 +84,15 @@ describe("project add evaluator code-based", () => { const appDir = join(projectRoot, "app", "answer_faithfulness"); const handler = await Bun.file(join(appDir, "lambda_function.py")).text(); - expect(handler).toContain("FaithfulnessMetric"); - expect(handler).toContain("AmazonBedrockModel"); - expect(handler).toContain("anthropic.claude-3-5-sonnet-20240620-v1:0"); + expect(handler).toContain("TODO"); + expect(handler).toContain("custom_code_based_evaluator"); + expect(await Bun.file(join(appDir, "README.md")).exists()).toBe(true); + expect(await Bun.file(join(appDir, "pyproject.toml")).exists()).toBe(true); expect(await Bun.file(join(appDir, "execution-role-policy.json")).exists()).toBe(true); expect(handler).not.toContain("{{"); }); - test("autoevals metric with default (non-bedrock) model", async () => { - const projectRoot = await inProject(); - await run([ - "add", - "evaluator", - "code-based", - "--name", - "factuality", - "--level", - "TRACE", - "--metric", - "autoevals.Factuality", - ]); - - expect( - (await evaluator(projectRoot, "factuality")).config.codeBased.managed.timeoutSeconds, - ).toBe(60); - const handler = await Bun.file( - join(projectRoot, "app", "factuality", "lambda_function.py"), - ).text(); - expect(handler).toContain("Factuality"); - expect(handler).not.toContain("{{"); - }); - - test("no metric, no lambda → empty managed stub", async () => { + test("no lambda → managed stub with the default timeout", async () => { const projectRoot = await inProject(); await run(["add", "evaluator", "code-based", "--name", "custom_eval", "--level", "TOOL_CALL"]); @@ -187,56 +162,7 @@ describe("project add evaluator code-based", () => { ["missing --name", ["--level", "SESSION"]], ["missing --level", ["--name", "x"]], [ - "--metric and --lambda-arn together", - [ - "--name", - "x", - "--level", - "SESSION", - "--metric", - "deepeval.FaithfulnessMetric", - "--lambda-arn", - "arn:aws:lambda:us-west-2:123456789012:function:f", - ], - ], - [ - "unknown metric library", - ["--name", "x", "--level", "SESSION", "--metric", "ragas.Faithfulness"], - ], - ["metric without a class", ["--name", "x", "--level", "SESSION", "--metric", "deepeval"]], - [ - "namespaced (multi-dot) metric class", - ["--name", "x", "--level", "SESSION", "--metric", "deepeval.metrics.Faithfulness"], - ], - [ - "non-Bedrock --model", - [ - "--name", - "x", - "--level", - "SESSION", - "--metric", - "deepeval.FaithfulnessMetric", - "--model", - "gpt-4o", - ], - ], - [ - "--model bedrock with no slash/id", - [ - "--name", - "x", - "--level", - "SESSION", - "--metric", - "autoevals.Factuality", - "--model", - "bedrock", - ], - ], - ["--model without --metric", ["--name", "x", "--level", "SESSION", "--model", "bedrock/foo"]], - [ - "managed flag with --lambda-arn", + "--timeout-seconds with --lambda-arn", [ "--name", "x", @@ -257,24 +183,27 @@ describe("project add evaluator code-based", () => { ); }); - test("accepts a bare Bedrock inference-profile model id and renders it into the source", async () => { + test.each([ + ["--metric", "deepeval.FaithfulnessMetric"], + ["--model", "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"], + ])("rejects removed %s before writing", async (removedFlag, value) => { const projectRoot = await inProject(); - await run([ - "add", - "evaluator", - "code-based", - "--name", - "prof", - "--level", - "SESSION", - "--metric", - "deepeval.FaithfulnessMetric", - "--model", - "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - ]); - expect(await evaluator(projectRoot, "prof")).toBeDefined(); - const src = await Bun.file(join(projectRoot, "app", "prof", "lambda_function.py")).text(); - expect(src).toContain("us.anthropic.claude-sonnet-4-5-20250929-v1:0"); + await expect( + run([ + "add", + "evaluator", + "code-based", + "--name", + "removed", + "--level", + "SESSION", + removedFlag, + value, + ]), + ).rejects.toMatchObject({ code: "commander.unknownOption" }); + + expect(await evaluator(projectRoot, "removed")).toBeUndefined(); + expect(await Bun.file(join(projectRoot, "app", "removed")).exists()).toBe(false); }); test("rejects a duplicate evaluator name", async () => { diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index 85e2b2f87..8b79d2019 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -1,17 +1,9 @@ import z from "zod"; import { createHandler, flag, ProjectKey } from "../../../../../router"; import { InputValidationError } from "../../../../../errors"; -import { - EvaluatorSchema, - EvaluationLevelSchema, - isValidBedrockModelId, -} from "../../../../../projectSchemas/evaluator"; +import { EvaluatorSchema, EvaluationLevelSchema } from "../../../../../projectSchemas/evaluator"; import { TagsSchema } from "../../../../../projectSchemas/tags"; -import { - EVALUATOR_LIBRARIES, - type EvaluatorLibrary, - type ManagedEvaluatorScaffoldInput, -} from "../../../types"; +import type { ManagedEvaluatorScaffoldInput } from "../../../types"; import { parseJsonFlagWithSchema } from "../../../../utils"; import type { AddProjectResourceConfig } from "../../types"; import { addProjectResource } from "../../shared"; @@ -20,24 +12,14 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon createHandler({ name: "code-based", description: - "add a code-based evaluator — a Lambda that scores a session. Pass a 3P metric, an existing Lambda, or neither to scaffold an empty evaluator you fill in", + "add a code-based evaluator — scaffold a Python Lambda with custom evaluation logic, or reference an existing Lambda with --lambda-arn", flags: [ flag("name", "the name of the evaluator", z.string().optional()), flag("level", "what to score: SESSION, TRACE, or TOOL_CALL", z.string().optional()), - flag( - "metric", - "3P metric to scaffold as , e.g. deepeval.FaithfulnessMetric or autoevals.Factuality", - z.string().optional(), - ), - flag( - "model", - "judge model for the 3P metric, e.g. bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", - z.string().optional(), - ), flag("lambda-arn", "ARN of an existing Lambda that scores a session", z.string().optional()), flag( "timeout-seconds", - "Lambda timeout in seconds (1-300)", + "evaluator timeout in seconds (1-300)", z.number().int().min(1).max(300).optional(), ), flag("description", "a description of what this evaluator measures", z.string().optional()), @@ -57,12 +39,7 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon if (!levelParsed.success) throw new InputValidationError(z.prettifyError(levelParsed.error)); const level = levelParsed.data; - const hasMetric = flags["metric"] !== undefined; const hasLambda = flags["lambda-arn"] !== undefined; - if (hasMetric && hasLambda) - throw new InputValidationError( - "provide either --metric (managed) or --lambda-arn (external), not both", - ); const tags = parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema); const base = { @@ -75,10 +52,8 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon const project = ctx.require(ProjectKey); if (hasLambda) { - if (flags["metric"] || flags["model"] || flags["timeout-seconds"] !== undefined) - throw new InputValidationError( - "--metric, --model, and --timeout-seconds are managed-only and not valid with --lambda-arn", - ); + if (flags["timeout-seconds"] !== undefined) + throw new InputValidationError("--timeout-seconds is not valid with --lambda-arn"); const parsed = EvaluatorSchema.safeParse({ ...base, config: { codeBased: { external: { lambdaArn: flags["lambda-arn"] } } }, @@ -97,12 +72,8 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon return; } - if (flags["model"] && !hasMetric) throw new InputValidationError("--model requires --metric"); - const scaffold: ManagedEvaluatorScaffoldInput = { ...base, - ...(hasMetric && { metric: parseMetric(flags["metric"]!) }), - ...(flags["model"] !== undefined && { model: resolveBedrockModel(flags["model"]) }), ...(flags["timeout-seconds"] !== undefined && { timeoutSeconds: flags["timeout-seconds"] }), }; @@ -117,37 +88,10 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon }, `added evaluator '${flags["name"]}' to '${project.name}'`, { - notes: hasMetric - ? [] - : [ - `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py`, - ], + notes: [ + `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py`, + ], }, ); }, }); - -function parseMetric(raw: string): { library: EvaluatorLibrary; metricClass: string } { - const dot = raw.indexOf("."); - const library = dot > 0 ? raw.slice(0, dot) : ""; - const metricClass = dot > 0 ? raw.slice(dot + 1) : ""; - if (!(EVALUATOR_LIBRARIES as readonly string[]).includes(library)) - throw new InputValidationError( - `invalid --metric "${raw}": expected where library is one of ${EVALUATOR_LIBRARIES.join(", ")} (e.g. deepeval.FaithfulnessMetric)`, - ); - if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(metricClass)) - throw new InputValidationError( - `invalid metric class "${metricClass}" in --metric "${raw}": expected a single class name like FaithfulnessMetric`, - ); - return { library: library as EvaluatorLibrary, metricClass }; -} - -function resolveBedrockModel(model: string | undefined): string | undefined { - if (!model) return undefined; - const id = model.startsWith("bedrock/") ? model.slice("bedrock/".length) : model; - if (!isValidBedrockModelId(id)) - throw new InputValidationError( - `invalid --model "${model}": expected a Bedrock model ID (e.g. anthropic.claude-3-5-sonnet-20240620-v1:0) or an inference-profile/foundation-model ARN, optionally prefixed with "bedrock/"`, - ); - return id; -} diff --git a/src/handlers/project/types.ts b/src/handlers/project/types.ts index c42d21f34..eb08483b9 100644 --- a/src/handlers/project/types.ts +++ b/src/handlers/project/types.ts @@ -25,9 +25,6 @@ type CreateProjectInputBase = { skipGit?: boolean; }; -export const EVALUATOR_LIBRARIES = ["deepeval", "autoevals"] as const; -export type EvaluatorLibrary = (typeof EVALUATOR_LIBRARIES)[number]; - /** Set of arguments needed to scaffold a managed code-based evaluator. */ export type ManagedEvaluatorScaffoldInput = { name: string; @@ -35,8 +32,6 @@ export type ManagedEvaluatorScaffoldInput = { description?: string; kmsKeyArn?: string; tags?: Record; - metric?: { library: EvaluatorLibrary; metricClass: string }; - model?: string; timeoutSeconds?: number; };