refactor(project): simplify code-based evaluator scaffolding

src/assets/evaluators/python-lambda/README.md

@@ -13,7 +13,11 @@ with your own logic.
 - `pyproject.toml` — Python dependencies, managed with
   [uv](https://docs.astral.sh/uv/).
 - `execution-role-policy.json` — extra IAM the evaluator Lambda gets at runtime.
-  Add statements here for anything your logic calls (DynamoDB, S3, …).
+  Add statements here for anything your logic calls (Bedrock, DynamoDB, S3, …).
+
+Third-party evaluation libraries are ordinary application dependencies. Add
+them to `pyproject.toml`, configure them in `lambda_function.py`, and grant any
+required permissions in `execution-role-policy.json`.
 
 ## Write your evaluator
 

src/core/project/templates/evaluator.ts

@@ -3,27 +3,12 @@ import type { AssetSource } from "../source";
 import type { Evaluator } from "../../../projectSchemas/evaluator";
 import type { TemplateRenderer, TemplateResolver } from "./types";
 import { toPythonPackageName } from "../fsUtils";
-import type {
-  EvaluatorLibrary,
-  ManagedEvaluatorScaffoldInput,
-} from "../../../handlers/project/types";
+import type { ManagedEvaluatorScaffoldInput } from "../../../handlers/project/types";
 
 const DEFAULT_TIMEOUT = 60;
-
-const EVALUATOR_ASSETS: Record<
-  EvaluatorLibrary,
-  { assetDir: string; defaultTimeoutSeconds: number }
-> = {
-  deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
-  autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
-};
-
-const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+const ASSET_DIR = "evaluators/python-lambda";
 
 function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
-  const timeoutSeconds =
-    input.timeoutSeconds ??
-    (input.metric ? EVALUATOR_ASSETS[input.metric.library].defaultTimeoutSeconds : DEFAULT_TIMEOUT);
   return {
     name: input.name,
     level: input.level,
@@ -33,7 +18,7 @@ function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evalua
         managed: {
           codeLocation: `app/${input.name}`,
           entrypoint: "lambda_function.handler",
-          timeoutSeconds,
+          timeoutSeconds: input.timeoutSeconds ?? DEFAULT_TIMEOUT,
           additionalPolicies: ["execution-role-policy.json"],
         },
       },
@@ -43,17 +28,6 @@ function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evalua
   };
 }
 
-function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
-  const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
-  if (input.metric) {
-    context["EvaluatorClass"] = input.metric.metricClass;
-    context["Model"] = input.model ?? "";
-    context["ModelProviderBedrock"] = input.model !== undefined;
-    context["EvaluatorParams"] = "";
-  }
-  return context;
-}
-
 type GetEvaluatorTemplateResolverConfig = {
   assetSource: AssetSource;
   templateRenderer: TemplateRenderer;
@@ -64,15 +38,13 @@ export function getEvaluatorTemplateResolver(
 ): TemplateResolver<ManagedEvaluatorScaffoldInput> {
   return {
     async resolve(input) {
-      const assetDir = input.metric
-        ? EVALUATOR_ASSETS[input.metric.library].assetDir
-        : EMPTY_ASSET_DIR;
       const tree = await FsTreeNode.fromAssetSource(
         { assetSource: config.assetSource },
-        { assetDir },
+        { assetDir: ASSET_DIR },
         {
           rootDirName: input.name,
-          transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+          transformContent: (raw) =>
+            config.templateRenderer.render(raw, { Name: toPythonPackageName(input.name) }),
         },
       );
       return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };