From 60f58b877fe01319e82e3f62c857082e39c139f3 Mon Sep 17 00:00:00 2001 From: 2233admin <2233admin@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:06:42 +0800 Subject: [PATCH] docs: consolidate development artifacts --- .claude-project.md | 83 ----- AGENTS.md | 12 + AUDIT-cybernetic-remediation.md | 143 -------- GOAL-2.md | 29 -- GOAL-3.md | 53 --- GOAL-4.md | 67 ---- GOAL-5.md | 92 ----- GOAL-6.md | 89 ----- GOAL-7.md | 97 ----- GOAL-agent-runtimes.md | 146 -------- GOAL.md | 76 ---- GRILL-KICKOFF.md | 49 --- HANDOFF-strangler-fig.md | 88 ----- LIVE_WEBHOOK_ACCEPTANCE.md | 100 ------ PLAN_adapter_selfheal.md | 44 --- PLAN_collection_nodes.md | 78 ---- PLAN_general_analysis_loop.md | 192 ---------- PLAN_native_nodes_loop.md | 332 ------------------ PLAN_plugin_system.md | 50 --- PLAN_product_positioning.md | 78 ---- PLAN_realtime_tool_capabilities.md | 51 --- PLAN_ui_reskin.md | 151 -------- PONYTAIL.md | 110 ------ README.md | 2 +- TODOS.md | 85 ----- backend/agent_runtimes/base.py | 7 +- backend/agent_runtimes/pi_adapter.py | 4 +- backend/agent_runtimes/registry.py | 2 +- backend/api/v1/browser_act.py | 2 +- backend/api/v1/chat.py | 2 +- backend/api/v1/model_defaults.py | 2 +- backend/api/v1/providers.py | 6 +- backend/browser_act/__init__.py | 2 +- backend/browser_act/cli.py | 8 +- backend/browser_act/scripts.py | 11 +- backend/browser_act_packs/VENDOR.md | 8 +- backend/browser_act_packs/catalog.py | 2 +- backend/browser_act_packs/manifest.py | 4 +- backend/channels/browser_act_channel.py | 6 +- backend/channels/crawl4ai_channel.py | 2 +- backend/channels/skill_channel.py | 2 +- backend/config.py | 2 +- backend/llm/__init__.py | 4 +- backend/llm/anthropic.py | 4 +- backend/llm/base.py | 6 +- backend/llm/catalog.py | 2 +- backend/llm/factory.py | 14 +- backend/llm/openai_compat.py | 4 +- backend/llm/resolver.py | 2 +- ...c9d0e1f2a3b4_add_runtimes_to_edge_nodes.py | 3 +- ..._add_provider_models_and_model_defaults.py | 2 +- backend/models/model_default.py | 4 +- backend/models/provider_model.py | 4 +- backend/pipeline/ai_processor.py | 4 +- backend/pipeline/pipeline.py | 2 +- backend/processors/claude_processor.py | 2 +- backend/processors/local_processor.py | 2 +- backend/processors/openai_processor.py | 2 +- backend/security/url_guard.py | 2 +- backend/services/provider_model_service.py | 4 +- backend/ws_agent_manager.py | 4 +- docs/CONTROL_THEORY_ARCHITECTURE.md | 3 +- docs/SYSTEM_ANALYSIS.md | 8 +- docs/WIRING_GAP_LEDGER.md | 4 +- docs/dify-p0-compatibility-runtime-PRD.md | 2 +- ...ify-manifest-intake-and-plugin-registry.md | 1 - frontend/lib/api/endpoints.ts | 4 +- frontend/lib/api/types.ts | 4 +- tests/integration/test_auth_ws.py | 2 +- .../integration/test_browser_act_packs_api.py | 2 +- tests/integration/test_browser_act_seeds.py | 2 +- tests/integration/test_model_defaults_api.py | 2 +- tests/integration/test_provider_models_api.py | 4 +- tests/unit/api/test_nodes_upsert.py | 3 +- tests/unit/browser_act/test_cli.py | 2 +- tests/unit/browser_act_packs/test_manifest.py | 2 +- tests/unit/channels/test_api_channel.py | 2 +- .../unit/channels/test_browser_act_channel.py | 4 +- .../unit/channels/test_web_scraper_channel.py | 4 +- tests/unit/llm/test_adapters.py | 2 +- tests/unit/llm/test_catalog.py | 2 +- tests/unit/llm/test_pr_e_consumers.py | 2 +- tests/unit/llm/test_resolver.py | 4 +- tests/unit/pipeline/test_ai_processor.py | 2 +- tests/unit/pipeline/test_pipeline_errors.py | 2 +- tests/unit/test_agent_server.py | 2 +- tests/unit/test_model_default.py | 2 +- tests/unit/test_provider_model.py | 2 +- tests/unit/test_runner.py | 4 +- tests/unit/test_schedule_service_redbeat.py | 2 +- tests/unit/worker/test_beat_schedule.py | 2 +- 91 files changed, 123 insertions(+), 2400 deletions(-) delete mode 100644 .claude-project.md delete mode 100644 AUDIT-cybernetic-remediation.md delete mode 100644 GOAL-2.md delete mode 100644 GOAL-3.md delete mode 100644 GOAL-4.md delete mode 100644 GOAL-5.md delete mode 100644 GOAL-6.md delete mode 100644 GOAL-7.md delete mode 100644 GOAL-agent-runtimes.md delete mode 100644 GOAL.md delete mode 100644 GRILL-KICKOFF.md delete mode 100644 HANDOFF-strangler-fig.md delete mode 100644 LIVE_WEBHOOK_ACCEPTANCE.md delete mode 100644 PLAN_adapter_selfheal.md delete mode 100644 PLAN_collection_nodes.md delete mode 100644 PLAN_general_analysis_loop.md delete mode 100644 PLAN_native_nodes_loop.md delete mode 100644 PLAN_plugin_system.md delete mode 100644 PLAN_product_positioning.md delete mode 100644 PLAN_realtime_tool_capabilities.md delete mode 100644 PLAN_ui_reskin.md delete mode 100644 PONYTAIL.md delete mode 100644 TODOS.md diff --git a/.claude-project.md b/.claude-project.md deleted file mode 100644 index b1038a3..0000000 --- a/.claude-project.md +++ /dev/null @@ -1,83 +0,0 @@ -# opencli-admin Claude Code 配置 - -> 遵循 Ponytail 理念：用最少的代码完成任务 - -## 项目规范 - -### 核心原则 - -1. **先问这个需要存在吗？** → 不需要就跳过 -2. **标准库能做吗？** → 用标准库 -3. **平台原生特性？** → 用原生 -4. **已安装的依赖能做吗？** → 用已有的 -5. **一行能搞定吗？** → 一行 -6. **最后才写最小可用的代码** - -### 永不动刀子的地方 - -- 输入验证 -- 错误处理 -- 安全性 -- 可访问性 - -## 开发规范 - -详细规范见 `./PONYTAIL.md` 和 `./docs/DEVELOPMENT_STANDARD.md` - -## 文件限制 - -- 单文件不超过 200 行 -- 超过 200 行才拆分 -- 目录结构扁平化 - -## 依赖规则 - -添加依赖前必须检查： -1. 标准库能做吗？ -2. 已有的依赖能做吗？ -3. 这个真的需要吗？ - -## Commit 规范 - -``` -feat(xxx): brief description -fix(xxx): brief description -chore: brief description -``` - -- 每行不超过 72 字符 -- 动词开头 -- 不写 "and" - -## 项目结构 - -``` -opencli-admin/ -├── apps/ -│ ├── web/ # Next.js 前端 -│ └── api/ # Hono API (待创建) -├── packages/ -│ └── shared/ # 共享类型 -├── docs/ # 文档 -├── frontend/ # 现有 Vite SPA -├── backend/ # FastAPI -├── iii/ # III 调度 -└── odp-rs/ # Rust 数据面 -``` - -## 技术栈 - -### 前端 -- Next.js 15 + React 19 + TypeScript 5 -- Tailwind CSS + shadcn/ui -- TanStack Query v5 -- Zod - -### 后端 -- Hono (替代 FastAPI) -- Drizzle ORM -- PostgreSQL - -### 数据面 -- Rust (odp-rs) -- Redis Streams diff --git a/AGENTS.md b/AGENTS.md index ca941d7..beec60a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,10 +9,22 @@ These rules apply only to this repository. They extend the workspace rules in affected scope, and the smallest check that can prove completion. - Keep changes small and reversible. Reuse existing project patterns and dependencies before adding abstractions or packages. +- Prefer deletion, the standard library, native platform features, and already + installed dependencies before adding code or dependencies. +- Never simplify away trust-boundary validation, data-loss protection, + security controls, or accessibility basics. - Do not claim completion without fresh evidence. Prefer targeted tests first, then typecheck, lint, build, integration, or browser smoke checks according to the risk of the change. +## Documentation Hygiene + +- Keep the repository root limited to durable product, architecture, testing, + and agent-instruction entry points. +- Track active plans and tasks in GitHub Issues. Record durable architecture + decisions in `docs/adr/`; do not add root-level `PLAN_*`, `GOAL*`, `HANDOFF*`, + or `AUDIT*` files. + ## Task-Aware Model Routing Use a Tian Ji horse-racing strategy: match the task to the least expensive model diff --git a/AUDIT-cybernetic-remediation.md b/AUDIT-cybernetic-remediation.md deleted file mode 100644 index 8b2bd72..0000000 --- a/AUDIT-cybernetic-remediation.md +++ /dev/null @@ -1,143 +0,0 @@ -# 系统工程审核 + 控制论修复方案 - -> 产出方式:5 路并行 Sonnet 子 agent 审计(架构/数据链路/安全/前端契约/测试运维), -> P0 级 + 锚定发现由主模型逐条读码复核。日期 2026-07-02。 -> 分支 `refactor/thin-channel-thick-runner`。 -> -> 标注:✅ = 已读码亲验 / ▫ = agent 报告,未独立复核。 - -## 部署信任模型(已定 2026-07-02) - -**局域网通信走 NetBird / WireGuard mesh,不做直接公网端口映射。** - -- **入站边界 = WG 层**:只有 fleet overlay 上已认证的 peer 能触达 API/MCP/odp-ingest。 - 应用层"无鉴权"由此获得真实网络边界兜底 → 相关发现(轮子 7)从 P1 降为 **P2 纵深防御备案**。 -- **出站威胁不受 WG 保护**:SSRF 类威胁是"服务器被恶意内容(爬到的网页/RSS/被改的 provider - `base_url`)骗着主动发出请求"。WG 挡入站不挡出站——出站可打 fleet overlay(100.80.x.x)、 - 本机 localhost 服务(redis 6379 等),或把真实 LLM key 外泄到公网。**轮子 4 仍必修(P1)。** -- **LLM agent 是可信 mesh 内的不可信因子**:MCP 把 `create_source`+`trigger_task` 暴露给 agent, - 而 agent 处理不可信内容(爬网页/RSS),prompt injection 可触发 cli 渠道任意二进制 → - **轮子 4 的 binary 白名单仍值做(P1 硬化),不因可信 mesh 而免**。 - ---- - -## 一、控制论诊断:这不是一堆孤立 bug,是四类回路病 - -维纳四原则拿来当透镜,本次绝大多数发现归到前两类——**反馈回路断裂**和**信息丢失**。 -这解释了为什么它们分散在 Python/Rust/React 三层却"长得一样"(同构性),也决定了修法: -不是逐点打补丁,是**装轮子**——一处机制,全层复用,让系统自己纠错(自组织)。 - -| 控制论原则 | 系统里对应的东西 | 违反它的发现 | -|---|---|---| -| **① 反馈是生命线**(感知差异→调整→再执行) | 重试分类、DLQ、错误上浮、CI 门禁 | #1 重试不触发、#3 DLQ 吞消息、#8 主循环 panic、shadow 错误没人读、前端缺 onError、CI 不测关键路径 | -| **② 信息即控制力**(信息丢=控制力衰减) | accepted 计数真实性、可观测性、凭证保密 | #2 摄入黑洞(假 accepted)、#4 毒消息与 trim 混同、明文 key 外泄、无 healthcheck、shadow 计数只进日志 | -| **③ 同构性**(同一模型跨领域复用) | 错误处理模式在三层重复出现 | 缺 onError(React)≡ 重试漏分类(Celery)≡ DLQ 丢信号(Rust)——**同一个病,该用同一个轮子治** | -| **④ 目的性/自组织**(局部规则+全局反馈→有序) | 校验器、契约、不变量在调用点就地强制 | SSRF 无统一校验器、env 开关散落、鉴权靠"部署层但愿"、游标非原子推进 | - ---- - -## 二、发现台账(记录) - -### P0 — 会真丢数/坏/被攻破 - -| # | 位置 | 问题 | 原则 | 验 | -|---|---|---|---|---| -| P0-1 | `backend/pipeline/error_taxonomy.py` + ODP sink 路径 | `httpx.HTTPStatusError` 不在 `_RETRYABLE`,ODP 回 5xx→判永久→`pipeline.py` 吞成失败结果正常返回→Celery `autoretry_for` 不触发。配 `odp_only` 策略=无处落、无自动重试。**机制现成**:有 `is_retryable_http_status()`+`RetryableHTTPStatus` sentinel,只是 ODP 路径没用 | ① | ✅ | -| P0-2 | `odp-rs/.../odp-ingest/src/state.rs:24` + `handlers.rs:99-101` | 漏配 Redis→`bus=None`→`else { accepted += 1 }`,返回 202 但没入流。纯黑洞,运行期零信号。(注:NDJSON 入口对解析失败是 400,黑洞只在 batch+无 bus) | ② | ✅ | -| P0-3 | `odp-rs/.../odp-store/src/reap.rs:56-62`(**本仓 `ef4828d` 引入**) | 毒消息(JSON 反序列化失败)走 DLQ 路径时 `read_entries_by_id` 静默丢弃→不进 `found_ids`→被误判"已 trim"直接 XACK,不写 DLQ 行=静默永久丢,且与"真被 trim"安全分支无法区分 | ①② | ✅ | -| P0-4 | `backend/channels/cli_channel.py:40,46` | `binary`+`command` 全来自 channel_config,`create_subprocess_exec(*full_cmd)`=任意二进制执行。配无鉴权 API/MCP=RCE。exec-form 无 shell 注入 | ④ | ✅ | - -### P1 — 严重债/潜伏 bug - -| # | 位置 | 问题 | 原则 | 验 | -|---|---|---|---|---| -| P1-1 | `backend/pipeline/storer.py:23,38` | 默认 `forward_to_odp=True`,裸 `ODP_INGEST_URL` 触发,绕开 `write_strategy` 状态机→legacy 源意外泄漏进 ODP + 双入口迷惑。(idempotency_key 去重挡掉字面重复,故非重复灌数)。`HANDOFF-strangler-fig.md` 标了"PR3 必收口" | ④ | ✅ | -| P1-2 | `backend/schemas/provider.py:34` + `api/v1/providers.py:15` | `GET /providers` 明文返回所有 LLM api_key;`ModelProvider.api_key` 明文存库,没走 `SourceCredential` 的 Fernet | ② | ✅ | -| P1-3 | `backend/main.py`(全局) | 全 API 无 AuthN/AuthZ(`api_key_enabled` 定义了没 `Depends` 校验);odp-ingest 无 auth 且默认 bind `0.0.0.0`;CORS debug 下 `["*"]`+`allow_credentials=True` | ④ | ✅ | -| P1-4 | 见下方 SSRF 枚举 | 响应回显型 SSRF(rss/web_scraper/api/crawl4ai 抓用户 URL→`GET /records` 读回=内网读原语);无 scheme 白名单/私网 IP 拦截/元数据防护;多处 `follow_redirects=True`;provider `base_url` 可改指外部主机→连真实 Bearer key 一起外泄 | ②④ | ▫ | -| P1-5 | `odp-rs/.../odp-store/src/main.rs:54,59,60` | 主循环裸 `?`,Redis/PG 瞬断→整进程 panic 退出(reap 有 log-continue,主循环没有) | ① | ✅ | -| P1-6 | `backend/pipeline/cursor_store.py` + `runner.py` | SELECT-then-write 无原子 UPSERT/行锁;只有按域名进程内信号量,非 per-source→并发触发同源丢更新(漏抓中间段) | ①④ | ▫ | -| P1-7 | `backend/pipeline/sinks/dual_sink.py:57` + `pipeline.py` | shadow 写失败塞进 `SinkResult.errors` 但 `pipeline.py` 从不读→影子模式 ODP 长期挂,任务仍显 completed,唯一线索是 worker 日志一行 warning | ② | ▫ | -| P1-8 | `.github/workflows/ci.yml` | 只 `pytest -m "not live" --no-cov`——无 `alembic upgrade head` 冒烟、无 `cargo test`(odp-rs 连已有单测都不跑)、coverage gate 关掉。迁移链坏能一路绿灯合并 | ① | ▫ | -| P1-9 | `tests/unit/pipeline/test_pipeline_errors.py:34,58,153` | 调用签名跟真实 `run_pipeline(task_id, source, parameters=None,...)` 对不上:传 `run_pipeline(db_session, source, task.id)`,`db_session` 当了 task_id(仅进日志 `%s` 侥幸不炸),`task.id` 当了 parameters(UUID 非 dict)。rss 源无 `session_affinity` capability 跳过 `params.get` 分支→侥幸通过=测了假东西 | ① | ✅ | -| P1-10 | `frontend/src/pages/BrowsersPage.tsx:778,528` | 重启 API 按钮 + 实例 agent_url 保存两个 mutation 缺 onError,失败全静默 | ① | ▫ | - -### P2 — 硬化项 - -- ▫ odp-ingest `dedup.rs` 无上限/TTL→投毒(自选幂等键让合法事件被当 duplicate 丢)+ 内存 DoS;`handlers.rs:71` 整批持单写锁 -- ▫ `handlers.rs:30` NDJSON 无显式请求体大小上限(仅靠 axum 默认 2MB) -- ▫ `odp-store/main.rs:46` 无优雅关闭(裸 `loop`,无 SIGTERM 处理)→部署强杀撞未 flush 批次 -- ▫ `docker-compose.yml` odp-ingest/odp-store 无 healthcheck;下游 `depends_on: service_started` 非 `service_healthy`→起容器时序竞争 -- ▫ `.env.example` 与实际 env 读取脱节:`ODP_INGEST_REQUIRED`(fail-open/closed 语义关键开关)、`ODP_INGEST_TIMEOUT`、odp-rs 侧 `ODP_DATABASE_URL/ODP_STORE_BATCH_SIZE/ODP_BUS_*` 全没进主文档 -- ▫ `frontend/src/pages/SourcesPage.tsx:301` 6 种完工渠道被硬编码标"(开发中)"=误导文案 -- ▫ `frontend/src/pages/SkillsPage.tsx` RecordWizard 修了点取消泄漏,但没堵路由跳转/后退卸载(缺 unmount cleanup)→录制中离开页面照旧泄漏 pool mutex -- ▫ `odp-store/src/reap.rs:38` `p.delivery_count > MAX_DELIVERIES`(=5)实为 6 次才进 DLQ,与注释"5"字面不符(应 `>=`) -- ▫ `external_http_processor.py:78` `os.path.expandvars(auth_header)`→配置者设 `$OPENAI_API_KEY` 可把宿主 env 注入出站 header -- ▫ `config.py:19,91` 弱默认密钥(`change-me-*`);本机 `.env` 已真实覆盖且已 gitignore,无提交泄密 -- ▫ `frontend` `connectivity_ok/connectivity_errors` 后端有、前端类型直接丢弃 - -### SSRF 完整出站枚举(20 点,详见安全 agent 原始报告) - -需统一校验器覆盖的用户/DB 供给 URL 出站点: -- 回显型(→records 可读回内网):rss_channel(2)、web_scraper_channel、api_channel、crawl4ai_channel、external_http_processor -- 探测 oracle:source_service `discover-feed`(2) -- 盲打:webhook/feishu/wecom notifier(3) -- **凭证外泄型**(base_url 可改+附真实 key):distill、skill_channel、crawl4ai LLMConfig、openai_processor -- health_check 变体(仅 bool):web_scraper/api/crawl4ai(3) - ---- - -## 三、修复方案 = 8 个"轮子"(可复用机制,非逐点补丁) - -排序 = 性价比。每个轮子标:治哪些发现 / 控制论原则 / 大致工作量。 - -### 轮子 1 — DLQ 毒消息可区分【P0-3】① ② · 小 -`reap.rs` 必须把"解析失败"与"真被 trim"分开:`read_entries_by_id` 对解析失败的条目保留原始 bytes 放进 DLQ payload(或单独计数+告警),绝不走"没 entry 就直接 ack"分支。 -**这是本仓上轮自己引入的丢数 bug,码在手边,最先修。** - -### 轮子 2 — 重试分类闭环【P0-1, P1-5】① · 小 -(a) ODP sink/`odp_client` 捕获 `httpx.HTTPStatusError`,对 5xx/429 走已有的 `is_retryable_http_status()`→设 `RetryableHTTPStatus` error_type,让负反馈回路真正闭合。 -(b) `odp-store/main.rs` 主循环三个裸 `?` 改 log-and-continue(沿用 reap 的模式),瞬断不再 panic 退出。 - -### 轮子 3 — 摄入信号真实性【P0-2】② · 小 -odp-ingest `bus=None` 时要么 fail-fast 拒绝启动,要么响应显式标 `degraded/no-op` 且**不计入 accepted**。假 accepted = 被污染的信息 = 假的控制力。 - -### 轮子 4 — 统一 URL 校验器【P0-4 部分, P1-4, 多个 P2】④ · 中 -一个 `safe_url(url)` 模块(仅 http/https;解析后拦 RFC1918/loopback/link-local/元数据 169.254;禁跳转到私网;pin 已解析 IP),在全部 20 个出站点就地调用。**同构性**:一处规则全层复用;**自组织**:每个调用点就地强制不变量,不需中央防火墙。cli_channel 额外加 binary 白名单。 - -### 轮子 5 — 错误上浮契约【P1-7, P1-10, 重试耗尽告警】① ② · 中 -定一条契约:任何失败都必须浮到可观测通道。 -- Python:`DualSink` shadow 错误接入 `events.emit(level=warning)` 或写回 `TaskRun`;Celery 重试耗尽发 dedicated 信号而非只写 DB 状态字段。 -- React:所有 `useMutation` 缺 onError 的补齐 toast(BrowsersPage 两处起)。 -**同构性**:三层同一个"别吞错误"模式,一份契约拉齐。 - -### 轮子 6 — CI 反馈门禁【P1-8, P0-3/P1-5 回归防护】① · 中 -这是元反馈回路——**捕捉断裂反馈回路的系统**。加: -- `alembic upgrade head`(fresh db)+ `downgrade -1 && upgrade head` 冒烟 job -- `cargo test --workspace`(odp-rs)job -- odp-store writer/reap 的 testcontainers 集成 job(redis+postgres,先 2-3 个关键用例:dead_letter 失败不 ack、reap 重复 claim 幂等、savepoint 隔离单条坏消息) -- 恢复 coverage gate - -### 轮子 7 — 鉴权边界【P1-3 → 降 P2 备案;P1-2 部分保留】④ · 视情 -**已定:WG mesh 承担入站边界**,故应用层鉴权降为纵深防御备案(非急):全局 API-key/session -`Depends`、CORS 修正、bind 收到 WG 接口/127.0.0.1(而非 0.0.0.0,防 split-tunnel 误配扩大暴露面)。 -**但 P1-2 的 provider key 加密+响应 mask 保留(挪进轮子 4 一起做)**——因为 key 外泄的真实路径 -是 SSRF 出站 exfil + 日志 + DB 备份,不是入站,WG 救不了。 - -### 轮子 8 — 游标原子推进 + strangler 收口【P1-1, P1-6】① ④ · 中 -(a) `source_cursors` 用 `INSERT ... ON CONFLICT (source_id) DO UPDATE` 或 `SELECT FOR UPDATE`;游标只在"这批已确认落盘"后推进。 -(b) `storer.py` 默认 `forward_to_odp=False`,删掉裸 env var 兜底路径,ODP forward 只由显式 `write_strategy` 触发(收口 HANDOFF 标注的 PR3)。 - ---- - -## 四、落地批次 - -| 批 | 内容 | 理由 | -|---|---|---| -| **B1(先)** | 轮子 1+2+3 | 三个数据链路 P0,全在既有职责内,码已读清,小改+补测,直接堵静默丢数 | -| **B2** | 轮子 6 | 装 CI 门禁,给后续所有改动兜底(含 B1 的回归) | -| **B3** | 轮子 4+5 | SSRF 校验器 + 错误上浮,面大但同构,一次机制多点收 | -| **B4** | 轮子 8 | strangler 收口 + 游标原子性,需设计确认 | -| **B5(降级备案)** | 轮子 7 | 部署面已定 WG mesh → 应用层鉴权降 P2 纵深防御,不进主线 | - -> 部署面已定(WG mesh),轮子 7 降级备案。B3 的 SSRF 校验器现含 provider key 加密+mask。 diff --git a/GOAL-2.md b/GOAL-2.md deleted file mode 100644 index 696b07e..0000000 --- a/GOAL-2.md +++ /dev/null @@ -1,29 +0,0 @@ -# GOAL-2 — opencli-admin Phase 2/3: AuthManager + session affinity 泛化 - -> `/loop` 自驱（接 strangler-fig GOAL 完成 `09e4860` 后的新里程碑）。同纪律： -> 每轮读本文件 → 下个未完 PR → 端到端做+测绿 → 自检 staged → auto-commit → 勾掉。 -> 命中真分叉 → 停问，别 big-bang。 -> 用户 2026-07-01 授权**每 PR 绿即 auto-commit**（仅限此 goal，显式 add 路径，push 等用户）。 - -## 坐标 -- repo `D:\projects\opencli-admin` 分支 `refactor/thin-channel-thick-runner`（接 `09e4860`） -- 测试闸 `uv run pytest tests/unit --no-cov -q`（基线 **379**）；PowerShell 跑 -- ⚠️ 永不 stage：`backend/api/v1/chat.py`、`PR-DESCRIPTION.md`、`HANDOFF-strangler-fig.md`、`GOAL.md`、`GOAL-2.md` - -## Track 1 — session affinity 泛化（Phase 3，低风险） -- [x] **PR-A** — pipeline 绑定 gate 改读 `capabilities.session_affinity`；opencli+skill 声明 `Capabilities(session_affinity=True)`（`073d391`，382 passed，行为零变）。 -- [x] **PR-B** `6e08d41`（404 passed）— 按域名并发上限 = **task 层进程内**（option ②，用户拍板）：`domain_limiter.domain_of(source)` 取 host + `domain_slot()` per-domain semaphore(`PER_DOMAIN_CONCURRENCY` 默认 3，registry 按 `(loop,domain)` 键)，runner Phase3 外套。全渠道覆盖（含 opencli/skill）。跨 worker 严格限 = 换 Redis（同插入点，缓）。 - -> ✅ **GOAL-2 完成**（2026-07-01）：Track-1(PR-A+PR-B) + Track-2(PR-C) 全落，379→**404 passed**，零回归，**已 push fork**（`09e4860..6e08d41`）。 - -## Track 2 — AuthManager + 加密凭据（Phase 2）✅ **DONE** -- [x] **PR-C** `d4aa324`（397 passed）— Fernet 加密凭据存储(`source_credentials` 表 + migration `q7l8m9n0o1p2`) + `AuthManager`(store/resolve/`resolve_context→AuthContext`) + channel_runner 注入 AuthContext(替占位) + api_channel 内联明文 **deprecation warning**。`cryptography` 提为直接依赖。已选方案=Fernet + env `CREDENTIAL_ENCRYPTION_KEY` + 表；接线深度到 AuthManager（未强迁 api_channel→`fetch()`=follow-up）。 -
原决策记录（已决议）： - 1. 加密方案：Fernet/AES（`cryptography` 依赖在否待查） - 2. master key 来源：env var - 3. 存储：新 `source_credentials` 表 / `DataSource` 字段 - 4. 迁移现有内联 secret - 5. 接线深度：api_channel 还没上 `fetch()`，`AuthContext` 走 runner 注入需 api_channel 迁厚契约（run_channel 已建 `AuthContext(kind=cap.auth_kind)` 占位） - -## 每 PR 验收 / 停止条件 / 提交策略 = 同 `GOAL.md` -行为零变（旧路径）、全 tests/unit 绿、显式 stage、auto-commit、真分叉停。 diff --git a/GOAL-3.md b/GOAL-3.md deleted file mode 100644 index dc97437..0000000 --- a/GOAL-3.md +++ /dev/null @@ -1,53 +0,0 @@ -# GOAL-3 — api_channel 厚契约迁移代码审查修复环 - -> `/loop` 自驱(接 GOAL-2 `6e08d41` 后,来自这轮 `/code-review xhigh` 的 14 条已验证 finding)。 -> 每轮读本文件 → 下个未 [x] 项 → 端到端做+测绿 → 自检 staged → auto-commit → 勾掉。 -> 命中真分叉/需重新设计游标语义等敏感决策 → 停问,别自己拍板架构。 -> 用户 2026-07-01 授权:每 PR(每条 finding)绿即 **auto-commit**(仅限此 goal,显式 `git add <路径>`,绝不 `add -A`;**push 仍等用户**)。 - -## 坐标 -- repo `D:\projects\opencli-admin` 分支 `refactor/thin-channel-thick-runner` -- 测试闸:`uv run pytest tests/unit tests/integration tests/skills --no-cov -q`(当前基线:unit 412 / integration 77 / skills 97,零失败) -- 跑测试用 PowerShell(`cd D:\projects\opencli-admin; uv run ...`);Bash 被 RTK hook 改写易炸("z: command not found") -- ⚠️ **永不 stage**:`backend/api/v1/chat.py`、`PR-DESCRIPTION.md`、`HANDOFF-strangler-fig.md`、`GOAL.md`、`GOAL-2.md`、`GOAL-3.md`(用户 dock WIP + 控制文件) -- 本 goal 源头改动(api_channel 厚契约 + credential endpoint,尚未 commit)已跑通:后端 tests 全绿 + 真实 uvicorn+encryption key 活验证过 store/list/delete 全链路 + `npx tsc -b` 前端类型检查干净。**这些改动本身先 commit 一刀(PR0),再逐条修 finding。** - -## PR0 — commit 本轮 api_channel 厚契约迁移源头改动(先做,别跳过) -把这些已验证但未提交的文件 add 并提交(排除上面永不 stage 清单): -`backend/api/v1/sources.py backend/auth/manager.py backend/channels/api_channel.py backend/channels/base.py backend/pipeline/channel_runner.py backend/pipeline/collector.py backend/schemas/credential.py frontend/src/api/endpoints.ts frontend/src/components/ChannelConfigForm.tsx frontend/src/pages/SourcesPage.tsx tests/integration/test_sources_api.py tests/unit/auth/test_manager.py tests/unit/channels/test_api_channel.py tests/unit/channels/test_rss_fetch.py tests/unit/pipeline/test_channel_runner.py tests/unit/pipeline/test_collector.py tests/unit/pipeline/test_collector_incremental.py` -测试闸跑一遍确认绿,再 commit。commit message 示例:`feat(api-channel): thick-contract fetch() + encrypted credential store endpoints`。 - -## 状态机(每轮更新) - -- [x] **PR0** — 见上,提交源头改动(`d15e4fd`,593 passed 7 skipped) -- [x] **PR1** — `api_channel.py` fetch() 转发 `timeout`(`8d0c6ad`,594 passed):`client.request(...)` 调用加 `timeout=timeout` kwarg(两处:owns_client 分支的 httpx.AsyncClient 已经隐式带了 client 级 timeout,但 `.request()` 显式传更保险且修的是 ctx.http 分支——共享 client 硬编码 30s,必须靠 per-request `timeout=` 覆盖)。加测试:mock ctx.http 记录 kwargs,断言 `timeout=` 被传入。 -- [x] **PR2** — `fetch()` 补 `except Exception` 兜底(`7de0191`,595 passed)(镜像 `collect()` 的 `except Exception as exc: ... "API request failed: {exc}"` 文案),包成 `ChannelFetchError` 抛出(别学 collect() 返回 ChannelResult.fail——fetch() 契约是抛异常)。加测试:mock client.request 抛 `OSError("connection refused")`,断言抛 `ChannelFetchError` 且消息含 "connection refused"。 -- [x] **PR3** — basic auth 两处实现合一(`2beefe8`,608 passed):抽一个共享 helper(建议 `backend/auth/header_builder.py` 或就近放 `backend/auth/manager.py` 顶层函数,如 `build_basic_auth_header(username, password) -> dict|None`——两者都空返回 None/不发头,而不是发空 Basic 头),`AuthManager.resolve_context()` 和 `ApiChannel._resolve_auth_headers()` 都改调用它。顺带把 bearer/api_key 的 header 构造也一并抽成共享 helper(消掉三处硬编码 key 名的问题:token/key/username+password 约定收敛到一处)。加测试覆盖"两条路径行为一致"的场景(空 creds → 都不发 Basic 头)。 -- [x] **PR4** — `AuthManager.store()` 防并发撞唯一约束(`79fc2e4`,609 passed):`session.commit()` 外包 `try/except IntegrityError`,冲突时 rollback + 重新 select + UPDATE(不是插第二行)。加测试:模拟并发双写同 `(source_id, key_name)`,断言最终只有一行、值是后写的。 -- [x] **PR5** — `CredentialCreate.key_name` 的 `max_length`(`91edcc0`,610 passed;有一次全量跑偶发单测flaky重跑绿,非本改动引入) 从 100 改成 64,和 `SourceCredential.key_name` 的 `String(64)` 对齐。加测试:65 字符 key_name 触发 Pydantic 422。 -- [x] **PR6** — `source_service.delete_source()`(`96511d9`,611 passed) 同一 session 里级联删对应 `source_credentials` 行(delete 前先 `DELETE FROM source_credentials WHERE source_id = ...` 或用 SQLAlchemy `delete(SourceCredential).where(...)`,和删 source 同一事务提交)。加测试:存凭据→删源→断言 `AuthManager().resolve(source_id)` 返回空字典(不再是孤儿)。 -- [x] **PR7** — 收窄 `run_channel()`/`collector.py` 在非增量渠道上的多余开销(`93c67db`,614 passed;RSS增量/限速路径零回归;isinstance检查换成`owns_client`布尔位,顺带更好mock):①`collector.py` 的 `_collect_via_runner` 在调用 `DBCursorStore().load()` 前先判 `channel.capabilities.incremental`,非增量渠道整段 db_cursor/staging 逻辑跳过(等价于 PR5b 之前的行为,只是路由仍统一走 `run_channel`);②`channel_runner.py` 的 `run_channel()` 只在 `chan.fetch is not AbstractChannel.fetch`(即渠道真正覆写了 fetch())时才构建 `RateLimitedClient`,否则 `client=None` 传给默认适配器(反正它不读 `ctx.http`)。**这条要仔细验证不破坏 PR5a/PR5b 的增量渠道行为**——RSS 仍要正常走 cursor+rate-limit。全量跑 tests/unit + tests/skills + tests/integration 确认零回归。 -- [x] **PR8** — `ApiChannel.collect()` 改成薄包装(`eafef17`,614 passed;老13个collect()测试原样过,顺带把fetch()的owns_client分支也改回`async with`保mock兼容+抽了个`_send`小helper去重),委托给 `fetch()`(构造一个不带 `http`/`source_id` 的 `FetchContext`,catch `ChannelFetchError` 转回 `ChannelResult.fail(str(exc))`,成功则 `ChannelResult.ok(result.items, **result.metadata)`)。**`tests/unit/channels/test_api_channel.py` 里所有 `test_collect_*` 断言必须原样通过不改**(这是这条 PR 的验收标准——旧接口行为零变,只是实现委托了)。 -- [x] **PR9** — `CredentialField` name 属性改派生自 `keyName`(已唯一+ASCII,没另加`fieldId` prop——`keyName`已经满足这个要求,加会是纯重复)(`1ee0cb0`,tsc干净+614 passed;⚠️本仓无React组件测试框架,只能typecheck验证)。 -- [x] **PR10** — `CredentialField` 的 `listSourceCredentials` 失败态(`5e0a344`,tsc干净):加 `loadError` state,`.catch()` 里 `setLoadError(true)`,placeholder 逻辑区分"确认未存储" vs "状态获取失败"(后者显示类似"⚠ 无法获取存储状态"而不是伪装成未配置)。 -- [x] **PR11** — `collector.py`/`pipeline.py` 的 `cursor_pending`/`cursor_source_id` 键名(`f5c3f76`,615 passed)改成防撞的保留前缀(如 `__cursor_pending__`/`__cursor_source_id__`),两处出现(`collector.py` 写入 + `pipeline.py` 的 `pop`)同步改。 -- [x] **PR12** — `channel_runner.py` 分页 metadata 合并策略加注释文档化(`b8b4e57`,615 passed)(明确"后页覆盖前页同名键"是有意行为,不是 bug),不改代码逻辑,只补 docstring/inline comment——因为目前无真实渠道 exercise 这条路径,别在没有真实用例前臆造合并语义。 -- [x] **PR13** — `run_channel()` 分页循环(`75db78a`,616 passed) - -> ✅ **GOAL-3 完成**(2026-07-01):PR0→PR13 全落,593→**616 passed**,零回归。`d15e4fd..75db78a` 共 14 个 commit,分支 `refactor/thin-channel-thick-runner`。**未 push**(push 等用户)。 `chan.fetch(ctx)` 外包 try/except,失败时 `log.warning(...)` 打出"第 N 页失败,已丢弃 M 条已抓 items,cursor 可能已推进到 X"再重新抛出(**不改变异常传播行为和 cursor 提交时机**——只加可观测性,cursor/一致性语义的真正修复留给以后,那是 GOAL.md 自己都判过的敏感决策点)。 - -## 每 PR 验收(DoD) -1. 全 `tests/unit` + `tests/integration` + `tests/skills` 绿(≥ 基线,PR0 后基线按新数字算) -2. 旧路径行为零变(尤其 PR7、PR8——这两条改的是既有生产路径,必须零回归) -3. commit(仅码+测路径,自检 staged 集,`git status --porcelain` 核对无 chat.py/GOAL*.md/HANDOFF*.md/PR-DESCRIPTION.md) -4. 更新本文件状态框 + 一行进度 - -## 停止条件(任一 → 停+报,别瞎猜) -- 全 PR 完 -- pytest 红且 2 轮内修不动 -- **真分叉**:PR7 发现会破坏 RSS 增量行为、PR13 发现必须动 cursor 提交时机才能修对、或任何一条修法出现多条不等价路径 -- 需要 push(push 永远等用户) - -## 参考 -- 本轮 finding 全文出自 `/code-review xhigh`(10 finder angles + 13 verify + 3 sweep,当前会话 transcript)。P1(PR1-4)/P2(PR5-9,含 PR0)/P3(PR10-13)按严重度排。 -- 已知刻意不修的架构级问题(记在这,别在 loop 里自己动):`ApiChannel` 从不声明 `Capabilities.auth_kind`,`ctx.auth` 对它架构性地永远无用——真正修法是把 `Capabilities.auth_kind` 泛化成可按 source 动态解析,这是设计级决策,不进本 goal。 diff --git a/GOAL-4.md b/GOAL-4.md deleted file mode 100644 index 3c867d9..0000000 --- a/GOAL-4.md +++ /dev/null @@ -1,67 +0,0 @@ -# GOAL-4 — 采集管线高可靠化(接 GOAL-3 之后) - -> `/loop` 自驱。每轮读本文件 → 下个未 [x] 项 → 端到端做+测绿 → auto-commit → 勾掉。 -> 命中真分叉 → 停问,别自己拍板架构。 -> 用户已批:每 PR 绿即 auto-commit(显式 `git add <路径>`,绝不 `add -A`,绝不碰 `backend/api/v1/chat.py`/`GOAL*.md`/`HANDOFF*.md`/`PR-DESCRIPTION.md`);push 仍等用户。 - -## 坐标 -- repo `D:\projects\opencli-admin` 分支 `refactor/thin-channel-thick-runner` -- 测试闸:`uv run pytest tests/unit tests/integration tests/skills --no-cov -q`(PowerShell,`cd D:\projects\opencli-admin`;Bash 工具在此环境被 RTK hook 改写会炸) -- 基线(PR0 已完成,commit `2e58cc3`):616 passed 7 skipped - -## 已完成背景(不用再做) -- PR0(本文件外,已提交 `2e58cc3`):`ChannelResult.error_type` 打标 + 6 渠道 catch-all 补齐 + `rss_channel.py:98` timeout 转发漏洞。 - -## 已锁定的架构决策(别重新问,直接照做) -1. **重试接入 celery**:`pipeline.py` 对 retryable 异常 re-raise,celery task 声明的 `max_retries=3` 借此自动生效——不改 `tasks.py`,不加显式 `self.retry()`。 -2. **调度器**:全套接通 redbeat 当 celery beat backend;`CronSchedule` CRUD(创建/改/删)时同步写/删 redbeat entry(不再是"改了要重启 beat 才生效");本地 `backend/scheduler.py` 那条轮询 loop 是否保留由你在做 PR-C 时看情况定(local executor 模式下可能还需要它,celery 模式下应该完全让位给 redbeat)——但两套调度语义不能同时活着互相打架,拿主意时记录在本文件里说明取舍,不用停下来问。 - -## 状态机 - -- [x] **PR-A — 错误分类法(taxonomy)+ 幂等性验证**(`e0c196f`,639 passed)。`error_taxonomy.py`落地;幂等性:序列重跑靠content_hash已经对(既有测试证实),但发现真gap——check-then-insert非原子,并发写同content_hash会IntegrityError丢整批,已修(rollback+recheck+逐条插survivors)+补测试。PR-B激活重试后这个race会变真实,不再是纯理论。 - 在 `backend/channels/base.py` 或新文件(建议 `backend/pipeline/error_taxonomy.py`)定义 `is_retryable(error_type: str) -> bool`: - - retryable:`TimeoutException`、`TimeoutError`、`ConnectionError`、`ConnectError`、`ReadError`、`RemoteProtocolError`、`OSError`(网络/subprocess 层瞬时故障) - - permanent:`ValueError`、`KeyError`、`FileNotFoundError`(二进制/配置缺失,重试没用)、`json.JSONDecodeError`、`ChannelFetchError` 本身(已经是包装过的语义错误,看 `__cause__` 才能细分——如果 cause 是 retryable 类型则 retryable,否则 permanent) - - `httpx.HTTPStatusError`:4xx(除 429,已经在 `RateLimitedClient` 层重试过、到这里说明重试也没用)→ permanent;5xx/429 理论上不该漏到这层(`RateLimitedClient` 已处理),如果漏到了按 retryable 处理兜底 - - 加单测覆盖每类判断。 - **幂等性验证**(不是新写,是确认现状):读 `backend/pipeline/sinks/`,确认 `collect()` 被重跑一次(同一批 items 再来一遍)时 `write_batch` 不会产生重复记录(应该靠 `identity()`/内容 hash 去重)。写一个测试用例证明"同一 task 的 collect→persist 跑两遍,`records_collected` 不翻倍"。如果发现不幂等,记录在本文件里,是否要在本 goal 内修还是记成已知风险——这是真分叉,停问。 - -- [x] **PR-B — pipeline.py 对 retryable 异常 re-raise,激活 celery 真重试**(`30ac9ae`,645 passed)。查清楚了:光re-raise不够,`tasks.py`必须加`autoretry_for=(Exception,)`(此前max_retries=3是死的,没人调self.retry());runner.py Phase 4只在run_pipeline正常返回时跑,re-raise会跳过它把TaskRun卡在running——加了except统一收口标failed再往上抛。`run_scheduled_collection`(cron那条兄弟task)现状没retry,不在这条PR范围,记了没动。 - `pipeline.py` 的 step1/collect 与 step2-3/sink 两处 `except Exception as exc:` 改成:先判 `is_retryable(type(exc).__name__)`(或 `channel_result.error_type` 那条路径),retryable 就 re-raise(让 `run_collection` celery task 函数本身抛出,`autoretry_for` 或 `self.retry()` 生效——先确认 celery task 装饰器需不需要加 `autoretry_for=(Exception,)` 或类似,`bind=True` 已经有了,可能只需要在 catch 到异常处显式 `raise self.retry(exc=exc, countdown=...)`,去 `worker/tasks.py` 确认清楚再改,别瞎猜);permanent 才照旧转 `PipelineResult(success=False)` 吞掉。 - **验证**:写测试模拟一次 retryable 失败,断言 celery task 走了 retry 路径(用 celery 的 eager/test 模式或 mock `self.retry`);模拟一次 permanent 失败,断言不重试、直接 `PipelineResult(success=False)`。 - -- [x] **PR-C — redbeat 接通(全套,见上面锁定决策)**(`77b4afb`,657 passed)。挖到底:celery beat压根没接过(`build_beat_schedule()`零调用方,已删,只留`parse_cron_expression`复用)。全套落地:celery-redbeat依赖+`beat_scheduler`配置+`redbeat_sync.py`(sync_entry/remove_entry/populate_all)+schedule CRUD同步(gate在`task_executor=="celery"`,fail不炸请求)+main.py启动populate_all。本地`scheduler.py`**保留不动**——local模式下唯一调度器,跟redbeat靠task_executor互斥不打架。踩坑并修:populate_all首版抄了tasks.py那套`new_event_loop().run_until_complete()`,但调用方main.py lifespan本身就在跑着的loop里,会炸"already running"——改成async函数直接await。本仓无真实redis,新测试走mock redbeat库边界。 - 1. `pyproject.toml`/`uv add celery-redbeat` - 2. `celery_app.py`:`beat_scheduler = "redbeat.RedBeatScheduler"`,`redbeat_redis_url` 配置(复用 `settings.redis_url` 或 `celery_broker_url`) - 3. 起始 populate:进程启动时(或一次性脚本)把现有 `CronSchedule` 表全量写成 redbeat entries(用 `worker/beat_schedule.py` 里已有的 `_get_enabled_schedules`/`parse_cron_expression` 复用,别重写) - 4. `backend/api/v1/schedules.py`(或对应 CRUD 文件——先找到)的创建/更新/删除/enable-toggle 端点里,同步调用 redbeat 的 `RedBeatSchedulerEntry(...).save()` / `.delete()`,不再只写 DB - 5. `backend/worker/beat_schedule.py` 的 `build_beat_schedule()` 现在有实际调用方了(populate 脚本)——如果它的逻辑跟 redbeat entry 构造重复,提取共享的 cron-parse 部分,别留两份平行实现 - 6. `backend/scheduler.py` 的本地 loop:决定去留(见上面架构决策),照实现,在本文件补一行说明取舍 - 7. **验证**:起真实 redis(本仓测试环境已有 fixture 大概率),写集成测试:建一个 schedule → 断言 redbeat 里有对应 entry;删 schedule → entry 消失;不需要真等 cron 触发(那是 celery beat 自己的事,不用集成测出"真的到点跑了") - -- [x] **PR-D — web_scraper 迁 fetch();opencli/cli/skill 评估后跳过**(`bbfd5b9`,660 passed)。web_scraper真迁了(拿限速+backoff),16个老collect()测试原样过。opencli:非HTTP client(subprocess+browser pool),迁了ctx.http也用不上,跳过。cli/skill:评估中发现PR-B的celery重试已经靠`error_type`taxonomy覆盖它们了(所有渠道failure都走`ChannelResult.fail(error_type=...)`,不只fetch()渠道)——单独retry wrapper纯重复,不加。 - 参照 GOAL-3 PR8 api_channel 的迁法:`collect()` 变薄包装委托给新写的 `fetch()`,**老的 `test_collect_*` 断言必须原样通过不改**(验收标准)。`web_scraper` 走 `ctx.http`(拿 `RateLimitedClient` 的 429/backoff);`opencli` 是 subprocess+浏览器池,不是 HTTP 请求,`fetch()` 迁移对它意义有限(它本来就没有走 HTTP client 这条路)——**先判断 opencli 值不值得迁,若"迁了但 ctx.http 完全用不上"就没必要,只把它记成"评估过、不迁,原因是 XXX"跳过,别为了凑数硬迁**。 - `cli`/`skill` 渠道如果评估后确实需要重试(比如 opencli/skill 的浏览器 flaky 场景),再加独立的 retry wrapper(不是 fetch() 迁移,是 collect() 外面包一层"失败重试 N 次"的装饰器)——先看有没有真实需求再动,没有就跳过记录原因。 - -- [x] **PR-E — 真健康探针(cheap liveness ≠ deep readiness,分两档)**(`a86c216`,675 passed)。`health_check()`签名从`()->bool`拓成`(config=None, source_id=None)->bool`(向后兼容,老0参调用不变),`source_service.py`透传。api真HEAD/GET+真auth头(走`_resolve_auth_headers`包括加密store);web_scraper两档(lxml驱动能用+目标真可达);opencli两档(二进制在+真打CDP `/json/version`,agent/bridge模式跳过深探针,pool未初始化兜底老行为)。health→dispatch gating按计划没做(feature不是fix)。 - - `api_channel.health_check()`:对 `config.get("base_url")` 发一个轻量 HEAD/GET(带 `_resolve_auth_headers`,真的带认证探活,不是随便connect一下),超时给短(如 5s),网络错误/4xx5xx → False - - `web_scraper_channel.health_check()`:探目标 URL 可达(HEAD/GET) + BeautifulSoup/lxml driver 能正常 import/实例化(这个基本不会挂,但按用户原话"driver 活着"补上) - - `opencli_channel.health_check()`:现在只查 `_OPENCLI_BIN` 存在;补上真正打 CDP endpoint(`GET {cdp_endpoint}/json/version`)确认浏览器起得来、够得着——注意这依赖 browser_pool 已 acquire 一个 endpoint,看现有 `pool.acquire()` 怎么用,别为了 health_check 常驻占一个浏览器槽位 - - **明确不做**:health_check 接入 dispatch gating(不健康就跳过任务)——那是 feature 不是 fix,本 goal 不做,写清楚原因 - -> ✅ **GOAL-4 完成**(2026-07-01):PR0(session外,`2e58cc3`)→PR-A→PR-E 全落,616→**675 passed**,零回归。`2e58cc3..a86c216` 共 6 个 commit,分支 `refactor/thin-channel-thick-runner`。**未 push**(push 等用户)。挖到的额外发现:celery beat 之前压根没接通(PR-C)、`run_scheduled_collection` 仍无重试(PR-B 范围外,已记录)、cli/skill 靠 PR-B 的 taxonomy 已间接拿到重试(PR-D 评估结论)。 - -## 每 PR 验收(DoD) -1. `tests/unit` + `tests/integration` + `tests/skills` 全绿(≥ 616 基线) -2. 老路径行为零回归(尤其 PR-B、PR-D——改的是生产路径) -3. commit 仅码+测路径,`git status --porcelain` 自检无 chat.py/GOAL*.md/HANDOFF*.md/PR-DESCRIPTION.md -4. 勾掉本文件对应项 + 一行进度(commit hash + 测试数) - -## 停止条件(真分叉才停,别瞎猜) -- 全 PR 完 -- pytest 红且 2 轮内修不动 -- PR-A 幂等性验证发现真的不幂等,要不要本 goal 内修 -- PR-B 里 celery 重试到底该用 `autoretry_for` 还是显式 `self.retry(exc=exc)`,两种语义不等价(前者装饰器声明式、每次都重试同样逻辑;后者能按错误类型定制 countdown/次数)——如果 `worker/tasks.py` 现状明显该用哪种就直接用,不明显再停问 -- PR-C 步骤 6(本地 scheduler.py 去留)如果发现 local executor 模式还有活人在用(不只是测试覆盖),别直接删,记下来问 -- PR-D 判断某渠道"迁不迁"本身有分歧(不确定算不算真分叉,判断错了也就是白评估一次,不算严重后果,不用为这个停) -- 需要 push(push 永远等用户) diff --git a/GOAL-5.md b/GOAL-5.md deleted file mode 100644 index 72f8c4a..0000000 --- a/GOAL-5.md +++ /dev/null @@ -1,92 +0,0 @@ -# GOAL-5 — Agent 接入(SKILL.md + RSS + REST)+ 内容分类 taxonomy(仿 AIHOT / Dify) - -> `/loop` 自驱。每轮读本文件 → 下个未 [ ] 项 → 端到端做+测绿 → auto-commit → 勾掉。 -> 命中真分叉 → 停问,别自己拍板架构。 -> 每 PR 绿即 auto-commit(显式 `git add <路径>`,绝不 `add -A`,绝不碰 `backend/api/v1/chat.py`/`GOAL*.md`/`HANDOFF*.md`/`AUDIT*.md`/`GRILL*.md`/`PR-DESCRIPTION.md`);push 仍等用户。 - -## 背景 - -对标 https://aihot.virxact.com/agent 的"Agent 接入"模式(Skill/RSS/REST 三轨,匿名免 token,按用户意图分流端点)。opencli-admin 现状(GOAL-5 之前): -- 内容模型 `CollectedRecord`(`backend/models/record.py`)全 JSON blob(raw_data/normalized_data/ai_enrichment),无真实分类/标签字段 -- `DataSource.tags`(`backend/models/source.py`)是自由 JSON list,不是本次要做的内容分类 -- 整个 `/api/v1/*` 无鉴权(admin 工具 style),无 public/private 区分 -- RSS 只有摄入(`rss_channel.py`),没有对外发布 -- 无 SKILL.md 生成能力(fork 上游有个同名"skill"概念但语义不同——浏览器录制→蒸馏的数据源渠道,和这次"agent 可装的 SKILL.md 包"是两回事,注意别混) - -## 坐标 -- repo `D:\projects\opencli-admin`,新分支 `feat/agent-access-taxonomy`(从当前 `main` 切出) -- 测试闸:`uv run pytest tests/unit tests/integration tests/skills --no-cov -q`(PowerShell,`cd D:\projects\opencli-admin`;Bash 工具在此环境被 RTK hook 改写会炸) -- 部署目标未定 —— 本 goal 只出设计落地的代码,不碰部署/域名 - -## 已锁定的架构决策(别重新问,直接照做) - -1. **鉴权**:公开接口匿名免 token(仿 AIHOT),限流用 IP-based 轻量中间件(内存 token bucket,不引入 Redis 依赖),不做 API Key 发放机制。 -2. **暴露范围**:`DataSource` 加 `public`(bool,默认 `False`)开关,只有显式打开的源的内容才可能进公开接口。 -3. **分类机制**:抄 Dify 的 `Tag` + `TagBinding` 模式,去掉 `tenant_id`(opencli-admin 无多租户概念)。`Tag(id, type, name, created_at)`,`type ∈ {"category", "subtag"}`;`TagBinding(id, tag_id, target_id, created_at)`,`target_id` 指向 `collected_records.id`,不建 DB 级 FK(照抄 Dify 做法,完整性靠服务层)。业务不变量:每条 record 最多绑 1 个 `type=category` 的 tag,`type=subtag` 可绑多个。 -4. **顶级分类闭集种子值**(占位提议,未被用户改,直接按此写 seed):`模型能力` / `产品动态` / `行业资讯` / `研究论文` / `工程实践` / `其它`。分类名不照抄 AIHOT(那套是仿 Dify 机制,不是仿 AIHOT 命名)。 -5. **分类来源**:`DataSource.default_category` 兜底(source 级默认);AI enrichment 阶段跑完后可用 LLM 输出覆盖式细化绑定。LLM 分类调用失败/超时 —— 不阻断 pipeline,直接落回 source 默认值,record 状态复用现有 `status` 枚举(`raw|normalized|ai_processed|notified|error`),不新增状态。 -6. **curated 精选**:`CollectedRecord` 加 `curated`(bool,默认 `False`),**v1 只做人工打标**(走现有 admin API 手动 PATCH)。自动/规则化 curate 是二期,本 goal 不做 —— 这是 YAGNI 裁决不是遗漏,别加。 -7. **PublicContentService**(`backend/services/public_content_service.py`)是唯一查询入口 —— 给定 `mode`(`selected`|`all`)/`category`/`since`/`q`/`take` 返回过滤后的 record 集合,REST 和 RSS 都调它,不重复写"哪些内容可对外"这条过滤逻辑(`source.public=True` 是硬性前提,`mode=selected` 再加 `curated=True`)。 -8. **RSS**:新增对外发布方向(现有 `rss_channel.py` 只是摄入,方向不同,不复用其摄入逻辑,只复用 `PublicContentService` 的查询结果),用 `feedgen` 库序列化成 Atom。 -9. **SKILL.md**:不手写。从 `taxonomy.py`(分类闭集)+ 路由定义脚本生成后 commit 进仓库,按静态文件路由 served。CI 加一致性检查:重新生成结果必须等于已提交文件,防止改分类忘同步文案。 -10. **Daily digest**:独立定时任务(复用现有 pipeline 调度基建,没有就加最简单 cron 任务),把当日 `public=True AND curated=True` 的 record 快照进新表 `daily_digests`(date, record_ids, 可选 LLM 摘要文案),不是实时计算。同一天重跑必须幂等(upsert,不重复插入)。 -11. **响应白名单**:新增 `PublicRecordRead` schema,只含 `id/title/url/summary/source_name/published_at/category/subtags`,显式排除 `raw_data`/`normalized_data`/内部 source 配置 —— 防止 admin 内部字段随手泄露到公开接口。 - -## 状态机 - -- [x] **PR-A — 数据模型:Tag/TagBinding + DataSource 扩展字段 + taxonomy 闭集定义**(`0f48495`,398→414 passed,零回归)。 - 新增 `backend/models/tag.py`(`Tag`/`TagBinding`,建表 migration,索引 `(type, name)`、`target_id`、`tag_id`)。`DataSource` 加 `public: bool = False`、`default_category: str | None` 两列(migration)。新增 `backend/taxonomy.py` 定义闭集分类常量(见架构决策 #4)+ 校验函数 `is_valid_category(name) -> bool`。 - 验收:migration 跑通,新表/新列存在,`taxonomy.py` 单测覆盖合法/非法分类名判断。 - -- [x] **PR-B — TagService**(`adaf55b`,414→424 passed,零回归)。 - `backend/services/tag_service.py`:`bind_category(record_id, category_name)`(校验闭集、强制"最多 1 个"覆盖式绑定,非法分类名报错)、`add_subtags(record_id, names: list[str])`(去重、允许新建)、`get_tags(record_id)`、`list_by_category(category_name)`。 - 验收:单测覆盖"重复绑定 category 是覆盖不是叠加"、"非法 category 名拒绝"、"subtags 去重"、"list_by_category 只返回该分类下的 record_id 集合"。 - -- [x] **PR-C — Enrichment 阶段接入分类**(`25d48f0`,424→439 passed,零回归)。走全LLM驱动+兜底:现有enrichment(`ai_processor.py`)本来就没有专门分类调用,只是把用户配置prompt的LLM JSON输出存进`ai_enrichment`——没造新LLM调用架构,机会性读该JSON里的`category`/`subtags`键(有校验),没有/非法/enrichment未跑/失败都落回`source.default_category`;`default_category`也空则记警告跳过分类,subtags仍照跑,不阻断pipeline、不新增status。 - 扩展现有 AI enrichment 阶段(找到 `backend/pipeline/` 里对应步骤):跑完常规 enrichment 后,调 `TagService.bind_category`(优先 LLM 输出,失败/超时/未跑则用 `source.default_category` 兜底)+ `TagService.add_subtags`(LLM 输出的细粒度标签)。 - 验收:LLM 分类成功路径、LLM 失败兜底路径、`source.default_category` 为空时的行为(记录警告,不崩)都有测试;确认 pipeline 整体状态机不受影响(现有 `status` 流转测试不回归)。 - -- [x] **PR-D — PublicContentService**(`68993be`,439→457 passed,零回归)。补了`CollectedRecord.curated`列(架构决策#6要但之前PR都没加,migration`n4i5j6k7l8m9`)。`since`过滤用`created_at`原生列不是`normalized_data.published_at`(那字段格式不统一,查了会错不只是慢)。`q`用`lower(cast(normalized_data as String)).contains()`(跟`record_service.py`现有写法一致,dialect通用非Postgres专属ILIKE)。`take`默认50上限200。安全测试(`public=False`永不泄露,含"给我全部"式恶意参数组合)已覆盖。 - `backend/services/public_content_service.py`:核心过滤(`source.public=True` 硬前提 + `mode`/`category`/`since`/`q`/`take`)。 - 验收(安全关键):`source.public=False` 的 record 无论调用方传什么过滤参数都不能出现在结果里 —— 这条测试必须覆盖"恶意/异常参数"场景,不只是正常路径。 - -- [x] **PR-E — REST API + 限流**(`b343146`,457→474 passed,零回归)。限流60次/分钟/IP内存token bucket,只挂public router非全局middleware。响应壳复用现有`ApiResponse[T]`(`backend/schemas/common.py`),白名单字段用显式mapper函数(非`model_validate`裸转)。`summary`取`ai_enrichment.summary`优先,没有则退`normalized_data.content`(合理外推非锁定决策原文)。泄露测试断言`raw_data`/`normalized_data`/`source_id`等键不在响应里,非仅信任schema。 - `backend/api/public/`(`router.py`/`items.py`/`schemas.py`/`throttle.py`),挂载 `/api/public/*`(与现有 `/api/v1/*` 并列,`main.py` 加一行 `include_router`)。`GET /api/public/items?mode=&category=&since=&q=&take=`。IP-based 内存限流中间件,只挂在 public router 上。非法 `category` 参数返回 400 + 合法值列表。 - 验收:端点集成测试(public/private 混合数据不泄露私有源)、限流触发 429+Retry-After、`PublicRecordRead` 字段白名单测试(响应体绝不含 `raw_data`/`normalized_data`)。 - -- [x] **PR-F — RSS 发布**(`457499b`,474→479 passed,零回归)。路由`/api/public/rss`,线路格式实为Atom(feedgen序列化),路径名"rss"照AIHOT命名对齐,docstring写清楚。新加`feedgen==1.0.0`依赖。复用PR-D查询+PR-E白名单mapper,不重写过滤逻辑。序列化失败兜底走手搭空Atom壳(不靠feedgen本身,防它自己炸时兜底也炸)。 - `backend/api/public/rss.py`:`GET /api/public/rss` 复用 `PublicContentService` 结果集,`feedgen` 序列化 Atom。序列化异常兜底吐合法空 `` 壳,不 500。 - 验收:产出 XML 用 `feedparser`(现有摄入用的同一库)反解 round-trip 测试通过;异常兜底路径有测试。 - -- [x] **PR-G — Daily digest**(`ce86256`,479→509 passed,零回归)。调度:本分支实际有`scheduler.py`+celery/beat但`CronSchedule.source_id`非空FK跟digest(不挂单一source)不契合,改动太侵入不值——改走混合:celery beat静态entry(`daily-digest-snapshot`,00:10 UTC,`task_executor=celery`时免费生效)+独立入口`backend/worker/digest_job.py`(`task_executor=local`时给外部OS调度器调),不新造进程内调度器。`PublicContentService`加`until`上界参数(可选,不改现有调用行为)。 - 新表 `daily_digests`(migration)+ `backend/services/digest_service.py` + 定时任务(复用现有调度基建;GOAL-4 已接通 redbeat,优先复用而不是新开一套调度)+ 端点 `GET /api/public/daily`、`/api/public/daily/{date}`、`/api/public/dailies?take=N`。 - 验收:同一天重跑两次不重复(幂等测试);空数据日的行为(无 curated 内容时不报错,返回空)。 - -- [x] **PR-H — SKILL.md 生成**(`cde20bf`,509→521 passed,零回归)。生成脚本`backend/scripts/generate_skill_md.py`直接内省PR-E/F/G的真实`APIRouter`(参数/默认值/description),不是手抄文档,配`--check`漂移模式。输出`backend/skills/agent_access/SKILL.md`,额外加`main.py`挂`StaticFiles`到`/skills`,真正served出`GET /skills/agent_access/SKILL.md`(spec原文要求的"静态文件路由 served")。CI真接了一步进`.github/workflows/ci.yml`的`backend-test`job,另加pytest漂移测试当本地权威闸(手动验证过篡改SKILL.md/taxonomy分类名都能触发失败)。**发现但没动的遗留问题**:`backend-test`job的`working-directory: backend`跟`pyproject.toml`/`tests/`实际在仓库根不一致,job本身像是已经跟别处改动脱节坏了,跟本PR无关,没有顺手改,记录在此。 - -> ✅ **GOAL-5 完成**(2026-07-08):PR-A→PR-H全落,398→**521 passed**(414/424/439/457/474/479/509/521 逐PR过点),零回归。`0f48495..cde20bf`共8个commit,分支`feat/agent-access-taxonomy`。**未push**(push等用户)。遗留:`.github/workflows/ci.yml`的`backend-test`job working-directory疑似已脱节坏了(见PR-H),不在本goal修。 - 生成脚本(从 `taxonomy.py` + 路由定义读取)产出 `backend/skills/agent_access/SKILL.md`,静态文件路由 served。CI 步骤:重新跑生成脚本,diff 已提交文件,不一致则失败。 - 验收:生成脚本单测 + CI 一致性检查文档化(写清楚怎么跑、什么时候会红)。 - -## 每 PR 验收(DoD) -1. `tests/unit` + `tests/integration` + `tests/skills` 全绿(不低于 PR-A 前的基线) -2. 老路径行为零回归(尤其 PR-C —— 改的是生产 pipeline 路径) -3. commit 仅码+测路径,`git status --porcelain` 自检无 `chat.py`/`GOAL*.md`/`HANDOFF*.md`/`AUDIT*.md`/`GRILL*.md`/`PR-DESCRIPTION.md` -4. 勾掉本文件对应项 + 一行进度(commit hash + 测试数) - -## 停止条件(真分叉才停,别瞎猜) -- 全 PR 完 -- pytest 红且 2 轮内修不动 -- 顶级分类种子值(架构决策 #4)如果实现中发现现有已采集内容明显套不进这 6 类,要不要现在改分类表 —— 停问,别自己加类目 -- 限流阈值具体数值(次数/窗口)没有强共识,给个保守默认(比如 60 req/min/IP)先落地,除非明显不合理不用为这个停 -- PR-G 复用哪套调度基建(redbeat vs 本地 scheduler.py)如果 GOAL-4 那两套还在互斥期,按 GOAL-4 已锁定的 `task_executor` gate 走,不重新纠结 -- 需要 push(push 永远等用户) - -## 后续排队(不在本 goal 内,已定顺序,未来各开一个 GOAL-N) -参照 https://github.com/langgenius/dify 的组件功能(后端+前端都要,不是只抄后端),按依赖链顺序: -1. 模型 Provider 管理(多 LLM provider 抽象) -2. Workflow 编排引擎 -3. 插件市场 -4. App 发布机制 - -这四项跟本 goal(Agent 接入 + 分类)相互独立,规模各自都够开一轮完整 brainstorming + spec,本文件不展开。 diff --git a/GOAL-6.md b/GOAL-6.md deleted file mode 100644 index 4012bca..0000000 --- a/GOAL-6.md +++ /dev/null @@ -1,89 +0,0 @@ -# GOAL-6 — 模型 Provider 管理(仿 Dify model-runtime,自研) - -> `/loop` 自驱。每轮读本文件 → 下个未 [ ] 项 → 端到端做+测绿 → auto-commit → 勾掉。 -> 命中真分叉 → 停问,别自己拍板架构。 -> 每 PR 绿即 auto-commit(显式 `git add <路径>`,绝不 `add -A`,绝不碰 `GOAL*.md`/`HANDOFF*.md`/`AUDIT*.md`/`GRILL*.md`/`PR-DESCRIPTION.md`);push 仍等用户。 - -## 背景 - -Dify 队列 item 1(见 GOAL-5 末尾排队)。main 上已有胚胎版,本 goal 升级成真子系统: - -- 已有:`ModelProvider` 表(name / provider_type∈{claude,openai,local} / base_url / api_key **已 Fernet 加密**(`backend/auth/crypto.py`,env `CREDENTIAL_ENCRYPTION_KEY`)/ default_model / enabled)+ CRUD API(`backend/api/v1/providers.py`,纯 CRUD 无 test/sync)+ 响应 masking(`has_api_key`/`api_key_preview`)+ Next.js providers 页(`frontend/app/(app)/providers/page.tsx`) -- 痛点 1:四个消费点各自为政拼 client —— `chat.py` agent 坞(私有 `_build_client`)、`skill_channel`、processors(openai/claude/local,config dict + env fallback)、`crawl4ai_channel`(litellm 间接) -- 痛点 2:`DataSource.ai_config` JSON 是绕开 ModelProvider 的平行凭证通道(双真相源) -- 痛点 3:无模型目录、无"系统默认模型"概念、无 test connection、无 failover -- 轮子边界:model-hotel(5080 自研网关)管跨 provider 聚合/凭证池/quota;app 内**只做** role 级候选顺序 failover,不重复造 - -## 坐标 - -- repo `D:\projects\opencli-admin`,新分支 `feat/model-provider-mgmt`,**从 main 切** -- **前置(用户手动)**:GOAL-5 分支 `feat/agent-access-taxonomy` 由用户 push + merge 进 main 后再开工;若开工时 main 里还没有 taxonomy 提交,停问 -- 测试闸:`uv run pytest tests/unit tests/integration tests/skills --no-cov -q`(PowerShell,`cd D:\projects\opencli-admin`;Bash 工具在此环境被 RTK hook 改写会炸) -- 基线测试数以切分支后 main 实测为准,PR-A 前先跑一遍记进本文件 - -## 已锁定的架构决策(别重新问,直接照做) - -1. **运行时自研,不引 litellm**。新包 `backend/llm/`:`base.py`(`ProviderAdapter` ABC:`chat()` / `list_models()` / `test_connection()`)、`openai_compat.py`、`anthropic.py`、`factory.py`、`resolver.py`、`catalog.py`(anthropic 硬编码模型目录常量)。 -2. **`provider_type` 枚举不动**(`openai|claude|local`),语义=adapter 族:`openai`/`local` → `OpenAICompatAdapter`,`claude` → `AnthropicAdapter`。不破坏 crawl4ai 的 litellm 前缀映射和存量数据。 -3. **`provider_models` 表**(模型目录):`id, provider_id(真 FK→model_providers, ondelete CASCADE), model_id, model_type(str, default "llm",闭集校验,v1 只有 "llm",列留 embedding/rerank 扩展位), capabilities(JSON 可空: tools/vision/context_window), source∈{discovered,manual}, enabled(bool default True), created_at`。唯一约束 `(provider_id, model_id)`。sync 是 upsert,`source=manual` 条目绝不被 sync 覆盖或删除。 -4. **`model_defaults` 表**(系统默认模型,按消费角色):`role`(闭集 `chat|executor|enrichment`,唯一)、`candidates`(JSON 有序列表 `[{provider_id, model_id}]`)。首位=主选,后位=failover 候选。role 对应:agent 坞对话 / skill_channel 便宜执行 / pipeline enrichment 兜底。 -5. **模型发现**:OpenAI-compat 打 `GET {base_url}/v1/models`(ollama/model-hotel/deepseek 等全通);anthropic 返回 `catalog.py` 常量。发现失败不崩,返回错误详情供前端展示,可手动登记兜底。 -6. **SSRF/key 外泄防护**:factory 建带 api_key 的 client 前必须过 `backend/security/url_guard.py`(`avalidate_public_url_and_ip` + `PinnedAsyncHTTPTransport`),与 main 现有 openai_processor/skill_channel 做法一致。本地地址(ollama/model-hotel)按 url_guard 现有豁免机制走,没有豁免机制则停问。加密/masking 复用现有,不重做。test connection 的错误响应绝不回显 api_key。 -7. **failover 语义**(resolver):`resolve(role)` 返回首位候选;`resolve_with_fallback(role)` 按 candidates 顺序试。**只有连接级错误(连不上/超时/5xx)才降级**;4xx 业务错误(如 401 key 错)不降级——那是配置错,降级会掩盖问题。坏 provider 进内存 cooldown(简单时间窗,进程内 dict,不引 Redis),窗口内跳过。 -8. **消费点收编范围**:`chat.py`(替 `_build_client`)、`skill_channel`、processors(openai/claude/local)三处走 factory;agent 级 `processor_config` 覆盖能力保留(provider 供底,agent config 覆盖)。**crawl4ai 例外**:client 是 crawl4ai 内部造的(litellm),只收编 provider/model/key 的**解析**走同一 resolver,litellm 调用保留,docstring 写明例外原因。 -9. **双轨收敛(软)**:`DataSource.ai_config` 支持 `provider_id` 引用;存量 inline `api_key`/`base_url` 继续能跑但记 deprecation 警告日志;新前端只给 provider 下拉。v1 不硬迁移、不删字段。`ai_agents.provider_id` 维持松散字符串列,不在本 goal 升 FK(改动收益比不值)。 -10. **API 面**(挂现有 providers router):`POST /providers/{id}/test`、`POST /providers/{id}/models/sync`、`GET|POST|PATCH|DELETE /providers/{id}/models`、`GET|PUT /model-defaults`。响应壳复用 `ApiResponse[T]`。 -11. **前端**:扩展 Next.js providers 页(shadcn/ui + react-query hooks 现有模式,不新增 zustand 用途):provider 行展开模型目录表格 + sync 按钮、test connection 按钮 + 状态徽章(延迟/失败原因)、defaults 卡(三 role 各配 candidates,可排序)、preset 列表加 model-hotel(prefill base_url)。 - -## 状态机 - -- [x] **PR-A — 数据模型**(基线 1504→**1530 passed**,+26 新测,零回归;GOAL-6 从 main `e60c473` 切 `feat/model-provider-mgmt`,该 main 已含 GOAL-7 browser-act 但**无 GOAL-5 taxonomy**——用户显式指示此序,taxonomy 非 GOAL-6 代码依赖)。`backend/models/provider_model.py`(`ProviderModel` 表 `provider_models`:**真 FK** `provider_id→model_providers.id ondelete CASCADE`+index、`model_id`、`model_type` default `llm`、`capabilities` JSON、`source`、`enabled`、唯一约束 `(provider_id, model_id)`)+ `backend/models/model_default.py`(`ModelDefault` 表 `model_defaults`:`role` 唯一、`candidates` JSON 有序)+ 注册进 `backend/models/__init__.py`。`backend/llm/__init__.py`(闭集 `VALID_MODEL_TYPES/ROLES/SOURCES`+校验 helper)+ `backend/llm/catalog.py`(`ANTHROPIC_CATALOG` 硬编码常量,决策 #5 无 /v1/models discovery:`claude-opus-4-8`/`claude-sonnet-5`/`claude-haiku-4-5-20251001`,ctx 200000)。Pydantic schema `provider_model.py`/`model_default.py`(field_validator 用 backend.llm helper 校闭集)。migration `d8e9f0a1b2c3`(down_revision=`a7v8w9x0y1z2`,scratch db 全链 base→head + downgrade round-trip 验通,repo db 被别进程锁故用 scratch)。闭集校验只在 Pydantic/backend.llm 层(SQLAlchemy 无 @validates,匹配现有 provider_type/channel_type 约定)。测试 `tests/unit/llm/` + `tests/unit/test_provider_model.py`/`test_model_default.py`(26 测:闭集 helper+schema 双层、唯一约束 IntegrityError、role 唯一、FK cascade、catalog 完整性)。 - ~~验收:migration 跑通;唯一约束生效测试;model_type/role 闭集校验单测;基线测试数记录进本文件。~~ 全达成。 - ⚠️ **PR-C/E 注意**:sqlite 默认不强制 FK,本 repo runtime(`backend/database.py`)从不发 `PRAGMA foreign_keys=ON`(已 grep 确认)——DB 级 cascade 只在 pragma 开时生效,生产删 `ModelProvider` 不会自动级联删 `provider_models`;PR-C 删 provider 时要显式清理 catalog 行或开 pragma,别假设 FK cascade 自动触发。 - -- [x] **PR-B — `backend/llm/` 运行时**(1530→**1551 passed**,+21 新测,零回归)。`base.py`(`ProviderAdapter` ABC:`chat(messages,*,model)->str` / `list_models()->list[str]`(失败 raise) / `test_connection()->ConnectionTestResult` TypedDict `{ok,latency_ms,error,models_sample}`(不 raise);`LlmAdapterError` 不含 secret;`redact_secret` helper)。`openai_compat.py`(`OpenAICompatAdapter` for openai|local,`avalidate_public_url_and_ip` 在建 `AsyncOpenAI` 前跑 + `PinnedAsyncHTTPTransport`,照 skill_channel/openai_processor)。`anthropic.py`(`AnthropicAdapter` for claude,`list_models`=`anthropic_catalog()` 决策 #5)。`factory.get_adapter(provider)` 按 provider_type 派发。**决策 #6 local-address 解**(关键):url_guard **无既有** localhost/私有豁免机制(读全模块+38测确认),故扩展 `backend/security/url_guard.py` 加 keyword `allow_private=False`(全层穿透:is_ip_blocked→_check_host_and_ips→validate*→PinnedAsyncHTTPTransport→guarded_async_client),**default False 全部现有调用行为不变**(38 url_guard 测 + 全量 1551 零回归验证);`unspecified/multicast/reserved` 恒 blocked、DNS-rebind pin 恒生效,`allow_private=True` 只放行 loopback/private/link-local/CGNAT;**唯一传 True 者=OpenAICompatAdapter 且仅 provider_type=="local"**(openai/claude 全守卫)。测试 `tests/unit/llm/test_adapters.py`(21:两 adapter chat/list_models/test_connection、url_guard 拒绝(恶意 base_url 建 client 前拒+SDK 从不被调)、api_key 不入 error(5测 redact 断言)、factory 派发、local 放行 loopback vs openai 拒同 URL + local 仍 pin)。 - ~~验收:...api_key 不出现在任何异常消息断言。~~ 全达成。 - ⚠️ **url_guard 是共享审计模块(AUDIT B3)**,本 PR 加了 `allow_private` 扩展——向后兼容(default False),但 review/合并时留意此跨切改动。 - -- [x] **PR-C — API:test + sync + 目录 CRUD + defaults**(1551→**1575 passed**,+24 新测,零回归)。`backend/services/provider_model_service.py`(薄端点,DB 逻辑在此,单一 mock 缝=`get_adapter`):`sync_models`(决策 #3:manual 行永不覆盖/删=`kept_manual`,discovered 已存=去重不违唯一约束,新=added,**stale discovered prune**=本 PR 设计选择已文档化,manual 永不 prune,幂等)、catalog CRUD(`add_manual_model` 结构上强制 source=manual)、`delete_provider_models`、`put_default`(role 闭集 + 每 candidate provider 存在 + model 在该 provider catalog,清晰错误无 key)、`test_connection`(转 adapter 结果已 sanitize)。端点(providers.py 扩展 + 新 `model_defaults.py` router 挂 __init__):`POST /providers/{id}/test`、`.../models/sync`(LlmAdapterError→502)、`GET|POST|PATCH|DELETE /providers/{id}/models[/{row}]`、`GET /model-defaults`、`PUT /model-defaults/{role}`(role 入 path,比 spec 的 GET|PUT 更 RESTful,判断改)。**provider-delete 清理**:现有 `DELETE /providers/{id}` 先调 `delete_provider_models`(PR-A 坑:sqlite 无 cascade)。测试 `tests/integration/test_provider_models_api.py`+`test_model_defaults_api.py`(24:test 成功/失败+api_key 不在 body、sync 幂等+manual 存活+stale prune、CRUD、defaults 校验 bad-role/nonexistent-provider/model-not-in-catalog、provider-delete 无孤儿)。 - ~~验收:...错误响应无 key 泄露断言。~~ 全达成(现有 providers 测试零回归)。 - -- [x] **PR-D — resolver + failover**(1575→**1599 passed**,+24 新测,零回归)。`backend/llm/resolver.py` `ProviderResolver`(注入式 monotonic clock、进程内 `_cooldown_until` dict 无 Redis、模块单例 `resolver`)。`resolve(db,role)->ResolvedModel|None`(首候选,无配置/空/provider 已删=None,不 failover)。`resolve_with_fallback(db,role,operation)`:顺序试候选——cooled→跳过不建 adapter、missing provider→跳过无 cooldown、成功→立即返、`LlmAdapterError retryable=True`(连接级)→cooldown+下一个、**`retryable=False`(4xx 业务)→立即 re-raise 不 cooldown 不 fallthrough(决策 #7 核心)**、全竭→`ResolverError`带 tried/cooled 计数无 key。`_set_cooldown` 同步读写无 await 协程安全。**错误分类**(改 PR-B 3 文件,向后兼容):`LlmAdapterError` 加 `retryable=False` kw、`base.classify_retryable(exc)` 三层(openai/anthropic APIConnection/Timeout/InternalServer→True,4xx/auth→False,`status_code>=500` 兜底,余 False),adapter chat/list_models except 传 `retryable=classify_retryable(exc)`。测试 `tests/unit/llm/test_resolver.py`(24:顺序降级+cooldown 记录、4xx 不降级+不 cooldown+B 不试、cooldown 窗口跳过+过期重试、全竭 ResolverError 无 key、20 并发一致、classify_retryable 15 例);adapters 测重跑 21/21 无回归。 - ~~验收:...并发调用下 cooldown dict 不炸。~~ 全达成。 - -- [x] **PR-E — 消费点收编**(1599→**1617 passed**,+18 新测,**零回归**——高危生产路径 PR,全量已跑验)。核心=去重 client 构造走 factory,**不改行为**。新 factory 助手 `build_openai_compat_adapter`/`build_anthropic_adapter`/`litellm_prefix_for` + `_provider_view`(SimpleNamespace,解两难:chat.py 的 live ORM provider 直接写 env-key 会被 autoflush 持久化进 DB→用抛弃视图规避;skill_channel/processors 是 dict 配置需属性访问)+ 各 adapter 加 `get_client()` 交出守卫 client(tool-loop 要裸 client)。收编:**chat.py**(`_build_client` 走 factory,`OPENAI_API_KEY` env fallback + `_pick_provider` + tool-loop 全保留;**原来无 SSRF 守卫→现在有**,决策 #6 顺带补洞,无测试覆盖故不回归)、**skill_channel**(走 factory,qwen3:4b + guard 保留)、**openai/claude processor**(走 factory + 各自 env fallback/usage 日志/JSON-mode 保留,裸 SDK response 仍自取)、**crawl4ai**(litellm 调用+LLMConfig 未动,只 `litellm_prefix_for` 收编映射)。**runner agent-override 未动**+新测锁定(provider 供底 agent processor_config 覆盖)。dead-code grep 净(无残留重复 client 构造)。测试 `tests/unit/llm/test_pr_e_consumers.py`+runner 测(18)。 - ~~验收:零回归闸...各消费点删掉的私有 client 拼装代码不残留。~~ 全达成,无改现有测试。 - ⚠️ **留白/偏差**(诚实记):(1)**resolver 未接入任何消费点**——PR-D 建好测好(24测)但 PR-E 未 wire,agent 保守判断:决策 #8 只要求"走 factory"非"走 resolver",接入会引入 role/model_defaults 选择轴改现有 provider 选择行为+风险回归;resolver 可 import 待未来 opt-in 消费点用(或另开收尾接入)。(2)**local_processor 未改**——ollama 原生 `/api/generate` 协议 + `timeout` 配置旋钮与冻结的 OpenAICompatAdapter 不匹配,强收会改 wire 行为/丢旋钮,判为不安全跳过。(3)chat.py 补了 SSRF 守卫(原缺)。 - -- [x] **PR-F — 双轨收敛(软)**(1617→**1622 passed**,+5 新测,零回归)。seam=`backend/pipeline/ai_processor.py` 新 `_resolve_llm_config(ai_config, source_id)`(process_with_ai 调):无 provider_id→`return ai_config` **字节不变**(仅 inline api_key/base_url 存在时 log deprecation 警告);provider_id 解析→`dict(ai_config)` copy 覆盖 processor_type/api_key/base_url/model(inline 也给时警告"provider_id 优先");provider_id 不解析→警告+回落 ai_config 不崩(fail-soft 同现有 posture)。DB session 仅 provider_id 存在时开(常路不碰 DB)。`ai_agents.provider_id` 未动(仍松散字符串,决策 #9)。判断:加 `resolve_provider=True` kwarg,pipeline.py 调用传 `resolve_provider=agent_config is None`(agent 级 config 经 runner 另路解析 provider_id,不走本 deprecation 逻辑避免每次 agent 运行误报)。测试 `tests/unit/pipeline/test_ai_processor.py`(5:provider_id 路径、inline 字节相同+warn(断言 `passed_config is ai_config`)、both→provider 赢、provider_id 不存在回落、resolve_provider=False 门控)。 - ~~验收:...警告日志断言。~~ 全达成(inline 路径行为字节相同)。 - -- [x] **PR-G — 前端**(决策 #11)。现状=providers 页原为纯只读 Card 网格(`createProvider/update/delete` 是死代码)。新建 `frontend/components/providers/`(`provider-form-dialog.tsx` 增改弹窗+3 预设 Claude/OpenAI/**model-hotel**、`provider-catalog-panel.tsx` 展开目录表+sync+手动添加、`model-defaults-card.tsx` 三角色候选卡可增删排序按角色保存)+ `types.ts`(修 `ModelProvider` 去假 api_key 加 `has_api_key/api_key_preview`,加 `ProviderModelRead/ConnectionTestResult/ModelRole/ModelDefault*` 等)+ `endpoints.ts`(8 端点)+ `hooks.ts`(11 react-query hooks 含 queryKey 失效)+ `page.tsx` 接线。四交互(目录展开/sync、test+徽章、defaults、preset)完整实现;**额外补了 create/edit/delete 弹窗**(超决策 #11 原文——preset 需 add-provider 表单才有落点,且后端一直有 provider CRUD 端点前端未接,接上使页面真可用)。api_key 全程 masked(密码框写入态、编辑只显 preview、page 只碰 has_api_key/api_key_preview,无处渲染原始 key)。 - ~~验收:...api_key 在 UI 全程 masked。~~ tsc `--noEmit` exit 0(独立重跑)+ eslint 净 + `next build` 成功 + 无碰 backend/tests。⚠️ **弹窗提交/mutation toast/Select 联动/惰性请求 需真实后端跑起来才能点击验证**——本轮无法起全栈(后端+DB+dev server),编译/构建/类型接线已证,合并前建议接后端手点一遍。 - -> ✅ **GOAL-6 完成**(2026-07-09):PR-A→PR-G 全落,1430→**1622 passed**(A~F 后端 1448/1467/1484... 至 1622;PR-G 前端 tsc/eslint/build 绿),零回归(既有 12 failed 全 main 遗留:opencli/workflow/nodes-install + 4 `*_live.py` 需真 Chrome)。分支 `feat/model-provider-mgmt`(从 main `e60c473` 切,7 commit `6cf3358..`)。**未 push**(push 等用户)。数据模型(provider_models+model_defaults 真 FK)+ 自研 LLM 运行时(OpenAICompat+Anthropic 双 adapter+factory,**不引 litellm**)+ url_guard `allow_private` local 豁免 + API(test/sync/目录 CRUD/defaults)+ resolver+failover(4xx 不降级+cooldown)+ 消费点收编(factory 去重 client 构造)+ ai_config 软收敛 + Next.js 前端。 -> **⚠️ 关键留白**(合并前须知):(1)**resolver+failover(PR-D)建好测好但未接入任何消费点**(PR-E 保守判断:接入改 provider 选择行为+风险回归)——model_defaults 表+API+resolver 全在,但目前无代码路径真正调用 `resolve_with_fallback`;要激活 failover 需另开收尾把消费点接上 resolver(或明确它是 opt-in 基建)。(2)**url_guard 加了 `allow_private`**(共享审计模块 AUDIT B3,向后兼容 default False,但跨切改动 review 留意)。(3)local_processor 未收编(ollama 协议+timeout 旋钮不匹配 adapter)。(4)前端交互需真实全栈点击验证。(5)`BROWSER_ACT_API_KEY`... 属 GOAL-7 无关。 - -## 每 PR 验收(DoD) - -1. `tests/unit` + `tests/integration` + `tests/skills` 全绿(不低于 PR-A 前基线) -2. 老路径行为零回归(尤其 PR-E —— 改的是生产 chat/pipeline 路径) -3. commit 仅码+测路径,`git status --porcelain` 自检无 `GOAL*.md`/`HANDOFF*.md`/`AUDIT*.md`/`GRILL*.md`/`PR-DESCRIPTION.md` -4. 勾掉本文件对应项 + 一行进度(commit hash + 测试数) - -## 停止条件(真分叉才停,别瞎猜) - -- 全 PR 完 -- 开工时 main 没有 GOAL-5 taxonomy 提交(前置未满足) -- pytest 红且 2 轮内修不动 -- url_guard 对本地地址(ollama/model-hotel)没有既有豁免机制(决策 #6) -- main 合并后发现 provider 相关结构与本设计冲突(比如别的分支也动了 ModelProvider) -- 需要 push(push 永远等用户) - -## 后续排队(不在本 goal 内,已定顺序,未来各开一个 GOAL-N) - -参照 https://github.com/langgenius/dify 组件功能,依赖链顺序: -1. ~~模型 Provider 管理~~(本 goal) -2. Workflow 编排引擎 -3. 插件市场 -4. App 发布机制 diff --git a/GOAL-7.md b/GOAL-7.md deleted file mode 100644 index 019c686..0000000 --- a/GOAL-7.md +++ /dev/null @@ -1,97 +0,0 @@ -# GOAL-7 — 内置 browser-act 采集包(vendor SKILL.md 包 + browser-act channel) - -> `/loop` 自驱。每轮读本文件 → 下个未 [ ] 项 → 端到端做+测绿 → auto-commit → 勾掉。 -> 命中真分叉 → 停问,别自己拍板架构。 -> 每 PR 绿即 auto-commit(显式 `git add <路径>`,绝不 `add -A`,绝不碰 `GOAL*.md`/`HANDOFF*.md`/`AUDIT*.md`/`GRILL*.md`/`PR-DESCRIPTION.md`);push 仍等用户。 - -## 背景 - -内置 https://github.com/browser-act/skills(MIT)—— BrowserAct 的浏览器自动化 CLI + ~30 个站点采集 SKILL.md 包(taobao/amazon/google-maps/youtube/reddit/微信/知乎 等,分 ecommerce/lead-generation/search-research/social-listening/video-platforms 五类)。作为 opencli-admin 的一个新采集 channel + vendored 包目录。 - -**上游仓三块**(参考克隆在 scratchpad,已看过): -- `browser-act` CLI —— 外部工具(`uv tool install browser-act-cli --python 3.12`),提供 session 制原语 `navigate`/`wait`/`eval`/`state`/`click N`/`input N`;`browser-act get-skills core` 出环境态。**本 goal 不 vendor CLI 本体**(它是外部 PyPI 工具),只 vendor 包 + 写 channel 壳调它。 -- `solutions/` ~30 个包 —— 每个 = `SKILL.md`(散文运行手册,给 agent 读)+ `scripts/*.py`(纯 JS 发射器:argparse → `print(js字符串)`,无 LLM/无网络/无文件读写)。 -- `browser-act-skill-forge`(包生成器)—— **本 goal 不碰**,YAGNI。 - -**命名雷(GOAL-5 已警告"别混")**:opencli-admin main 上现有 `backend/skills/` + `Skill(domain, capability)` DB 表是**完全不同的东西**(record→distill→执行环,DB 存储,无文件包)。browser-act 的 SKILL.md 是**文件包**。本 goal 的 vendored 包与现有 DB Skill 子系统**完全隔离**,不共用表、不共用 `skill_channel`,不写 pack→DB 导入器(架构决策 #2)。 - -**执行契约(已实测确认)**:SKILL.md 的确定性部分 = `navigate {url}` → `wait stable` → `eval "$(python scripts/x.py {params})"` → 收 JSON。散文里的 `$(...)` 是 shell 替换语法(注入风险);本 channel **绝不走 shell**,两跳都 argv-only(见架构决策 #6)。登录/反爬是散文里的 LLM/人工部分,本 channel 故意不做,遇到就 fail loudly(架构决策 #4)。 - -## 坐标 - -- repo `D:\projects\opencli-admin`,新分支 `feat/browser-act-channel`,**从 main 切**(main 有 channel 子系统全套:`AbstractChannel`/registry/`cli_channel`/`opencli_channel`/`crawl4ai_channel`;本 worktree 的 `feat/agent-access-taxonomy` 没有,别在这切) -- 排期:GOAL-6(模型 Provider)先跑;本 goal 正交于 Provider,选了 script-runner 不依赖 GOAL-6,可并行/后排 -- 上游参考克隆:scratchpad `browser-act-skills/`(实施时重新 `git clone --depth 1` 拿最新,别信旧副本) -- 测试闸:`uv run pytest tests/unit tests/integration tests/skills --no-cov -q`(PowerShell,`cd D:\projects\opencli-admin`;Bash 工具在此环境被 RTK hook 改写会炸) -- 基线(feat/browser-act-channel 从 main edbb1ca 切,PR-A 前实测):**1430 passed, 12 failed, 6 skipped**。12 failed 全是 main 既有(opencli channel/workflow/nodes-install + 4 个 `tests/skills/*_live.py` 需真 Chrome),与本 goal 无关,DoD"全绿"=不新增失败 - -## 已锁定的架构决策(别重新问,直接照做) - -1. **执行模型 = 确定性 script-runner,不引 LLM**。channel 按 manifest 步骤序列驱动 browser-act 子进程:`navigate` → `wait` → 跑 `python scripts/x.py ` 拿 JS → `browser-act ... eval ` → 解析 JSON → 收 items。无 perceive/gate/act 环,不碰 ModelProvider。(LLM-runner 是未来可选二期,本 goal 不做。) - -2. **与现有 DB Skill 子系统完全隔离**。vendored 包是**文件**,不进 `skills` 表,不复用 `backend/skills/`(那是 DB 的 record→distill 系统)。新建独立包目录 + `PackCatalog`(扫目录)。不写 pack→Skill 导入器(两种格式强映射阻抗大,YAGNI)。命名统一用 "pack"/"browser_act_pack" 而非 "skill",避免与现有概念撞名。 - -3. **vendored 包目录**:`backend/browser_act_packs///`,内含**原样** `SKILL.md` + `scripts/*.py`(不改上游内容,作出处 + 人类参考 + 上游 `git pull` 刷新)+ **我们新写的** `channel.manifest.json`(机器可读执行契约,见 #5)。保留上游 `LICENSE`(MIT)+ 顶层 `backend/browser_act_packs/VENDOR.md` 记来源 commit/URL/署名(MIT 合规)。 - -4. **登录/反爬 = fail loudly**,不自动化。scripts 吐的 JSON 若含 `{error: true, message: "...login..."}` 或页面判据失败,channel 返回 `ChannelResult(success=False, error_type="needs_human", error=<原因>)`,不吞、不猜、不重试绕过。这是采集边界=用户手动能看到的数据(照抄上游 SKILL.md 的"operational boundary"声明),不越权破鉴权。 - -5. **`channel.manifest.json` schema**(我们新写,每包一份):`{domain, capability, param_schema:[{name, required, default, enum?}], steps:[{op: "navigate"|"wait"|"eval_script"|"click"|"input", ...}], pagination:{mode, url_template?, page_param?, stop_when?}, success:{min_count, required_field?}}`。channel = 这份 manifest 的通用解释器,不为每包写死代码。手写 SKILL.md 散文→manifest 的翻译过程记进 VENDOR.md。 - -6. **子进程安全**(复用 `cli_channel`/`opencli_channel` 的 `asyncio.create_subprocess_exec` + `wait_for(timeout)` + `TimeoutError→kill()` 模式,别新造): - - **两跳全 argv-only,绝不 shell**:`python scripts/x.py ...`(argv 列表)拿 stdout JS;再 `browser-act ... eval `(argv)。用户参数经 argv 传,永不字符串插值进 shell/JS 模板。 - - browser-act 二进制走专属 env `BROWSER_ACT_BIN`(默认 `browser-act`),照 `opencli_channel` 的 `OPENCLI_BIN` 做法(固定二进制,无需 `CLI_CHANNEL_ALLOWED_BINARIES` 那种任意二进制 allowlist)。 - - vendored 包 = 信任边界(vendor 时人工审 + git 钉死);v1 用户**不能**上传/新增任意包(那是 skill-forge,不在本 goal)。 - - 超时:navigate/eval 每步 env `browser_act_timeout`(默认 120s,进 `backend/config.py` Settings,与现有 `opencli_timeout` 同款)。 - -7. **browser mode / 凭证**:server 端默认 `chrome-direct`(CDP,免 signup);`stealth` 模式需 BrowserAct API key,存 `SourceCredential`(加密,复用 `AuthManager.store/resolve`,别明文别新造凭证系统),key_name=`browser_act_api_key`。channel `collect()` 前经 `AuthManager.resolve` 取 key 注入子进程 env,错误响应绝不回显 key。 - -8. **channel 契约**:实现 `AbstractChannel`(`channel_type="browser_act"`,`collect(config, parameters)` + `validate_config(config)` + `health_check`)。`channel_config` schema:`{pack: "/" 或 domain+capability, params: {...}, mode: "chrome-direct"|"stealth", max_pages?}`。`validate_config` 校验 pack 存在于 PackCatalog + 必填 param 齐 + mode 合法。`health_check` = `browser-act --version` 子进程探活。`capabilities = Capabilities(paginated=True, session_affinity=True)`(browser-act session 有状态)。 - -9. **前端上架**(照 main 现有硬编码模式,别新造 API):`browser-act` 加进 `frontend/lib/api/types.ts` 的 `channel_type` union + `frontend/app/(app)/sources/page.tsx` 的 `CHANNEL_LABEL`(中文标签"浏览器采集/BrowserAct");preset 从 `PackCatalog`(新 `GET /api/v1/browser-act/packs` 端点)拉包列表填一键配置。 - -10. **CLI agent 化摩擦(记录,不硬解)**:上游 browser-act SKILL.md 要求"每条命令前先 `get-skills core`、别截断输出"—— 那是给 agent 的指令。本 channel 把 CLI 当受控子进程(固定 session、我们管生命周期),按需在 session 开头调一次 `get-skills core` 取环境/browser 选择态,不把它当交互 agent。若实测发现 CLI 强依赖交互确认(browser 创建确认门)无法非交互跑通 —— **停问**,别硬灌 yes。 - -## 状态机 - -- [x] **PR-A — vendor 包 + PackCatalog**(1430→**1448 passed**,+18 新测,零回归)。上游 `a23131e`(browser-act/skills)`solutions/**` 原样 vendor 进 `backend/browser_act_packs///`(**78 个包** 194 文件,Get-FileHash 逐文件核对 0 mismatch,非"~30"—— 上游实际 78,决策已用 `>=20` 断言不写死)+ `LICENSE`(MIT)+ `VENDOR.md`(commit/URL/署名)+ `SOLUTIONS-README.md`。`catalog.py`=`PackCatalog`(rglob SKILL.md,YAML frontmatter 取 name/description,domain=category/capability=pack 目录名从布局派生;**发现并修**:`social-listening/reddit-warmup/SKILL.md` 带 UTF-8 BOM 致 `startswith("---")` 漏包,改 `utf-8-sig` 读,只动 catalog.py 非 vendored 字节)。`manifest.py`=`PackManifest` schema(决策 #5)+`load_manifest`,无 manifest 内容(留 PR-D)。测试 `tests/unit/browser_act_packs/`(catalog 扫 ≥20 + BOM/坏 frontmatter skip + get_pack + manifest 校验,18 测)。 - ~~验收:catalog 扫出 ~30 包(数量断言)、frontend 解析不崩、非法/缺 frontmatter 包被跳过并记警告的单测;`VENDOR.md` 存在且含 commit hash;基线测试数记录进本文件。~~ 全达成。 - -- [x] **PR-B — browser-act CLI 封装**(1448→**1467 passed**,+19 新测,零回归)。新包 `backend/browser_act/`(≠现有 `backend/cli.py` opencli HTTP client、≠ `browser_act_packs/`):`cli.py` = `_run`(`create_subprocess_exec` + `wait_for(timeout)` + `TimeoutError→kill()+wait()`,照 `cli_channel.py`)、`version()`/`get_skills()`、`session(name, env)` async ctx mgr → `BrowserActSession`(navigate/wait/eval/state/click/input/run,每条前置 `--session `)。binary 走 `BROWSER_ACT_BIN` env 调用时读(照 opencli `OPENCLI_BIN`),`browser_act_timeout=120` 进 Settings。**全 argv-only**(#6):仅 `create_subprocess_exec` 无 shell,用户值(URL/JS/input)恒单 argv 元素;`BrowserActError` 错误文本不含 env(secret 走 env 不入 argv/日志)。session `__aexit__` no-op(#10:browser open/close 归 PR-C,不对称清理会误拆调用方仍需的 session,已注释)。CLI 命令面已对上游 `docs/commands.md` 核实。测试 `tests/unit/browser_act/test_cli.py`(19 测:argv 精确断言、注入安全断言 `create_subprocess_shell` 从不被调+危险串单 argv 逐字、timeout kill、非零退出错误不含注入的 api_key、`BROWSER_ACT_BIN` 覆盖、真 `sys.executable` 往返)。 - ~~验收:mock 子进程...参数含 shell 元字符时不注入的测试(断言走 argv 非 shell)。~~ 全达成。 - -- [x] **PR-C — BrowserActChannel + manifest 解释器**(1467→**1484 passed**,+17 新测,零回归,全量已跑验)。`backend/channels/browser_act_channel.py`=`BrowserActChannel(AbstractChannel)` `channel_type="browser_act"` `@register_channel`,`Capabilities(paginated=True, session_affinity=True)`;通用 manifest 解释器(无 per-pack 码,#1/#5):resolve pack(PackCatalog)→`load_manifest`→逐 step 驱动(navigate/wait/eval_script/click/input),`eval_script`两跳=`run_pack_script`(新 `backend/browser_act/scripts.py`,`create_subprocess_exec(sys.executable,...)` argv-only #6,`ScriptError`)拿 JS→`sess.eval`拿 JSON。**登录/反爬**(#4):JSON `{error:true,message}` 经 `_classify_error` 命中 auth 关键词(login/captcha/verify/登录/验证/人机…)→`error_type="needs_human"`立即停不重试;其余 error 归 `"error"`。分页仅解释 `url_page`+`result_count ✅ **GOAL-7 完成**(2026-07-08):PR-A→PR-E 全落,1430→**1504 passed**(1448/1467/1484/1492/1504 逐 PR 过点),零回归(既有 12 failed 全 main 遗留:opencli/workflow/nodes-install + 4 `*_live.py` 需真 Chrome,与本 goal 无关)。`d72c0e8..` 共 5 commit,分支 `feat/browser-act-channel`(从 main `edbb1ca` 切)。**未 push**(push 等用户)。vendor 78 包(byte 不改)+ browser_act CLI 封装 + BrowserActChannel manifest 解释器 + 2 seed manifest + 凭证/端点/前端。**留白**:~76 包无 manifest(schema 是扩展点,增量补)、api-skill 包异形态(脚本直打 API 无 browser session)本 interpreter 不建模、google 多页需 `start=(page-1)*num` 算术当前不支持、interpreter 不 URL-encode 参数、前端无 source 创建 UI 故 packs hook 暂未消费、`BROWSER_ACT_API_KEY` env 名是假设(上游无文档)。 - -## 每 PR 验收(DoD) - -1. `tests/unit` + `tests/integration` + `tests/skills` 全绿(不低于 PR-A 前基线) -2. 老路径行为零回归(尤其 PR-C/E —— 动 channel registry + API 路由) -3. commit 仅码+测+vendored 包路径,`git status --porcelain` 自检无 `GOAL*.md`/`HANDOFF*.md`/`AUDIT*.md`/`GRILL*.md`/`PR-DESCRIPTION.md` -4. 勾掉本文件对应项 + 一行进度(commit hash + 测试数) -5. vendored `SKILL.md`/`scripts` 内容零改动(git diff 自检:只新增,不改上游文件字节) - -## 停止条件(真分叉才停,别瞎猜) - -- 全 PR 完 -- 从 main 切分支时 channel 子系统结构与本设计冲突(别的分支也动了 registry/AbstractChannel) -- browser-act CLI 强依赖交互确认门,非交互跑不通(#10) -- pytest 红且 2 轮内修不动 -- seed 包全都需登录、无一能在无凭证下端到端验证(PR-D 选包卡住) -- 需要 push(push 永远等用户) - -## 明确不做(YAGNI 裁决,别加) - -- 不 vendor browser-act CLI 本体(外部 PyPI 工具) -- 不做 skill-forge(用户上传/生成任意包) -- 不做 LLM-runner(登录/反爬自愈) -- 不写 pack→DB Skill 表导入器 -- 不改上游 SKILL.md/scripts 字节 -- 不给全 ~30 包写 manifest(seed 2-3 个验证管道,余量增量补) diff --git a/GOAL-agent-runtimes.md b/GOAL-agent-runtimes.md deleted file mode 100644 index 94775d3..0000000 --- a/GOAL-agent-runtimes.md +++ /dev/null @@ -1,146 +0,0 @@ -# GOAL: Pluggable Agent Runtimes on Fleet Edge Nodes - -Status: PROPOSAL (2026-07-03) — research done, design settled, awaiting implementation green-light. -Owner intent (user, 2026-07-03): the edge agent (Docker agent / agent_server.py) should be able to -run agentic workflows built on multiple frameworks — LangGraph, VoltAgent, pi — behind ONE -decoupled, reusable abstraction. Framework choice must never leak past the adapter boundary. -Reference patterns: OpenAlice `feature/openalice-dev` (`src/workspaces/` CLI-adapter layer). - ---- - -## 1. Why process-level, not library-level - -The three frameworks span two runtimes: - -| Framework | Lang | Native external-invocation surface | Weight | -|---|---|---|---| -| LangGraph (langchain-ai) | Python | `langgraph dev` local HTTP server — Assistants/Threads/Runs API + SSE (open-source, not Platform-gated) | heavy (langchain-core stack) | -| VoltAgent | TS/Node | embedded REST server (Hono/Elysia): `POST /agents/:id/stream` SSE, OpenAPI 3.1 spec | moderate (Node + ai-sdk) | -| pi (earendil-works) | TS/Node | `--mode rpc`: stdio JSONL RPC, purpose-built for subprocess embedding; `--mode json`; `-p` one-shot | lightest (pinned npm shrinkwrap) | - -A Python `import`-based abstraction can only ever cover LangGraph. Therefore the adapter contract -is a **process/protocol contract**: each runtime runs as a subprocess or local sidecar in its own -env (venv / node_modules), and the adapter translates its native stream (SSE / JSONL) into one -normalized event set. This is exactly OpenAlice's proven split: subprocess CLI adapters -(`workspaces/`) kept separate from in-process SDK providers (`ai-providers/`) — different failure -modes, different contracts. We build the subprocess layer. - -## 2. Core contract (new module `backend/agent_runtimes/`) - -Mirrors `backend/channels/` conventions (AbstractChannel / Capabilities / registry decorator): - -```python -@dataclass(frozen=True) -class RuntimeCapabilities: - transport: str # "stdio" | "http" - streaming: bool = True - resume_by_id: bool = False # can reopen a session by launcher-assigned id - checkpoint: str = "none" # none | memory | sqlite | postgres - concurrent_sessions: bool = True - -@dataclass -class AgentTask: - task_id: str - workflow: str # runtime-native workflow/agent identifier - input: dict[str, Any] - config: dict[str, Any] # runtime-specific (model, tools, cwd, ...) - session_id: str | None = None # resume handle - -# Closed tagged-union event set — adapters normalize INTO this, callers never -# see framework-native shapes. (OpenAlice lesson: normalize the protocol, not -# the output; keep the set tiny and closed.) -RuntimeEvent = {"type": "started" | "text" | "tool_call" | "tool_result" - | "state" | "done" | "error", ...} - -class RuntimeAdapter(ABC): - runtime_type: str # "pi" | "langgraph" | "voltagent" - capabilities: RuntimeCapabilities # data flags — callers branch on these, never isinstance - - @abstractmethod - async def invoke(self, task: AgentTask) -> AsyncIterator[dict]: ... # yields RuntimeEvents - @abstractmethod - async def health(self) -> bool: ... - @abstractmethod - def validate_config(self, config: dict) -> list[str]: ... - async def bootstrap(self) -> None: ... # one-time env/config setup (OpenAlice bootstrap()) -``` - -Registry: same decorator pattern as `channels/registry.py`; discovery import in `__init__`. - -Split-by-concern composition (from OpenAlice `CliAdapter`): argv composition, env composition, -provider-config translation, and session-id acquisition are separate small methods on stdio -adapters — never one monolithic `spawn()`. - -Session identity ⊥ process lifetime (OpenAlice registry/pool split): a small durable -`RuntimeSessionRegistry` (session_id, runtime_type, resume hint, state) survives agent restarts; -the ephemeral process/connection pool does not. - -## 3. Per-runtime adapters - -- **`pi`** (P0, first): subprocess `pi --mode rpc`, LF-delimited JSONL over stdio. Same shape as - our existing shell-out pattern — least new plumbing. Steal from OpenAlice `adapters/pi.ts`: - `--session-id` create-or-reopen (launcher assigns ids → resume_by_id=True), provider override - via `PI_CODING_AGENT_DIR` redirect, skills injection into `/.pi/skills`, tools via - CLI-shim-on-PATH since pi speaks no MCP. -- **`langgraph`** (P1): local sidecar server (open-source `langgraph-api`, the `langgraph dev` - machinery) on a loopback port; adapter = httpx client speaking Assistants/Threads/Runs + SSE. - Best checkpoint story (Postgres/SQLite checkpointers) → the runtime for long/resumable - workflows. Sidecar lifecycle owned by our supervisor (below). -- **`voltagent`** (P2): Node sidecar with `@voltagent/server`, REST+SSE per its OpenAPI 3.1 spec. - Bespoke schema (no A2A/OpenAI-compat confirmed) → adapter does the translation, nothing else does. - -Sidecar supervisor in agent_server: spawn on first use, health-probe, restart-with-backoff, -SIGTERM watchdog + kill grace (OpenAlice headless-task pattern; plain subprocess, never PTY — -PTY mangles JSON streams). - -## 4. Wire protocol extension (center ⇄ edge) - -Today: ws reverse channel carries `collect` → single `result` (request/response, -`ws_agent_manager.resolve_response`). Agent runs are long and streaming: - -- New message types: `agent_task` (center→edge), `agent_event` (edge→center, many, - carries request_id + one RuntimeEvent), final `agent_result`. -- `ws_agent_manager` grows a per-request event callback/queue alongside the existing - single-shot future (existing collect path untouched). -- **Runtime advertisement**: register handshake gains `runtimes: ["pi", ...]` — the center - learns node capabilities the same way it learns mode/node_type today; scheduler can route - agent tasks only to nodes advertising the runtime (analog of `session_affinity`). -- Fleet auth: already covered — ws handshake carries the bearer token (8fab4fe). - -## 5. Center side - -- Thin `agent_channel` (AbstractChannel impl): declares capabilities, one fetch() = one agent - run dispatched via the reverse channel; runner keeps owning retry/rate/cursor. Agent runs - emit the same evidence (accepted/error_kind → SourceMeasurement) so the control loop - (PR-Control-*) covers agent tasks with zero new sensor machinery — agent runtimes are just - another 被控对象 class with different observability/controllability. -- MCP callback surface (P1+): expose center MCP endpoint to edge runtimes so agents report - structured results by calling back (OpenAlice `inbox_push` inversion) instead of us parsing - heterogeneous stdout. opencli-admin already ships `backend/mcp_server.py` — reuse. - -## 6. Docker image layering - -Base image = agent_server only (unchanged). Runtimes are opt-in layers/build-args: -`INSTALL_RUNTIME_PI=true` (adds node + pinned pi), `INSTALL_RUNTIME_LANGGRAPH=true` -(pip layer), `INSTALL_RUNTIME_VOLTAGENT=true` (node layer). Image advertises what it has -via the register handshake — no config drift. LLM API keys: node-local env first (P0); -center-side encrypted distribution later (provider api_key store already exists, `06684a7`). - -## 7. Phasing - -- **P0**: contract + registry + pi adapter (stdio) + ws `agent_task`/`agent_event` protocol + - runtime advertisement + tests. Proves the seam end-to-end with the lightest runtime. -- **P1**: LangGraph sidecar adapter + sidecar supervisor + center `agent_channel` + - session registry (resume) + MCP callback. -- **P2**: VoltAgent adapter + control-loop evidence integration + credential distribution + - UI (node runtime badges on the topology canvas). - -## 8. Non-goals / rejected - -- In-process Python embedding of LangGraph as the primary path (couples versions, blocks - TS runtimes from ever being first-class; kept possible later as an optimization behind the - same adapter contract). -- Normalizing framework semantics (graph vs supervisor vs tool-loop) — we normalize only the - run/stream/result protocol; workflow definitions stay runtime-native in `AgentTask.config`. -- A2A/AG-UI as the wire format now — only LangGraph speaks AG-UI today; revisit if a second - runtime adopts it. diff --git a/GOAL.md b/GOAL.md deleted file mode 100644 index c6a909d..0000000 --- a/GOAL.md +++ /dev/null @@ -1,76 +0,0 @@ -# GOAL — opencli-admin strangler-fig 自动环 - -> `/loop` 自驱目标文件。**每轮重读本文件** → 取下个未完 PR → 端到端做(码+测) → -> pytest 绿闸 → 自动 commit(仅列出路径) → 勾掉状态+记一行进度 → 下一轮。 -> 命中任一停止条件 → 停+报，**别瞎猜**。 - ---- - -## 北极星 -接一个正经数据源 ≈ 100 行，只写它独有的「发一次请求 + 解析成条目」。 -横切脏活(刷 token / 翻页 / 限速 / 存游标 / 写目的地)全归框架，渠道不碰。 -迁移 = **Strangler Fig**：旧路径不破坏、新路径旁路验证、逐源切主路。 -**绝不为新架构破坏旧行为。** - -## 坐标 -- repo: `D:\projects\opencli-admin` 分支: `refactor/thin-channel-thick-runner` -- 测试闸: `uv run pytest tests/unit --no-cov -q`（须全绿，当前基线 347） -- 跑测试用 PowerShell（`cd D:\projects\opencli-admin; uv run ...`）；Bash 被 RTK hook 改写易炸。 -- ⚠️ **永不 stage**: `backend/api/v1/chat.py`、`PR-DESCRIPTION.md`、`HANDOFF-strangler-fig.md`、`GOAL.md`（用户 dock WIP + 本控制文件）。 - -## 提交策略（用户 2026-07-01 授权，仅限此 goal） -每 PR 绿即**自动 commit**。granular，一 PR 一 commit。提交前**显式 `git add <精确路径>` + `git diff --cached --name-only` 自检**，绝不 `git add -A`/`add .`。**push 仍等用户**。 - ---- - -## 状态机（每轮更新） -- [x] **PR1** — LegacyDbSink 写缝（`b33416a`，行为零变） -- [x] **PR2** — 锁旧 ODP 契约 + `backend/odp/`{schemas,mapper} + odp_client 走 mapper（`532291b`） -- [x] **PR3** — ODP forward→`OdpSink` + `LegacyDbSink(forward_to_odp)` gate + `DualSink` 不双发（`74ac704`，355 passed） -- [x] **PR4** — `write_strategy` 状态机→选 sink + column/migration `o5j6k7l8m9n0`（`db10450`，365 passed） -- [x] **PR5a** — DB cursor 表+migration `p6k7l8m9n0o1` + `DBCursorStore` + RSS `fetch()` etag/304 增量 + `identity()`=item id（`9cbfb80`，374 passed；纯加性，未碰 live pipeline） -- [x] **PR5b** — collect `collect()`→`run_channel()`(incremental opt-in) + cursor 后置 commit(sink durable 后)（`09e4860`，379 passed） - -> ✅ **GOAL 完成**（2026-07-01）：strangler-fig 重构 PR2→PR5b 全落，325→**379 passed**，零回归，每刀行为零变。**未 push**（push 等用户）。范围外后续：AuthManager+加密凭据、session affinity 泛化。 - -### ✅ PR5b 分叉（已决议 2026-07-01：安全切——仅 incremental opt-in + cursor 后置 commit） -1. **路由范围**：(a) 仅 `capabilities.incremental` 渠道走 run_channel(opt-in strangler，RSS先)；(b) 全渠道切 run_channel；(c) 仅 RSS 显式特判。 -2. **cursor 前进时机**：现 `run_channel` 每页 fetch 后立即 save(channel_runner.py:82-84)。规则要「只在进可靠写入层才前进」→ 需重构：(a) run_channel 返回 (items, pending_cursor)，pipeline sink 写成功后才 commit cursor；(b) 把 sink 注入 run_channel，每页写+commit(保翻页 resumability 但耦合 runner↔sink)。 -3. **durability 判据**：odp_only/odp_primary 下「durable」= ODP 真落(Redis Stream queued 算不算？)；memory-only 假 202 不能让 cursor 前进。 - -## 每 PR 验收（DoD） -1. 全 `tests/unit` 绿（≥ 上一基线） -2. 旧路径行为零变 / 新路径有 characterization 或新测护栏 -3. commit（仅码+测路径，自检 staged 集） -4. 更新本文件状态框 + 一行进度 - -## 停止条件（任一 → 停+报） -- 全 PR 完 -- pytest 红且 2 轮内修不动 -- **真分叉**：设计有多条不等价路 / 必须破坏旧行为 / 缺凭据或外部依赖 / 要碰 WIP 文件 -- 需要 push（push 永远等用户） - ---- - -## PR 详细规格（出自 HANDOFF §4 + memory `opencli-admin-channel-runner-refactor`） - -### PR3 — OdpSink + 双发陷阱解 -- **双发陷阱**：`storer.py:34-45` 在 `ODP_INGEST_URL` 设了时已 forward 到 ODP（上游既有 shadow）。所以 `LegacyDbSink` 现含此 forward。将来 `DualSink(LegacyDbSink+OdpSink)` 会**双发** → 污染 shadow。 -- **做**： - 1. `LegacyDbSink(forward_to_odp: bool = True)` 加 gate；storer 的 forward 受此控制（默认 True = 行为零变）。 - 2. 新 `backend/pipeline/sinks/odp_sink.py` `OdpSink`：normalize → `odp_client.forward_triples`（复用 PR2 mapper），`SinkResult.records=[]`（forward-only，AI/notify no-op）。accepted=queued、duplicates、rejected 按 ODP 响应。 - 3. 新 `DualSink(legacy=LegacyDbSink(forward_to_odp=False), odp=OdpSink)`：legacy 写 DB（不 forward）+ OdpSink 发**一次**；ODP 失败不阻断 legacy。 -- **验收**：legacy 模式同 PR1；odp_shadow=legacy 写 DB + ODP 发一次（非两次）；ODP 失败 legacy 照常。 - -### PR4 — write_strategy 状态机 -- `data_sources.write_strategy` ∈ {legacy / odp_shadow / odp_dual_required / odp_primary / odp_only} → 选 sink。 -- 一旦显式策略，ODP forward 不能再藏 storer 里（PR3 已把它收进 sink）。 -- 选 sink 的工厂 + pipeline 注入点（`run_pipeline(sink=)` 已存在）。 - -### PR5 — RSS 真实竖切（含原 Phase 1b） -- `source_cursors` 表 + alembic migration + `DBCursorStore`（实现 PR1a 的 `CursorStore` Protocol）。 -- `RSSChannel.fetch()` 走 etag/If-None-Match 增量（304 = 无新条目）；`identity()` = item id。 -- **规则**：cursor **只在数据进了可靠写入层才前进**；prod 的 odp-ingest 不能 memory-only 假 202。 - -## 其后（不在本 goal 范围，到此停） -AuthManager + 加密凭据（堵 `channel_config` 明文 key）；会话亲和 `pipeline.py:45-56` 特判 → `Capabilities.session_affinity` 泛化 + 按域名并发上限。 diff --git a/GRILL-KICKOFF.md b/GRILL-KICKOFF.md deleted file mode 100644 index 6f58e22..0000000 --- a/GRILL-KICKOFF.md +++ /dev/null @@ -1,49 +0,0 @@ -# Grill Kickoff — opencli-admin 收口定调 - -> 用法:新 session,cwd=`D:\projects\opencli-admin`,跑 `/grill-with-docs`,把本文件当输入喂进去。 -> 目的:对齐收口目标 → 裁剪残留清单 → 出 CONTEXT.md/ADR 收口宪法。收口 = 裁剪,不是做完。 - -## 现成输入(盘问前先读) - -- `AUDIT-cybernetic-remediation.md` — 控制论审核残留账 -- `docs/CONTROL_THEORY_ARCHITECTURE.md` — 控制回路架构 -- Memory: `opencli-admin-cybernetic-audit`(Control-4 硬规格:recovery rate 阈值 per-state gate automatic mode)、`opencli-admin-channel-runner-refactor`(留白清单) - -## 已定事实(不许在盘问里重新翻案) - -- PR-Control-3(advisory 决策引擎)+ 3.5(证据台账 `control_actions` + outcome 判定 + recovery 报表)已推 fork = 4f3b2fe -- 控制回路语义已定:Advisory-Gated Automatic Execution;recovery 阈值 per-state 门禁 automatic -- PR#4(薄渠道+厚 runner)已合 main = f731897 -- 回测/画布等其他摊子与本收口无关 - -## 盘问必须逼出答案的问题 - -1. **收口线定义**:Control-4 actuator 落地算完?还是 advisory 攒证据阶段就封版?证据要攒几天(天然分界)? -2. **残留裁剪**(每项:进收口 / 弃权写 ADR): - - crawl4ai call-time SSRF - - per-source objective 存储 - - odp-store 心跳 producer(Rust) - - error_kinds histogram - - trend fallback - - 前端 3 处遗留 / CLI ctrl+c / opencli_channel 路由(channel-runner 留白) -3. **悬而未决**: - - 部署面:纯本机 vs LAN(→鉴权 P1/P2) - - gitea push 凭证 - - topology ODP 节点 -4. **每项验收标准**:测试断言级,不是"做了"。 - -## 收口后流程(同一窗,不 compact 不断窗) - -grill → `/to-prd` → `/to-issues`。多 session build(Control-4 单 PR 都嫌大)。 - -## Issue 模板硬规则(写死进每个 issue,喂 Sonnet 5 子 agent) - -- 契约 pin 死:endpoint schema、复用 `control_actions` 表(mode=automatic / executed=True)、零变异测试:原断言不许动 -- HARD RULE:禁 Agent 工具、禁 commit(主模型验收后统一 commit) -- 验收闸门写进 issue:全量 pytest + cov≥80 + alembic 单 head -- 复用现成机制,不手搓(memory: feedback-reuse-wheels) - -## 不走的岔路 - -- `/triage` 不用(issue 全自产) -- `/prototype` 不用(控制回路语义已定,没有跑起来才能答的问题) diff --git a/HANDOFF-strangler-fig.md b/HANDOFF-strangler-fig.md deleted file mode 100644 index 0de2b97..0000000 --- a/HANDOFF-strangler-fig.md +++ /dev/null @@ -1,88 +0,0 @@ -# HANDOFF — opencli-admin 渠道系统 Strangler Fig 重构 - -> 用法：开一个**新 session**（本窗已过 smart zone），先读这份文件 + 读 memory -> `opencli-admin-channel-runner-refactor`，然后从 **PR2** 接着干。一口气做完一刀再 commit。 - ---- - -## 0. 一句话北极星 - -接一个新数据源 ≈ 100 行，只写它独有的「发一次请求 + 解析成条目」。刷 token / 翻页 / 限速 / 存游标 / 写目的地 —— 全归框架，渠道不碰。 - -迁移打法 = **Strangler Fig**：旧路径不破坏、新路径旁路验证、逐源切主路。**绝不为新架构破坏旧行为。** - ---- - -## 1. 坐标 - -- 机器：5090，`D:\projects\opencli-admin`（FastAPI Python） -- 分支：`refactor/thin-channel-thick-runner` -- remotes：`origin`=xjh1994(上游，**零接触**)、`fork`=2233admin(我们的，已推)、`gitea`=Curry 镜像 -- 测试：`uv run --directory D:\projects\opencli-admin pytest tests/unit --no-cov -q`（现 **325 passed**） -- ⚠️ 同仓 `backend/api/v1/chat.py`(M) + `PR-DESCRIPTION.md`(??) = 用户 dock WIP，**不碰、不提交、不 stage**。 - ---- - -## 2. 已落（提交在分支上，已推 fork） - -| commit | 内容 | -|---|---| -| `b33416a` | **PR1** — LegacyDbSink 写缝（行为零变） | -| `56fa0c4` | Phase 1a — 厚 runner 地基（cursor store + 限速重试 client + 翻页） | -| `de52c25` | Phase 0 — 加厚渠道契约（Capabilities/FetchContext/FetchResult） | - -「Phase 0/1a」= runner 层（`channel_runner.py`/`cursor_store.py`/`http_client.py`），仍有效复用。 - -### PR1 装了什么（写缝） - -`backend/pipeline/sinks/`： -- `base.py` — `ItemSink` Protocol（一个方法 `write_batch(ctx, items) -> SinkResult`）+ `RunContext`(task_id/source_id/provider/ingest_mode/run_id) + `SinkResult`(accepted/duplicates/rejected/normalized/**records**/errors)。 - - `SinkResult.records` 回带 ORM 行 —— `pipeline.py` 后续 ai/notify 依赖它。**必须存在**，否则 PR1 不是行为零变。 - - accepted/duplicates/rejected 语义按**各 sink 自己的 durable 边界**写死（legacy accepted=已插入行 ≠ odp accepted=已入队）。 -- `legacy_db_sink.py` — `LegacyDbSink` 包现有 `normalizer.normalize_items` + `storer.store_records`，行为照旧。 -- `pipeline.py` — step2+3 改成 `active_sink.write_batch()`；`run_pipeline` 加 `sink=` 注入口（默认 LegacyDbSink）。 - ---- - -## 3. ⚠️ 双发陷阱（最大坑，PR3 必解） - -`backend/pipeline/storer.py:34-45` 在 `ODP_INGEST_URL` 设了时**已经 forward 到 ODP**（这是上游 fork 里既有的 shadow，不是我们加的）。 - -所以 **`LegacyDbSink` 现在不是纯 legacy** —— 它经 storer，带着这个 ODP forward。PR1 故意保留（行为零变）。 - -后果：将来 `DualSink(LegacyDbSink + OdpSink)` 会**对 ODP 双发** → 污染 shadow 对比指标。 - -解法（PR3）：`LegacyDbSink(forward_to_odp: bool = True)` 加 gate；DualSink 用 `forward_to_odp=False` + `OdpSink`。`legacy_db_sink.py` 已留 `TODO(PR3)` 在 storer 调用处。 - ---- - -## 4. 剩余路线（**顺序依赖，不是独立可抢** —— 别拆 /to-issues） - -- **PR2（下一刀）** = 锁旧契约 + mapper，**不搬 forward**： - 1. 读死 `storer.py` 当前 ODP forward 的 payload shape（经 `odp_client.forward_triples`）。 - 2. 给该 forward 加 **characterization test**：`ODP_INGEST_URL` 设了时 `store_records()` 会 forward，payload 与当前一致。锁旧行为，证明 PR3 搬迁前后等价。 - 3. 新增 `backend/odp/schemas.py`：`RecordEvent` / `OdpIngestResponse`，字段对齐 Rust `odp-rs/crates/odp-contracts`（RecordEvent v2）+ `IngestBatchResponse`(accepted/duplicates/rejected/errors)。 - 4. 新增 `RecordEventMapper`，输入**对齐 normalized record**（沿用现有 normalizer 结果），**别从 raw collector item 另起一套语义**（否则 legacy DB 字段语义 ≠ ODP payload 语义）。 - 5. **暂不搬** storer 的 forward；只把 mapper/client 备好。`backend/pipeline/odp_client.py` **已存在**（commit 97b8d93）→ 扩展，别重建。 - 6. 全 `tests/unit` 保持绿。 -- **PR3** = ODP forward 从 storer 搬进 `OdpSink` + `LegacyDbSink(forward_to_odp)` gate + DualSink 不双发。验收：legacy 模式同 PR1；odp_shadow=legacy 写 DB + ODP 发**一次**；ODP 失败不阻断 legacy。 -- **PR4** = `data_sources.write_strategy` 状态机(legacy / odp_shadow / odp_dual_required / odp_primary / odp_only)→选 sink。一旦显式策略，ODP forward 不能再藏 storer 里。 -- **PR5** = RSS 真实竖切。并入原计划「Phase 1b」：`source_cursors` 表 + alembic migration + `DBCursorStore`；`RSSChannel.fetch()` 走 etag/If-None-Match 增量(304=无新条目)；`identity()`=item id。规则：cursor **只在数据进了可靠写入层才前进**；prod 的 odp-ingest 不能 memory-only 假 202。 -- 其后：AuthManager + 加密凭据（堵 `channel_config` 明文 key）；会话亲和 `pipeline.py:45-56` 特判 → `Capabilities.session_affinity` 泛化 + 按域名并发上限。 - ---- - -## 5. 工作纪律（用户定，硬约束） - -- 回复中文、代码/路径/commit 英文；caveman 简洁。 -- **接到明确方向就端到端做完**（自己 build/test/真验证再交），中途不一步一问、不开菜单挑下一步；只真分叉才问。 -- **只在用户说 "commit" 时提交**；stage 时显式列路径，**绝不** stage `chat.py` / `PR-DESCRIPTION.md`。 -- 不向上游 PR，自己 fork 开发。「PR1/PR2」只是每刀的叫法 = 本地 commit。 - ---- - -## 6. 指针 - -- 路线 + 双发陷阱全文：memory `opencli-admin-channel-runner-refactor`（新窗自动 recall）。 -- odp-rs Rust 热路径子系统简报（给 GPT 调研）：本机 scratchpad `odp-rs-briefing.md`（2 二进制 odp-ingest:8040/odp-store + 3 crate contracts/bus/store；Redis Streams + Postgres odp_records；两层去重；缺 XAUTOCLAIM/DLQ/outbox）。 -- 关键源：`backend/pipeline/{pipeline,collector,normalizer,storer,odp_client,channel_runner,cursor_store,http_client}.py`、`backend/pipeline/sinks/`、`backend/channels/base.py`、`backend/models/{record,source}.py`。 diff --git a/LIVE_WEBHOOK_ACCEPTANCE.md b/LIVE_WEBHOOK_ACCEPTANCE.md deleted file mode 100644 index 71ce2ca..0000000 --- a/LIVE_WEBHOOK_ACCEPTANCE.md +++ /dev/null @@ -1,100 +0,0 @@ -# Live Webhook Acceptance - -Date: 2026-07-07 - -## WSL Test Environment - -- Repo: `/mnt/c/c/Users/Administrator/projects/opencli-admin-backend` -- WSL distro: Ubuntu -- Working Python: `3.12.13` -- WSL venv: `/root/.cache/codex/venvs/opencli-admin-backend-py312` -- Install command: - -```bash -uv venv /root/.cache/codex/venvs/opencli-admin-backend-py312 \ - --python /root/.local/share/uv/python/cpython-3.12-linux-x86_64-gnu/bin/python3.12 - -uv pip install \ - --python /root/.cache/codex/venvs/opencli-admin-backend-py312/bin/python \ - -e '.[dev]' -``` - -Notes: - -- The repo-local `.venv` is a Windows venv (`Scripts/python.exe`), not usable from WSL. -- Ubuntu's system Python is `3.14.4`; it was not used for acceptance because `lxml==5.4.0` does not build cleanly on this image. -- `uv python install 3.11` did not complete in this WSL session, but uv already had Python `3.12.13`, which satisfies the project `>=3.11` requirement. - -## Baseline Pytest Acceptance - -Command: - -```bash -cd /mnt/c/c/Users/Administrator/projects/opencli-admin-backend -/root/.cache/codex/venvs/opencli-admin-backend-py312/bin/python \ - -m pytest -q -m 'not live' --maxfail=20 -``` - -Result: - -```text -1430 passed, 1 skipped, 9 deselected, 92 warnings in 235.18s -Required test coverage of 80% reached. Total coverage: 89.72% -``` - -One test adjustment was needed: compile API binding assertions now allow the runtime binding to include the new `contract` manifest while still asserting the original stable binding fields and matching `contract.bindingId`. - -## Generic Webhook Live Acceptance - -Added test: - -```text -tests/integration/test_generic_webhook_live.py -``` - -Behavior: - -- If `OPENCLI_GENERIC_WEBHOOK_LIVE_URL` is set, the test posts to that URL. -- If unset, the test creates a temporary Webhook.site token with `POST https://webhook.site/token`. -- For Webhook.site URLs, the test reads `request/latest/raw` and verifies the captured payload. -- The test exercises the real project path: `execute_workflow_webhook_delivery()` -> `WebhookNotifier` -> public HTTPS POST. - -Command: - -```bash -cd /mnt/c/c/Users/Administrator/projects/opencli-admin-backend -/root/.cache/codex/venvs/opencli-admin-backend-py312/bin/python \ - -m pytest -q -m live tests/integration/test_generic_webhook_live.py --no-cov -``` - -Result: - -```text -1 passed, 27 warnings in 2.89s -``` - -Manual smoke also passed before the pytest was added: - -```text -delivery_result.delivered=true -captured_event=workflow.evidence_batch.ready -captured_title=WSL live webhook acceptance -``` - -## Secret Handling - -No Feishu, DingTalk, WeCom, Hookdeck, or other private keys were added to the repo or written into this document. - -For provider-specific live checks, inject secrets through WSL environment variables or an external secret manager: - -```bash -export OPENCLI_FEISHU_WEBHOOK_URL='...' -export OPENCLI_DINGTALK_WEBHOOK_URL='...' -export OPENCLI_WECOM_WEBHOOK_URL='...' -``` - -Then add or run provider-specific `pytest -m live` tests that skip unless the matching env var is present. - -## Next Step - -After generic webhook live is green, wire Feishu/DingTalk/WeCom live smoke tests behind env-var skips. Keep their keys out of git, shell history, docs, and chat. diff --git a/PLAN_adapter_selfheal.md b/PLAN_adapter_selfheal.md deleted file mode 100644 index a4ccd3b..0000000 --- a/PLAN_adapter_selfheal.md +++ /dev/null @@ -1,44 +0,0 @@ -# PLAN: 适配器生态 + 自愈 (调研综合, 2026-07-19) - -两路调研 (采集生态参照 / 自愈) 综合。核心结论: **自愈的核心已存在**, 适配器生态的注册骨架也已有雏形 —— 缺的是补齐, 不是从零。 - -## 一、既有资产 (别重造) - -### 两套自愈系统已在 -1. **skill 层** (`backend/skills/`, agentic 通道): perception 每步现场重导元素 ref → **天然抗 selector 漂移** (无持久 selector 可烂); correction=re_distill (整体重生+版本化+回滚, ADR-0003); `maybe_propose_correction` 连续 3 次真失败 propose-only; 完整 API `/skills/{id}/redistill|dismiss|rollback` -2. **control-theory 层** (`backend/control/`, 19 文件, ADR-0004/5/7, **通道无关**): coverage 传感器诚实门 → recorder/aggregation (每 run 一 SourceMeasurement→Trend) → error_kinds (映射 error_taxonomy, 含 SCHEMA_DRIFT) → evaluator 状态机 → policies 反馈律 → gate 安全门 (默认 advisory) → actuator 3 白名单可逆动作 (increase_interval/pause/require_review) → outcomes 恢复率经验闭环 → cycle 独立 tick。**8 个 channel 统一走这套** - -### 适配器注册雏形已在 -- `browser_act_packs/{manifest.py,catalog.py}`: PackManifest (param_schema/pagination/success) + PackCatalog._scan() **扫描式声明目录** —— 已是声明式注册的工作原型, 只是圈在 browser_act 一个通道 -- `channels/base.py`: AbstractChannel + Capabilities (incremental/paginated/auth_kind/...) 契约已好 -- `auth/manager.py` + source_credential: 加密凭证存储已有 (缺的是声明式 auth_schema) - -## 二、最高 ROI: schema-drift 传感器 (Gap 1) - -control 层的 **SCHEMA_DRIFT 状态/policy/ledger/actuator 全建好在等, 但没有任何 channel 产生这个信号**。selector/schema 烂了今天是"更少/更空 items 无 error" → 假 HEALTHY (DEAD 要零接受+真 error; DEGRADED 要 error 率上升; 都不触发)。**80% 机器已有, 只缺传感器。** - -- **Step A (传感器, 最省)**: 各内容通道 (web_scraper/crawl4ai CSS/api result_path/rss) 比对字段填充率 vs 滚动 baseline, 骤降 (>50% 相对, 或验证非空页面上归零) → `ChannelResult.fail(error_type="SchemaDriftError")`。下游**零改动** (error_taxonomy→error_kinds→evaluator→policies→actuator 全已处理)。几乎纯 channel 层追加 -- **Step B (提议修复)**: 复制 skill re_distill 模式到 `DataSource.channel_config`: last_failing_sample 存失败页 → 复用 distill.call_llm 重生 selector/schema → **dry-run 验证匹配率恢复后才 surface** → 版本化 diff → 人审 apply (revision 乐观锁 ADR-0025)。cheap 前置: Healenium 式 LCS selector 相似度重匹配, 省 LLM 调用 -- Gap 3 (auth): 401 → 强刷 cookie 重试一次再升级 pause (channel/http_client 层小改) -- Gap 5 (证据流): skill evidence + control ledger 两条, 缺统一 inbox (ADR-0024 已设计未实现) — 最小: skill correction_proposed 也发 require_review 进 control ledger - -护栏 (repo 已有原则): LLM 产出是提议非权威 (ADR-0003 D7/D8 + ADR-0004 数据变更排除自动白名单); 验证先于 surface; 单一 kill_switch/control_mode 门; PIT 安全 (只改未来抓取, 历史 CollectedRecord 不可变) - -## 三、适配器生态补全 (调研 A backlog, value/effort 排序) - -1. **声明式 channel 注册** — 把 PackCatalog 的 manifest+scan 从 browser_act 推广成仓库级 channel 注册, 替 registry.py 硬编码 8 import (= PLAN_plugin_system.md「最小起步」, 消"改一次动三处") -2. **trigger_kind 字段 + polling-diff trigger** (changedetection.io 式快照 diff, 层叠现有 fetch) — "监测页面变化"最高频缺口 -3. **inbound webhook trigger** (generic /webhooks/ → 起 run, Dify Trigger/n8n Webhook) -4. **声明式 auth_schema** (挂现有 AuthManager, n8n ICredentialType 式) — catalog 配置表单前提 -5. cloud-storage/drive datasource; 6. TurboPush sink 泛化成 Extension 类; 7. 通知走 Apprise (照 EXTERNAL_WHEEL_AUDIT 先例); 8. schedule trigger 首类目录项; 9. DB/sheet source; 10. skill agent-strategy 泛化 - -## 四、分阶段 - -- **P1 (最高 ROI, 最省)**: schema-drift 传感器 Step A (点亮已建好的 control SCHEMA_DRIFT 全链) + 声明式 channel 注册 (生态骨架) -- **P2**: Step B 提议修复 (skill 模式复制到 channel_config) + polling-diff/webhook trigger + auth_schema -- **P3**: 统一 recovery inbox (ADR-0024 落地) + datasource 广度 + Extension/通知泛化 - -分类映射 Dify 六类型 (Model 除外): Tool/Datasource/Trigger/Extension/AgentStrategy, 对齐前端 node-catalog.ts 已有 category enum + epic #25 统一插件中心。 - -## 参照 -Airbyte (schema-change 分级/connector catalog), changedetection.io (polling-diff), Dify (插件类型/Trigger/Marketplace), n8n (trigger vs credential), RSSHub (PR-式自描述路由=本仓 PackCatalog 已证), ScrapeGraphAI/Firecrawl (LLM 重生 extraction), Healenium (非 LLM selector 重匹配), Skyvern/browser-use (验证 skill perception 的无持久 selector 选择是业界共识)。 diff --git a/PLAN_collection_nodes.md b/PLAN_collection_nodes.md deleted file mode 100644 index 9f8cfde..0000000 --- a/PLAN_collection_nodes.md +++ /dev/null @@ -1,78 +0,0 @@ -# 采集节点页面重构计划 - -> **状态：已完成** (2026-03-21) -> -> 最终实现与原计划略有差异：没有使用三个 Tab，而是采用"采集模式"卡片切换（本地模式 / Agent 模式）， -> 本地模式展示本地 Chrome 端点（Bridge/CDP 切换），Agent 模式展示节点列表 + 新增节点向导。 -> Docker Chrome 实例区已从 UI 移除。`/browsers` 重定向到 `/nodes`，`BrowsersPage.tsx` 不再使用。 - -## 背景 - -当前路由/页面混乱： -- `/browsers` 路由名称来自早期只有浏览器实例的时候，现在有了 agent 节点概念后显得模糊 -- 我把 `/browsers` 映射回 `BrowsersPage` 后，`NodesPage`（管理 EdgeNode/agent 注册）失去路由入口，导致"添加 agent"入口丢失 -- 浏览器（Browser）是归属于节点（Node）的，应该是节点包含浏览器，不是并列 - -## 目标架构 - -### 路由改动 - -| 旧路由 | 新路由 | 说明 | -|--------|--------|------| -| `/browsers` | `/nodes` | 重命名，语义更准确 | - -### 新"采集节点"页面（`/nodes`）— 三个 Tab - -**Tab 1：本地直连** -- 适用场景：shell 部署，中心直连本地 Chrome，不经过 agent -- 内容：`BrowsersPage` 里"本地浏览器"卡片区（`instanceIndex === null` 的端点） -- 功能：Bridge/CDP 模式切换、站点绑定 - -**Tab 2：本地 Agent** -- 适用场景：本地运行 agent_server.py，中心通过 localhost agent 采集 -- 内容：`NodesPage` 里 `agent_url` 为 localhost 的节点 -- 功能：查看状态、切换模式、查看事件 - -**Tab 3：远端 Agent** -- 适用场景：多机分布式部署，远端 agent 注册到中心 -- 内容：`NodesPage` 里 `agent_url` 为非 localhost 的节点 -- 功能：查看状态、添加 agent（安装脚本/手动注册）、删除、查看事件 - -**保留**：Docker 实例卡片（`instanceIndex !== null` 的端点，即 agent-N 容器）放在"远端 Agent" tab 里或单独的 Docker tab - -### Docker 实例去向 -Docker Chrome 容器（`agent-1:19222` 格式）是 Docker 模式下的"本地 agent"，放在 Tab 2 或单独 Tab 4"Docker 实例"均可。优先简单，先合并到 Tab 2。 - -## 文件改动清单 - -1. **`frontend/src/App.tsx`** - - 删除 `BrowsersPage` import（或保留） - - 新增统一页面（或直接复用 NodesPage + 新 tab） - - 路由：`"browsers"` → `"nodes"`，element 改为新页面 - -2. **`frontend/src/components/Layout.tsx`** - - nav 链接：`/browsers` → `/nodes` - - ROUTE_LABELS 更新 - -3. **`frontend/src/pages/NodesPage.tsx`**（主力页面） - - 加入 Tab 切换（本地直连 / 本地 Agent / 远端 Agent） - - Tab 1 内容：从 BrowsersPage 抽取本地端点卡片 + 模式切换 - - Tab 2/3 内容：现有 NodesPage 内容按 localhost 过滤分组 - -4. **`frontend/src/pages/BrowsersPage.tsx`** - - 可能保留 Docker 实例管理部分，或将 Docker 实例卡片也迁移到 NodesPage - - 若迁移完毕可废弃此文件 - -## 执行顺序 - -1. 先在 Layout.tsx 和 App.tsx 改路由 `/browsers` → `/nodes`，同时把 element 改为 NodesPage（先恢复"添加 agent"功能） -2. 在 NodesPage 里加 Tab 切换 -3. 把 BrowsersPage 的"本地直连"内容（本地端点卡片 + 模式切换）移植到 NodesPage Tab 1 -4. 处理 Docker 实例的归属（暂时放 Tab 1 或 Tab 2） -5. 更新 i18n、文档注释 - -## 注意事项 - -- `BrowsersPage` 的 `ModeToggle`、`InstanceCard`、`SiteDropdown` 等子组件可能需要提取到独立文件复用 -- `NodesPage` 里已经有 `collection_mode` 切换（local/agent），确保 Tab 切换不与此冲突 -- Docker 实例的"新增实例"/"重启 API"功能要保留入口 diff --git a/PLAN_general_analysis_loop.md b/PLAN_general_analysis_loop.md deleted file mode 100644 index 4bf852f..0000000 --- a/PLAN_general_analysis_loop.md +++ /dev/null @@ -1,192 +0,0 @@ -# 通用 Deep Research / Analysis：研究闭环与 A 股盘前 PTT - -## 目标 - -交付一个真实可运行、可审计、严格有界的研究闭环： - -```text -ResearchBrief - -> 多源 EvidenceBatch / recordCandidate - -> Claim Projection - -> Coverage Audit - -> Counter Thesis - -> Scenario Simulation - -> Revision Diff - -> Publish Gate -> Existing Record Acceptance / Sink - -> ResearchRevision - -> approved collect_more -> immutable child WorkflowRun -``` - -- A 股盘前研究是首个 PTT；通用算子不写入 A 股字段，垂直场景只存在于可导入 Workflow fixture。 -- Workflow 图保持 DAG；不在编译图中制造反馈边。 -- `Coverage Audit` 输出 `finalize | collect_more | stop_incomplete` 和有界 Collection Proposal。 -- 补搜需显式调用 research continuation API；每轮创建不可变子 Run，服务端推进预算并复用父图快照。 -- 每个 Claim 必须绑定当前 Run 的 EvidenceBatch 和稳定 Evidence ID；反方结论与修订保留可追溯 Evidence ID。 -- 首轮复用现有 Workflow Compiler、Runtime Registry、Capability Catalog、WorkflowRun/Event、EvidenceBatch、Data Operator 和 AgentDrawer proposal 链路。 -- 合成测试证明跨 Run 补搜闭环；A 股 PTT 证明 2026-07-28 官方源快照可走完整通用链，仍不冒充持续在线采集。 - -## 首轮非目标 - -- 不宣称“任意网页”已经达到生产可用。 -- 不新增数据库表、队列、向量库、依赖或另一套 Artifact/Run 系统。 -- 不实现后台无限循环或动态改图。 -- 不允许 Agent 绕过 Capability/Runtime/权限 Gate。 -- 不包含交易、下单、通知、发帖等外部写操作。 -- 不把 A 股字段、日报格式或交易策略写进通用算子。 - -## 最小契约 - -首轮继续使用 `recordCandidate[]`。研究结构写入 -`recordCandidate.normalizedData`，并保留原有 lineage。 - -### EvidenceRef - -```json -{ - "evidenceId": "stable source item id", - "itemKey": "item key inside the EvidenceBatch", - "batchId": "current-run EvidenceBatch id", - "runId": "current workflow run id", - "nodeId": "EvidenceBatch producer node", - "manifestUri": "EvidenceBatch detail endpoint", - "odpRef": "stable ODP batch reference", - "sourceId": "optional source id", - "url": "optional source URL" -} -``` - -只引用 Batch 而不引用单条 Evidence 不能支撑 Claim。 - -### Claim - -```json -{ - "claimId": "stable hash", - "statement": "claim text", - "disposition": "supported|contradicted|mixed|unverified", - "supportingEvidenceIds": [], - "contradictingEvidenceIds": [], - "qualifyingEvidenceIds": [], - "evidenceIds": [], - "evidenceRefs": [], - "dimensions": [] -} -``` - -### CoverageReport - -```json -{ - "claimSetHash": "fingerprint of the exact audited claim set", - "semanticClaimSetHash": "fingerprint excluding run-local EvidenceRef locations", - "requiredDimensions": [], - "coveredDimensions": [], - "gaps": [], - "satisfied": false, - "decision": "finalize|collect_more|stop_incomplete", - "stopReason": "coverage_satisfied|max_iterations_reached|max_additional_collections_reached", - "continuationProposal": { - "proposalId": "stable id", - "action": "collect_more", - "gaps": [], - "nextIteration": 2, - "nextAdditionalCollectionCount": 1 - } -} -``` - -### CounterThesis - -```json -{ - "counterThesisId": "stable hash", - "statement": "contrary or qualifying evidence summary", - "targetClaimIds": [], - "evidenceIds": [] -} -``` - -### ResearchRevision - -```json -{ - "claimSetHash": "fingerprint of the exact revised claim set", - "scenarioSetHash": "fingerprint of scenario assessments", - "added": [], - "changed": [], - "removed": [], - "evidenceIds": [] -} -``` - -## 有界执行与后续闭环 - -当前实际强制： - -- `maxIterations <= 5` -- `maxAdditionalCollections <= 3` -- `iteration` 不得超过 `maxIterations` -- `additionalCollectionCount` 不得超过 `maxAdditionalCollections` -- Workflow Compiler 沿用现有 DAG 环检测，不允许反馈边 -- Demand Draft 受现有双输入 merge 契约约束，一次最多组装两个 source -- Coverage 只输出有界决策：`finalize | collect_more | stop_incomplete` -- Publish Gate 要求当前 Claim 集合的 Coverage/Revision 指纹一致，并且每个 Evidence ID 有当前 Run 的 EvidenceRef -- continuation 每次最多 200 个新增 evidence item / 2 MiB -- 单个 ledger 最多累计 1000 个 evidence item / 8 MiB -- continuation 只接受现有 source node ID，不接受客户端 project 或预算覆盖 -- 相同 idempotency key 重放同一个子 Run；不同输入冲突关闭 -- 重复证据不创建子 Run；语义 Claim 集与 coverage gaps 无进展时 ledger 标记 `no_progress` -- `maxGraphMutations = 0`，子 Run 只更新 Coverage/Revision 的运行态配置 - -Agent 权限边界： - -1. 读取 Capability Catalog 和上一轮 Run/Event/EvidenceBatch。 -2. 提议新增或调整 source、研究算子配置。 -3. 提交 Workflow patch preview。 -4. 只有显式 continuation 调用可把已批准 `sourceOutputs` 带入下一轮；不会自动选择任意网站。 - -## 分布式工作包 - -所有 Agent 共用工作树；每个文件只有一个 owner，任何人不得回滚别人的修改。 - -| 工作包 | Owner | 文件范围 | 依赖 | 交付 | -|---|---|---|---|---| -| WP0 架构与边界 | Lead/Architect | 本计划 | 无 | 目标、契约、停止条件 | -| WP1 研究算子 | Research semantics Agent | `backend/workflow/research_operators.py`、最小注册改动 | WP0 | Claim、Coverage、Counter、Scenario、Revision、Publish Gate | -| WP2 单元与集成 | Test Agent | 新增 research 专用测试 | WP1 契约 | 确定性、预算、DAG/API 运行 | -| WP3 前端入口验收 | Frontend Agent | Run Trace 与 API proxy | 现有 UI | Revision Ledger、EvidenceRef、补搜输入与子 Run | -| WP4 编排入口 | Backend Agent | 需求组装/HDA 的最小接入文件 | WP1、WP2 | Agent 可组装研究节点 | -| WP5 独立审查 | Critic/Verifier | 只读 | WP1-WP4 | 范围、证据链、测试复核 | - -当前用户已有改动必须保留，首轮避免触碰： - -- `backend/api/v1/nodes.py` -- `backend/workflow/opencli_adapter_nodes.py` -- `tests/unit/test_opencli_adapter_nodes.py` -- `tests/unit/test_realtime_market_executor.py` - -## 验收 - -1. `builtin.research@1.0.0` 在现有 Capability Catalog 中可发现并有真实 runtime binding。 -2. 六个研究算子可作为现有 data-operator 节点编译、执行和记录 metrics。 -3. 相同输入得到相同 Claim ID、Evidence ID、Coverage、Counter Thesis 和 Revision Diff。 -4. 每个可发布 Claim 的 EvidenceRef 都绑定当前 Run 的 EvidenceBatch；旧 Coverage/Revision 指纹不能解锁 Gate。 -5. 缺引用 Claim 是 `unverified`；Coverage 的成功、`collect_more` 决策和预算终止路径都有算子测试。 -6. Publish Gate 的通过和阻断两条路径都跑过 Existing Record Acceptance / Sink；阻断路径写入零条研究记录。 -7. 合成跨源研究证据可跑三轮不可变 Run，预算、幂等、重复证据拒绝和 ledger ancestry 可验证。 -8. Agent 修改仍是 proposal-only；接受前不应用 patch。 -9. 无新依赖、无迁移、无外部写操作。 -10. 可导入的 A 股盘前 Workflow 使用上交所休市/公告、深交所公告和国家统计局发布四类官方证据快照，逐 Claim EvidenceRef 可导航。 - -目标命令： - -```powershell -uv run pytest tests/unit/test_research_operators.py tests/unit/test_data_operators.py --no-cov -q -uv run pytest tests/integration/test_workflow_deep_research_api.py --no-cov -q -uv run pytest tests/integration/test_a_share_premarket_research_ptt.py --no-cov -q -uv run pytest tests/integration/test_dataflow_operator_pipeline_api.py tests/integration/test_workflow_patch_api.py --no-cov -q -``` - -持续在线 A 股 PTT 仍需把四个已核验官方端点物化为已批准的只读 source capability， -并记录限速、抓取时间、HTTP 元数据与截断。当前仓库验收是 2026-07-28 -`verified_snapshot`，不会把 fixture 冒充持续在线采集。 diff --git a/PLAN_native_nodes_loop.md b/PLAN_native_nodes_loop.md deleted file mode 100644 index 3d7da86..0000000 --- a/PLAN_native_nodes_loop.md +++ /dev/null @@ -1,332 +0,0 @@ -# OpenCLI Admin Native Nodes Loop - -> Status: completed first native-runtime loop -> Scope: first-loop native node system -> Related ADR: `docs/adr/0010-native-runtime-nodes-and-managed-external-graphs.md` - -## Goal - -Build the first durable loop of OpenCLI Admin's native node system: a real workflow can be assembled from packaged nodes, validated through capability availability and typed ports, run through the backend, merged with lineage, accepted into records through a gate, and inspected through run trace. - -This loop is not a throwaway demo. It is the foundation that later LangGraph, LangChain, Pi, and other external runtimes must enter through. - -## Definition of Done - -A real workflow can run end to end: - -`Packaged Source Node -> Normalize Transform -> Merge Node -> Record Acceptance Gate -> Record Sink -> Run Trace` - -The first loop is done when all of these are true: - -- Capability Catalog is the authoritative source for runnable node capabilities. -- Capability Manifest declares each packaged capability's schema, resources, permissions, runtime binding, trace mapping, and probes. -- Capability Availability reports whether each declared capability is ready, blocked, missing resources, missing dependencies, permission-gated, or probe-failed. -- Packaged Node Presets are discoverable by the Canvas and grouped by Node Preset Family. -- Typed Ports prevent incompatible edges and let AI assemble plans without glue code. -- At least one real Source preset runs against a real collection source. -- Normalize Transform produces Record Candidates from source output. -- Merge Node supports typed fan-in and preserves lineage. -- Record Acceptance Gate decides whether Record Candidates become Records. -- Record Sink writes accepted Records into the existing records system. -- Run Trace records node lifecycle, tool call events, artifacts, timing, errors, and lineage pointers. -- AI can generate a Plan Draft from an operator intent, but cannot run it until it is materialized and approved on the Canvas. - -## First-Loop Node Families - -### Source - -- One real source preset backed by an existing OpenCLI Admin adapter or OpenCLI command. -- It must emit a typed Record Candidate stream or Runtime Artifact stream that can be transformed into candidates. - -### Transform - -- Normalize Transform. -- Artifact Transform only if the first source emits HTML, screenshots, or raw artifacts before candidates. - -### Flow - -- Merge Node with concat as the minimum strategy. -- The implementation must preserve lineage even for concat. - -### Control - -- Record Acceptance Gate. -- Basic schema and lineage checks are required; quality and manual review rules may start minimal. - -### Sink - -- Record Sink into the existing records table/system. - -### Runtime Package - -- Placeholder only in the first loop. -- LangGraph, LangChain, Pi, and other imported runtimes are future extension points and must not drive the first implementation path. - -## Original Non-Goals For First Loop - -These were the original boundaries before the long-task scope was expanded by -the OpenCLI Admin native-runtime decisions. They remain useful as historical -guardrails for the first minimal loop, but the current active scope now includes -external-runtime import as native OpenCLI nodes and durable run recovery. - -- No LangGraph importer in the minimal first-loop slice. -- No LangChain importer in the minimal first-loop slice. -- No Pi executor. -- No full external-runtime checkpoint implementation in the minimal first-loop slice. -- No large site-node library. -- No frontend-only fake nodes. -- No direct primitive tool access for imported runtimes. - -## Implementation Order - -1. Audit the current workflow capability, registry, node catalog, trace, and records paths. -2. Align existing code with the domain model in `CONTEXT.md`. -3. Establish the Capability Manifest and Capability Availability shape. -4. Project the Capability Catalog to the frontend node palette. -5. Add or adapt Packaged Node Presets for the first source, normalize, merge, record acceptance, and record sink. -6. Enforce Typed Ports at plan compile/materialization time. -7. Implement Merge Node lineage. -8. Implement Record Acceptance Gate. -9. Wire Record Sink and Run Trace evidence for the full loop. -10. Add AI Plan Draft generation only after the catalog and typed ports are real. - -## Verification - -- Unit or integration tests must prove capability projection and availability. -- Compile/materialization tests must reject incompatible typed edges and unresolved resources. -- Runtime tests must execute the first-loop workflow and assert Records, lineage, and Run Trace events. -- Frontend smoke should show packaged presets from backend metadata rather than hardcoded fake palette nodes. - -## Operating Rule - -When implementation pressure creates a choice, prefer the smallest real end-to-end loop over adding more node types. A node that cannot be traced, typed, and materialized is not part of the first loop. - -## Progress - -### 2026-07-05 - -- Added first-loop native node vocabulary to `CONTEXT.md` and recorded ADR-0010. -- Added backend workflow schema support for `flow`, `control`, `sink`, `merge`, and `accept`. -- Added catalog/runtime bindings for: - - `intelligence.flow.merge` -> `workflow.flow.merge` - - `intelligence.control.record-acceptance` -> `workflow.gate.record-acceptance` - - `intelligence.sink.records` -> `workflow.record-sink.records` -- Added frontend Packaged Node Presets and port contracts for Merge, Record Acceptance Gate, and Record Sink. -- Added compile tests proving the new native nodes project to runtime bindings and Plan IR ports. -- Verified focused backend tests, frontend typecheck, and ruff for touched Python files. - -### 2026-07-05 Runtime Trace Slice - -- Added standalone native Normalize runtime binding: - - `intelligence.processing.normalize` -> `workflow.transform.normalize` -- Added run-trace behavior for native first-loop nodes: - - Source fixture output emits typed `items[]` with lineage pointers. - - Normalize consumes upstream items and emits `recordCandidate[]` trace details. - - Merge consumes upstream candidates, emits strategy, typed input/output, and lineage-preservation details. - - Record Acceptance Gate consumes merged candidates, emits schema, dedupe, lineage, accepted count, and review count details. - - Record Sink consumes accepted records and emits target, write mode, stored refs, stored count, and lineage details. -- Updated HDA run projection so internal normalize is no longer a blocked missing-runtime node. -- Added `/api/v1/workflows/runs` integration coverage for a non-package native first-loop workflow. -- Verified deterministic item propagation through: - - `Source fixtureItems -> Normalize -> Merge -> Record Acceptance Gate -> Record Sink` -- Verified focused backend tests, frontend typecheck, and ruff for the runtime trace slice. - -### 2026-07-05 Record Sink Persistence Slice - -- Wired `/api/v1/workflows/runs` to pass the request DB session into workflow run execution. -- Materialized workflow Source nodes into real `DataSource` rows when Record Sink needs ownership and no existing `sourceId` is bound. -- Materialized one `CollectionTask(trigger_type="workflow")` per origin Source node and workflow run. -- Updated Record Sink execution to group accepted Records by origin Source lineage, then persist through the existing `store_records` path. -- Preserved workflow lineage in stored record raw data and in Record Sink `storedRefs`. -- Added integration coverage proving: - - `CollectedRecord` rows are written. - - each row has real `source_id` and `task_id` ownership. - - workflow lineage survives the sink boundary. - -### 2026-07-05 AI Plan Draft Slice - -- Extended reviewable workflow patch operations with `add_adapter` so AI drafts can add Source presets with real adapter bindings instead of hidden executor assumptions. -- Changed demand draft assembly from an OpenCLI HDA package shortcut to native OpenCLI Admin nodes: - - OpenCLI Source preset - - Normalize Transform - - Merge Node - - Record Acceptance Gate - - Record Sink -- Demand drafts remain patches for Canvas review; they are compiled for validation but not dispatched by the draft endpoint. -- Added integration coverage for: - - single-source demand draft generation into native nodes. - - multi-source demand draft fan-in through Merge. - - runtime bindings for normalize, merge, record acceptance, and record sink. - -### 2026-07-05 Typed Port Gate Slice - -- Added backend typed-port contracts for catalog-owned packaged presets: - - OpenCLI Source Slot - - Source Pool - - Normalize - - Dedupe - - Merge - - Record Acceptance Gate - - Record Sink - - Collection Output - - Inbox Output -- Compile now rejects catalog-owned edges with invalid source ports, invalid target ports, or incompatible port types. -- Package internals are validated too, so locked HDA graphs cannot hide incompatible internal edges. -- Updated OpenCLI HDA template internals to use real port ids (`out -> in`) instead of type labels as port names. -- Aligned Collection Output to consume `recordCandidate[]`, matching native Normalize output. -- Added integration coverage proving: - - incompatible `recordCandidate[] -> record[]` edges are rejected before materialization. - - invalid typed port ids are rejected. - - valid native first-loop and AI demand-draft graphs still compile. - -### 2026-07-05 Source Batch Ingest Slice - -- Added `WorkflowRunStartRequest.sourceOutputs` as an explicit runtime batch-ingest seam for external worker/source outputs. -- Workflow run execution now consumes `sourceOutputs[nodeId]` as typed `items[]` before checking node fixtures, bound tasks, or OpenCLI dispatch. -- Runtime source output lineage is marked with the `sourceOutputs` artifact so traces can distinguish external batch input from node-param fixtures and persisted task records. -- Added a workflow source-output ingestion boundary for Source nodes bound to an existing `taskId`, `collectionTaskId`, or `boundTaskId`. -- When a Source node has a bound task and a DB session is available, workflow run execution now loads that task's `CollectedRecord` rows as typed `items[]` before dispatching a new OpenCLI batch. -- Loaded source records carry lineage with: - - source node id - - source group - - `collected_records` artifact marker - - record id - - task id - - source id -- Downstream Normalize, Merge, Record Acceptance Gate, and Record Sink consume those bound task records through the same native node path as fixture-driven items. -- Added integration coverage proving a non-fixture workflow run can read real persisted task records, propagate them through the native first-loop chain, and write workflow-owned records through Record Sink. -- Added integration coverage proving request-level runtime source outputs can drive the same first-loop chain without `fixtureItems` in node params. -- Added `/api/v1/workflows/runs/{run_id}/source-outputs` continuation for source batches that arrive after a run has already started. -- Continuation merges late `sourceOutputs`, reruns the same WorkflowProject with the same run id and trace id, and appends new node events after the existing sequence. -- Added integration coverage proving a dispatch-first empty run can later continue with source outputs, append trace events, and persist records through the native sink chain. - -### 2026-07-05 Capability Manifest Slice - -- Extended capability projection rows with a structured `manifest` field. -- Added first-loop manifests declaring: - - schema id - - input and output typed ports - - required resources - - permissions - - runtime binding - - trace event mapping - - probes -- Added manifests for: - - Collection Need / demand draft - - OpenCLI Source Slot - - Source Pool - - Normalize Transform - - Merge - - Record Acceptance Gate - - Collection Output - - Record Sink -- Updated frontend capability typing to accept capability manifests. -- Added capability API integration coverage proving manifest runtime binding, ports, resources, permissions, trace mapping, and probes are present for first-loop nodes. - -### 2026-07-05 Persisted Run Trace Slice - -- Added persisted workflow run trace tables: - - `workflow_runs` - - `workflow_run_events` -- Workflow runtime now persists: - - original `WorkflowRunStartRequest` - - latest `WorkflowRunProjection` - - replayable node event payloads ordered by sequence -- `/api/v1/workflows/runs/{run_id}` and `/events` can recover from the database when the process-local run cache is empty. -- `/api/v1/workflows/runs/{run_id}/source-outputs` can continue a run from the persisted request/projection/events, then append the new replay sequence. -- Added integration coverage proving: - - run trace rows are written. - - projection and event replay survive clearing the in-memory cache. - - late source-output continuation works after DB recovery. -- Added Alembic migration coverage with a temporary SQLite `upgrade head` run. - -### 2026-07-05 Checkpoint And Trace Query Slice - -- Added a durable workflow run checkpoint descriptor derived from persisted request, projection, and event sequence. -- Added `/api/v1/workflows/runs/{run_id}/checkpoint` so Canvas and AI agents can recover: - - run id - - trace id - - checkpoint id - - latest global sequence - - node states - - source-output resume summary - - continuation and trace query paths -- Added `/api/v1/workflows/runs/{run_id}/trace` with query filters: - - `afterSequence` - - `nodeId` - - `eventType` - - `limit` -- Extended `/api/v1/workflows/runs/{run_id}/events` with the same event filters for lightweight replay. -- Added frontend proxy routes and TypeScript client helpers for checkpoint, trace query, event filtering, and late source-output continuation. -- Added integration coverage proving: - - checkpoints recover from DB after clearing the in-memory run cache. - - trace query uses a global sequence cursor. - - event filtering works for node, event type, cursor, and limit. - -### 2026-07-05 External Runtime Import Slice - -- Added `external.tool.capability` as an OpenCLI Admin catalog capability for imported external-runtime tool nodes. -- Registered its manifest with: - - unknown typed input/output ports - - external workflow origin resource - - canvas review permission - - pending OpenCLI executor binding -- Added `/api/v1/workflows/import/external-runtime` for LangGraph and LangChain graph import. -- Import preserves external graph structure by mapping: - - external nodes to OpenCLI Admin WorkflowProject nodes. - - external edges to WorkflowProject edges. - - external ids/types into `ui.externalWorkflow` and `params.externalWorkflow`. -- External runtime nodes do not carry raw executors or direct tool calls into Canvas. -- Tool-like/unknown external nodes import as `external.tool.capability`. -- External merge/join nodes import as native `intelligence.flow.merge`. -- Parser/transform-like nodes import as native `intelligence.processing.normalize`. -- Added frontend proxy and TypeScript helper for external runtime imports. -- Added integration coverage proving: - - LangGraph list-node graphs import as OpenCLI catalog nodes. - - LangChain dict-node graphs and edge-only nodes import without losing structure. - - imported tool nodes compile with an OpenCLI Admin Tool Capability binding requirement instead of pretending to have an external executor. - -### 2026-07-05 Tool Capability Executor Seam - -- Added `workflow.external-tool.capability` runtime binding for imported OpenCLI Admin Tool Capability nodes. -- `external.tool.capability` is now a runnable catalog node type, but each node still needs an explicit node-level `params.toolCapability` binding. -- Unbound imported tool nodes now block with `missing_tool_capability_binding`. -- Bound imported tool nodes can run through a guarded fixture executor: - - `params.toolCapability.id` - - `params.toolCapability.executor.mode = fixture` - - `params.toolCapability.executor.output` or `outputs` -- Runtime trace now emits node lifecycle events plus explicit `tool_call_started` and `tool_call_completed` events for bound OpenCLI Tool Capability nodes. -- Fixture outputs preserve external workflow provenance and append OpenCLI Admin lineage. -- Importer now carries explicit `toolCapability` or `toolCapabilityId` + `executor` declarations from LangGraph/LangChain node metadata into native node params. -- Added integration coverage proving: - - unbound imported tool nodes remain blocked at runtime binding. - - bound imported tool nodes compile to `workflow.external-tool.capability`. - - bound imported tool nodes run through the backend and feed downstream native Normalize. - -### 2026-07-05 Tool Capability Registry Slice - -- Added an OpenCLI Admin Tool Capability registry. -- Added `/api/v1/workflows/tool-capabilities` for registered tool capability discovery. -- Registered `tool.search.fixture` as the first deterministic fixture-backed Tool Capability for external-runtime import review. -- Projected registered Tool Capabilities into `/api/v1/workflows/capabilities` as resource capabilities so the main capability surface can see available tool resources. -- Runtime now requires `params.toolCapability.id` to resolve against the registry before binding. -- Unknown tool capability ids block with `unknown_tool_capability` even when a fixture executor is present. -- `external.tool.capability` manifest now declares `tool_capability_registry` as a required resource. -- `external.tool.capability` and registered tool capability manifests now declare explicit tool-call trace events, so tool calls remain runtime events instead of Canvas nodes. -- Added frontend proxy and TypeScript client helper for workflow tool capability discovery. -- Added integration coverage proving: - - registered tool capabilities are discoverable through the API. - - registered tool capabilities are visible through the main capability projection. - - known tool capabilities bind and run. - - unknown tool capabilities remain blocked. - -Remaining first-loop work: - -- Production hardening: add richer probe execution rather than manifest-declared probe names only. -- Production hardening: add real guarded executor adapters beyond the deterministic fixture-backed Tool Capability seam. - -These are follow-on hardening items. They do not block the completed first -native-runtime loop because the current goal is proven through native catalog -capabilities, typed ports, backend runtime binding, persisted trace/checkpoint -recovery, external-runtime import into OpenCLI nodes, Tool Capability registry -binding, and explicit runtime tool-call trace events. diff --git a/PLAN_plugin_system.md b/PLAN_plugin_system.md deleted file mode 100644 index 8812e48..0000000 --- a/PLAN_plugin_system.md +++ /dev/null @@ -1,50 +0,0 @@ -# PLAN: Dify 式插件系统 (方向记录, 待讨论) - -2026-07-18 口述方向, 未排期。本文只固化共识, 不是实施承诺。 - -## 方向 - -系统学 Dify 的插件架构: **工作流管线里的每种能力都是插件** —— 工具节点 (JoyAI-VL、OKX 行情、opencli 采集)、Agent (对话/富化/研判)、通知渠道、数据源渠道, 全部同一套插件契约装卸。 - -现状已经半路在这个形态上: - -| 现状机制 | 插件化后 | -|---|---| -| `backend/workflow/tool_capabilities.py` 硬编码注册表 | 插件 manifest 声明 capability, 装载时注册 | -| executor mode = schema Literal 白名单 (改一次动三处: schema/registry/tracer) | executor 随插件包提供, 运行时发现 | -| chat.py agent 工具面硬编码 4 类 | agent = 插件, 工具面由已装插件聚合 (全局对话 agent 控制整个工作台的前提) | -| channel_type (rss/opencli/cli/...) if-else 分发 | 渠道插件 | - -## 为什么是 Dify 模式 - -- 已有 Dify 工作流导入 (`import/external-runtime`), 概念对齐降低映射成本 -- Dify 插件 = manifest (声明 capability/权限/配置 schema) + 运行时包, 与本仓 tool capability `manifest` 字段现有形状接近 -- 装卸不动核心: 新增一个 JoyAI 级别的节点不应该再改 `schemas/workflow.py` 的 Literal (本次 PR #19 实际改了 5 处 — 这就是要消掉的摩擦) - -## 最小起步 (讨论后再动) - -1. executor mode 从 schema Literal 放开为 str + 运行时注册校验 (消掉三处联动) -2. tool capability 注册表改声明式加载 (entry-point / 目录扫描) -3. chat agent 工具面从插件注册表聚合, 而不是手写 TOOLS 列表 - -## P0 已定边界 (2026-07-21) - -- 插件边界: 安装阶段只导入 Dify manifest / `.difypkg` 元数据, 不解压到可执行目录, - 不加载第三方 Python。后续执行接 Dify Plugin Daemon/独立运行时, 不在 API 进程内 - `import` 插件代码。 -- 前端 palette: 后端注册表投影带安装 ID、版本来源的 locked node definition；只有已注册 - OpenCLI runtime adapter 的 capability 才能变为 `READY`, 其余能力保留可见并标为 - `BLOCKED`, 不静默隐藏。 -- 权限模型: 插件声明 `resource.permission` 和 credential schema；注册表只展示声明, - 工作流运行仍由 OpenCLI 项目权限与资源配置显式批准。导入插件不等于授权执行。 -- 目录与实例继续分离: `/api/v1/plugins` 是已安装/内置能力的统一事实源；Provider 凭证、 - 数据源实例和工作流节点仍是独立领域对象。 - -## UI 分组铁律 (2026-07-18 走查补充) - -**目录 (可安装的生态) 与实例 (已接入的配置) 必须分开成组** — Dify 插件市场 vs 应用内引用的关系: - -- 「RSS 源库」= provider 目录 → 独立源库/生态分组, 不挂在自动化与 Agent 页 (issue #22) -- 「项目内模板推荐」= 模板目录 → 独立模板分组, 项目内只留轻量引用入口 (issue #23) -- 「项目内任务通知」= 跨项目全局信息流 → 独立通知中心分组, 项目内最多未读角标 (issue #24) -- 判定法: 一个面板回答"有什么可装" → 目录组; 回答"我装了什么/跑得怎样" → 实例组; 回答"系统在告诉我什么" → 通知组。混在一页 = 违反本原则 diff --git a/PLAN_product_positioning.md b/PLAN_product_positioning.md deleted file mode 100644 index 21dfb85..0000000 --- a/PLAN_product_positioning.md +++ /dev/null @@ -1,78 +0,0 @@ -# OpenCLI Admin — 产品定位与架构原则（Fork） - -> **状态**：方向锁定（2026-06-18） -> **Obsidian 副本**：`Documents/Obsidian/Research/opencli-admin/开发调研-产品与架构-2026-06-18.md` - -## 一句话定位 - -**带登录态的采集调度器 + 结构化数据出口**；补 n8n 抓不了的那段；**不是** Mac Mini 舰队控制台，**不是** n8n 替代品。 - -## 分工 - -| OpenCLI Admin | n8n | -|---------------|-----| -| 浏览器会话（CDP/Bridge） | Webhook 下游编排 | -| Adapter 采集与 normalize | 去重、通知、写第三方 | -| SQLite `records` | 集成与自动化 | - -## 第一性原理（最小闭环） - -1. Session — LAN 上谁方便挂 Chrome（**Windows / Linux 均可**） -2. Adapter — 站点 → 结构化记录 -3. Schedule — 何时采 -4. Egress — webhook / export / 知识库 - -Mac Mini 集群是规模化选项，**不是**个人 NAS lab 默认路径。 - -## 控制论 + OODA（反馈优先） - -先闭合：**采到了吗 → 下游 ACK 了吗 → 会话还有效吗**；再扩张多节点执行器。 -**OODA 现状**：Observe/Orient 弱，Decide 无输入，Act（多 Agent）过重 — 操作者完不成一圈「观察→调整」。 -当前欠账：质量仪表、webhook 投递反馈、去重/异常 Orient、规则化 Decide、数据 egress。 - -## Fork 优先级 - -1. 文档：NAS + LAN Session + n8n 模板 -2. 单机/LAN UI 模式（隐藏舰队 IA） -3. Records 导出 API -4. UI reskin（见 `PLAN_ui_reskin.md`） -5. Obsidian / 媒体（按需） - -## X / 中文社区市场叙事（2026-06-18 调研） - -### 声量结构 - -| 项目 | X 上的角色 | 典型叙事 | -|------|------------|----------| -| **OpenCLI**（jackwener） | 主声源 [@jakevin7](https://x.com/jakevin7) | Agent 读写基础设施、Browser Bridge、站点 CLI、自愈 adapter | -| **OpenCLI Admin**（xjh1994） | **二次传播**，非作者主导 campaign | 舆情监控、多 Chrome 多账号、Mac Mini 集群、Docker、AI 打标 + 飞书 | - -**高曝光帖**（公开可读）： - -- [@codesth](https://x.com/codesth/status/2034660940139167956) — 2026-03-19，~302K views，README 式摘要 -- [@liangwenhao3](https://x.com/liangwenhao3/status/2036756586048463328) — 2026-03-25，~8.1K views，`#AI #采集` - -**中文长文**（与 X 叙事一致）：[zymn.cc 舆情实战](https://zymn.cc/2026/03/28/opencli-admin-monitoring-dashboard/) — Mac Mini A/B/C 分工、飞书推送、AI 摘要打标。 - -**竞品参照**：[@wey_gu](https://x.com/wey_gu/status/2052443011632791888) 将 Printing Press 与 OpenCLI 类比（CLI-factory / 登录态浏览器），说明「网站→CLI」赛道在 2026 仍热，但 **Admin 层几乎无人单独讲叙事**。 - -### 市场默认故事 vs Fork 差异 - -| 维度 | 上游 / 社区默认 | 本 Fork 主张 | -|------|-----------------|--------------| -| 硬件 | Mac Mini 舰队、多机多账号 | **任意 LAN 会话机**（Windows/Linux）；Mac 仅规模化选项 | -| 能力 | 浏览器 + opencli 为主 | **多执行面**：RSS/API/Web/CLI/脚本；浏览器是 capability 之一 | -| 价值句 | 舆情面板 + AI 打标 + IM 推送 | **采到了然后呢**：normalize → egress → OODA 反馈（ACK、质量、会话健康） | -| 编排 | 产品内通知为主 | Admin 抓 + **n8n 接**；不替代 n8n | - -### Fork 对外话术候选（A/B） - -1. **个人 lab**：「NAS 调度 + LAN 登录态 — 带会话的采集调度器，不是爬虫脚本堆。」 -2. **执行面**：「浏览器只是节点之一；RSS/API/脚本同样进同一套 records 出口。」 -3. **缺口**：「opencli 把数据搬下来；我们补 **结构化落库 + webhook + 可观测闭环**。」 - -## 待用户确认 - -- 数据终态：Obsidian / Postgres / 仅 n8n？ -- 是否需要非文本资产管道？ -- 对外主推哪条话术（上表 1/2/3）？ \ No newline at end of file diff --git a/PLAN_realtime_tool_capabilities.md b/PLAN_realtime_tool_capabilities.md deleted file mode 100644 index 5edf819..0000000 --- a/PLAN_realtime_tool_capabilities.md +++ /dev/null @@ -1,51 +0,0 @@ -# OpenCLI Admin Realtime Tool Capabilities - -> Status: active tool-capability package -> Builds on: `PLAN_native_nodes_loop.md` - -## Position - -Realtime collection, rolling computation, state cache, feature computation, and -signal emission are **tools**, not Canvas nodes and not a new product axis. - -Canvas nodes remain business/capability nodes. Realtime work appears as runtime -tool calls through OpenCLI Admin Tool Capability bindings, then enters trace, -state, checkpoint, and replay like every other governed tool call. - -## Registered Tools - -- `tool.realtime.stream.subscribe` - - `trigger -> event[]` - - stream subscribe/replay/poll acquisition. -- `tool.realtime.event.normalize` - - `event[] -> event[]` - - event-time/source/raw/lineage normalization. -- `tool.realtime.window.rolling` - - `event[] -> window[]` - - event-time windowing, watermark, dedupe boundary. -- `tool.realtime.state.cache` - - `window[] -> stateSnapshot[]` - - incremental state cache and checkpointable state. -- `tool.realtime.feature.compute` - - `stateSnapshot[] -> feature[]` - - quant and situation-awareness feature computation. -- `tool.realtime.signal.emit` - - `feature[] -> signal[]` - - traceable signal output. It must not directly place orders. - -## Runtime Rule - -- Tool calls are runtime events, not Canvas nodes. -- These tools are projected as `resource.tool-capability.*` resources. -- A business node, imported external tool node, preset, or managed executor may - bind to these tools through `params.toolCapability`. -- Trace must record `tool_call_started`, partial output evidence, and - `tool_call_completed`. - -## Next Slice - -- Add guarded non-fixture executors for these registered tools. -- Add durable stream offset/window/state/feature/signal storage. -- Add first adapter executor for OKX ETH market stream replay/live subscribe. -- Keep the Canvas surface centered on business nodes and presets, not low-level - realtime tool internals. diff --git a/PLAN_ui_reskin.md b/PLAN_ui_reskin.md deleted file mode 100644 index f117cfb..0000000 --- a/PLAN_ui_reskin.md +++ /dev/null @@ -1,151 +0,0 @@ -# OpenCLI Admin 前端视觉改版（立项） - -> **状态：立项**（2026-06-18） -> **优先级**：P2 — 功能稳定后再做，不阻塞 NAS 部署与采集链路 -> **仓库**：本地 fork `opencli-admin`（当前 `origin` 仍指向上游 `xjh1994/opencli-admin`，改版分支建议在自有 remote 上开） - -## 背景 - -完整版已在 NAS 跑通（`8030` / `8031`），数据库已从精简版迁入。 -当前前端**能用但观感差**：默认 Tailwind 蓝灰后台、shadcn token 未贯通、页面样式两套混写、看板/stat 卡偏「内部运维面板」气质。 - -我们已 fork，具备长期维护自己的 UI 的前提；**本期只立项，不实施**。 - -## 设计方向（已锁定） - -**参考**：Linear（结构与交互克制）+ SpaceX（任务控制台气质）+ Grok（深色、干净、偏未来感）。 - -一句话：**深色为主的任务控制台 — 信息密度高、装饰极少、强调色只出现在「可点击 / 当前态」。** - -### 气质对照 - -| 来源 | 吸收什么 | 明确不要什么 | -|------|----------|--------------| -| **Linear** | 侧栏层级清晰、紧凑间距、细边框分层、单一强调色用于 active/focus | 大面积紫渐变、过度圆角糖果风 | -| **SpaceX** | 黑白灰主调、数字/状态像遥测读数、表格像任务清单 | 工业风过度粗犷、纯黑刺眼无层次 | -| **Grok** | 默认深色、面板干净、正文清晰、少量高光 | 花哨动效、拟物、彩虹 KPI 卡 | - -### 默认主题 - -- **默认暗色**（首次访问 / 无 localStorage 时）；亮色作为可选，不是主设计稿。 -- 背景分层（示意，落地进 CSS token）： - - `canvas` `#09090b`（近 Grok 底） - - `surface` `#111113` - - `elevated` `#18181b` - - `border` `rgba(255,255,255,0.08)`（Linear 式 hairline） -- **强调色**：冷紫 `hsl(252 56% 58%)` — 仅 nav active、primary button、focus ring、图表主线；**禁止** stat 卡底部 `bg-blue-500` 一类色块。 - -### 字体 - -| 用途 | 字体 | -|------|------| -| UI | **Inter**（或 Geist Sans 二选一，全站统一） | -| 遥测/ID | **JetBrains Mono** — `task_id`、时间戳、端点 URL、日志片段 | - -字号偏小一档（`text-sm` 为正文默认），行高略紧，对齐 Linear 密度。 - -### 组件规则 - -- **侧栏**：窄、图标 + 文案；active = 左侧 2px 强调条 + 微亮背景，不用整块 `bg-blue-600`。 -- **页头**：标题 `text-xl font-semibold tracking-tight`；描述 `text-muted` 一行即可。 -- **卡片**：`border` + 微透明底，**不用** `shadow-lg`；stat 数字大号单色，图标灰阶。 -- **表格**：斑马纹可选但极淡；hover 一行高亮；表头 `text-xs uppercase tracking-wider text-muted`。 -- **图表**：网格线 `#ffffff10`；系列色 ≤2（主线强调紫 + 次要灰）；去掉 Recharts 默认Legend 大块色。 -- **品牌区**：字标 `OpenCLI` + 小写 `Admin` 或几何 monogram；**去掉** ⚡ emoji。 - -### 反模式清单（改版时对照删） - -- [ ] Tailwind 默认 `blue-600` 当主色铺满 -- [ ] 彩虹 stat 图标底（蓝/紫/绿/红各一块） -- [ ] 页面级手写 `inputCls` 与 shadcn `Input` 并存 -- [ ] 面包屑/导航中文硬编码与 i18n 两套文案 - -## 目标 - -1. **统一设计语言**：shadcn CSS 变量 ↔ `tailwind.config` ↔ 组件，消灭页面级 `bg-blue-600` / 手写 `inputCls` 混用。 -2. **提升产品感**：侧栏、字体、间距、表格/空状态、图表配色 — 一眼不像脚手架 demo。 -3. **保持功能等价**：不改 API 契约、不删路由；i18n（中/英）继续可用。 -4. **可部署到 NAS**：`frontend` 自建镜像替换 `xjh1994/opencli-admin-frontend:0.3.6`。 - -## 非目标（本期不做） - -- 不重写业务逻辑 / 不拆 `SourcesPage` 800 行（可放到 Phase 2+） -- 不追求与上游 PR 合并（fork 自用为主；可选 cherry-pick 上游功能） -- 不做移动端专用适配（响应式修好即可） -- 不引入新框架（继续 Vite + React 18 + Tailwind 3 + 现有 shadcn 组件） - -## 现状摘要（改版前基线） - -| 项 | 现状 | -|----|------| -| 组件库 | shadcn/ui 已装，大量页面仍用原生 class | -| 主题 | `index.css` 有 token，`tailwind.config.js` 未接 `hsl(var(--primary))` | -| 布局 | `Layout.tsx` 侧栏 `gray-900` + emoji logo；面包屑中文硬编码 | -| 看板 | 彩虹 stat 图标底 + Recharts 默认色 | -| 暗色模式 | `useEffect` 仅 mount 同步，存在刷新后主题不一致风险 | -| 部署 | 官方 frontend 镜像；本地 `npm run build` 已通过 | - -## 成功标准（验收） - -- [ ] 全站主色/圆角/边框/ focus 环来自同一套 token -- [ ] 侧栏 + 页头有明确品牌区（非 emoji 占位） -- [ ] 视觉符合上文「设计方向」；看板 stat/图表 ≤2 强调色 -- [ ] 默认暗色；遥测字段使用等宽字体 -- [ ] 暗色模式切换后刷新仍正确 -- [ ] `npm run build` 无报错；NAS 上自建镜像 UI 200、核心流程可点通（数据源列表、任务、节点） -- [ ] 截图对比：改版前后各 1 张存档于 `docs/ui-reskin/` - -## 分期计划 - -### Phase 0 — 准备（0.5d，开工时做） - -- [ ] 确认 fork remote（Gitea/GitHub）与默认分支策略 -- [ ] 建分支 `feat/ui-reskin` -- [ ] 抓基线截图：`dashboard` / `sources` / `nodes` 亮暗各一套 -- [x] 选定参考方向：**Linear + SpaceX + Grok**（见「设计方向」） -- [ ] 在 `frontend/index.html` 引入 Inter + JetBrains Mono（或 npm `@fontsource`） -- [ ] 起草 token 草稿：`frontend/src/index.css` + `tailwind.config.js` 对照表 - -### Phase 1 — 设计系统落地（1–2d） - -- [ ] `tailwind.config.js` 接入 shadcn 语义色 + `fontFamily` -- [ ] `Layout.tsx`：侧栏、nav 激活态、底栏控件改用 `Button` / token -- [ ] 修复暗色模式持久化逻辑 -- [ ] 面包屑走 i18n，去掉 `ROUTE_LABELS` 硬编码 - -### Phase 2 — 高频页面抛光（2–3d） - -- [ ] `DashboardPage`：stat 卡、图表主题、表格样式 -- [ ] `SourcesPage` / `TasksPage`：表单与列表统一组件 -- [ ] `EmptyState` / `PageHeader` / `DataTable` 统一间距 rhythm - -### Phase 3 — 部署与文档（0.5d） - -- [ ] `frontend/Dockerfile` 本地 build tag，例如 `opencli-admin-frontend:curry-ui` -- [ ] NAS `docker-compose` 改镜像名；保留官方 tag 可回滚 -- [ ] 本文件状态改为 **已完成** 或拆 `CHANGELOG` 条目 - -## 风险与约束 - -- **上游合并**：UI 大 diff 难 upstream；功能修复尽量独立 commit，UI 单独分支。 -- **NAS**：Compose v2.20.1，继续避免 `env_file: path/required` 新语法。 -- **范围蔓延**：Phase 1 只做「看起来像一套产品」；巨型页面重构单列 backlog。 - -## Backlog（改版后可选） - -- 拆分 `SourcesPage.tsx` 为 wizard + 列表子模块 -- 导航分组（采集 / 运行 / 系统） -- 上游有 UI 更新时 diff 合并策略 - -## 相关路径 - -- 前端源码：`frontend/src/` -- NAS 完整版：`/volume1/docker/opencli-admin/` -- 本地 env 模板：`~/.omc/opencli-admin-nas.env` - -## 变更记录 - -| 日期 | 说明 | -|------|------| -| 2026-06-18 | 立项，NAS 完整版已上线，UI 改版推迟 | -| 2026-06-18 | 设计方向锁定：Linear + SpaceX + Grok，默认暗色任务控制台 | \ No newline at end of file diff --git a/PONYTAIL.md b/PONYTAIL.md deleted file mode 100644 index f4a04cc..0000000 --- a/PONYTAIL.md +++ /dev/null @@ -1,110 +0,0 @@ -# Ponytail Style Guide for opencli-admin - -> Read this before writing any code. - -## The Ladder - -Before writing code, stop at the first rung that satisfies: - -1. **Does this need to exist?** → No: skip it -2. **Stdlib?** → Use it -3. **Native platform feature?** → Use it -4. **Installed dependency?** → Use it -5. **One line?** → One line -6. **Only then: minimum that works** - -## Never Cut - -- Trust-boundary validation -- Data-loss handling -- Security -- Accessibility - -## Code Rules - -### Prefer Native - -```tsx -// ❌ over-engineer - -useEffect(() => { fetchData() }, []) - -// ✅ ponytail - -``` - -### Prefer Stdlib - -```tsx -// ❌ dayjs -dayjs().format('YYYY-MM-DD') - -// ✅ stdlib -new Date().toISOString().split('T')[0] -``` - -### Prefer One Line - -```tsx -// ❌ 50 lines for a status color -switch(status) { case 'active': return 'green' } - -// ✅ one line -const color = { active: 'green', inactive: 'red' }[status] ?? 'gray' -``` - -### Prefer Installed - -```tsx -// ❌ install new package -import { useLocalStorage } from 'usehooks-ts' - -// ✅ use what's already in package.json -import { useLocalStorage } from '@/hooks' -``` - -## File Rules - -- Max 200 lines per file -- Split only when necessary (test, reused 3x, >200 lines) -- Flat directory structure -- Group by feature, not by type - -## Import Rules - -Order: -1. Node built-ins (`path`, `fs`) -2. External packages -3. Internal packages (`@opencli/shared`) -4. Relative imports (`./`, `../`) - -## Type Rules - -- Infer when obvious -- Explicit when crossing boundaries (API, props) -- Don't type everything - -## API Rules - -- One endpoint per resource concern -- Don't over-wrap responses -- Add metadata only when list pagination needs it - -## Commit Rules - -- Verb first, under 72 chars -- No "and" -- feat/fix/chore/docs/refactor/test - -## Dependency Rules - -Before adding `npm install`: - -1. Stdlib works? -2. Already installed? -3. Really needed? -4. Will it be reused? - ---- - -*Less code. More ship.* diff --git a/README.md b/README.md index 9370fc4..e9cfc5a 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,7 @@ v0.4.0 同时发布 linux/amd64 和 linux/arm64： - [开发规范](docs/DEVELOPMENT_STANDARD.md) - [架构决策记录](docs/adr) -Issue 与 PR 均在本仓库公开协作。提交功能前请先运行与改动范围对应的最小测试，再运行前端 lint / typecheck / build 或后端测试。 +Issue 与 PR 均在本仓库公开协作；开发计划和任务统一进入 GitHub Issues，长期架构决策进入 `docs/adr`。提交功能前请先运行与改动范围对应的最小测试，再运行前端 lint / typecheck / build 或后端测试。 ## License diff --git a/TODOS.md b/TODOS.md deleted file mode 100644 index 4991c3e..0000000 --- a/TODOS.md +++ /dev/null @@ -1,85 +0,0 @@ -# TODOs - -## Workflow Runtime Conformance - -### Maintain config-blocked conformance cases - -- **Status:** First taxonomy-backed fixtures are implemented for missing webhook URL, missing source credential, and missing runtime resource. -- **What:** Keep `config-blocked` conformance cases current as new runtime bindings add config, credential, or resource gates. -- **Why:** The first conformance slice covers happy path, permission-blocked, and missing-binding evidence; config absence is a separate failure class that open-source users must be able to diagnose without guessing. -- **Pros:** Extends the drift gate beyond permissions and missing bindings, and makes live/preview/simulated claims harder to overstate. -- **Cons:** New bindings must register stable reason taxonomy entries before tests can assert exact failures without churn. -- **Context:** `Administrator-codex-opinion-monitor-quickstart-design-20260706-182303.md` defines runtime truth as registry declaration plus executable fixture plus observed event transcript. Config-blocked cases were intentionally deferred from the first PR-sized slice to keep the initial harness small. -- **Depends on / blocked by:** Future binding-specific config gates and their stable block reason definitions. - -### Maintain SSE event stream smoke coverage - -- **Status:** Canonical happy-path `/events/stream` smoke coverage now reuses the snapshot expected transcript matcher. -- **What:** Keep `/api/v1/workflows/runs/{runId}/events/stream` smoke coverage aligned with the snapshot-based conformance matcher as event shapes evolve. -- **Why:** The conformance matcher should use the deterministic `/events` snapshot as its primary evidence source, but the live UI still depends on the stream endpoint. -- **Pros:** Protects the live event-stream user path without making the main conformance gate depend on polling windows or SSE timing. -- **Cons:** SSE tests are more timing-sensitive than snapshot tests and should stay out of the primary conformance matcher. -- **Context:** The `/plan-eng-review` decision for performance was to use `/events` snapshots for conformance evidence and reserve `/events/stream` for a later smoke test only. -- **Depends on / blocked by:** Future event shape changes must update both snapshot expected transcripts and SSE parser expectations. - -### Maintain ODP/Redis-stream conformance after the interim harness - -- **Status:** Workflow-run event mirror conformance now publishes stable event facts through the Redis stream interface and reads them back with the shared transcript matcher. -- **What:** Keep ODP/Redis-stream conformance cases current as the event mirror moves from fixture Redis clients to deployment Redis and later ODP consumers. -- **Why:** The first conformance slice certifies the current `/events` API, but the longer-term architecture makes ODP/event streams the runtime source of truth. -- **Pros:** Prevents the interim snapshot harness from becoming the permanent definition of runtime truth, and keeps the open-source conformance story aligned with the event-stream-first architecture. -- **Cons:** Expands scope into executor/source-of-truth migration and should wait until the transcript schema is stable. -- **Context:** The current design deliberately separates interim workflow-run event evidence from long-term ODP/Redis event-stream evidence. -- **Depends on / blocked by:** Deployment Redis/ODP consumer coverage. - -### Maintain real node I/O contract coverage - -- **Status:** Runtime bindings now declare stable input shape, output shape, permission gate, config gate, event shape, and fixture coverage through `backend/workflow/runtime_contracts.py`. -- **What:** Keep each new runtime binding in the contract table before exposing it as runnable in compile output or Canvas capability/status surfaces. -- **Why:** A binding is not runtime-certified just because compile can produce a node or the UI can place it on the Canvas; it needs a stable I/O contract first. -- **Pros:** Prevents runnable/status drift and keeps resource internals out of user-entered fields. -- **Cons:** New runtime bindings must update both the contract table and focused fixture coverage before they can honestly appear runnable. -- **Context:** The contract is projected into both registry metadata and capability manifests; webhook delivery now has its own deterministic request-capture fixture. -- **Depends on / blocked by:** Future runtime bindings must add contract declarations and fixture evidence before being exposed as runnable. - -### Maintain webhook real delivery fixtures - -- **Status:** `workflow.notifier.webhook.send` now sends through the registered webhook notifier when send permission, configured URL, and upstream EvidenceBatch projection are present. -- **What:** Keep the success request-capture fixture and negative missing-permission, missing-URL, and missing-projection fixtures aligned with the delivery payload. -- **Why:** Webhook delivery is the final runtime layer; regressions here would silently turn real delivery back into a projection-only claim. -- **Pros:** Confirms actual POST construction while keeping SSRF-safe notifier plumbing and blocked preconditions visible. -- **Cons:** Payload schema changes must update both request-capture assertions and expected transcript evidence. -- **Context:** Capability/status surfaces remain blocked by default because each run still needs user configuration and upstream projection input, but the backend delivery path is now executable. -- **Depends on / blocked by:** Future webhook payload schema or notifier security changes. - -## Consolidated follow-ups from completed goals - -### Activate or remove model-provider failover - -- **Status:** `ProviderResolver.resolve_with_fallback()` is implemented and tested but has no production caller. -- **What:** Either route one explicit provider role through the resolver or remove the unused failover surface. -- **Why:** Defaults and cooldown currently imply runtime failover that the application does not perform. - -### Live-test the model-provider UI - -- **Status:** Type checking, linting, and production build passed; create/edit/delete, sync, connection test, defaults ordering, and toast behavior were not exercised against a live backend. -- **What:** Run one focused full-stack acceptance pass and record or fix the observed result. -- **Why:** Compile-time checks do not validate mutation wiring or response handling. - -### Expand Browser Act support only from real demand - -- **Status:** 78 vendored packs exist, but only 2 have manifests; the interpreter does not URL-encode parameters or model arithmetic pagination. -- **What:** Add manifests and interpreter features when onboarding a concrete pack, starting with correct URL encoding for any URL-bound parameter. -- **Why:** Bulk-promising support for every vendored pack would overstate runtime capability; demand-led onboarding keeps each claim testable. - -### Execute capability probes - -- **Status:** Capability manifests expose probe names, but the workflow capability layer does not execute richer runtime probes. -- **What:** Add probe execution when a concrete capability needs environment-level readiness evidence. -- **Why:** A declared probe name alone cannot prove that a capability is runnable in the current deployment. - -### Add guarded production tool executors - -- **Status:** `workflow.external-tool.capability` accepts only the deterministic fixture executor seam. -- **What:** Add a real executor adapter only when onboarding the first production external Tool Capability. -- **Why:** Imported tools should remain blocked until a guarded, observable executor exists; speculative adapters are unnecessary. diff --git a/backend/agent_runtimes/base.py b/backend/agent_runtimes/base.py index 502e736..5bc4f1c 100644 --- a/backend/agent_runtimes/base.py +++ b/backend/agent_runtimes/base.py @@ -2,16 +2,15 @@ Mirrors ``backend/channels/base.py``'s split: a frozen ``Capabilities`` dataclass the caller branches on (never ``isinstance``), a small task/result value-object -pair, and an ``ABC`` adapters implement. See ``GOAL-agent-runtimes.md`` §2 for -the design rationale — the short version: the three target frameworks +pair, and an ``ABC`` adapters implement. The three target frameworks (LangGraph, VoltAgent, pi) span two languages and three transport shapes, so the abstraction has to be a *process/protocol* contract, not a Python import. Each adapter owns exactly one framework's native stream and translates it into the closed event set below; nothing outside the adapter ever sees a framework-native shape. -Event design note (OpenAlice lesson, cited in the GOAL doc): normalize the -*protocol*, not the output. The event set is intentionally tiny and closed — +Event design note: normalize the *protocol*, not the output. The event set is +intentionally tiny and closed — ``EVENT_TYPES`` — so adapters cannot invent new shapes ad hoc. Use the ``event_*`` helper constructors below rather than hand-building event dicts; that is what prevents typos in ``type`` strings and missing ``task_id`` fields diff --git a/backend/agent_runtimes/pi_adapter.py b/backend/agent_runtimes/pi_adapter.py index 42e974d..92972dd 100644 --- a/backend/agent_runtimes/pi_adapter.py +++ b/backend/agent_runtimes/pi_adapter.py @@ -36,12 +36,12 @@ ``text_delta`` such as ``thinking_delta``/``toolcall_delta``) is skipped with a debug log — NOT invented into a new event type, per the closed event-set rule in ``base.py``. - * ``PI_CODING_AGENT_DIR`` (assumed in the original GOAL doc as a "provider + * ``PI_CODING_AGENT_DIR`` (assumed in the original design as a "provider override" env var) DOES NOT EXIST in pi's actual settings/providers docs. The real, documented env var for redirecting pi's persistent state is ``PI_CODING_AGENT_SESSION_DIR`` (overrides the ``sessionDir`` setting; default is under the user's home directory). This adapter's - ``provider_dir`` config key is kept (GOAL doc's name) but is wired to the + ``provider_dir`` config key is kept (design notes's name) but is wired to the real ``PI_CODING_AGENT_SESSION_DIR`` env var — see ``_compose_env``. If a future pi release adds a true provider/model-config directory override, fix the one line in ``_compose_env``. diff --git a/backend/agent_runtimes/registry.py b/backend/agent_runtimes/registry.py index dcd94ec..dc74a43 100644 --- a/backend/agent_runtimes/registry.py +++ b/backend/agent_runtimes/registry.py @@ -34,7 +34,7 @@ def available_runtimes() -> list[str]: sync ``is_available()`` classmethod. This is what the ws register handshake advertises to the center — never the full registry, since a node may not have every runtime's binary installed (Docker image - layering, GOAL doc §6).""" + layering, design notes §6).""" available: list[str] = [] for runtime_type, instance in _REGISTRY.items(): is_available = getattr(type(instance), "is_available", None) diff --git a/backend/api/v1/browser_act.py b/backend/api/v1/browser_act.py index f9f6cde..b1b2062 100644 --- a/backend/api/v1/browser_act.py +++ b/backend/api/v1/browser_act.py @@ -1,5 +1,5 @@ """GET /api/v1/browser-act/packs — read-only catalog listing for the -BrowserAct channel's frontend surfacing (GOAL-7 PR-E, decision #9). +BrowserAct channel's frontend surfacing (Browser Act integration PR-E, decision #9). Reuses ``PackCatalog`` (PR-A) + ``load_manifest`` (PR-A schema / PR-D seeds) — no new catalog mechanism, no DB table. Read-only, no auth (matches the diff --git a/backend/api/v1/chat.py b/backend/api/v1/chat.py index 6e49fd8..ee4f636 100644 --- a/backend/api/v1/chat.py +++ b/backend/api/v1/chat.py @@ -212,7 +212,7 @@ async def _pick_provider(db: AsyncSession, provider_id: Optional[str]) -> ModelP async def _build_client(provider: ModelProvider): """Build the agent dock's OpenAI-compatible tool-calling client. - GOAL-6 PR-E: consolidates what used to be a private ``AsyncOpenAI(...)`` + model-provider runtime PR-E: consolidates what used to be a private ``AsyncOpenAI(...)`` construction here into :class:`~backend.llm.openai_compat.OpenAICompatAdapter` via :func:`~backend.llm.factory.build_openai_compat_adapter` — the same guarded client :class:`OpenAICompatAdapter` gives every other PR-E diff --git a/backend/api/v1/model_defaults.py b/backend/api/v1/model_defaults.py index 765cb2a..2e79d72 100644 --- a/backend/api/v1/model_defaults.py +++ b/backend/api/v1/model_defaults.py @@ -1,4 +1,4 @@ -"""GET/PUT endpoints for ``model_defaults`` (GOAL-6 PR-C, decision #10). +"""GET/PUT endpoints for ``model_defaults`` (model-provider runtime PR-C, decision #10). A top-level resource, not nested under ``/providers/{id}`` — a role's candidate list can reference any provider, so it doesn't belong under one diff --git a/backend/api/v1/providers.py b/backend/api/v1/providers.py index 694e364..0a9ff0f 100644 --- a/backend/api/v1/providers.py +++ b/backend/api/v1/providers.py @@ -1,4 +1,4 @@ -"""CRUD endpoints for model providers, plus GOAL-6 PR-C's provider-scoped +"""CRUD endpoints for model providers, plus model-provider runtime PR-C's provider-scoped API: test-connection, model-catalog sync, and model catalog CRUD (decision #10). DB logic for the PR-C additions lives in ``backend.services.provider_model_service`` (thin-endpoint convention); this @@ -235,7 +235,7 @@ async def delete_provider(provider_id: str, db: AsyncSession = Depends(get_db)) provider = result.scalar_one_or_none() if not provider: raise HTTPException(status_code=404, detail="Provider not found") - # GOAL-6 PR-C (decision #3 / PR-A note): sqlite here never runs with + # model-provider runtime PR-C (decision #3 / PR-A note): sqlite here never runs with # PRAGMA foreign_keys=ON, so provider_models' ondelete=CASCADE never # fires at runtime -- clean up the catalog explicitly or it orphans. await provider_model_service.delete_provider_models(db, provider_id) @@ -245,7 +245,7 @@ async def delete_provider(provider_id: str, db: AsyncSession = Depends(get_db)) # --------------------------------------------------------------------------- -# GOAL-6 PR-C: test connection / model catalog sync + CRUD (decision #10) +# model-provider runtime PR-C: test connection / model catalog sync + CRUD (decision #10) # --------------------------------------------------------------------------- diff --git a/backend/browser_act/__init__.py b/backend/browser_act/__init__.py index b764f9b..54aadf3 100644 --- a/backend/browser_act/__init__.py +++ b/backend/browser_act/__init__.py @@ -1,4 +1,4 @@ -"""backend.browser_act — browser-act CLI subprocess wrapper (GOAL-7 PR-B). +"""backend.browser_act — browser-act CLI subprocess wrapper (Browser Act integration PR-B). Distinct from both: - ``backend/cli.py`` — the opencli-skill HTTP client (unrelated tool). diff --git a/backend/browser_act/cli.py b/backend/browser_act/cli.py index 3d8bf13..0364a7f 100644 --- a/backend/browser_act/cli.py +++ b/backend/browser_act/cli.py @@ -1,4 +1,4 @@ -"""browser-act CLI subprocess wrapper (GOAL-7 PR-B). +"""browser-act CLI subprocess wrapper. Distinct from both ``backend/cli.py`` (the opencli-skill HTTP client) and ``backend/browser_act_packs/`` (PR-A's vendored SKILL.md packs). This module @@ -9,8 +9,8 @@ ``TimeoutError`` -> ``proc.kill()`` + ``await proc.wait()`` so a timed-out child is never orphaned. -Architecture decision #6 (GOAL-7.md): both hops (this CLI, and the -scripts/*.py -> eval-js hop the future channel drives) are argv-only. Never +Both hops (this CLI, and the scripts/*.py -> eval-js hop the future channel +drives) are argv-only. Never ``asyncio.create_subprocess_shell``, never ``shell=True``, never a string-joined command. User-controlled values (URLs, JS, input text) always travel as individual argv elements so they can never be interpreted by a @@ -187,7 +187,7 @@ async def session( ) -> AsyncIterator[BrowserActSession]: """Scope a browser-act session name/env for a block of calls. - Deliberately MINIMAL lifecycle (GOAL-7 architecture decision #10): this + Deliberately MINIMAL lifecycle (Browser Act integration architecture decision #10): this does NOT call ``browser open`` on enter nor ``session close`` on exit. Browser creation/deletion carries BrowserAct's own confirmation-gate concern, and owning that gate is PR-C's job (the manifest-interpreter diff --git a/backend/browser_act/scripts.py b/backend/browser_act/scripts.py index 71ec0ba..22d668d 100644 --- a/backend/browser_act/scripts.py +++ b/backend/browser_act/scripts.py @@ -1,6 +1,7 @@ -"""run_pack_script — the SECOND subprocess hop for a browser-act pack -(GOAL-7 PR-C). The first hop is browser-act's own CLI (backend/browser_act/ -cli.py, PR-B); this one runs a vendored pack's scripts/*.py, which are pure +"""run_pack_script — the second subprocess hop for a browser-act pack. + +The first hop is browser-act's own CLI (backend/browser_act/cli.py); this one +runs a vendored pack's scripts/*.py, which are pure JS emitters (argparse -> print(js_string), no LLM/no network/no file I/O -- see backend/browser_act_packs/VENDOR.md). @@ -9,8 +10,8 @@ asyncio.wait_for(timeout) + TimeoutError -> proc.kill() + await proc.wait() so a timed-out script is never orphaned. -Architecture decision #6 (GOAL-7.md): BOTH subprocess hops are argv-only, -never shell. This hop invokes ``sys.executable `` via +Both subprocess hops are argv-only, never shell. This hop invokes +``sys.executable `` via create_subprocess_exec -- caller params travel as separate argv elements, never string-interpolated into a shell command. """ diff --git a/backend/browser_act_packs/VENDOR.md b/backend/browser_act_packs/VENDOR.md index 3b228ec..8a147d8 100644 --- a/backend/browser_act_packs/VENDOR.md +++ b/backend/browser_act_packs/VENDOR.md @@ -30,7 +30,7 @@ time; see `catalog.py`'s scan for the live count). (kept out of the category tree so it isn't mistaken for a pack). The upstream `LICENSE` was copied unchanged to `backend/browser_act_packs/LICENSE`. -**Not vendored** (see GOAL-7.md, YAGNI list): +**Not vendored**: - the `browser-act` CLI itself (external PyPI tool, `uv tool install browser-act-cli`) - `browser-act-skill-forge` (pack generator, out of scope) @@ -45,7 +45,7 @@ re-copy `solutions/**`), or override in code outside this tree. - `catalog.py`, `manifest.py`, `__init__.py` — new code, not vendored. - `channel.manifest.json` files (one per pack, machine-readable execution - contract) are **our addition**, authored in PR-D of GOAL-7 — they do not + contract) are **our addition**, authored in PR-D of Browser Act integration — they do not exist upstream and are not part of the BrowserAct skills repo. PR-A (this vendor drop) defines only the `PackManifest` schema/loader; it does not write any `channel.manifest.json` content. @@ -63,7 +63,7 @@ To pull upstream updates: ## Seed manifests (PR-D) Two of the 78 vendored packs got a hand-authored `channel.manifest.json` -(GOAL-7 decision #5), chosen as the pair that can be validated end-to-end +(Browser Act integration decision #5), chosen as the pair that can be validated end-to-end without a logged-in session: `search-research/google-search-serp` (a login-free public search) and `ecommerce/taobao-keyword-search` (a login-free public listing search page). Both translations below were @@ -154,7 +154,7 @@ SKILL.md prose → manifest: enhancement (e.g. a `pagination.mode` that lets a manifest declare an offset formula), not something this seed pretends to support. 3. **The other ~76 vendored packs have no `channel.manifest.json` yet.** - Seeding is intentionally incremental (GOAL-7 PR-D scope: 2 packs to prove + Seeding is intentionally incremental (Browser Act integration PR-D scope: 2 packs to prove the pipeline, not full coverage) — `backend/browser_act_packs/manifest.py` (`PackManifest`) is the extension point; add a `channel.manifest.json` next to a pack's `SKILL.md` as each one is needed, translating its prose diff --git a/backend/browser_act_packs/catalog.py b/backend/browser_act_packs/catalog.py index 2c29539..896c1de 100644 --- a/backend/browser_act_packs/catalog.py +++ b/backend/browser_act_packs/catalog.py @@ -1,6 +1,6 @@ """PackCatalog — scans the vendored browser-act pack tree for SKILL.md files. -Domain/capability derivation (kept simple, per GOAL-7 decision #2/#5): the +Domain/capability derivation (kept simple, per Browser Act integration decision #2/#5): the SKILL.md frontmatter itself only declares ``name``/``description``, it does not declare a domain/capability split. Rather than invent a second taxonomy, this catalog derives domain/capability straight from the directory layout diff --git a/backend/browser_act_packs/manifest.py b/backend/browser_act_packs/manifest.py index be7d245..e561047 100644 --- a/backend/browser_act_packs/manifest.py +++ b/backend/browser_act_packs/manifest.py @@ -1,4 +1,4 @@ -"""PackManifest — the ``channel.manifest.json`` schema (GOAL-7 decision #5). +"""PackManifest — the ``channel.manifest.json`` schema (Browser Act integration decision #5). This is the machine-readable execution contract that ``BrowserActChannel`` (PR-C) interprets generically: ``steps`` is a sequence of browser-act @@ -33,7 +33,7 @@ class ParamSpec(BaseModel): enum: Optional[list[str]] = None -#: The browser-act operations a manifest step can drive. See GOAL-7 decision +#: The browser-act operations a manifest step can drive. See Browser Act integration decision #: #1/#6: the channel drives these deterministically, argv-only, no shell. StepOp = Literal["navigate", "wait", "eval_script", "click", "input"] diff --git a/backend/channels/browser_act_channel.py b/backend/channels/browser_act_channel.py index 1bba72b..b31c8dd 100644 --- a/backend/channels/browser_act_channel.py +++ b/backend/channels/browser_act_channel.py @@ -1,6 +1,4 @@ -"""BrowserActChannel — the generic browser-act pack manifest interpreter -(GOAL-7 PR-C/PR-E; see GOAL-7.md architecture decisions #1, #4, #5, #6, #7, -#8, #10). +"""BrowserActChannel — the generic browser-act pack manifest interpreter. Drives a vendored pack (backend/browser_act_packs///, PR-A) through its channel.manifest.json (PR-A schema, backend/ @@ -138,7 +136,7 @@ def _stop_when_triggered(stop_when: str | None, page_item_count: int) -> bool: is NOT recognized as a stop signal here -- callers must still rely on ``max_pages`` and the "a page returned 0 items" fallback to terminate pagination for stop_when expressions this can't interpret (documented - limitation, GOAL-7 PR-C -- full stop_when grammar is out of scope). + limitation, Browser Act integration PR-C -- full stop_when grammar is out of scope). """ if not stop_when: return False diff --git a/backend/channels/crawl4ai_channel.py b/backend/channels/crawl4ai_channel.py index 457ed7c..e2476b1 100644 --- a/backend/channels/crawl4ai_channel.py +++ b/backend/channels/crawl4ai_channel.py @@ -175,7 +175,7 @@ async def _resolve_llm_config(provider_id: str | None) -> Any: """Same autonomous-default convention as backend.pipeline.runner: an explicit provider_id wins, otherwise the first enabled ModelProvider. - GOAL-6 PR-E exception (decision #8): crawl4ai's LLM calls go through + model-provider runtime PR-E exception (decision #8): crawl4ai's LLM calls go through its own ``litellm``-backed ``LLMExtractionStrategy`` / ``AsyncWebCrawler`` — an internal client this module has no clean seam to route through ``backend.llm``'s adapters/resolver, so that diff --git a/backend/channels/skill_channel.py b/backend/channels/skill_channel.py index 48643b0..577a578 100644 --- a/backend/channels/skill_channel.py +++ b/backend/channels/skill_channel.py @@ -210,7 +210,7 @@ async def _build_model_call(provider: dict[str, Any]) -> Any: model. ``reply`` is the raw OpenAI chat object the loop already knows how to normalize (both ``tool_calls`` and the Qwen XML ```` path). - GOAL-6 PR-E: client construction is consolidated through + model-provider runtime PR-E: client construction is consolidated through :class:`~backend.llm.openai_compat.OpenAICompatAdapter` (via :func:`~backend.llm.factory.build_openai_compat_adapter`) — the same guarded ``AsyncOpenAI`` construction ``chat.py`` and the ``openai`` diff --git a/backend/config.py b/backend/config.py index 4cd4220..5b19992 100644 --- a/backend/config.py +++ b/backend/config.py @@ -143,7 +143,7 @@ def cdp_endpoints(self) -> list[str]: # Managed acquisition runtime. The commit/version are code-owned pins; # this path merely locates the installed checkout on every platform. ohmyopencli_root: str = "/opt/ohmyopencli" - # browser-act CLI subprocess execution timeout, per call (GOAL-7 PR-B). + # browser-act CLI subprocess execution timeout, per call (Browser Act integration PR-B). # Binary path is BROWSER_ACT_BIN env var (mirrors OPENCLI_BIN), not a # Settings field — this is only the per-call timeout default. browser_act_timeout: int = 120 diff --git a/backend/llm/__init__.py b/backend/llm/__init__.py index c7565c9..8d91c7c 100644 --- a/backend/llm/__init__.py +++ b/backend/llm/__init__.py @@ -1,4 +1,4 @@ -"""backend.llm — self-built model-provider runtime (GOAL-6 decision #1). +"""backend.llm — self-built model-provider runtime (model-provider runtime decision #1). No litellm: this package owns provider-agnostic building blocks for chat/list_models/test_connection dispatch across ``model_providers`` rows. @@ -29,7 +29,7 @@ #: a plain string so future embedding/rerank rows don't need a migration. VALID_MODEL_TYPES = frozenset({"llm"}) -#: model_defaults.role — the three consumption points GOAL-6 collapses onto +#: model_defaults.role — the three consumption points model-provider runtime collapses onto #: ModelProvider (decision #4): agent dock chat, skill_channel's cheap #: executor model, pipeline enrichment fallback. VALID_ROLES = frozenset({"chat", "executor", "enrichment"}) diff --git a/backend/llm/anthropic.py b/backend/llm/anthropic.py index ba0f171..29b89bc 100644 --- a/backend/llm/anthropic.py +++ b/backend/llm/anthropic.py @@ -1,4 +1,4 @@ -"""Anthropic adapter (GOAL-6 PR-B) — ``provider_type == "claude"`` (decision +"""Anthropic adapter (model-provider runtime PR-B) — ``provider_type == "claude"`` (decision #2).""" from __future__ import annotations @@ -87,7 +87,7 @@ async def aclose(self) -> None: self._pinned_http_client = None async def get_client(self) -> Any: - """Public accessor for the guarded ``AsyncAnthropic`` client (GOAL-6 + """Public accessor for the guarded ``AsyncAnthropic`` client (model-provider runtime PR-E) — mirrors :meth:`OpenAICompatAdapter.get_client`. Used by ``claude_processor`` to consolidate client construction while keeping its own per-record loop + usage-token logging (which needs the raw diff --git a/backend/llm/base.py b/backend/llm/base.py index b231f32..893d62f 100644 --- a/backend/llm/base.py +++ b/backend/llm/base.py @@ -1,4 +1,4 @@ -"""``ProviderAdapter`` ABC (GOAL-6 PR-B). +"""``ProviderAdapter`` ABC (model-provider runtime PR-B). Every concrete adapter (:class:`~backend.llm.openai_compat.OpenAICompatAdapter`, :class:`~backend.llm.anthropic.AnthropicAdapter`) implements the same three @@ -55,7 +55,7 @@ class LlmAdapterError(Exception): bug in the adapter that raised it, not something this class can enforce on its own. - ``retryable`` (GOAL-6 PR-D, decision #7) tells + ``retryable`` (model-provider runtime PR-D, decision #7) tells :class:`~backend.llm.resolver.ProviderResolver` whether this failure is connection-level (connect error / timeout / 5xx — the provider is unreachable or broken right now, worth failing over to the next @@ -75,7 +75,7 @@ def __init__(self, message: str, *, retryable: bool = False) -> None: def classify_retryable(exc: BaseException) -> bool: """Classify a caught SDK/transport exception as connection-level - (``True``) vs business-level (``False``) for GOAL-6 decision #7. + (``True``) vs business-level (``False``) for model-provider runtime decision #7. Connection-level (worth a failover): stdlib/httpx transport timeouts and connection failures, and each SDK's own connection/timeout/5xx exception diff --git a/backend/llm/catalog.py b/backend/llm/catalog.py index 19e770d..f83957d 100644 --- a/backend/llm/catalog.py +++ b/backend/llm/catalog.py @@ -1,4 +1,4 @@ -"""Anthropic model catalog (GOAL-6 decision #5). +"""Anthropic model catalog (model-provider runtime decision #5). Anthropic has no ``GET /v1/models``-style discovery endpoint the way OpenAI-compatible providers do (PR-B's ``OpenAICompatAdapter`` hits diff --git a/backend/llm/factory.py b/backend/llm/factory.py index ddc215a..b553e34 100644 --- a/backend/llm/factory.py +++ b/backend/llm/factory.py @@ -1,4 +1,4 @@ -"""Adapter factory (GOAL-6 PR-B, decision #6): dispatch a +"""Adapter factory (model-provider runtime PR-B, decision #6): dispatch a :class:`~backend.models.provider.ModelProvider` row to its :class:`~backend.llm.base.ProviderAdapter` by ``provider_type``. @@ -29,7 +29,7 @@ } #: provider_type -> litellm provider-name prefix (crawl4ai_channel's -#: LLMConfig, GOAL-6 PR-E decision #8's crawl4ai exception). Mirrors +#: LLMConfig, model-provider runtime PR-E decision #8's crawl4ai exception). Mirrors #: _ADAPTERS' family grouping 1:1 (openai/local share the openai wire #: protocol, claude is Anthropic's own) so the two mappings can't quietly #: drift apart by being hand-maintained in two files. @@ -59,7 +59,7 @@ def get_adapter(provider: Any) -> ProviderAdapter: def litellm_prefix_for(provider_type: str | None) -> str: """Map a ``ModelProvider.provider_type`` to the litellm provider-name - prefix ``crawl4ai_channel``'s ``LLMConfig`` needs (GOAL-6 PR-E, decision + prefix ``crawl4ai_channel``'s ``LLMConfig`` needs (model-provider runtime PR-E, decision #8's crawl4ai exception): the litellm client/call itself stays untouched there, but *which* prefix a given ``provider_type`` maps to is now decided here — the same place :func:`get_adapter` dispatches from — @@ -81,7 +81,7 @@ def _provider_view( ) -> Any: """Build a minimal read-only stand-in for a :class:`~backend.models.provider.ModelProvider` row from already-resolved - field values (GOAL-6 PR-E). + field values (model-provider runtime PR-E). ``OpenAICompatAdapter``/``AnthropicAdapter`` only ever read ``provider.provider_type`` / ``.base_url`` / ``.api_key`` / @@ -102,7 +102,7 @@ def _provider_view( Each PR-E call site keeps resolving its OWN fields first (attribute vs dict-get, its own env-var fallback name, its own default) exactly as it - did before GOAL-6 — this only removes the duplicated *client + did before model-provider runtime — this only removes the duplicated *client construction* step, not each caller's field-resolution rules. """ return SimpleNamespace( @@ -121,7 +121,7 @@ def build_openai_compat_adapter( provider_type: str | None = None, ) -> OpenAICompatAdapter: """Build an :class:`OpenAICompatAdapter` from already-resolved field - values (GOAL-6 PR-E) — for ``chat.py``/``skill_channel``/the ``openai`` + values (model-provider runtime PR-E) — for ``chat.py``/``skill_channel``/the ``openai`` processor, which need the guarded ``AsyncOpenAI`` client construction this adapter implements, but whose ``provider`` is either a live ORM row (chat.py) or a plain config ``dict`` (skill_channel, the processors), not @@ -155,7 +155,7 @@ def build_anthropic_adapter( default_model: str | None = None, ) -> AnthropicAdapter: """Build an :class:`AnthropicAdapter` from already-resolved field values - (GOAL-6 PR-E) — for the ``claude`` processor. See + (model-provider runtime PR-E) — for the ``claude`` processor. See :func:`build_openai_compat_adapter` / :func:`_provider_view` for why this takes field values rather than a real provider object. """ diff --git a/backend/llm/openai_compat.py b/backend/llm/openai_compat.py index e2bf7ce..88b8922 100644 --- a/backend/llm/openai_compat.py +++ b/backend/llm/openai_compat.py @@ -1,4 +1,4 @@ -"""OpenAI-compatible adapter (GOAL-6 PR-B) — ``provider_type in {"openai", +"""OpenAI-compatible adapter (model-provider runtime PR-B) — ``provider_type in {"openai", "local"}`` (decision #2: both are the same wire protocol, just different trust levels for the target address). """ @@ -106,7 +106,7 @@ async def aclose(self) -> None: self._pinned_http_client = None async def get_client(self) -> Any: - """Public accessor for the guarded ``AsyncOpenAI`` client (GOAL-6 + """Public accessor for the guarded ``AsyncOpenAI`` client (model-provider runtime PR-E). ``chat.py``'s agent-dock tool-calling loop and ``skill_channel``'s diff --git a/backend/llm/resolver.py b/backend/llm/resolver.py index 8483812..9cb0f53 100644 --- a/backend/llm/resolver.py +++ b/backend/llm/resolver.py @@ -1,4 +1,4 @@ -"""Provider resolver + failover (GOAL-6 PR-D, decision #7). +"""Provider resolver + failover (model-provider runtime PR-D, decision #7). Reads a role's ``model_defaults.candidates`` (PR-A) — an ordered list of ``{"provider_id", "model_id"}`` dicts — and dispatches through diff --git a/backend/migrations/versions/c9d0e1f2a3b4_add_runtimes_to_edge_nodes.py b/backend/migrations/versions/c9d0e1f2a3b4_add_runtimes_to_edge_nodes.py index 93ce2e0..e62c0b2 100644 --- a/backend/migrations/versions/c9d0e1f2a3b4_add_runtimes_to_edge_nodes.py +++ b/backend/migrations/versions/c9d0e1f2a3b4_add_runtimes_to_edge_nodes.py @@ -4,8 +4,7 @@ Revises: b8c9d0e1f2a3 Create Date: 2026-07-03 -P0 work package B (GOAL-agent-runtimes.md §4): the ws register handshake now -optionally advertises the agent-runtime types available on that edge node +The ws register handshake optionally advertises the agent-runtime types available on that edge node (e.g. ["pi"]) via ``backend.agent_runtimes.registry.available_runtimes()``. Persisted as a nullable JSON list so older nodes / handshakes that omit the field leave the column NULL rather than forcing an empty-list default. diff --git a/backend/migrations/versions/d8e9f0a1b2c3_add_provider_models_and_model_defaults.py b/backend/migrations/versions/d8e9f0a1b2c3_add_provider_models_and_model_defaults.py index 0580464..be6e116 100644 --- a/backend/migrations/versions/d8e9f0a1b2c3_add_provider_models_and_model_defaults.py +++ b/backend/migrations/versions/d8e9f0a1b2c3_add_provider_models_and_model_defaults.py @@ -4,7 +4,7 @@ Revises: a7v8w9x0y1z2 Create Date: 2026-07-09 -GOAL-6 PR-A (model-provider-mgmt, decisions #3/#4): the model catalog +model-provider runtime PR-A (model-provider-mgmt, decisions #3/#4): the model catalog (``provider_models`` — one row per model a provider exposes, sourced from discovery sync or manual entry) and system default candidates per consumption role (``model_defaults``). Adds ONLY these two tables — no diff --git a/backend/models/model_default.py b/backend/models/model_default.py index 1c5200f..c00ae82 100644 --- a/backend/models/model_default.py +++ b/backend/models/model_default.py @@ -5,14 +5,14 @@ class ModelDefault(TimestampMixin): - """System default model candidates for one consumption role (GOAL-6 + """System default model candidates for one consumption role (model-provider runtime decision #4). One row per role (``role`` is UNIQUE): ``candidates`` is an ordered list of ``{"provider_id": ..., "model_id": ...}`` dicts — index 0 is the primary pick, the rest are failover order tried in sequence by the resolver (PR-D; this table only defines the shape, no resolve logic - lives here). Roles map to the three consumption points GOAL-6 collapses + lives here). Roles map to the three consumption points model-provider runtime collapses onto ModelProvider (decision #4): ``chat`` (agent dock conversation), ``executor`` (skill_channel's cheap execution model), ``enrichment`` (pipeline processor fallback). diff --git a/backend/models/provider_model.py b/backend/models/provider_model.py index 5722356..b56a7ee 100644 --- a/backend/models/provider_model.py +++ b/backend/models/provider_model.py @@ -7,7 +7,7 @@ class ProviderModel(TimestampMixin): - """One model in a provider's catalog (GOAL-6 decision #3). + """One model in a provider's catalog (model-provider runtime decision #3). Populated either by discovery sync (``source="discovered"`` — OpenAI-compat ``GET {base_url}/v1/models``, or the Anthropic hardcoded catalog for @@ -16,7 +16,7 @@ class ProviderModel(TimestampMixin): delete a ``manual`` row (PR-C concern; this table only defines the shape). ``provider_id`` is a real FK — unlike ``AIAgent.provider_id``, which stays - a loose string column per GOAL-6 decision #9 (the migration cost isn't + a loose string column per model-provider runtime decision #9 (the migration cost isn't worth it there) — so deleting a ``ModelProvider`` cascades its whole catalog away instead of leaving orphan rows. """ diff --git a/backend/pipeline/ai_processor.py b/backend/pipeline/ai_processor.py index c5d6638..071d793 100644 --- a/backend/pipeline/ai_processor.py +++ b/backend/pipeline/ai_processor.py @@ -10,7 +10,7 @@ async def _resolve_llm_config(ai_config: dict[str, Any], source_id: Any) -> dict[str, Any]: - """GOAL-6 PR-F (decision #9): soft dual-track convergence between + """model-provider runtime PR-F (decision #9): soft dual-track convergence between ``DataSource.ai_config``'s legacy inline ``api_key``/``base_url`` and the governed ``ModelProvider`` catalog (``backend.models.provider``). @@ -86,7 +86,7 @@ async def process_with_ai( processor_type: claude | openai | local model: model name prompt_template: Jinja2 template - provider_id: GOAL-6 PR-F — governed ModelProvider reference; wins + provider_id: model-provider runtime PR-F — governed ModelProvider reference; wins over inline api_key/base_url when both are present (decision #9) ...processor-specific options diff --git a/backend/pipeline/pipeline.py b/backend/pipeline/pipeline.py index 7d18d26..f151db0 100644 --- a/backend/pipeline/pipeline.py +++ b/backend/pipeline/pipeline.py @@ -430,7 +430,7 @@ async def run_pipeline( new_records, effective_ai_config, source_id=source.id, - # GOAL-6 PR-F decision #9 dual-track resolution only applies to + # model-provider runtime PR-F decision #9 dual-track resolution only applies to # DataSource.ai_config; an agent_config override already went # through its own (untouched) ai_agents.provider_id resolution # in backend.pipeline.runner phase 2, so it's used as-is. diff --git a/backend/processors/claude_processor.py b/backend/processors/claude_processor.py index d824e9c..7124970 100644 --- a/backend/processors/claude_processor.py +++ b/backend/processors/claude_processor.py @@ -63,7 +63,7 @@ async def process( model, max_tokens, len(records), request_timeout, max_concurrency, ) - # GOAL-6 PR-E: client construction consolidated through + # model-provider runtime PR-E: client construction consolidated through # backend.llm.anthropic.AnthropicAdapter (via # backend.llm.factory.build_anthropic_adapter). This processor never # configured a base_url (Anthropic's endpoint is effectively fixed), diff --git a/backend/processors/local_processor.py b/backend/processors/local_processor.py index dec7480..3e832fa 100644 --- a/backend/processors/local_processor.py +++ b/backend/processors/local_processor.py @@ -1,6 +1,6 @@ """Local model processor via Ollama/vLLM compatible API. -GOAL-6 PR-E note: deliberately NOT routed through +model-provider runtime PR-E note: deliberately NOT routed through ``backend.llm.factory``/``OpenAICompatAdapter`` like the openai/claude processors. Two real incompatibilities, not just "not bothered yet": diff --git a/backend/processors/openai_processor.py b/backend/processors/openai_processor.py index 9a9c136..c4100c8 100644 --- a/backend/processors/openai_processor.py +++ b/backend/processors/openai_processor.py @@ -66,7 +66,7 @@ async def process( request_timeout, max_concurrency, ) - # GOAL-6 PR-E: client construction (SSRF guard + DNS-rebind pinning) + # model-provider runtime PR-E: client construction (SSRF guard + DNS-rebind pinning) # is consolidated through backend.llm.openai_compat.OpenAICompatAdapter # (via backend.llm.factory.build_openai_compat_adapter) — this used to # be a verbatim duplicate of the same wiring in chat.py/skill_channel. diff --git a/backend/security/url_guard.py b/backend/security/url_guard.py index 8b57b8a..cbb2d0e 100644 --- a/backend/security/url_guard.py +++ b/backend/security/url_guard.py @@ -135,7 +135,7 @@ def is_ip_blocked( ``reserved`` are still always blocked — no legitimate provider endpoint lives there, so there is no reason to ever allow them. - This is a narrow, explicit opt-in (GOAL-6 PR-B, decision #6). It exists + This is a narrow, explicit opt-in (model-provider runtime PR-B, decision #6). It exists because ``url_guard`` had **no** existing localhost/private-IP exemption mechanism, yet self-hosted LLM providers (``ModelProvider.provider_type == "local"`` — ollama on loopback, model-hotel on the NetBird fleet mesh) diff --git a/backend/services/provider_model_service.py b/backend/services/provider_model_service.py index b4bcaba..e3915e6 100644 --- a/backend/services/provider_model_service.py +++ b/backend/services/provider_model_service.py @@ -1,4 +1,4 @@ -"""Service-layer logic for GOAL-6 PR-C: provider model catalog (sync + CRUD) +"""Service-layer logic for model-provider runtime PR-C: provider model catalog (sync + CRUD) and model_defaults (get/put), plus provider-delete catalog cleanup. Kept out of ``backend/api/v1/providers.py`` / ``backend/api/v1/model_defaults.py`` @@ -124,7 +124,7 @@ async def delete_provider_models(db: AsyncSession, provider_id: str) -> int: """Wipe a provider's whole catalog. Returns the number of rows deleted. Called from the providers router's ``DELETE /providers/{id}`` BEFORE the - provider row itself is deleted (GOAL-6 PR-A note, decision #3): this + provider row itself is deleted (model-provider runtime PR-A note, decision #3): this repo's runtime engine (``backend/database.py``) never issues ``PRAGMA foreign_keys=ON``, so ``provider_models.provider_id``'s ``ondelete= CASCADE`` clause never actually fires against the production sqlite file diff --git a/backend/ws_agent_manager.py b/backend/ws_agent_manager.py index af0a07b..6188380 100644 --- a/backend/ws_agent_manager.py +++ b/backend/ws_agent_manager.py @@ -8,7 +8,7 @@ - ``collect`` / ``result`` — single-shot opencli collection tasks (unchanged). - ``agent_task`` / ``agent_event`` (0..N) / ``agent_result`` — streaming - agent-runtime task dispatch (GOAL-agent-runtimes.md §4, P0 work package B). + agent-runtime task dispatch. Wire protocol — every reverse-channel message type, one-line field shapes: @@ -183,7 +183,7 @@ def resolve_response(request_id: str, result: dict[str, Any]) -> None: # ── Streaming agent-task dispatch ─────────────────────────────────────────── # Alongside the collect/result single-shot path above: agent_task/agent_event/ # agent_result support a long-running streaming task with N intermediate -# events before the terminal result (GOAL-agent-runtimes.md §4). +# events before the terminal result. async def send_agent_task( diff --git a/docs/CONTROL_THEORY_ARCHITECTURE.md b/docs/CONTROL_THEORY_ARCHITECTURE.md index ec977bc..1dbefee 100644 --- a/docs/CONTROL_THEORY_ARCHITECTURE.md +++ b/docs/CONTROL_THEORY_ARCHITECTURE.md @@ -1,8 +1,7 @@ # opencli-admin as a Data Acquisition Control System > 工程控制论(钱学森)视角的架构原则 + 落地路线。 -> 与 2026-07-02 系统审计(`AUDIT-cybernetic-remediation.md`)是同一透镜的两端: -> 审计做**诊断**(哪些反馈回路断了),本文档做**运行**(如何把系统建成自稳定控制系统)。 +> 本文同时覆盖**诊断**(哪些反馈回路断了)与**运行**(如何把系统建成自稳定控制系统)。 > > 核心命题:opencli-admin 不是"多数据源采集平台",而是 > **可观测、可反馈、可自稳定的数据采集控制系统**。RSS/API/CLI/Web/skill 看似不同, diff --git a/docs/SYSTEM_ANALYSIS.md b/docs/SYSTEM_ANALYSIS.md index 4df3774..00c65f9 100644 --- a/docs/SYSTEM_ANALYSIS.md +++ b/docs/SYSTEM_ANALYSIS.md @@ -140,10 +140,10 @@ Step 5: notifier_dispatch.py → Webhook/飞书/钉钉/企微/Email | 文档 | 说明 | |------|------| -| `PLAN_product_positioning.md` | 产品定位与市场叙事 | -| `PLAN_ui_reskin.md` | UI 改版计划 | -| `PLAN_collection_nodes.md` | 节点页面重构（已完成） | -| `ARCHITECTURE.md` | 架构设计详细文档 | +| `README.md` | 产品定位、能力与安装入口 | +| `CONTEXT.md` | 当前代码与系统上下文 | +| `DESIGN.md` | 架构与产品设计 | +| `docs/adr/` | 长期架构决策记录 | | `docs/PROJECT_MANAGEMENT.md` | 项目管理规范 | --- diff --git a/docs/WIRING_GAP_LEDGER.md b/docs/WIRING_GAP_LEDGER.md index 5aca606..7d49b72 100644 --- a/docs/WIRING_GAP_LEDGER.md +++ b/docs/WIRING_GAP_LEDGER.md @@ -33,7 +33,7 @@ ADR 0010-0025 (企业工作流平台愿景) 大片 UNIMPLEMENTED, 这些是**没 - ADR-0022 Ephemeral Execution Grants (无 Connection/Binding/Grant 链, 现是扁平 per-source 凭证) - ADR-0021 Delivery 业务结果 vs 提交 (无 BusinessOutcome, webhook_delivery 只到传输层, 200 ≠ 业务成功) - ADR-0024 Recovery Cases 统一 inbox (无 RecoveryCase model; /inbox 页存在但活查三端点, 没串成案子) — **半接线, 比其他愿景更接近** -- ADR-0014/0015/0019 plugin 系统 (DataFoundry/声明式 plugin UI/locked node def 均无) — PLAN_plugin_system 已点最小起步 (executor mode Literal→str) +- ADR-0014/0015/0019 plugin 系统 (DataFoundry/声明式 plugin UI/locked node def 均无) - ADR-0016 Setup Center (无 readiness 聚合面, 相对便宜) - ADR-0023 幂等 Operation ID (仅 acquisition 有 idempotency_key, 未泛化成跨 plugin 契约) - ADR-0025 乐观锁 guard (模式已在 source_cursor 证明, 但目标对象 Project/Proposal 未存在, 无处可挂) @@ -41,7 +41,7 @@ ADR 0010-0025 (企业工作流平台愿景) 大片 UNIMPLEMENTED, 这些是**没 ## D. 文档 stale (低成本清理, 防误导) - `CONTROL_THEORY_ARCHITECTURE.md` **低估**: 写"拟新增 control 层", 实际 19 文件已建好在跑 -- `PLAN_ui_reskin.md` / `PLAN_collection_nodes.md` 描述已整体换掉的 Vite+React18 树 (现 Next.js 16+React19) +- 旧 Vite+React18 阶段计划已删除，当前实现为 Next.js 16+React19 - ADR-0001 (FlowGram) / 0002 (Radix) 已被 xyflow / base-ui **superseded**, ADR 文件无标注 --- diff --git a/docs/dify-p0-compatibility-runtime-PRD.md b/docs/dify-p0-compatibility-runtime-PRD.md index c424b19..66d51d3 100644 --- a/docs/dify-p0-compatibility-runtime-PRD.md +++ b/docs/dify-p0-compatibility-runtime-PRD.md @@ -12,7 +12,7 @@ Grounding: - Workflow compile seam: `backend/workflow/compiler.py:121-200` - Runtime binding registry: `backend/workflow/runtime_registry.py:36-53,85-150` - Run/event projection: `backend/workflow/opencli_hda_tracer.py:213-390,1240-1363` -- Plugin direction: `PLAN_plugin_system.md:5-34` +- Plugin direction: `docs/adr/0014-datafoundry-backed-analysis-as-an-optional-plugin.md`, `docs/adr/0015-plugin-ui-is-declarative-and-platform-rendered.md`, `docs/adr/0019-locked-plugin-node-definitions-and-project-owned-derivatives.md` - External runtime boundary: `docs/adr/0010-native-runtime-nodes-and-managed-external-graphs.md:1-17` - Architecture decision: `docs/adr/0026-run-dify-workflows-through-a-pinned-graphon-sidecar.md` diff --git a/docs/dify-p0-compatibility-runtime-issues/05-dify-manifest-intake-and-plugin-registry.md b/docs/dify-p0-compatibility-runtime-issues/05-dify-manifest-intake-and-plugin-registry.md index aa716cc..087001d 100644 --- a/docs/dify-p0-compatibility-runtime-issues/05-dify-manifest-intake-and-plugin-registry.md +++ b/docs/dify-p0-compatibility-runtime-issues/05-dify-manifest-intake-and-plugin-registry.md @@ -32,7 +32,6 @@ Modify: - `backend/api/v1/__init__.py:5-63` - `backend/workflow/capability_projection.py` - `backend/workflow/node_registry.py` -- `PLAN_plugin_system.md:24-34` after implementation, to close decided questions ## Persistence diff --git a/frontend/lib/api/endpoints.ts b/frontend/lib/api/endpoints.ts index dd898cd..cfe9742 100644 --- a/frontend/lib/api/endpoints.ts +++ b/frontend/lib/api/endpoints.ts @@ -654,7 +654,7 @@ export const buildFeedProviderWorkflowNode = ( ) .then((r) => r.data.data) -// ── Model defaults (GOAL-6 — role-based failover candidate lists) ─────────────── +// ── Model defaults (model-provider runtime — role-based failover candidate lists) ─────────────── // A role can be entirely absent from the list on a fresh install — that's a // legitimate "no candidates configured yet" state, not an error. export const listModelDefaults = () => @@ -763,7 +763,7 @@ export const updateChromeEndpointMode = (endpoint: string, mode: 'bridge' | 'cdp export const listPresets = () => apiClient.get>('/presets').then((r) => r.data.data) -// ── BrowserAct packs (GOAL-7 PR-E, decision #9) ───────────────────────────────── +// ── BrowserAct packs (Browser Act integration PR-E, decision #9) ───────────────────────────────── // Read-only vendored-pack catalog for the 'browser_act' channel's one-click // config preset (pack picker) — never carries a credential/api_key. export const listBrowserActPacks = () => diff --git a/frontend/lib/api/types.ts b/frontend/lib/api/types.ts index cabbfb8..d9963e3 100644 --- a/frontend/lib/api/types.ts +++ b/frontend/lib/api/types.ts @@ -21,7 +21,7 @@ export interface WorkspaceSettingsRead { updated_at: string | null } -// Self-hosted LLM-provider runtime (GOAL-6, backend/llm/, no litellm) — mirrors +// Self-hosted LLM-provider runtime (model-provider runtime, backend/llm/, no litellm) — mirrors // backend.schemas.provider.ModelProviderRead.from_model exactly. The raw // api_key is NEVER returned by the backend (from_model explicitly masks it) — // only has_api_key / api_key_preview (e.g. "sk-...wxyz", null if unset) ever @@ -916,7 +916,7 @@ export interface PlanHealthRead { recorded_at: string } -// ── BrowserAct packs (GOAL-7 PR-E, decision #9) ────────────────────────────── +// ── BrowserAct packs (Browser Act integration PR-E, decision #9) ────────────────────────────── // Mirrors backend.api.v1.browser_act.BrowserActPackRead. Read-only vendored // pack catalog for the 'browser_act' channel's config preset — never carries // any credential/api_key (the BrowserAct API key is a SourceCredential, diff --git a/tests/integration/test_auth_ws.py b/tests/integration/test_auth_ws.py index 9afb8d6..942ac43 100644 --- a/tests/integration/test_auth_ws.py +++ b/tests/integration/test_auth_ws.py @@ -124,7 +124,7 @@ def test_nodes_ws_query_token_accepted(test_client, auth_enabled): assert reply["type"] == "registered" -# ── runtime advertisement (P0 work package B, GOAL-agent-runtimes.md §4) ─── +# ── runtime advertisement ───────────────────────────────────────────────── # # nodes.py's node_ws_endpoint opens its own backend.database.AsyncSessionLocal # session per-call (not the `get_db` dependency TestClient normally overrides diff --git a/tests/integration/test_browser_act_packs_api.py b/tests/integration/test_browser_act_packs_api.py index 7750c42..5f0d7a0 100644 --- a/tests/integration/test_browser_act_packs_api.py +++ b/tests/integration/test_browser_act_packs_api.py @@ -1,4 +1,4 @@ -"""HTTP-seam tests for GET /api/v1/browser-act/packs (GOAL-7 PR-E, decision +"""HTTP-seam tests for GET /api/v1/browser-act/packs (Browser Act integration PR-E, decision #9): the real vendored catalog (backend/browser_act_packs/), no fixture override -- this is a static file-tree scan, deterministic regardless of test order/host, same as PackCatalog's own unit tests. diff --git a/tests/integration/test_browser_act_seeds.py b/tests/integration/test_browser_act_seeds.py index 3964ffd..3263dcc 100644 --- a/tests/integration/test_browser_act_seeds.py +++ b/tests/integration/test_browser_act_seeds.py @@ -1,4 +1,4 @@ -"""Integration tests for GOAL-7 PR-D seed manifests. +"""Integration tests for Browser Act integration PR-D seed manifests. Exercises the two hand-authored ``channel.manifest.json`` seeds (``search-research/google-search-serp``, ``ecommerce/taobao-keyword-search``) diff --git a/tests/integration/test_model_defaults_api.py b/tests/integration/test_model_defaults_api.py index 6844334..c6a47de 100644 --- a/tests/integration/test_model_defaults_api.py +++ b/tests/integration/test_model_defaults_api.py @@ -1,4 +1,4 @@ -"""Integration tests for GOAL-6 PR-C's ``GET|PUT /model-defaults`` (decision +"""Integration tests for model-provider runtime PR-C's ``GET|PUT /model-defaults`` (decision #10): role closed-set validation and candidate (provider_id, model_id) existence validation, both enforced before a row is ever stored. """ diff --git a/tests/integration/test_provider_models_api.py b/tests/integration/test_provider_models_api.py index 482658d..0626d5b 100644 --- a/tests/integration/test_provider_models_api.py +++ b/tests/integration/test_provider_models_api.py @@ -1,4 +1,4 @@ -"""Integration tests for GOAL-6 PR-C's provider-scoped API (decision #10): +"""Integration tests for model-provider runtime PR-C's provider-scoped API (decision #10): ``POST /providers/{id}/test``, ``POST /providers/{id}/models/sync``, and ``GET|POST|PATCH|DELETE /providers/{id}/models``. @@ -154,7 +154,7 @@ async def test_sync_creates_discovered_rows(client, provider_data): @pytest.mark.asyncio async def test_sync_idempotent_and_manual_preserved(client, provider_data): - """The scenario from GOAL-6 PR-C's spec: seed provider, sync [m1, m2] -> + """The scenario from model-provider runtime PR-C's spec: seed provider, sync [m1, m2] -> 2 discovered rows; add manual m3; sync again with only [m1] -> m3 (manual) survives untouched, m2 (stale discovered) is pruned, m1 is not duplicated (idempotent).""" diff --git a/tests/unit/api/test_nodes_upsert.py b/tests/unit/api/test_nodes_upsert.py index fd9f7d7..bd4e13f 100644 --- a/tests/unit/api/test_nodes_upsert.py +++ b/tests/unit/api/test_nodes_upsert.py @@ -1,5 +1,4 @@ -"""Unit tests for backend/api/v1/nodes.py's `_upsert_node` runtimes handling -(P0 work package B, GOAL-agent-runtimes.md §4). +"""Unit tests for backend/api/v1/nodes.py's `_upsert_node` runtimes handling. Uses the shared async `db_session` fixture (tests/conftest.py) directly — no WS/HTTP layer involved — so persistence of the new nullable diff --git a/tests/unit/browser_act/test_cli.py b/tests/unit/browser_act/test_cli.py index 3bb097c..5ee55e2 100644 --- a/tests/unit/browser_act/test_cli.py +++ b/tests/unit/browser_act/test_cli.py @@ -1,4 +1,4 @@ -"""Unit tests for backend.browser_act.cli (GOAL-7 PR-B). +"""Unit tests for backend.browser_act.cli (Browser Act integration PR-B). Mirrors tests/unit/channels/test_cli_channel.py conventions: patch asyncio.create_subprocess_exec / asyncio.wait_for with AsyncMock, and use a diff --git a/tests/unit/browser_act_packs/test_manifest.py b/tests/unit/browser_act_packs/test_manifest.py index 1783b83..edc507b 100644 --- a/tests/unit/browser_act_packs/test_manifest.py +++ b/tests/unit/browser_act_packs/test_manifest.py @@ -1,5 +1,5 @@ """Unit tests for PackManifest (backend/browser_act_packs/manifest.py) — -GOAL-7 decision #5 schema + loader. PR-A defines schema only, no seed content.""" +Browser Act integration decision #5 schema + loader. PR-A defines schema only, no seed content.""" import json diff --git a/tests/unit/channels/test_api_channel.py b/tests/unit/channels/test_api_channel.py index 9898a57..9de8d9f 100644 --- a/tests/unit/channels/test_api_channel.py +++ b/tests/unit/channels/test_api_channel.py @@ -704,7 +704,7 @@ async def test_fetch_client_404_classified_permanent(channel): assert exc_info.value.error_type == "PermanentHTTPStatus" -# ── health_check (GOAL-4 PR-E: real per-source probe) ─────────────────────────── +# ── health_check (runtime-hardening phase PR-E: real per-source probe) ─────────────────────────── @pytest.mark.asyncio async def test_health_check_no_config_is_liveness_only(channel): diff --git a/tests/unit/channels/test_browser_act_channel.py b/tests/unit/channels/test_browser_act_channel.py index ef4a445..96d77b3 100644 --- a/tests/unit/channels/test_browser_act_channel.py +++ b/tests/unit/channels/test_browser_act_channel.py @@ -1,4 +1,4 @@ -"""Unit tests for BrowserActChannel (GOAL-7 PR-C/PR-E). +"""Unit tests for BrowserActChannel (Browser Act integration PR-C/PR-E). Builds a SYNTHETIC pack under tmp_path so these tests never depend on a real PR-D manifest or a real browser: the browser-act CLI hop is mocked at @@ -356,7 +356,7 @@ async def test_run_pack_script_injection_safety(tmp_path): assert output.strip() == dangerous -# ── credentials (GOAL-7 PR-E, decision #7) ──────────────────────────────── +# ── credentials (Browser Act integration PR-E, decision #7) ──────────────────────────────── # # AuthManager.store()/.resolve() open their own session via # backend.database.AsyncSessionLocal -- these tests point that at a per-test diff --git a/tests/unit/channels/test_web_scraper_channel.py b/tests/unit/channels/test_web_scraper_channel.py index 9c16800..c044126 100644 --- a/tests/unit/channels/test_web_scraper_channel.py +++ b/tests/unit/channels/test_web_scraper_channel.py @@ -349,7 +349,7 @@ def fake_client_constructor(**kwargs): assert "User-Agent" in captured_headers -# ── GOAL-4 PR-D: fetch() thick contract (ctx.http path — rate limit/retry) ────── +# ── runtime-hardening phase PR-D: fetch() thick contract (ctx.http path — rate limit/retry) ────── @pytest.mark.asyncio @@ -414,7 +414,7 @@ async def test_collect_still_delegates_to_fetch_without_ctx_http(channel): assert result.success is True -# ── health_check (GOAL-4 PR-E: real per-source probe) ─────────────────────────── +# ── health_check (runtime-hardening phase PR-E: real per-source probe) ─────────────────────────── @pytest.mark.asyncio diff --git a/tests/unit/llm/test_adapters.py b/tests/unit/llm/test_adapters.py index fa740c7..277904c 100644 --- a/tests/unit/llm/test_adapters.py +++ b/tests/unit/llm/test_adapters.py @@ -1,4 +1,4 @@ -"""Unit tests for backend.llm's runtime adapters (GOAL-6 PR-B). +"""Unit tests for backend.llm's runtime adapters (model-provider runtime PR-B). Every SDK client (openai.AsyncOpenAI / anthropic.AsyncAnthropic) is mocked at the class level — these tests never make a real network call. The two diff --git a/tests/unit/llm/test_catalog.py b/tests/unit/llm/test_catalog.py index c581335..c84bd83 100644 --- a/tests/unit/llm/test_catalog.py +++ b/tests/unit/llm/test_catalog.py @@ -1,4 +1,4 @@ -"""Unit tests for backend.llm (GOAL-6 PR-A): closed-set vocabulary shared by +"""Unit tests for backend.llm (model-provider runtime PR-A): closed-set vocabulary shared by the model-provider data layer, plus the hardcoded Anthropic model catalog (decision #5 — Anthropic has no /v1/models discovery endpoint).""" diff --git a/tests/unit/llm/test_pr_e_consumers.py b/tests/unit/llm/test_pr_e_consumers.py index a56e083..53d9ead 100644 --- a/tests/unit/llm/test_pr_e_consumers.py +++ b/tests/unit/llm/test_pr_e_consumers.py @@ -1,4 +1,4 @@ -"""GOAL-6 PR-E: consumer client-construction consolidation tests. +"""model-provider runtime PR-E: consumer client-construction consolidation tests. Covers chat.py / skill_channel.py / the openai+claude processors' switch from each hand-rolling its own AsyncOpenAI/AsyncAnthropic + SSRF-guard wiring to diff --git a/tests/unit/llm/test_resolver.py b/tests/unit/llm/test_resolver.py index c3eddbf..cefd950 100644 --- a/tests/unit/llm/test_resolver.py +++ b/tests/unit/llm/test_resolver.py @@ -1,4 +1,4 @@ -"""Unit tests for GOAL-6 PR-D: ``backend.llm.resolver`` (failover) and +"""Unit tests for model-provider runtime PR-D: ``backend.llm.resolver`` (failover) and ``backend.llm.base.classify_retryable`` (decision #7's connection-vs-business error split). @@ -30,7 +30,7 @@ class FakeClock: - """Injectable monotonic clock (GOAL-6 PR-D): starts at 0.0, only moves + """Injectable monotonic clock (model-provider runtime PR-D): starts at 0.0, only moves when the test calls :meth:`advance` — no real sleeps anywhere here.""" def __init__(self) -> None: diff --git a/tests/unit/pipeline/test_ai_processor.py b/tests/unit/pipeline/test_ai_processor.py index 5879797..29f2ca1 100644 --- a/tests/unit/pipeline/test_ai_processor.py +++ b/tests/unit/pipeline/test_ai_processor.py @@ -59,7 +59,7 @@ async def test_process_with_ai_enriches_records(): assert result == 2 -# ─── GOAL-6 PR-F (decision #9): DataSource.ai_config <-> ModelProvider ───── +# ─── model-provider runtime PR-F (decision #9): DataSource.ai_config <-> ModelProvider ───── # soft dual-track convergence at the ai_config -> processor-config seam. diff --git a/tests/unit/pipeline/test_pipeline_errors.py b/tests/unit/pipeline/test_pipeline_errors.py index 5235299..d1cb0dd 100644 --- a/tests/unit/pipeline/test_pipeline_errors.py +++ b/tests/unit/pipeline/test_pipeline_errors.py @@ -107,7 +107,7 @@ async def test_pipeline_with_ai_failure_still_returns_success(db_session): assert result.success is True -# ── GOAL-4 PR-B: retryable errors propagate instead of being swallowed ────────── +# ── runtime-hardening phase PR-B: retryable errors propagate instead of being swallowed ────────── @pytest.mark.asyncio async def test_pipeline_collect_retryable_exception_propagates(db_session): diff --git a/tests/unit/test_agent_server.py b/tests/unit/test_agent_server.py index 8c9e6ee..21a3696 100644 --- a/tests/unit/test_agent_server.py +++ b/tests/unit/test_agent_server.py @@ -1,5 +1,5 @@ """Unit tests for backend/agent_server.py fleet-auth header attachment (ADR-0005) -and streaming agent-task dispatch (`_handle_ws_agent_task`, GOAL-agent-runtimes.md §4). +and streaming agent-task dispatch (`_handle_ws_agent_task`). Covers `_auth_headers()`, the Authorization header on `_register_with_center`'s httpx POST calls, and the `additional_headers` -> `extra_headers` fallback in diff --git a/tests/unit/test_model_default.py b/tests/unit/test_model_default.py index 47c36de..d36aaf8 100644 --- a/tests/unit/test_model_default.py +++ b/tests/unit/test_model_default.py @@ -1,4 +1,4 @@ -"""Unit tests for ModelDefault (GOAL-6 PR-A, decision #4): the per-role +"""Unit tests for ModelDefault (model-provider runtime PR-A, decision #4): the per-role system default candidates table (role UNIQUE) and the Pydantic schemas' closed-set validation for role. """ diff --git a/tests/unit/test_provider_model.py b/tests/unit/test_provider_model.py index a568f62..cd2345f 100644 --- a/tests/unit/test_provider_model.py +++ b/tests/unit/test_provider_model.py @@ -1,4 +1,4 @@ -"""Unit tests for ProviderModel (GOAL-6 PR-A, decision #3): the model catalog +"""Unit tests for ProviderModel (model-provider runtime PR-A, decision #3): the model catalog table, its real FK to ModelProvider (ondelete CASCADE — unlike AIAgent.provider_id, which stays a loose string per decision #9), the (provider_id, model_id) uniqueness constraint, and the Pydantic schemas' diff --git a/tests/unit/test_runner.py b/tests/unit/test_runner.py index 3a6f3cc..d13cd3e 100644 --- a/tests/unit/test_runner.py +++ b/tests/unit/test_runner.py @@ -504,7 +504,7 @@ def phase2_get(model, obj_id): @pytest.mark.asyncio async def test_run_pipeline_agent_processor_config_overrides_provider_base(): - """GOAL-6 PR-E / decision #8: provider supplies BASE fields (api_key, + """model-provider runtime PR-E / decision #8: provider supplies BASE fields (api_key, base_url) via provider_config; agent.processor_config is layered on top and WINS on any overlapping key. test_run_pipeline_with_agent_id (above) only asserts the pipeline succeeds — it never inspects the merged @@ -590,7 +590,7 @@ async def fake_run_pipeline(**kwargs): assert agent_config["model"] == "claude-3-haiku" -# ── GOAL-4 PR-B: retryable run_pipeline failure records then re-propagates ────── +# ── runtime-hardening phase PR-B: retryable run_pipeline failure records then re-propagates ────── @pytest.mark.asyncio diff --git a/tests/unit/test_schedule_service_redbeat.py b/tests/unit/test_schedule_service_redbeat.py index c965edc..49a040c 100644 --- a/tests/unit/test_schedule_service_redbeat.py +++ b/tests/unit/test_schedule_service_redbeat.py @@ -1,4 +1,4 @@ -"""Unit tests for schedule_service's redbeat sync gating (GOAL-4 PR-C). +"""Unit tests for schedule_service's redbeat sync gating (runtime-hardening phase PR-C). Existing tests in test_schedules_crud.py run under the default task_executor="local" and never touch redbeat at all (the gate short-circuits) diff --git a/tests/unit/worker/test_beat_schedule.py b/tests/unit/worker/test_beat_schedule.py index 0ff021c..d265427 100644 --- a/tests/unit/worker/test_beat_schedule.py +++ b/tests/unit/worker/test_beat_schedule.py @@ -1,6 +1,6 @@ """Unit tests for backend/worker/beat_schedule.py — the surviving parse_cron_expression helper (its former build_beat_schedule() was dead code, -removed in GOAL-4 PR-C, superseded by worker/redbeat_sync.py).""" +removed in runtime-hardening phase PR-C, superseded by worker/redbeat_sync.py).""" import pytest from celery.schedules import crontab