From 54284b529b6b8bdd9c793d8ea3a9fbd88b2bf5d1 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Wed, 22 Jul 2026 23:56:04 -0700 Subject: [PATCH 01/13] feat(world): WORLD-MODEL design + smesh-world kernel Signed organizational world models: expert roster with type-enforced separation of powers, CDM with dual-anchor provenance, grounded-citation candidates, type-state ratification lifecycle, Meridian benchmark types. 28 tests, clippy clean. --- Cargo.toml | 2 + WORLD-MODEL.md | 213 +++++++++++++++++++++++ smesh-world/Cargo.toml | 14 ++ smesh-world/src/candidate.rs | 269 ++++++++++++++++++++++++++++ smesh-world/src/cdm.rs | 210 ++++++++++++++++++++++ smesh-world/src/corpus.rs | 246 ++++++++++++++++++++++++++ smesh-world/src/delta.rs | 328 +++++++++++++++++++++++++++++++++++ smesh-world/src/lib.rs | 51 ++++++ smesh-world/src/ontology.rs | 144 +++++++++++++++ smesh-world/src/role.rs | 180 +++++++++++++++++++ 10 files changed, 1657 insertions(+) create mode 100644 WORLD-MODEL.md create mode 100644 smesh-world/Cargo.toml create mode 100644 smesh-world/src/candidate.rs create mode 100644 smesh-world/src/cdm.rs create mode 100644 smesh-world/src/corpus.rs create mode 100644 smesh-world/src/delta.rs create mode 100644 smesh-world/src/lib.rs create mode 100644 smesh-world/src/ontology.rs create mode 100644 smesh-world/src/role.rs diff --git a/Cargo.toml b/Cargo.toml index b0c1fec..7e0bf9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "smesh-agent", "smesh-bounty", "smesh-cli", + "smesh-world", ] [workspace.package] @@ -33,6 +34,7 @@ async-trait = "0.1" # Crypto sha2 = "0.10" ed25519-dalek = { version = "2.1", features = ["rand_core", "serde", "pkcs8"] } +blake3 = "1.5" rand = "0.8" uuid = { version = "1.6", features = ["v4", "serde"] } diff --git a/WORLD-MODEL.md b/WORLD-MODEL.md new file mode 100644 index 0000000..c8f8dd7 --- /dev/null +++ b/WORLD-MODEL.md @@ -0,0 +1,213 @@ +# WORLD-MODEL — Signed Organizational World Models via SMESH × AION + +**Status:** Design accepted, scaffold in `smesh-world/` +**Depends on:** `smesh-core` (signal field, reputation, trust), `smesh-agent` (LLM backends, tool contracts), [aion-context](https://github.com/aion-context/aion-context) (signed graph substrate) + +--- + +## 1. Problem + +An agent without a structured model of the organization it serves does not say +"I don't know your PTO policy." It **confabulates context**: it imports the +nearest-neighbor organization from pretraining and speaks fluent Generic Corp — +wrong verbiage, wrong policy, wrong org chart, delivered with full confidence. + +The fix is not more prompt. It is converting "fill in the blank" into +"look up the node" — and making the *absence* of a node itself signed and +queryable (**verifiable negative space**). An agent grounded in a signed world +model can refuse to invent what the world does not contain. + +## 2. Thesis + +> **SMESH is how the world model earns its edges. AION is how the world model +> proves its identity. The revision loop is how it stays alive. The human is +> the merge authority.** + +- **AION = the truth plane.** Content-addressed nodes (idempotent ingestion), + Ed25519-signed revision chain (an agent verifies it loaded *this org's* + world, untampered), append-only history (replay any decision against the + world *as it was*), deny/allow (retraction without deletion). +- **SMESH = the refinery.** Expert agents emit candidate nodes/edges as + signals. Reinforcement = independent corroboration. Decay = unsupported + claims fade. Contested edges escalate to a human instead of being resolved + by the mesh. +- **Governing principle — separation of powers:** extractors propose, + verifiers judge, one non-LLM curator signs. No expert holds two of those + powers. + +## 3. Revision loop + +``` +raw corpus ──► Tier 0 intake ──► CDM docs ──► Tier 1 experts ──► candidate signals + │ reinforce / decay + ▼ + Tier 2 verification (adversarial) + │ + ▼ + staged changeset (rev N → proposed rev N+1) + │ + ▼ + HUMAN RATIFICATION (dashboard, tiered lanes) + │ + ▼ + Curator signs ratified changeset ──► world rev N+1 + ▲ + new docs / drift ────────────────────────────┘ +``` + +Nothing reaches the signed graph without passing `Ratified`. The Curator's +signing key is only ever applied to a changeset carrying a human ratification +record. + +## 4. Expert roster + +### Tier 0 — Intake ("Registrar of Record") + +Any artifact in → **Canonical Document Model (CDM)** out. Tier 1 never sees a +raw format; adding a format touches Tier 0 only. + +| Concern | Design | +|---|---| +| Identity | `doc_id = BLAKE3(bytes)` — content-addressed, idempotent re-ingestion | +| Detection | Deterministic sniffing first; LLM triage only for ambiguous blobs | +| Provenance | **Dual anchoring**: extractor citations anchor to CDM spans; every CDM span maps back to native coordinates (PDF page/rect, email message-id + line, markdown heading path). The dashboard highlights the *original* artifact | +| Metadata | Email headers are native provenance — `From:` + `Date:` yield attribution and temporal edges for free | + +Tooling (mostly assembled from the existing portfolio): `scry` for format +triage, AkamaiForms PDF kernel as the PDF adapter, `whatthediff` for +cross-format structural unification and near-duplicate detection, Prévue's +pdf-extract path as text fallback. New work: `.eml`/mbox adapter. + +### Tier 1 — Extraction (read CDM → emit candidate signals) + +| Expert | Lens | Tools | +|---|---|---| +| **Lexicon** | Org verbiage: defined terms, acronyms, canonical names — department-scoped | `corpus_read`, `concordance`, `emit_candidate(Term)` | +| **Policy** | Normative statements (must/shall/may), effective dates, supersession | `corpus_read`, `emit_candidate(PolicyRule)` | +| **Structure** | Org chart, roles, systems, ownership | `corpus_read`, `emit_candidate(Entity\|Relation)` | +| **Process** | Workflows, lifecycles, temporal ordering | `corpus_read`, `emit_candidate(ProcessStep)` | + +### Tier 2 — Verification (adversarial; may corroborate/refute, **cannot emit facts**) + +| Expert | Lens | Tools | +|---|---|---| +| **Grounding Auditor** | Does the cited span *entail* the claim? Runs a **different model family** than the extractor — always | `span_fetch`, `verdict(corroborate\|refute)` | +| **Contradiction Sentinel** | Candidate deltas vs current rev: `aion_contradictions` + typed-edge constraints (one `reports_to` per person per timeslice; `supersedes` acyclic) | `graph_read`, `aion_contradictions`, `escalate` | + +### Tier 3 — Stewardship + +| Expert | Lens | Tools | +|---|---|---| +| **Ontologist** | Owns the edge vocabulary (rev-0 ontology node). Only role that may propose new edge kinds — via human-approved escalation only | `graph_read`, `ontology_map`, `propose_edge_kind` | +| **Curator** | Sole AION writer. Batches consensus survivors, runs final gates, signs. **Deliberately not an LLM** — signing authority is code | `aion_annotate`, `aion_link`, `aion_snapshot` | + +## 5. Hard rules + +1. **Provenance is enforced at the tool layer.** `emit_candidate` + schema-requires `{doc_id, span, quote}`. A citation-free candidate is + rejected by the tool contract before it becomes a signal. +2. **Capability-scoped tools per role.** Extractors: corpus read only. + Verifiers: graph read, no corpus write. Curator alone writes AION. +3. **Reputation-weighted amplitude.** Auditor verdicts feed + `smesh-core::reputation`; a repeatedly-refuted extractor emits weaker + signals. +4. **Model diversity is a requirement.** Extraction: `moonshotai/kimi-k3` + (1M ctx — whole-corpus passes, no chunking). Verification: a different + family (Claude, or minimally `kimi-k2-thinking`). Same-model verification + lets correlated hallucinations survive. +5. **Consensus thresholds are per-edge-kind.** `defines_term` with one + corroboration may pass; `governed_by` needs ≥2 independent corroborations + and zero contradictions. + +## 6. Human ratification protocol + +The mesh never publishes — it stages. A staged changeset is a PR against the +world; the dashboard is the review screen. + +### Delta lifecycle + +``` +Draft ──► Staged ──┬── approve ──► Ratified ──► Signed (rev N+1) + ├── edit ─────► HumanAmended ──► Ratified (human version wins) + ├── reject ───► Killed (+reason → expert reputation hit) + └── defer ────► Parked (decays back into the field) +``` + +### Reviewer identity + +Each reviewer holds an Ed25519 identity; ratification is a signed act recorded +in the graph. Every edge answers four questions forever: what it says, where +it came from (span), which experts corroborated it, **which human approved it +and when**. + +### Provenance classes + +- `corpus_derived` — extracted, span-cited, mesh-corroborated. +- `human_attested` — no document exists; the provenance *is* the human's + signature. Highest-trust class. The dashboard is the tribal-knowledge + capture instrument: reviewers author facts the corpus never contained. + +Human edits to staged candidates are labeled error signals — fed back into +expert reputation, retained as future tuning data. + +### Attention lanes (ratification is total; attention is tiered) + +| Lane | Contents | Interaction | +|---|---|---| +| **Green** | High-consensus, exact-quote, low-risk (terms, org structure) | Summarized batch, spot-checkable | +| **Amber** | Normative/policy edges; single corroborator | Individually surfaced with evidence | +| **Red** | Contested edges, contradictions, ontology changes | Mandatory individual decision | + +Dashboard write path is a thin Axum layer over the prevue-api registry +pattern (browse, drill into cited evidence) plus Solv's cited-cards queue UX. + +## 7. Showcase corpus: Meridian Mutual + +**Author the gold graph first; render documents from it.** The answer key is +exact by construction; validation is scored, not vibes. + +Meridian Mutual: fictional regional insurance company. Departments: Claims, +Clinical Policy, Finance, HR, IT/Security, Legal/Compliance. Formats: +markdown handbooks, PDF policy manuals, `.eml` threads, YAML +config-as-policy, CSV org roster, wiki export, scanned-memo PDFs. + +### Planted-defect manifest + +| Planted challenge | Exercises | +|---|---| +| "Claim" means different things in Legal / Claims / Finance | Lexicon + department-scoped terms | +| Two departments with contradictory rules | Contradiction Sentinel → red lane → human | +| Policy PDF superseded by a later email memo | Cross-format supersession chains | +| Same policy as PDF and markdown, slightly divergent | Consensus + divergence detection | +| Rule existing only in an email thread | Tribal-knowledge analog; email provenance | +| Question the corpus deliberately never answers | **Confabulation trap** — must yield negative space | +| Attached vendor policy document | Attribution scoping — external ≠ Meridian | +| Generic-corp boilerplate contradicting Meridian's quirky rule | The founding failure mode | + +### Scorecard (also the permanent regression gate) + +- **Edge precision / recall** vs gold graph +- **Contradiction detection rate** (caught / planted) +- **Confabulation rate** — edges in no document with no human signature. Target: **zero**. Headline number for the thesis +- **Provenance integrity** — cross-model audit that every `corpus_derived` span entails its claim +- **Escalation quality** — red-lane items reached the human; green-lane items deserved green + +## 8. Scaffold map (`smesh-world/`) + +| Module | Contents | +|---|---| +| `role.rs` | `WorldRole` taxonomy (Tiers 0–3), capability sets, per-role tool inventories, model assignment | +| `cdm.rs` | Canonical Document Model: content-addressed docs, blocks, dual-anchor spans | +| `ontology.rs` | Typed edge vocabulary + per-edge-kind consensus thresholds + structural constraints | +| `candidate.rs` | Candidate node/edge emissions with schema-mandatory citations | +| `delta.rs` | Changeset lifecycle state machine; type-state ratification (unsigned changeset cannot reach `Signed`) | +| `corpus.rs` | Gold graph, planted-defect manifest, scorecard types for Meridian | + +## 9. Phasing + +- **Phase 0 (this scaffold):** types + state machine + gates green. +- **Phase 1:** Tier 0 intake adapters (markdown, PDF via AkamaiForms path, `.eml`) → CDM fixtures. +- **Phase 2:** Meridian gold graph + deterministic renderer + planted defects. +- **Phase 3:** Wire Tier 1/2 experts over `smesh-agent` (K3 extract, cross-model verify), run the field, emit first staged changeset. +- **Phase 4:** Ratification dashboard (Axum; prevue-api pattern) + Curator signing into AION. +- **Phase 5:** Scorecard harness; Meridian becomes the regression gate. diff --git a/smesh-world/Cargo.toml b/smesh-world/Cargo.toml new file mode 100644 index 0000000..47f7a27 --- /dev/null +++ b/smesh-world/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "smesh-world" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +description = "Signed organizational world models: expert roster, CDM, delta lifecycle, Meridian benchmark" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +blake3 = { workspace = true } diff --git a/smesh-world/src/candidate.rs b/smesh-world/src/candidate.rs new file mode 100644 index 0000000..a0a177d --- /dev/null +++ b/smesh-world/src/candidate.rs @@ -0,0 +1,269 @@ +//! Candidate emissions and verdicts. +//! +//! Provenance is enforced at the type layer, not the prompt layer: a +//! `Citation` can only be constructed against a real document with a valid +//! span whose text matches the quote, and `CandidateEdge` construction checks +//! role capabilities. You cannot prompt your way past this module. + +use crate::cdm::{CdmDocument, CdmSpan, DocId}; +use crate::ontology::EdgeKind; +use crate::role::{Capability, WorldRole}; +use crate::WorldError; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Citation { + pub doc: DocId, + pub span: CdmSpan, + pub quote: String, +} + +impl Citation { + /// The only constructor: the quote is extracted from the document, so a + /// citation whose quote diverges from its span cannot exist. + pub fn grounded(doc: &CdmDocument, span: CdmSpan) -> Result { + let quote = doc.span_text(span)?.to_owned(); + Ok(Self { + doc: doc.id, + span, + quote, + }) + } + + /// Re-verify against the (possibly re-fetched) document — the Grounding + /// Auditor's first check before any semantic entailment work. + pub fn verify_against(&self, doc: &CdmDocument) -> Result<(), WorldError> { + if doc.span_text(self.span)? == self.quote { + Ok(()) + } else { + Err(WorldError::QuoteMismatch) + } + } +} + +/// How an edge earns its place in the graph. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum ProvenanceClass { + /// Extracted, span-cited, mesh-corroborated. + CorpusDerived { citations: Vec }, + /// No document exists; the provenance IS the human's signature. + /// Highest-trust class — captured tribal knowledge. + HumanAttested { reviewer: crate::delta::ReviewerId }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Judgment { + Corroborate, + Refute, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Verdict { + pub by: WorldRole, + pub judgment: Judgment, + pub rationale: String, +} + +impl Verdict { + pub fn new(by: WorldRole, judgment: Judgment, rationale: String) -> Result { + if !by.has_capability(Capability::Verdict) { + return Err(WorldError::CapabilityDenied(by, Capability::Verdict)); + } + Ok(Self { + by, + judgment, + rationale, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CandidateEdge { + pub subject: String, + pub kind: EdgeKind, + pub object: String, + pub provenance: ProvenanceClass, + pub emitted_by: WorldRole, + pub verdicts: Vec, +} + +impl CandidateEdge { + pub fn emit( + emitted_by: WorldRole, + subject: String, + kind: EdgeKind, + object: String, + citations: Vec, + ) -> Result { + if !emitted_by.has_capability(Capability::EmitCandidate) { + return Err(WorldError::CapabilityDenied( + emitted_by, + Capability::EmitCandidate, + )); + } + if citations.is_empty() { + return Err(WorldError::MissingCitation); + } + Ok(Self { + subject, + kind, + object, + provenance: ProvenanceClass::CorpusDerived { citations }, + emitted_by, + verdicts: Vec::new(), + }) + } + + /// Stable identity used by ratification records and dedup. + pub fn key(&self) -> String { + format!("{}|{:?}|{}", self.subject, self.kind, self.object) + } + + pub fn record(&mut self, verdict: Verdict) { + self.verdicts.push(verdict); + } + + pub fn corroborations(&self) -> u32 { + self.count(Judgment::Corroborate) + } + + pub fn refutations(&self) -> u32 { + self.count(Judgment::Refute) + } + + /// A contested edge has live evidence on both sides — it must reach a + /// human, never be resolved by the mesh. + pub fn is_contested(&self) -> bool { + self.corroborations() > 0 && self.refutations() > 0 + } + + pub fn meets_consensus(&self) -> bool { + let policy = self.kind.consensus_policy(); + self.corroborations() >= policy.min_corroborations + && self.refutations() <= policy.max_refutations + } + + fn count(&self, judgment: Judgment) -> u32 { + u32::try_from( + self.verdicts + .iter() + .filter(|v| v.judgment == judgment) + .count(), + ) + .unwrap_or(u32::MAX) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cdm::{DocMetadata, SourceFormat}; + + fn doc() -> CdmDocument { + CdmDocument::ingest( + b"handbook-v1", + SourceFormat::Markdown, + "The Claims department must file within 30 days.".to_owned(), + vec![], + DocMetadata::default(), + ) + } + + fn citation(d: &CdmDocument) -> Citation { + Citation::grounded(d, CdmSpan::new(4, 21)).unwrap() + } + + #[test] + fn citation_quote_is_extracted_not_asserted() { + let d = doc(); + let c = citation(&d); + assert_eq!(c.quote, "Claims department"); + assert!(c.verify_against(&d).is_ok()); + } + + #[test] + fn citation_verification_catches_document_drift() { + let d = doc(); + let c = citation(&d); + let mutated = CdmDocument::ingest( + b"handbook-v2", + SourceFormat::Markdown, + "The Claims division must file within 30 days.".to_owned(), + vec![], + DocMetadata::default(), + ); + assert_eq!(c.verify_against(&mutated), Err(WorldError::QuoteMismatch)); + } + + #[test] + fn only_extractors_emit_and_citations_are_mandatory() { + let d = doc(); + let err = CandidateEdge::emit( + WorldRole::GroundingAuditor, + "Claims".into(), + EdgeKind::GovernedBy, + "30-day filing rule".into(), + vec![citation(&d)], + ) + .unwrap_err(); + assert!(matches!(err, WorldError::CapabilityDenied(..))); + + let err = CandidateEdge::emit( + WorldRole::Policy, + "Claims".into(), + EdgeKind::GovernedBy, + "30-day filing rule".into(), + vec![], + ) + .unwrap_err(); + assert_eq!(err, WorldError::MissingCitation); + } + + #[test] + fn only_verifiers_issue_verdicts() { + let err = Verdict::new( + WorldRole::Policy, + Judgment::Corroborate, + "looks right".into(), + ) + .unwrap_err(); + assert!(matches!(err, WorldError::CapabilityDenied(..))); + assert!(Verdict::new( + WorldRole::GroundingAuditor, + Judgment::Corroborate, + "span entails claim".into() + ) + .is_ok()); + } + + #[test] + fn consensus_and_contested_semantics() { + let d = doc(); + let mut edge = CandidateEdge::emit( + WorldRole::Policy, + "Claims".into(), + EdgeKind::GovernedBy, + "30-day filing rule".into(), + vec![citation(&d)], + ) + .unwrap(); + assert!(!edge.meets_consensus()); + + let corroborate = |r| Verdict::new(r, Judgment::Corroborate, "entailed".into()).unwrap(); + edge.record(corroborate(WorldRole::GroundingAuditor)); + edge.record(corroborate(WorldRole::ContradictionSentinel)); + assert!(edge.meets_consensus()); + assert!(!edge.is_contested()); + + edge.record( + Verdict::new( + WorldRole::ContradictionSentinel, + Judgment::Refute, + "conflicts with Finance rule".into(), + ) + .unwrap(), + ); + assert!(edge.is_contested()); + assert!(!edge.meets_consensus()); + } +} diff --git a/smesh-world/src/cdm.rs b/smesh-world/src/cdm.rs new file mode 100644 index 0000000..7cdc8c4 --- /dev/null +++ b/smesh-world/src/cdm.rs @@ -0,0 +1,210 @@ +//! Canonical Document Model: Tier 0 output. Extractors never see raw formats. +//! +//! Every CDM span is dual-anchored — addressable in canonical text for +//! machine citation, and mapped back to native coordinates so the human +//! dashboard highlights the original artifact. + +use crate::WorldError; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fmt; + +/// Content-addressed document identity: `BLAKE3(original bytes)`. +/// Re-ingesting identical bytes is idempotent by construction. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct DocId([u8; 32]); + +impl DocId { + pub fn from_bytes(original: &[u8]) -> Self { + Self(*blake3::hash(original).as_bytes()) + } + + pub fn as_hex(&self) -> String { + self.0.iter().map(|b| format!("{b:02x}")).collect() + } +} + +impl fmt::Debug for DocId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "DocId({}…)", &self.as_hex()[..12]) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SourceFormat { + Markdown, + Pdf, + Eml, + Yaml, + Json, + Csv, + Html, + Unknown, +} + +/// Native coordinates in the original artifact, per format. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum NativeAnchor { + PdfPage { + page: u32, + }, + EmailLine { + message_id: String, + line: u32, + }, + MarkdownHeading { + heading_path: Vec, + line: u32, + }, + Line { + line: u32, + }, + ByteOffset { + offset: usize, + }, +} + +/// A half-open byte range `start..end` into a document's canonical text. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct CdmSpan { + pub start: usize, + pub end: usize, +} + +impl CdmSpan { + pub fn new(start: usize, end: usize) -> Self { + Self { start, end } + } +} + +/// Structured metadata harvested at intake. Email headers are native +/// provenance: `From:` + `Date:` yield attribution and temporal edges free. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DocMetadata { + pub title: Option, + pub author: Option, + pub date: Option, + pub department: Option, + pub extra: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CdmDocument { + pub id: DocId, + pub format: SourceFormat, + pub canonical_text: String, + pub anchors: Vec<(CdmSpan, NativeAnchor)>, + pub metadata: DocMetadata, +} + +impl CdmDocument { + pub fn ingest( + original_bytes: &[u8], + format: SourceFormat, + canonical_text: String, + anchors: Vec<(CdmSpan, NativeAnchor)>, + metadata: DocMetadata, + ) -> Self { + Self { + id: DocId::from_bytes(original_bytes), + format, + canonical_text, + anchors, + metadata, + } + } + + pub fn span_text(&self, span: CdmSpan) -> Result<&str, WorldError> { + let len = self.canonical_text.len(); + if span.start >= span.end + || span.end > len + || !self.canonical_text.is_char_boundary(span.start) + || !self.canonical_text.is_char_boundary(span.end) + { + return Err(WorldError::InvalidSpan { + start: span.start, + end: span.end, + len, + }); + } + Ok(&self.canonical_text[span.start..span.end]) + } + + /// Resolve a canonical span back to its native coordinates: the innermost + /// anchor whose span contains it. + pub fn native_anchor(&self, span: CdmSpan) -> Option<&NativeAnchor> { + self.anchors + .iter() + .filter(|(s, _)| s.start <= span.start && span.end <= s.end) + .min_by_key(|(s, _)| s.end - s.start) + .map(|(_, a)| a) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn doc() -> CdmDocument { + CdmDocument::ingest( + b"raw-bytes", + SourceFormat::Markdown, + "Claims must be filed within 30 days.".to_owned(), + vec![ + ( + CdmSpan::new(0, 36), + NativeAnchor::MarkdownHeading { + heading_path: vec!["Claims".into(), "Filing".into()], + line: 12, + }, + ), + (CdmSpan::new(0, 6), NativeAnchor::Line { line: 12 }), + ], + DocMetadata::default(), + ) + } + + #[test] + fn doc_id_is_content_addressed_and_idempotent() { + assert_eq!(DocId::from_bytes(b"same"), DocId::from_bytes(b"same")); + assert_ne!(DocId::from_bytes(b"same"), DocId::from_bytes(b"other")); + } + + #[test] + fn span_text_extracts_and_rejects_out_of_bounds() { + let d = doc(); + assert_eq!(d.span_text(CdmSpan::new(0, 6)).unwrap(), "Claims"); + assert!(matches!( + d.span_text(CdmSpan::new(10, 200)), + Err(WorldError::InvalidSpan { .. }) + )); + assert!(d.span_text(CdmSpan::new(5, 5)).is_err()); + } + + #[test] + fn span_text_rejects_non_char_boundaries() { + let d = CdmDocument::ingest( + b"x", + SourceFormat::Markdown, + "Prévue".to_owned(), + vec![], + DocMetadata::default(), + ); + assert!(d.span_text(CdmSpan::new(0, 3)).is_err()); + assert_eq!(d.span_text(CdmSpan::new(0, 4)).unwrap(), "Pré"); + } + + #[test] + fn native_anchor_prefers_innermost_containing_span() { + let d = doc(); + assert!(matches!( + d.native_anchor(CdmSpan::new(0, 6)), + Some(NativeAnchor::Line { line: 12 }) + )); + assert!(matches!( + d.native_anchor(CdmSpan::new(7, 11)), + Some(NativeAnchor::MarkdownHeading { .. }) + )); + assert!(d.native_anchor(CdmSpan::new(0, 999)).is_none()); + } +} diff --git a/smesh-world/src/corpus.rs b/smesh-world/src/corpus.rs new file mode 100644 index 0000000..5f7276f --- /dev/null +++ b/smesh-world/src/corpus.rs @@ -0,0 +1,246 @@ +//! Meridian Mutual: the instrumented showcase corpus. +//! +//! The gold graph is authored first; documents are rendered from it with a +//! planted-defect manifest. The answer key is exact by construction, so every +//! mesh run is scored — and the scorecard is the permanent regression gate. + +use crate::candidate::{CandidateEdge, ProvenanceClass}; +use crate::ontology::EdgeKind; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct GoldEdge { + pub subject: String, + pub kind: EdgeKind, + pub object: String, +} + +impl GoldEdge { + pub fn key(&self) -> String { + format!("{}|{:?}|{}", self.subject, self.kind, self.object) + } +} + +/// Each planted defect targets a specific subsystem of the mesh. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PlantedDefect { + /// Same term, different meaning per department — Lexicon scoping. + TermCollision { + term: String, + departments: Vec, + }, + /// Two departments with directly contradictory rules — Sentinel → red lane. + CrossDepartmentContradiction { edge_a: GoldEdge, edge_b: GoldEdge }, + /// Policy PDF superseded by a later email memo — cross-format supersession. + CrossFormatSupersession { + superseded: GoldEdge, + superseding: GoldEdge, + }, + /// Same policy in two formats, slightly divergent — consensus mechanics. + DivergentDuplicate { canonical: GoldEdge }, + /// Rule existing only in an email thread — tribal-knowledge analog. + EmailOnlyRule { edge: GoldEdge }, + /// A question the corpus deliberately never answers — confabulation trap. + NegativeSpace { question: String }, + /// Attached vendor policy that must not be ingested as Meridian's. + VendorAttachment { vendor: String }, + /// Generic-corp boilerplate contradicting Meridian's actual quirky rule. + BoilerplateTrap { meridian_rule: GoldEdge }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DefectManifest { + pub defects: Vec, +} + +impl DefectManifest { + pub fn planted_contradictions(&self) -> usize { + self.defects + .iter() + .filter(|d| { + matches!( + d, + PlantedDefect::CrossDepartmentContradiction { .. } + | PlantedDefect::BoilerplateTrap { .. } + ) + }) + .count() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GoldGraph { + pub edges: Vec, +} + +/// The headline numbers. Confabulation rate targets zero — an edge asserted +/// in no document and carrying no human signature is the founding failure +/// mode this whole system exists to eliminate. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Scorecard { + pub true_positives: usize, + pub false_positives: usize, + pub false_negatives: usize, + pub confabulated: usize, + pub contradictions_planted: usize, + pub contradictions_caught: usize, +} + +impl Scorecard { + pub fn evaluate( + gold: &GoldGraph, + observed: &[CandidateEdge], + manifest: &DefectManifest, + contradictions_caught: usize, + ) -> Self { + let gold_keys: BTreeSet = gold.edges.iter().map(GoldEdge::key).collect(); + let observed_keys: BTreeSet = observed.iter().map(CandidateEdge::key).collect(); + + let true_positives = observed_keys.intersection(&gold_keys).count(); + let false_positives = observed_keys.difference(&gold_keys).count(); + let false_negatives = gold_keys.difference(&observed_keys).count(); + let confabulated = observed + .iter() + .filter(|e| { + !gold_keys.contains(&e.key()) + && match &e.provenance { + ProvenanceClass::CorpusDerived { citations } => citations.is_empty(), + ProvenanceClass::HumanAttested { .. } => false, + } + }) + .count(); + + Self { + true_positives, + false_positives, + false_negatives, + confabulated, + contradictions_planted: manifest.planted_contradictions(), + contradictions_caught, + } + } + + pub fn precision(&self) -> f64 { + ratio( + self.true_positives, + self.true_positives + self.false_positives, + ) + } + + pub fn recall(&self) -> f64 { + ratio( + self.true_positives, + self.true_positives + self.false_negatives, + ) + } + + pub fn contradiction_detection_rate(&self) -> f64 { + ratio(self.contradictions_caught, self.contradictions_planted) + } + + pub fn passes_gate(&self) -> bool { + self.confabulated == 0 && self.contradiction_detection_rate() >= 1.0 + } +} + +fn ratio(num: usize, den: usize) -> f64 { + if den == 0 { + 1.0 + } else { + num as f64 / den as f64 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::candidate::Citation; + use crate::cdm::{CdmDocument, CdmSpan, DocMetadata, SourceFormat}; + use crate::role::WorldRole; + + fn gold() -> GoldGraph { + GoldGraph { + edges: vec![ + GoldEdge { + subject: "Claims".into(), + kind: EdgeKind::ReportsTo, + object: "Ops".into(), + }, + GoldEdge { + subject: "Finance".into(), + kind: EdgeKind::Owns, + object: "Ledger".into(), + }, + ], + } + } + + fn manifest() -> DefectManifest { + DefectManifest { + defects: vec![ + PlantedDefect::CrossDepartmentContradiction { + edge_a: gold().edges[0].clone(), + edge_b: gold().edges[1].clone(), + }, + PlantedDefect::NegativeSpace { + question: "What is the remote-work policy?".into(), + }, + ], + } + } + + fn observed(subject: &str, kind: EdgeKind, object: &str) -> CandidateEdge { + let doc = CdmDocument::ingest( + b"gold-rendered", + SourceFormat::Markdown, + "Claims reports to Ops.".to_owned(), + vec![], + DocMetadata::default(), + ); + CandidateEdge::emit( + WorldRole::Structure, + subject.into(), + kind, + object.into(), + vec![Citation::grounded(&doc, CdmSpan::new(0, 6)).unwrap()], + ) + .unwrap() + } + + #[test] + fn scorecard_computes_precision_and_recall() { + let hits = vec![ + observed("Claims", EdgeKind::ReportsTo, "Ops"), + observed("HR", EdgeKind::Owns, "Handbook"), + ]; + let card = Scorecard::evaluate(&gold(), &hits, &manifest(), 1); + assert_eq!(card.true_positives, 1); + assert_eq!(card.false_positives, 1); + assert_eq!(card.false_negatives, 1); + assert_eq!(card.precision(), 0.5); + assert_eq!(card.recall(), 0.5); + assert_eq!(card.contradiction_detection_rate(), 1.0); + } + + #[test] + fn cited_false_positives_are_wrong_but_not_confabulated() { + let hits = vec![observed("HR", EdgeKind::Owns, "Handbook")]; + let card = Scorecard::evaluate(&gold(), &hits, &manifest(), 1); + assert_eq!(card.false_positives, 1); + assert_eq!(card.confabulated, 0); + assert!(card.passes_gate()); + } + + #[test] + fn gate_fails_when_planted_contradictions_slip_through() { + let hits = vec![observed("Claims", EdgeKind::ReportsTo, "Ops")]; + let card = Scorecard::evaluate(&gold(), &hits, &manifest(), 0); + assert!(!card.passes_gate()); + } + + #[test] + fn manifest_counts_planted_contradictions() { + assert_eq!(manifest().planted_contradictions(), 1); + } +} diff --git a/smesh-world/src/delta.rs b/smesh-world/src/delta.rs new file mode 100644 index 0000000..4c7a904 --- /dev/null +++ b/smesh-world/src/delta.rs @@ -0,0 +1,328 @@ +//! Changeset lifecycle: the mesh never publishes — it stages. +//! +//! A staged changeset is a PR against the world. Type-state makes the +//! ratification protocol unskippable: `sign` exists only on +//! `Changeset`, and a `Ratified` value can only be produced by +//! `ratify`, which demands a reviewer decision for every candidate. + +use crate::candidate::CandidateEdge; +use crate::ontology::EdgeKind; +use crate::WorldError; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct ReviewerId(pub String); + +/// Ed25519 signature bytes. Key handling lives with the dashboard identity +/// layer; the kernel only requires that a signature is present and bound. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Signature(pub Vec); + +/// Ratification is always total; attention is tiered. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Lane { + /// Summarized batch, spot-checkable. + Green, + /// Individually surfaced with evidence. + Amber, + /// Mandatory individual decision — why the human is in the loop. + Red, +} + +impl Lane { + pub fn assign(edge: &CandidateEdge) -> Lane { + if edge.is_contested() { + return Lane::Red; + } + let normative = matches!( + edge.kind, + EdgeKind::GovernedBy | EdgeKind::Requires | EdgeKind::Triggers | EdgeKind::Supersedes + ); + if normative || edge.corroborations() < 2 { + Lane::Amber + } else { + Lane::Green + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum ReviewDecision { + Approve, + /// The human version wins; the mesh-vs-human delta is a labeled error + /// signal fed back into expert reputation. + Edit { + amended: CandidateEdge, + }, + Reject { + reason: String, + }, + /// Decays back into the field for more evidence. + Defer, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RatificationRecord { + pub reviewer: ReviewerId, + pub decisions: BTreeMap, + pub signature: Signature, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Staged; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Ratified; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Signed; + +#[derive(Debug, Clone, PartialEq)] +pub struct Changeset { + pub base_rev: String, + pub edges: Vec, + state: std::marker::PhantomData, +} + +/// What ratification produced besides the ratified changeset itself. +#[derive(Debug, Clone, PartialEq)] +pub struct RatificationOutcome { + pub changeset: Changeset, + pub ratification: RatificationRecord, + pub killed: Vec<(CandidateEdge, String)>, + pub parked: Vec, +} + +impl Changeset { + pub fn stage(base_rev: String, edges: Vec) -> Result { + if edges.is_empty() { + return Err(WorldError::EmptyChangeset); + } + Ok(Self { + base_rev, + edges, + state: std::marker::PhantomData, + }) + } + + pub fn lanes(&self) -> Vec<(Lane, &CandidateEdge)> { + self.edges.iter().map(|e| (Lane::assign(e), e)).collect() + } + + /// Every candidate must carry a decision — ultimate sign-off is total. + pub fn ratify(self, record: RatificationRecord) -> Result { + for edge in &self.edges { + if !record.decisions.contains_key(&edge.key()) { + return Err(WorldError::UnreviewedCandidate(edge.key())); + } + } + let mut retained = Vec::new(); + let mut killed = Vec::new(); + let mut parked = Vec::new(); + for edge in self.edges { + match record.decisions.get(&edge.key()).cloned() { + Some(ReviewDecision::Approve) => retained.push(edge), + Some(ReviewDecision::Edit { amended }) => retained.push(amended), + Some(ReviewDecision::Reject { reason }) => killed.push((edge, reason)), + Some(ReviewDecision::Defer) => parked.push(edge), + None => unreachable!("coverage checked above"), + } + } + if retained.is_empty() { + return Err(WorldError::EmptyChangeset); + } + Ok(RatificationOutcome { + changeset: Changeset { + base_rev: self.base_rev, + edges: retained, + state: std::marker::PhantomData, + }, + ratification: record, + killed, + parked, + }) + } +} + +/// The Curator's output: a new revision whose identity commits to the base +/// revision, the surviving edges, and the human ratification that authorized +/// them. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SignedChangeset { + pub base_rev: String, + pub new_rev: String, + pub edges: Vec, + pub ratification: RatificationRecord, +} + +impl Changeset { + /// Only reachable through `ratify` — the type system is the protocol. + pub fn sign(self, ratification: RatificationRecord) -> SignedChangeset { + let mut hasher = blake3::Hasher::new(); + hasher.update(self.base_rev.as_bytes()); + hasher.update(&ratification.signature.0); + for edge in &self.edges { + hasher.update(edge.key().as_bytes()); + } + SignedChangeset { + base_rev: self.base_rev, + new_rev: hasher.finalize().to_hex().to_string(), + edges: self.edges, + ratification, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::candidate::{Citation, Judgment, Verdict}; + use crate::cdm::{CdmDocument, CdmSpan, DocMetadata, SourceFormat}; + use crate::role::WorldRole; + + fn doc() -> CdmDocument { + CdmDocument::ingest( + b"handbook", + SourceFormat::Markdown, + "Claims reports to Operations. Finance owns the ledger.".to_owned(), + vec![], + DocMetadata::default(), + ) + } + + fn edge(d: &CdmDocument, subject: &str, kind: EdgeKind, object: &str) -> CandidateEdge { + CandidateEdge::emit( + WorldRole::Structure, + subject.into(), + kind, + object.into(), + vec![Citation::grounded(d, CdmSpan::new(0, 6)).unwrap()], + ) + .unwrap() + } + + fn corroborated(mut e: CandidateEdge, times: u32) -> CandidateEdge { + for _ in 0..times { + e.record( + Verdict::new( + WorldRole::GroundingAuditor, + Judgment::Corroborate, + "ok".into(), + ) + .unwrap(), + ); + } + e + } + + fn record(reviewer: &str, decisions: Vec<(String, ReviewDecision)>) -> RatificationRecord { + RatificationRecord { + reviewer: ReviewerId(reviewer.into()), + decisions: decisions.into_iter().collect(), + signature: Signature(vec![7; 64]), + } + } + + #[test] + fn staging_rejects_empty_changesets() { + assert_eq!( + Changeset::stage("rev0".into(), vec![]).unwrap_err(), + WorldError::EmptyChangeset + ); + } + + #[test] + fn lane_assignment_tiers_attention() { + let d = doc(); + let green = corroborated(edge(&d, "Claims", EdgeKind::MemberOf, "Meridian"), 2); + let amber = corroborated(edge(&d, "Claims", EdgeKind::GovernedBy, "30-day rule"), 2); + let mut red = corroborated(edge(&d, "Claims", EdgeKind::ReportsTo, "Ops"), 1); + red.record( + Verdict::new( + WorldRole::ContradictionSentinel, + Judgment::Refute, + "conflicting org chart".into(), + ) + .unwrap(), + ); + assert_eq!(Lane::assign(&green), Lane::Green); + assert_eq!(Lane::assign(&amber), Lane::Amber); + assert_eq!(Lane::assign(&red), Lane::Red); + } + + #[test] + fn ratification_must_cover_every_candidate() { + let d = doc(); + let a = edge(&d, "Claims", EdgeKind::ReportsTo, "Ops"); + let b = edge(&d, "Finance", EdgeKind::Owns, "Ledger"); + let staged = Changeset::stage("rev0".into(), vec![a.clone(), b]).unwrap(); + let partial = record("dj", vec![(a.key(), ReviewDecision::Approve)]); + assert!(matches!( + staged.ratify(partial), + Err(WorldError::UnreviewedCandidate(_)) + )); + } + + #[test] + fn decisions_route_to_retained_killed_parked() { + let d = doc(); + let a = edge(&d, "Claims", EdgeKind::ReportsTo, "Ops"); + let b = edge(&d, "Finance", EdgeKind::Owns, "Ledger"); + let c = edge(&d, "HR", EdgeKind::MemberOf, "Meridian"); + let amended = edge(&d, "Finance", EdgeKind::Owns, "General Ledger"); + let staged = + Changeset::stage("rev0".into(), vec![a.clone(), b.clone(), c.clone()]).unwrap(); + let outcome = staged + .ratify(record( + "dj", + vec![ + (a.key(), ReviewDecision::Approve), + ( + b.key(), + ReviewDecision::Edit { + amended: amended.clone(), + }, + ), + (c.key(), ReviewDecision::Defer), + ], + )) + .unwrap(); + assert_eq!(outcome.changeset.edges, vec![a, amended]); + assert!(outcome.killed.is_empty()); + assert_eq!(outcome.parked, vec![c]); + } + + #[test] + fn fully_rejected_changesets_cannot_be_signed() { + let d = doc(); + let a = edge(&d, "Claims", EdgeKind::ReportsTo, "Ops"); + let staged = Changeset::stage("rev0".into(), vec![a.clone()]).unwrap(); + let outcome = staged.ratify(record( + "dj", + vec![( + a.key(), + ReviewDecision::Reject { + reason: "stale org chart".into(), + }, + )], + )); + assert_eq!(outcome.unwrap_err(), WorldError::EmptyChangeset); + } + + #[test] + fn signing_commits_to_base_edges_and_ratification() { + let d = doc(); + let a = edge(&d, "Claims", EdgeKind::ReportsTo, "Ops"); + let staged = Changeset::stage("rev0".into(), vec![a.clone()]).unwrap(); + let outcome = staged + .ratify(record("dj", vec![(a.key(), ReviewDecision::Approve)])) + .unwrap(); + let signed = outcome.changeset.clone().sign(outcome.ratification.clone()); + assert_eq!(signed.base_rev, "rev0"); + assert_eq!(signed.new_rev.len(), 64); + + let mut other_sig = outcome.ratification.clone(); + other_sig.signature = Signature(vec![9; 64]); + let resigned = outcome.changeset.sign(other_sig); + assert_ne!(signed.new_rev, resigned.new_rev); + } +} diff --git a/smesh-world/src/lib.rs b/smesh-world/src/lib.rs new file mode 100644 index 0000000..80acd6a --- /dev/null +++ b/smesh-world/src/lib.rs @@ -0,0 +1,51 @@ +//! # SMESH World +//! +//! Signed organizational world models built by a mesh of experts and ratified +//! by humans. See `WORLD-MODEL.md` at the workspace root for the full design. +//! +//! - **AION is the truth plane**: signed, append-only, content-addressed. +//! - **SMESH is the refinery**: experts propose, the field filters. +//! - **The human is the merge authority**: nothing is signed unratified. +//! +//! Governing principle — separation of powers: extractors propose, verifiers +//! judge, one non-LLM curator signs. No role holds two of those powers, and +//! the type system enforces it. + +pub mod candidate; +pub mod cdm; +pub mod corpus; +pub mod delta; +pub mod ontology; +pub mod role; + +pub use candidate::{CandidateEdge, Citation, Judgment, ProvenanceClass, Verdict}; +pub use cdm::{CdmDocument, CdmSpan, DocId, NativeAnchor, SourceFormat}; +pub use corpus::{DefectManifest, GoldEdge, GoldGraph, PlantedDefect, Scorecard}; +pub use delta::{ + Changeset, Lane, RatificationRecord, Ratified, ReviewDecision, ReviewerId, Signature, Signed, + Staged, +}; +pub use ontology::{ConsensusPolicy, EdgeKind, StructuralConstraint, TransportKind}; +pub use role::{Capability, ModelPolicy, Tier, WorldRole}; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum WorldError { + #[error("span {start}..{end} is invalid for document of length {len}")] + InvalidSpan { + start: usize, + end: usize, + len: usize, + }, + #[error("quote does not match the cited span text")] + QuoteMismatch, + #[error("role {0:?} lacks capability {1:?}")] + CapabilityDenied(role::WorldRole, role::Capability), + #[error("corpus-derived candidates require at least one citation")] + MissingCitation, + #[error("human-attested candidates require a reviewer signature")] + MissingAttestation, + #[error("changeset is empty; nothing to stage")] + EmptyChangeset, + #[error("ratification record does not cover candidate {0}")] + UnreviewedCandidate(String), +} diff --git a/smesh-world/src/ontology.rs b/smesh-world/src/ontology.rs new file mode 100644 index 0000000..6dcfe0d --- /dev/null +++ b/smesh-world/src/ontology.rs @@ -0,0 +1,144 @@ +//! The typed edge vocabulary — rev 0 of the world model. +//! +//! AION's four edge kinds (semantic/causal/temporal/provenance) are the +//! transport layer; the org world model speaks a typed vocabulary on top. +//! Consensus thresholds are per-edge-kind, not global. + +use serde::{Deserialize, Serialize}; + +/// AION transport-layer edge kind a typed edge lowers to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum TransportKind { + Semantic, + Causal, + Temporal, + Provenance, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub enum EdgeKind { + DefinesTerm, + GovernedBy, + ReportsTo, + Owns, + Operates, + MemberOf, + Precedes, + Requires, + Triggers, + Supersedes, + ScopedTo, +} + +/// Structural invariants the Contradiction Sentinel enforces mechanically. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum StructuralConstraint { + /// At most one such edge per subject per timeslice (e.g. `reports_to`). + UniquePerSubjectPerTimeslice, + /// The relation must form no cycles (e.g. `supersedes`). + Acyclic, +} + +/// How much independent agreement an edge kind must earn before the Curator +/// will include it in a changeset. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ConsensusPolicy { + pub min_corroborations: u32, + pub max_refutations: u32, +} + +impl EdgeKind { + pub const ALL: [EdgeKind; 11] = [ + EdgeKind::DefinesTerm, + EdgeKind::GovernedBy, + EdgeKind::ReportsTo, + EdgeKind::Owns, + EdgeKind::Operates, + EdgeKind::MemberOf, + EdgeKind::Precedes, + EdgeKind::Requires, + EdgeKind::Triggers, + EdgeKind::Supersedes, + EdgeKind::ScopedTo, + ]; + + pub fn transport(&self) -> TransportKind { + match self { + EdgeKind::DefinesTerm | EdgeKind::MemberOf | EdgeKind::ScopedTo => { + TransportKind::Semantic + } + EdgeKind::GovernedBy + | EdgeKind::ReportsTo + | EdgeKind::Owns + | EdgeKind::Operates + | EdgeKind::Requires + | EdgeKind::Triggers => TransportKind::Causal, + EdgeKind::Precedes => TransportKind::Temporal, + EdgeKind::Supersedes => TransportKind::Provenance, + } + } + + /// Normative and identity-bearing edges demand more agreement than + /// vocabulary edges. + pub fn consensus_policy(&self) -> ConsensusPolicy { + let min_corroborations = match self { + EdgeKind::DefinesTerm | EdgeKind::MemberOf | EdgeKind::ScopedTo => 1, + EdgeKind::ReportsTo | EdgeKind::Owns | EdgeKind::Operates | EdgeKind::Precedes => 2, + EdgeKind::GovernedBy + | EdgeKind::Requires + | EdgeKind::Triggers + | EdgeKind::Supersedes => 2, + }; + ConsensusPolicy { + min_corroborations, + max_refutations: 0, + } + } + + pub fn structural_constraint(&self) -> Option { + match self { + EdgeKind::ReportsTo => Some(StructuralConstraint::UniquePerSubjectPerTimeslice), + EdgeKind::Supersedes | EdgeKind::Precedes => Some(StructuralConstraint::Acyclic), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_edge_kind_lowers_to_a_transport() { + for kind in EdgeKind::ALL { + let _ = kind.transport(); + } + } + + #[test] + fn normative_edges_demand_more_agreement_than_vocabulary() { + assert!( + EdgeKind::GovernedBy.consensus_policy().min_corroborations + > EdgeKind::DefinesTerm.consensus_policy().min_corroborations + ); + } + + #[test] + fn no_edge_kind_tolerates_unresolved_refutations() { + for kind in EdgeKind::ALL { + assert_eq!(kind.consensus_policy().max_refutations, 0); + } + } + + #[test] + fn supersession_is_acyclic_and_reporting_is_unique() { + assert_eq!( + EdgeKind::Supersedes.structural_constraint(), + Some(StructuralConstraint::Acyclic) + ); + assert_eq!( + EdgeKind::ReportsTo.structural_constraint(), + Some(StructuralConstraint::UniquePerSubjectPerTimeslice) + ); + } +} diff --git a/smesh-world/src/role.rs b/smesh-world/src/role.rs new file mode 100644 index 0000000..65903b7 --- /dev/null +++ b/smesh-world/src/role.rs @@ -0,0 +1,180 @@ +//! The expert roster: Tiers 0–3, capability sets, model policy. +//! +//! Separation of powers is encoded here and asserted by tests: no role may +//! both emit candidates and issue verdicts; only the Curator writes AION; +//! the Curator is not an LLM. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Tier { + Intake, + Extraction, + Verification, + Stewardship, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Capability { + CorpusRead, + GraphRead, + EmitCandidate, + Verdict, + Escalate, + OntologyMap, + ProposeEdgeKind, + AionWrite, +} + +/// Which model a role runs, and the diversity constraint that keeps +/// correlated hallucinations from surviving verification. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ModelPolicy { + /// Deterministic code path; no model involved. + None, + /// LLM used only to triage ambiguous inputs; fidelity work is code. + TriageOnly(&'static str), + Fixed(&'static str), + /// Must run a model family distinct from every Extraction-tier model. + DistinctFamilyFromExtraction(&'static str), +} + +pub const EXTRACTION_MODEL: &str = "moonshotai/kimi-k3"; +pub const VERIFICATION_MODEL: &str = "anthropic/claude-sonnet-4.5"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum WorldRole { + /// Tier 0 — any artifact in, CDM out. + Registrar, + /// Tier 1 — org verbiage: defined terms, acronyms, canonical names. + Lexicon, + /// Tier 1 — normative statements, effective dates, supersession. + Policy, + /// Tier 1 — org chart, roles, systems, ownership. + Structure, + /// Tier 1 — workflows, lifecycles, temporal ordering. + Process, + /// Tier 2 — does the cited span entail the claim? + GroundingAuditor, + /// Tier 2 — candidate deltas vs the current world revision. + ContradictionSentinel, + /// Tier 3 — owns the edge vocabulary; new kinds only via escalation. + Ontologist, + /// Tier 3 — sole AION writer; deliberately not an LLM. + Curator, +} + +impl WorldRole { + pub const ALL: [WorldRole; 9] = [ + WorldRole::Registrar, + WorldRole::Lexicon, + WorldRole::Policy, + WorldRole::Structure, + WorldRole::Process, + WorldRole::GroundingAuditor, + WorldRole::ContradictionSentinel, + WorldRole::Ontologist, + WorldRole::Curator, + ]; + + pub fn tier(&self) -> Tier { + match self { + WorldRole::Registrar => Tier::Intake, + WorldRole::Lexicon | WorldRole::Policy | WorldRole::Structure | WorldRole::Process => { + Tier::Extraction + } + WorldRole::GroundingAuditor | WorldRole::ContradictionSentinel => Tier::Verification, + WorldRole::Ontologist | WorldRole::Curator => Tier::Stewardship, + } + } + + pub fn capabilities(&self) -> &'static [Capability] { + use Capability::*; + match self { + WorldRole::Registrar => &[CorpusRead], + WorldRole::Lexicon | WorldRole::Policy | WorldRole::Structure | WorldRole::Process => { + &[CorpusRead, EmitCandidate] + } + WorldRole::GroundingAuditor => &[CorpusRead, Verdict], + WorldRole::ContradictionSentinel => &[GraphRead, Verdict, Escalate], + WorldRole::Ontologist => &[GraphRead, OntologyMap, ProposeEdgeKind, Escalate], + WorldRole::Curator => &[GraphRead, AionWrite], + } + } + + pub fn has_capability(&self, cap: Capability) -> bool { + self.capabilities().contains(&cap) + } + + pub fn model_policy(&self) -> ModelPolicy { + match self { + WorldRole::Registrar => ModelPolicy::TriageOnly(EXTRACTION_MODEL), + WorldRole::Lexicon | WorldRole::Policy | WorldRole::Structure | WorldRole::Process => { + ModelPolicy::Fixed(EXTRACTION_MODEL) + } + WorldRole::GroundingAuditor | WorldRole::ContradictionSentinel => { + ModelPolicy::DistinctFamilyFromExtraction(VERIFICATION_MODEL) + } + WorldRole::Ontologist => ModelPolicy::Fixed(VERIFICATION_MODEL), + WorldRole::Curator => ModelPolicy::None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_role_both_proposes_and_judges() { + for role in WorldRole::ALL { + assert!( + !(role.has_capability(Capability::EmitCandidate) + && role.has_capability(Capability::Verdict)), + "{role:?} violates separation of powers" + ); + } + } + + #[test] + fn only_the_curator_writes_aion() { + for role in WorldRole::ALL { + assert_eq!( + role.has_capability(Capability::AionWrite), + role == WorldRole::Curator + ); + } + } + + #[test] + fn signing_authority_is_code_not_a_model() { + assert_eq!(WorldRole::Curator.model_policy(), ModelPolicy::None); + } + + #[test] + fn extractors_cannot_touch_the_graph() { + for role in WorldRole::ALL + .iter() + .filter(|r| r.tier() == Tier::Extraction) + { + assert!(!role.has_capability(Capability::GraphRead)); + assert!(!role.has_capability(Capability::AionWrite)); + } + } + + #[test] + fn verifiers_run_a_distinct_model_family() { + for role in WorldRole::ALL + .iter() + .filter(|r| r.tier() == Tier::Verification) + { + match role.model_policy() { + ModelPolicy::DistinctFamilyFromExtraction(m) => { + let extraction_family = EXTRACTION_MODEL.split('/').next().unwrap(); + assert_ne!(m.split('/').next().unwrap(), extraction_family); + } + other => panic!("{role:?} must verify cross-family, got {other:?}"), + } + } + } +} From 091263d79c123271f655369785c8478511b168f2 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Wed, 22 Jul 2026 23:57:49 -0700 Subject: [PATCH 02/13] feat(world): Tier 0 intake - Registrar sniffing + markdown/.eml adapters Deterministic format detection (magic bytes, header-block structure, text fallback). Markdown adapter anchors every line with its heading path; .eml adapter harvests headers as native provenance (From/Date/Subject/ X-Department) and anchors body lines to message-id coordinates. Grounded citations round-trip through both adapters. 41 tests. --- WORLD-MODEL.md | 5 +- smesh-world/src/intake/eml.rs | 151 +++++++++++++++++++++++++++++ smesh-world/src/intake/markdown.rs | 133 +++++++++++++++++++++++++ smesh-world/src/intake/mod.rs | 109 +++++++++++++++++++++ smesh-world/src/lib.rs | 6 ++ 5 files changed, 402 insertions(+), 2 deletions(-) create mode 100644 smesh-world/src/intake/eml.rs create mode 100644 smesh-world/src/intake/markdown.rs create mode 100644 smesh-world/src/intake/mod.rs diff --git a/WORLD-MODEL.md b/WORLD-MODEL.md index c8f8dd7..be6b818 100644 --- a/WORLD-MODEL.md +++ b/WORLD-MODEL.md @@ -202,11 +202,12 @@ config-as-policy, CSV org roster, wiki export, scanned-memo PDFs. | `candidate.rs` | Candidate node/edge emissions with schema-mandatory citations | | `delta.rs` | Changeset lifecycle state machine; type-state ratification (unsigned changeset cannot reach `Signed`) | | `corpus.rs` | Gold graph, planted-defect manifest, scorecard types for Meridian | +| `intake/` | Tier 0 Registrar: deterministic format sniffing; markdown adapter (heading-path anchors) and `.eml` adapter (header harvest, body-line anchors); PDF pending (AkamaiForms path) | ## 9. Phasing -- **Phase 0 (this scaffold):** types + state machine + gates green. -- **Phase 1:** Tier 0 intake adapters (markdown, PDF via AkamaiForms path, `.eml`) → CDM fixtures. +- **Phase 0 — DONE:** types + state machine + gates green. +- **Phase 1 — IN PROGRESS:** Tier 0 intake adapters → CDM. Markdown + `.eml` shipped; PDF (AkamaiForms path) pending. - **Phase 2:** Meridian gold graph + deterministic renderer + planted defects. - **Phase 3:** Wire Tier 1/2 experts over `smesh-agent` (K3 extract, cross-model verify), run the field, emit first staged changeset. - **Phase 4:** Ratification dashboard (Axum; prevue-api pattern) + Curator signing into AION. diff --git a/smesh-world/src/intake/eml.rs b/smesh-world/src/intake/eml.rs new file mode 100644 index 0000000..71cbbbf --- /dev/null +++ b/smesh-world/src/intake/eml.rs @@ -0,0 +1,151 @@ +//! RFC 5322 `.eml` → CDM. Single-part text bodies only for now — the +//! Meridian corpus controls its own email shape; MIME multipart lands with +//! the attachment-scoping work (vendor-attachment defect). +//! +//! Headers are native provenance: `From:` + `Date:` become attribution and +//! temporal metadata without any model in the loop. `X-Department` is +//! honored because Meridian's renderer stamps it. + +use crate::cdm::{CdmDocument, CdmSpan, DocMetadata, NativeAnchor, SourceFormat}; +use crate::WorldError; +use std::collections::BTreeMap; + +pub fn ingest(bytes: &[u8]) -> Result { + let text = std::str::from_utf8(bytes) + .map_err(|e| WorldError::Malformed(format!("eml is not UTF-8: {e}")))?; + let normalized = text.replace("\r\n", "\n"); + let (head, body) = normalized + .split_once("\n\n") + .ok_or_else(|| WorldError::Malformed("missing header/body separator".into()))?; + + let headers = parse_headers(head); + let message_id = headers + .get("message-id") + .map(|v| v.trim_matches(['<', '>']).to_owned()) + .ok_or_else(|| WorldError::Malformed("missing Message-ID header".into()))?; + + let canonical = body.to_owned(); + let mut anchors = Vec::new(); + let mut offset = 0usize; + for (idx, line) in canonical.split('\n').enumerate() { + if !line.trim().is_empty() { + anchors.push(( + CdmSpan::new(offset, offset + line.len()), + NativeAnchor::EmailLine { + message_id: message_id.clone(), + line: u32::try_from(idx + 1).unwrap_or(u32::MAX), + }, + )); + } + offset += line.len() + 1; + } + + let mut extra = BTreeMap::new(); + extra.insert("message-id".to_owned(), message_id); + if let Some(to) = headers.get("to") { + extra.insert("to".to_owned(), to.clone()); + } + + Ok(CdmDocument::ingest( + bytes, + SourceFormat::Eml, + canonical, + anchors, + DocMetadata { + title: headers.get("subject").cloned(), + author: headers.get("from").cloned(), + date: headers.get("date").cloned(), + department: headers.get("x-department").cloned(), + extra, + }, + )) +} + +/// Lowercased header map with RFC 5322 folding (continuation lines start +/// with whitespace) unfolded. +fn parse_headers(head: &str) -> BTreeMap { + let mut headers: BTreeMap = BTreeMap::new(); + let mut current: Option = None; + for line in head.lines() { + if line.starts_with([' ', '\t']) { + if let Some(key) = ¤t { + if let Some(value) = headers.get_mut(key) { + value.push(' '); + value.push_str(line.trim()); + } + } + } else if let Some((key, value)) = line.split_once(':') { + let key = key.trim().to_ascii_lowercase(); + headers.insert(key.clone(), value.trim().to_owned()); + current = Some(key); + } + } + headers +} + +#[cfg(test)] +mod tests { + use super::*; + + const MEMO: &str = "From: Dana Reyes \nTo: claims-all@meridianmutual.example\nSubject: Filing window change,\n effective immediately\nDate: Tue, 21 Jul 2026 09:14:00 -0700\nMessage-ID: \nX-Department: Claims\n\nTeam,\n\nThe filing window is now 45 days, superseding the handbook's 30.\n\nDana\n"; + + #[test] + fn headers_become_metadata_including_folded_subject() { + let doc = ingest(MEMO.as_bytes()).unwrap(); + assert_eq!( + doc.metadata.title.as_deref(), + Some("Filing window change, effective immediately") + ); + assert_eq!( + doc.metadata.author.as_deref(), + Some("Dana Reyes ") + ); + assert_eq!(doc.metadata.department.as_deref(), Some("Claims")); + assert_eq!( + doc.metadata.extra.get("message-id").map(String::as_str), + Some("memo-4417@meridianmutual.example") + ); + } + + #[test] + fn body_lines_anchor_to_message_id_coordinates() { + let doc = ingest(MEMO.as_bytes()).unwrap(); + let start = doc.canonical_text.find("45 days").unwrap(); + match doc.native_anchor(CdmSpan::new(start, start + 7)).unwrap() { + NativeAnchor::EmailLine { message_id, line } => { + assert_eq!(message_id, "memo-4417@meridianmutual.example"); + assert_eq!(*line, 3); + } + other => panic!("unexpected anchor {other:?}"), + } + } + + #[test] + fn headers_are_not_part_of_canonical_text() { + let doc = ingest(MEMO.as_bytes()).unwrap(); + assert!(doc.canonical_text.starts_with("Team,")); + assert!(!doc.canonical_text.contains("Message-ID")); + } + + #[test] + fn missing_separator_and_missing_message_id_are_malformed() { + assert!(matches!( + ingest(b"From: a@b.c\nSubject: x"), + Err(WorldError::Malformed(_)) + )); + assert!(matches!( + ingest(b"From: a@b.c\nSubject: x\n\nbody\n"), + Err(WorldError::Malformed(_)) + )); + } + + #[test] + fn crlf_emails_normalize_identically() { + let crlf = MEMO.replace('\n', "\r\n"); + let doc = ingest(crlf.as_bytes()).unwrap(); + assert_eq!( + doc.canonical_text, + ingest(MEMO.as_bytes()).unwrap().canonical_text + ); + } +} diff --git a/smesh-world/src/intake/markdown.rs b/smesh-world/src/intake/markdown.rs new file mode 100644 index 0000000..3446106 --- /dev/null +++ b/smesh-world/src/intake/markdown.rs @@ -0,0 +1,133 @@ +//! Markdown → CDM. Canonical text is the LF-normalized source; every +//! non-empty line is anchored with its heading path so a citation renders as +//! "Handbook › Claims › Filing, line 12" on the dashboard. + +use crate::cdm::{CdmDocument, CdmSpan, DocMetadata, NativeAnchor, SourceFormat}; +use crate::WorldError; + +pub fn ingest(bytes: &[u8]) -> Result { + let text = std::str::from_utf8(bytes) + .map_err(|e| WorldError::Malformed(format!("markdown is not UTF-8: {e}")))?; + let canonical = text.replace("\r\n", "\n"); + + let mut anchors = Vec::new(); + let mut heading_stack: Vec<(u8, String)> = Vec::new(); + let mut title = None; + let mut offset = 0usize; + + for (idx, line) in canonical.split('\n').enumerate() { + let line_no = u32::try_from(idx + 1).unwrap_or(u32::MAX); + if let Some((level, heading)) = parse_heading(line) { + while heading_stack.last().is_some_and(|(l, _)| *l >= level) { + heading_stack.pop(); + } + heading_stack.push((level, heading.clone())); + if level == 1 && title.is_none() { + title = Some(heading); + } + } + if !line.trim().is_empty() { + anchors.push(( + CdmSpan::new(offset, offset + line.len()), + NativeAnchor::MarkdownHeading { + heading_path: heading_stack.iter().map(|(_, h)| h.clone()).collect(), + line: line_no, + }, + )); + } + offset += line.len() + 1; + } + + Ok(CdmDocument::ingest( + bytes, + SourceFormat::Markdown, + canonical, + anchors, + DocMetadata { + title, + ..DocMetadata::default() + }, + )) +} + +fn parse_heading(line: &str) -> Option<(u8, String)> { + let hashes = line.bytes().take_while(|b| *b == b'#').count(); + if !(1..=6).contains(&hashes) { + return None; + } + let rest = line[hashes..].strip_prefix(' ')?; + let heading = rest.trim().trim_end_matches('#').trim(); + if heading.is_empty() { + return None; + } + Some(( + u8::try_from(hashes).expect("hashes <= 6"), + heading.to_owned(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::candidate::Citation; + + const HANDBOOK: &str = "# Meridian Handbook\n\n## Claims\n\n### Filing\n\nClaims must be filed within 30 days.\n\n## Finance\n\nFinance owns the general ledger.\n"; + + #[test] + fn title_comes_from_first_h1() { + let doc = ingest(HANDBOOK.as_bytes()).unwrap(); + assert_eq!(doc.metadata.title.as_deref(), Some("Meridian Handbook")); + } + + #[test] + fn heading_paths_track_nesting_and_sibling_resets() { + let doc = ingest(HANDBOOK.as_bytes()).unwrap(); + let filing = doc.canonical_text.find("Claims must").unwrap(); + let span = CdmSpan::new( + filing, + filing + "Claims must be filed within 30 days.".len(), + ); + match doc.native_anchor(span).unwrap() { + NativeAnchor::MarkdownHeading { heading_path, line } => { + assert_eq!(heading_path, &["Meridian Handbook", "Claims", "Filing"]); + assert_eq!(*line, 7); + } + other => panic!("unexpected anchor {other:?}"), + } + + let ledger = doc.canonical_text.find("Finance owns").unwrap(); + match doc.native_anchor(CdmSpan::new(ledger, ledger + 7)).unwrap() { + NativeAnchor::MarkdownHeading { heading_path, .. } => { + assert_eq!(heading_path, &["Meridian Handbook", "Finance"]); + } + other => panic!("unexpected anchor {other:?}"), + } + } + + #[test] + fn crlf_input_is_normalized_but_identity_is_original_bytes() { + let crlf = HANDBOOK.replace('\n', "\r\n"); + let doc = ingest(crlf.as_bytes()).unwrap(); + assert_eq!(doc.canonical_text, HANDBOOK); + assert_ne!(doc.id, ingest(HANDBOOK.as_bytes()).unwrap().id); + } + + #[test] + fn grounded_citation_round_trips_through_the_adapter() { + let doc = ingest(HANDBOOK.as_bytes()).unwrap(); + let start = doc.canonical_text.find("30 days").unwrap(); + let citation = Citation::grounded(&doc, CdmSpan::new(start, start + 7)).unwrap(); + assert_eq!(citation.quote, "30 days"); + assert!(citation.verify_against(&doc).is_ok()); + } + + #[test] + fn hash_runs_and_missing_space_are_not_headings() { + assert!(parse_heading("#######").is_none()); + assert!(parse_heading("#NoSpace").is_none()); + assert_eq!( + parse_heading("## Trailing ##"), + Some((2, "Trailing".into())) + ); + } +} diff --git a/smesh-world/src/intake/mod.rs b/smesh-world/src/intake/mod.rs new file mode 100644 index 0000000..3f10d34 --- /dev/null +++ b/smesh-world/src/intake/mod.rs @@ -0,0 +1,109 @@ +//! Tier 0 — the Registrar of Record. +//! +//! Any artifact in, CDM out. Detection and extraction are deterministic +//! code; per the model policy, an LLM is reserved for triaging genuinely +//! ambiguous blobs and never touches fidelity. Tier 1 experts only ever see +//! the CDM this module produces. + +mod eml; +mod markdown; + +use crate::cdm::{CdmDocument, SourceFormat}; +use crate::WorldError; + +pub struct Registrar; + +impl Registrar { + /// Deterministic format detection. Order matters: magic bytes first, + /// then structural sniffs on the text, markdown as the text fallback. + pub fn sniff(bytes: &[u8]) -> SourceFormat { + if bytes.starts_with(b"%PDF-") { + return SourceFormat::Pdf; + } + let Ok(text) = std::str::from_utf8(bytes) else { + return SourceFormat::Unknown; + }; + if looks_like_email(text) { + return SourceFormat::Eml; + } + let trimmed = text.trim_start(); + if trimmed.starts_with('{') || trimmed.starts_with('[') { + return SourceFormat::Json; + } + let lower = trimmed.get(..14).unwrap_or(trimmed).to_ascii_lowercase(); + if lower.starts_with(" Result { + match Self::sniff(bytes) { + SourceFormat::Markdown => markdown::ingest(bytes), + SourceFormat::Eml => eml::ingest(bytes), + other => Err(WorldError::UnsupportedFormat(other)), + } + } +} + +/// An email is a header block (`Key: value` lines, folding allowed) that +/// includes `From:` plus `Subject:` or `Message-ID:` before the first blank +/// line. +fn looks_like_email(text: &str) -> bool { + let mut saw_from = false; + let mut saw_subject_or_id = false; + for line in text.lines() { + if line.trim().is_empty() { + break; + } + let lower = line.to_ascii_lowercase(); + if lower.starts_with("from:") { + saw_from = true; + } else if lower.starts_with("subject:") || lower.starts_with("message-id:") { + saw_subject_or_id = true; + } else if !line.starts_with([' ', '\t']) && !line.contains(':') { + return false; + } + } + saw_from && saw_subject_or_id +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sniff_detects_by_magic_and_structure() { + assert_eq!(Registrar::sniff(b"%PDF-1.7 ..."), SourceFormat::Pdf); + assert_eq!(Registrar::sniff(b"{\"a\": 1}"), SourceFormat::Json); + assert_eq!( + Registrar::sniff(b""), + SourceFormat::Html + ); + assert_eq!( + Registrar::sniff(b"From: a@b.c\nSubject: hi\n\nbody"), + SourceFormat::Eml + ); + assert_eq!( + Registrar::sniff(b"# Handbook\n\ntext"), + SourceFormat::Markdown + ); + assert_eq!(Registrar::sniff(&[0xff, 0xfe, 0x00]), SourceFormat::Unknown); + } + + #[test] + fn prose_with_colons_is_not_an_email() { + assert_eq!( + Registrar::sniff(b"From: the desk of the CEO\nnote that follows\n"), + SourceFormat::Markdown + ); + } + + #[test] + fn unsupported_formats_are_an_explicit_error() { + assert!(matches!( + Registrar::ingest(b"%PDF-1.7"), + Err(WorldError::UnsupportedFormat(SourceFormat::Pdf)) + )); + } +} diff --git a/smesh-world/src/lib.rs b/smesh-world/src/lib.rs index 80acd6a..1253e16 100644 --- a/smesh-world/src/lib.rs +++ b/smesh-world/src/lib.rs @@ -15,6 +15,7 @@ pub mod candidate; pub mod cdm; pub mod corpus; pub mod delta; +pub mod intake; pub mod ontology; pub mod role; @@ -25,6 +26,7 @@ pub use delta::{ Changeset, Lane, RatificationRecord, Ratified, ReviewDecision, ReviewerId, Signature, Signed, Staged, }; +pub use intake::Registrar; pub use ontology::{ConsensusPolicy, EdgeKind, StructuralConstraint, TransportKind}; pub use role::{Capability, ModelPolicy, Tier, WorldRole}; @@ -46,6 +48,10 @@ pub enum WorldError { MissingAttestation, #[error("changeset is empty; nothing to stage")] EmptyChangeset, + #[error("no adapter for format {0:?}")] + UnsupportedFormat(cdm::SourceFormat), + #[error("malformed artifact: {0}")] + Malformed(String), #[error("ratification record does not cover candidate {0}")] UnreviewedCandidate(String), } From 1720c3135cbfb3070d3107a303003ff613824c61 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Thu, 23 Jul 2026 00:06:09 -0700 Subject: [PATCH 03/13] feat(world): PDF adapter + Meridian instrumented corpus (Phases 1-2) PDF: pdf-extract per-page text to page-anchored CDM spans; textless PDFs are loudly Malformed (OCR is future work); deterministic lopdf renderer (rayon off, no Info dict) keeps DocIds stable across runs. Meridian Mutual: gold graph authored first (11 edges), documents rendered from it - 5 markdown, 3 .eml, 1 two-page PDF - with an 8-defect manifest (term collision, cross-dept contradiction, cross-format supersession, divergent duplicate, email-only rule, negative space, vendor attachment, boilerplate trap) and an evidence table binding gold edges to artifact quotes. Tests prove: every artifact ingests through the Registrar, every evidence quote grounds a citation, the divergent duplicate actually diverges, negative space is truly absent, rendering is byte-deterministic. 53 tests. --- Cargo.toml | 6 + WORLD-MODEL.md | 7 +- smesh-world/Cargo.toml | 2 + smesh-world/src/intake/mod.rs | 7 +- smesh-world/src/intake/pdf.rs | 117 +++++++ smesh-world/src/intake/pdf_render.rs | 74 +++++ smesh-world/src/lib.rs | 1 + smesh-world/src/meridian.rs | 440 +++++++++++++++++++++++++++ 8 files changed, 649 insertions(+), 5 deletions(-) create mode 100644 smesh-world/src/intake/pdf.rs create mode 100644 smesh-world/src/intake/pdf_render.rs create mode 100644 smesh-world/src/meridian.rs diff --git a/Cargo.toml b/Cargo.toml index 7e0bf9e..17813b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,12 @@ async-trait = "0.1" sha2 = "0.10" ed25519-dalek = { version = "2.1", features = ["rand_core", "serde", "pkcs8"] } blake3 = "1.5" + +# PDF (Tier 0 intake + Meridian renderer) +# default-features off: lopdf's `rayon` feature parses objects on the +# global thread pool, which is nondeterministic under test parallelism. +lopdf = { version = "0.36", default-features = false } +pdf-extract = "0.10" rand = "0.8" uuid = { version = "1.6", features = ["v4", "serde"] } diff --git a/WORLD-MODEL.md b/WORLD-MODEL.md index be6b818..86a5033 100644 --- a/WORLD-MODEL.md +++ b/WORLD-MODEL.md @@ -202,13 +202,14 @@ config-as-policy, CSV org roster, wiki export, scanned-memo PDFs. | `candidate.rs` | Candidate node/edge emissions with schema-mandatory citations | | `delta.rs` | Changeset lifecycle state machine; type-state ratification (unsigned changeset cannot reach `Signed`) | | `corpus.rs` | Gold graph, planted-defect manifest, scorecard types for Meridian | -| `intake/` | Tier 0 Registrar: deterministic format sniffing; markdown adapter (heading-path anchors) and `.eml` adapter (header harvest, body-line anchors); PDF pending (AkamaiForms path) | +| `intake/` | Tier 0 Registrar: deterministic format sniffing; markdown adapter (heading-path anchors), `.eml` adapter (header harvest, body-line anchors), PDF adapter (pdf-extract per-page → page anchors; textless PDFs loudly malformed), deterministic lopdf renderer | +| `meridian.rs` | The instrumented corpus: authored gold graph (11 edges), 8-defect manifest, 9 rendered artifacts (5 md / 3 eml / 1 two-page PDF), evidence table binding gold edges to artifact quotes | ## 9. Phasing - **Phase 0 — DONE:** types + state machine + gates green. -- **Phase 1 — IN PROGRESS:** Tier 0 intake adapters → CDM. Markdown + `.eml` shipped; PDF (AkamaiForms path) pending. -- **Phase 2:** Meridian gold graph + deterministic renderer + planted defects. +- **Phase 1 — DONE:** Tier 0 intake adapters → CDM (markdown, `.eml`, PDF). MIME multipart + OCR deferred to the vendor-attachment / scanned-doc work. +- **Phase 2 — DONE:** Meridian gold graph + deterministic renderer + planted defects; every artifact ingests through the Registrar, every evidence quote grounds a citation, negative space verified truly absent. - **Phase 3:** Wire Tier 1/2 experts over `smesh-agent` (K3 extract, cross-model verify), run the field, emit first staged changeset. - **Phase 4:** Ratification dashboard (Axum; prevue-api pattern) + Curator signing into AION. - **Phase 5:** Scorecard harness; Meridian becomes the regression gate. diff --git a/smesh-world/Cargo.toml b/smesh-world/Cargo.toml index 47f7a27..316eb7c 100644 --- a/smesh-world/Cargo.toml +++ b/smesh-world/Cargo.toml @@ -12,3 +12,5 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } blake3 = { workspace = true } +lopdf = { workspace = true } +pdf-extract = { workspace = true } diff --git a/smesh-world/src/intake/mod.rs b/smesh-world/src/intake/mod.rs index 3f10d34..a1c61de 100644 --- a/smesh-world/src/intake/mod.rs +++ b/smesh-world/src/intake/mod.rs @@ -7,6 +7,8 @@ mod eml; mod markdown; +mod pdf; +pub(crate) mod pdf_render; use crate::cdm::{CdmDocument, SourceFormat}; use crate::WorldError; @@ -41,6 +43,7 @@ impl Registrar { match Self::sniff(bytes) { SourceFormat::Markdown => markdown::ingest(bytes), SourceFormat::Eml => eml::ingest(bytes), + SourceFormat::Pdf => pdf::ingest(bytes), other => Err(WorldError::UnsupportedFormat(other)), } } @@ -102,8 +105,8 @@ mod tests { #[test] fn unsupported_formats_are_an_explicit_error() { assert!(matches!( - Registrar::ingest(b"%PDF-1.7"), - Err(WorldError::UnsupportedFormat(SourceFormat::Pdf)) + Registrar::ingest(b"{\"not\": \"yet\"}"), + Err(WorldError::UnsupportedFormat(SourceFormat::Json)) )); } } diff --git a/smesh-world/src/intake/pdf.rs b/smesh-world/src/intake/pdf.rs new file mode 100644 index 0000000..2c7c60f --- /dev/null +++ b/smesh-world/src/intake/pdf.rs @@ -0,0 +1,117 @@ +//! PDF → CDM via `pdf-extract` per-page text (the Prévue/AkamaiForms +//! lineage). Each page's text becomes a page-spanning anchor, so a citation +//! resolves to "policy-manual.pdf, page 3" on the dashboard. +//! +//! A PDF with no extractable text (pure scan) is `Malformed` rather than a +//! silently empty document — the OCR path is future Tier 0 work and its +//! absence must be loud. + +use crate::cdm::{CdmDocument, CdmSpan, DocMetadata, NativeAnchor, SourceFormat}; +use crate::WorldError; + +pub fn ingest(bytes: &[u8]) -> Result { + let pages = pdf_extract::extract_text_from_mem_by_pages(bytes) + .map_err(|e| WorldError::Malformed(format!("pdf extraction failed: {e}")))?; + + let mut canonical = String::new(); + let mut anchors = Vec::new(); + for (idx, page) in pages.iter().enumerate() { + let text = page.replace("\r\n", "\n"); + let trimmed = text.trim(); + if trimmed.is_empty() { + continue; + } + if !canonical.is_empty() { + canonical.push_str("\n\n"); + } + let start = canonical.len(); + canonical.push_str(trimmed); + anchors.push(( + CdmSpan::new(start, canonical.len()), + NativeAnchor::PdfPage { + page: u32::try_from(idx + 1).unwrap_or(u32::MAX), + }, + )); + } + + if canonical.is_empty() { + return Err(WorldError::Malformed( + "pdf has no extractable text (scanned image? OCR not yet supported)".into(), + )); + } + + Ok(CdmDocument::ingest( + bytes, + SourceFormat::Pdf, + canonical, + anchors, + DocMetadata::default(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::candidate::Citation; + use crate::intake::pdf_render::render_pdf; + + fn fixture() -> Vec { + render_pdf(&[ + vec![ + "Meridian Clinical Policy Manual".to_owned(), + "Prior authorization is required for specialty drugs.".to_owned(), + ], + vec!["Claims must be filed within 30 days.".to_owned()], + ]) + } + + #[test] + fn pages_become_page_anchored_spans() { + let doc = ingest(&fixture()).unwrap(); + assert_eq!(doc.format, SourceFormat::Pdf); + + let p1 = doc.canonical_text.find("Prior authorization").unwrap(); + match doc.native_anchor(CdmSpan::new(p1, p1 + 19)).unwrap() { + NativeAnchor::PdfPage { page } => assert_eq!(*page, 1), + other => panic!("unexpected anchor {other:?}"), + } + + let p2 = doc.canonical_text.find("30 days").unwrap(); + match doc.native_anchor(CdmSpan::new(p2, p2 + 7)).unwrap() { + NativeAnchor::PdfPage { page } => assert_eq!(*page, 2), + other => panic!("unexpected anchor {other:?}"), + } + } + + #[test] + fn grounded_citation_round_trips_through_pdf() { + let doc = ingest(&fixture()).unwrap(); + let start = doc.canonical_text.find("specialty drugs").unwrap(); + let citation = Citation::grounded(&doc, CdmSpan::new(start, start + 15)).unwrap(); + assert_eq!(citation.quote, "specialty drugs"); + assert!(citation.verify_against(&doc).is_ok()); + } + + #[test] + fn rendering_is_deterministic_so_doc_ids_are_stable() { + assert_eq!(fixture(), fixture()); + assert_eq!( + ingest(&fixture()).unwrap().id, + ingest(&fixture()).unwrap().id + ); + } + + #[test] + fn textless_pdf_is_loudly_malformed() { + let empty = render_pdf(&[vec![]]); + assert!(matches!(ingest(&empty), Err(WorldError::Malformed(_)))); + } + + #[test] + fn garbage_bytes_are_malformed_not_a_panic() { + assert!(matches!( + ingest(b"%PDF-1.4 garbage"), + Err(WorldError::Malformed(_)) + )); + } +} diff --git a/smesh-world/src/intake/pdf_render.rs b/smesh-world/src/intake/pdf_render.rs new file mode 100644 index 0000000..040b1fe --- /dev/null +++ b/smesh-world/src/intake/pdf_render.rs @@ -0,0 +1,74 @@ +//! Deterministic PDF rendering for fixtures and the Meridian corpus. +//! +//! lopdf with rayon off and no Info dictionary (no timestamps) renders +//! byte-identical output for identical input — which keeps `DocId`s stable +//! across runs, the property the whole content-addressed pipeline leans on. + +use lopdf::content::{Content, Operation}; +use lopdf::{dictionary, Document, Object, Stream}; + +/// Render one page per entry; each entry is the page's lines. Text must be +/// plain ASCII prose (Helvetica, standard encoding) — all Meridian needs. +pub fn render_pdf(pages: &[Vec]) -> Vec { + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + + let mut kids: Vec = Vec::new(); + for lines in pages { + let mut operations = vec![ + Operation::new("BT", vec![]), + Operation::new("Tf", vec!["F1".into(), 12.into()]), + Operation::new("TL", vec![16.into()]), + Operation::new("Td", vec![72.into(), 720.into()]), + ]; + for line in lines { + operations.push(Operation::new( + "Tj", + vec![Object::string_literal(line.as_str())], + )); + operations.push(Operation::new("T*", vec![])); + } + operations.push(Operation::new("ET", vec![])); + + let content = Content { operations }; + let content_id = doc.add_object(Stream::new( + dictionary! {}, + content.encode().expect("static content encodes"), + )); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + }); + kids.push(page_id.into()); + } + + let count = i64::try_from(kids.len()).expect("page count fits i64"); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => kids, + "Count" => count, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + + let mut bytes = Vec::new(); + doc.save_to(&mut bytes).expect("in-memory save cannot fail"); + bytes +} diff --git a/smesh-world/src/lib.rs b/smesh-world/src/lib.rs index 1253e16..d7ab8fd 100644 --- a/smesh-world/src/lib.rs +++ b/smesh-world/src/lib.rs @@ -16,6 +16,7 @@ pub mod cdm; pub mod corpus; pub mod delta; pub mod intake; +pub mod meridian; pub mod ontology; pub mod role; diff --git a/smesh-world/src/meridian.rs b/smesh-world/src/meridian.rs new file mode 100644 index 0000000..b6266cc --- /dev/null +++ b/smesh-world/src/meridian.rs @@ -0,0 +1,440 @@ +//! Meridian Mutual — the instrumented showcase corpus. +//! +//! The gold graph is authored first; every document below is rendered *from* +//! it, so the answer key is exact by construction. Each planted defect +//! targets a specific mesh subsystem (see `WORLD-MODEL.md` §7). Rendering is +//! fully deterministic: same call, same bytes, same `DocId`s. + +use crate::corpus::{DefectManifest, GoldEdge, GoldGraph, PlantedDefect}; +use crate::intake::pdf_render::render_pdf; +use crate::ontology::EdgeKind; +use serde::Serialize; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Artifact { + pub name: &'static str, + pub bytes: Vec, +} + +/// Ties a gold edge to the artifact and quote that prove it — the ground +/// truth for the Phase 5 provenance-integrity score. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Evidence { + pub edge_key: String, + pub artifact: &'static str, + pub quote: &'static str, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct MeridianCorpus { + pub gold: GoldGraph, + pub manifest: DefectManifest, + pub artifacts: Vec, + pub evidence: Vec, +} + +fn edge(subject: &str, kind: EdgeKind, object: &str) -> GoldEdge { + GoldEdge { + subject: subject.to_owned(), + kind, + object: object.to_owned(), + } +} + +pub fn corpus() -> MeridianCorpus { + let claim_legal = edge( + "Claim (Legal)", + EdgeKind::DefinesTerm, + "a formal demand for coverage under a policy", + ); + let claim_finance = edge( + "Claim (Finance)", + EdgeKind::DefinesTerm, + "an expense reimbursement entry in the general ledger", + ); + let filing_45 = edge("Claims", EdgeKind::GovernedBy, "45-day filing window"); + let filing_30 = edge("Claims", EdgeKind::GovernedBy, "30-day filing window"); + let supersession = edge( + "45-day filing window", + EdgeKind::Supersedes, + "30-day filing window", + ); + let portal_only = edge("Claims", EdgeKind::GovernedBy, "portal-only intake"); + let pto_rule = edge( + "HR", + EdgeKind::GovernedBy, + "PTO accrues 1 day per fortnight worked, capped at 26 days", + ); + let ledger = edge("Finance", EdgeKind::Owns, "general ledger"); + let prior_auth = edge( + "Clinical Policy", + EdgeKind::GovernedBy, + "prior authorization required for specialty drugs", + ); + let token_rotation = edge("IT-Security", EdgeKind::GovernedBy, "90-day token rotation"); + let hr_equipment = edge( + "HR", + EdgeKind::GovernedBy, + "equipment returned within 14 days of departure", + ); + let it_equipment = edge( + "IT-Security", + EdgeKind::GovernedBy, + "equipment returned within 7 days of departure", + ); + + let gold = GoldGraph { + edges: vec![ + claim_legal.clone(), + edge("Claim (Legal)", EdgeKind::ScopedTo, "Legal"), + claim_finance.clone(), + edge("Claim (Finance)", EdgeKind::ScopedTo, "Finance"), + filing_45.clone(), + supersession, + portal_only.clone(), + pto_rule.clone(), + ledger.clone(), + prior_auth.clone(), + token_rotation.clone(), + ], + }; + + let manifest = DefectManifest { + defects: vec![ + PlantedDefect::TermCollision { + term: "Claim".to_owned(), + departments: vec!["Legal".to_owned(), "Finance".to_owned()], + }, + PlantedDefect::CrossDepartmentContradiction { + edge_a: hr_equipment, + edge_b: it_equipment, + }, + PlantedDefect::CrossFormatSupersession { + superseded: filing_30, + superseding: filing_45.clone(), + }, + PlantedDefect::DivergentDuplicate { + canonical: pto_rule.clone(), + }, + PlantedDefect::EmailOnlyRule { + edge: portal_only.clone(), + }, + PlantedDefect::NegativeSpace { + question: "What is the travel reimbursement mileage rate?".to_owned(), + }, + PlantedDefect::VendorAttachment { + vendor: "Acme Retention Systems".to_owned(), + }, + PlantedDefect::BoilerplateTrap { + meridian_rule: pto_rule.clone(), + }, + ], + }; + + let artifacts = vec![ + Artifact { + name: "employee-handbook.md", + bytes: EMPLOYEE_HANDBOOK.as_bytes().to_vec(), + }, + Artifact { + name: "it-security-policy.md", + bytes: IT_SECURITY_POLICY.as_bytes().to_vec(), + }, + Artifact { + name: "legal-definitions.md", + bytes: LEGAL_DEFINITIONS.as_bytes().to_vec(), + }, + Artifact { + name: "finance-glossary.md", + bytes: FINANCE_GLOSSARY.as_bytes().to_vec(), + }, + Artifact { + name: "wiki-export.md", + bytes: WIKI_EXPORT.as_bytes().to_vec(), + }, + Artifact { + name: "memo-4417.eml", + bytes: MEMO_4417.as_bytes().to_vec(), + }, + Artifact { + name: "ops-bulletin.eml", + bytes: OPS_BULLETIN.as_bytes().to_vec(), + }, + Artifact { + name: "vendor-notice.eml", + bytes: VENDOR_NOTICE.as_bytes().to_vec(), + }, + Artifact { + name: "clinical-policy-manual.pdf", + bytes: clinical_policy_manual_pdf(), + }, + ]; + + let evidence = vec![ + Evidence { + edge_key: claim_legal.key(), + artifact: "legal-definitions.md", + quote: "a Claim means a formal demand for coverage under a policy", + }, + Evidence { + edge_key: claim_finance.key(), + artifact: "finance-glossary.md", + quote: "a Claim means an expense reimbursement entry in the general ledger", + }, + Evidence { + edge_key: filing_45.key(), + artifact: "memo-4417.eml", + quote: "the claims filing window is 45 days", + }, + Evidence { + edge_key: portal_only.key(), + artifact: "ops-bulletin.eml", + quote: "All claims arrive via the provider portal only", + }, + Evidence { + edge_key: pto_rule.key(), + artifact: "employee-handbook.md", + quote: "PTO accrues at 1 day per fortnight worked, capped at 26 days", + }, + Evidence { + edge_key: ledger.key(), + artifact: "finance-glossary.md", + quote: "Finance owns the general ledger", + }, + Evidence { + edge_key: prior_auth.key(), + artifact: "clinical-policy-manual.pdf", + quote: "Prior authorization is required for specialty drugs.", + }, + Evidence { + edge_key: token_rotation.key(), + artifact: "it-security-policy.md", + quote: "All access tokens rotate every 90 days", + }, + ]; + + MeridianCorpus { + gold, + manifest, + artifacts, + evidence, + } +} + +const EMPLOYEE_HANDBOOK: &str = "\ +# Meridian Mutual Employee Handbook + +## Claims + +Claims must be filed within 30 days of the date of service. + +## Human Resources + +PTO accrues at 1 day per fortnight worked, capped at 26 days. + +Departing employees must return company equipment within 14 days. + +## Finance + +Expense reports are reviewed by the Finance department monthly. +"; + +const IT_SECURITY_POLICY: &str = "\ +# Meridian IT-Security Policy + +All access tokens rotate every 90 days. + +Departing employees must return company equipment within 7 days. +"; + +const LEGAL_DEFINITIONS: &str = "\ +# Meridian Legal and Compliance Definitions + +In Legal usage, a Claim means a formal demand for coverage under a policy. +"; + +const FINANCE_GLOSSARY: &str = "\ +# Meridian Finance Glossary + +In Finance usage, a Claim means an expense reimbursement entry in the general ledger. + +Finance owns the general ledger. +"; + +const WIKI_EXPORT: &str = "\ +# Meridian Wiki - Benefits FAQ + +Employees accrue PTO at the industry standard rate of 10 days per year. +"; + +const MEMO_4417: &str = "\ +From: Dana Reyes +To: claims-all@meridianmutual.example +Subject: Filing window change, effective immediately +Date: Tue, 21 Jul 2026 09:14:00 -0700 +Message-ID: +X-Department: Claims + +Team, + +Effective immediately the claims filing window is 45 days, superseding +the 30-day window stated in the handbook and the clinical policy manual. + +Dana +"; + +const OPS_BULLETIN: &str = "\ +From: Ops Desk +To: claims-all@meridianmutual.example +Subject: Fax intake decommissioned +Date: Wed, 08 Jul 2026 15:02:00 -0700 +Message-ID: +X-Department: Claims + +As of this week, fax intake is decommissioned. +All claims arrive via the provider portal only. + +Ops Desk +"; + +const VENDOR_NOTICE: &str = "\ +From: Acme Retention +To: procurement@meridianmutual.example +Subject: Updated Acme retention terms +Date: Mon, 06 Jul 2026 11:30:00 -0700 +Message-ID: + +Per our updated terms, Acme Retention Systems retains customer data for +10 years from the date of collection. + +Acme Retention Systems +"; + +fn clinical_policy_manual_pdf() -> Vec { + render_pdf(&[ + vec![ + "Meridian Clinical Policy Manual".to_owned(), + "Prior authorization is required for specialty drugs.".to_owned(), + "Claims must be filed within 30 days of the date of service.".to_owned(), + ], + vec![ + "Appendix A - Benefits summary".to_owned(), + "PTO accrues at 1 day per fortnight worked, capped at 25 days.".to_owned(), + ], + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::candidate::Citation; + use crate::cdm::{CdmSpan, SourceFormat}; + use crate::intake::Registrar; + + #[test] + fn every_artifact_ingests_through_the_registrar() { + for artifact in corpus().artifacts { + let doc = Registrar::ingest(&artifact.bytes) + .unwrap_or_else(|e| panic!("{} failed: {e}", artifact.name)); + let expected = match artifact.name.rsplit('.').next().unwrap() { + "md" => SourceFormat::Markdown, + "eml" => SourceFormat::Eml, + "pdf" => SourceFormat::Pdf, + other => panic!("unexpected extension {other}"), + }; + assert_eq!(doc.format, expected, "{}", artifact.name); + } + } + + #[test] + fn every_evidence_quote_grounds_a_citation() { + let corpus = corpus(); + for ev in &corpus.evidence { + let artifact = corpus + .artifacts + .iter() + .find(|a| a.name == ev.artifact) + .unwrap_or_else(|| panic!("evidence names unknown artifact {}", ev.artifact)); + let doc = Registrar::ingest(&artifact.bytes).unwrap(); + let start = doc + .canonical_text + .find(ev.quote) + .unwrap_or_else(|| panic!("{:?} not found in {}", ev.quote, ev.artifact)); + let citation = + Citation::grounded(&doc, CdmSpan::new(start, start + ev.quote.len())).unwrap(); + assert_eq!(citation.quote, ev.quote); + assert!( + doc.native_anchor(citation.span).is_some(), + "{}", + ev.artifact + ); + } + } + + #[test] + fn every_gold_edge_with_corpus_provenance_has_evidence() { + let corpus = corpus(); + let evidenced: Vec<&str> = corpus + .evidence + .iter() + .map(|e| e.edge_key.as_str()) + .collect(); + for gold_edge in &corpus.gold.edges { + let structural = matches!(gold_edge.kind, EdgeKind::ScopedTo | EdgeKind::Supersedes); + if !structural { + assert!( + evidenced.contains(&gold_edge.key().as_str()), + "gold edge {} has no supporting evidence", + gold_edge.key() + ); + } + } + } + + #[test] + fn negative_space_is_truly_absent() { + for artifact in corpus().artifacts { + if let Ok(doc) = Registrar::ingest(&artifact.bytes) { + assert!( + !doc.canonical_text.to_ascii_lowercase().contains("mileage"), + "{} answers the negative-space question", + artifact.name + ); + } + } + } + + #[test] + fn divergent_duplicate_actually_diverges() { + let corpus = corpus(); + let handbook = corpus + .artifacts + .iter() + .find(|a| a.name == "employee-handbook.md") + .unwrap(); + let manual = corpus + .artifacts + .iter() + .find(|a| a.name == "clinical-policy-manual.pdf") + .unwrap(); + let handbook_doc = Registrar::ingest(&handbook.bytes).unwrap(); + let manual_doc = Registrar::ingest(&manual.bytes).unwrap(); + assert!(handbook_doc.canonical_text.contains("capped at 26 days")); + assert!(manual_doc.canonical_text.contains("capped at 25 days")); + } + + #[test] + fn rendering_is_deterministic_across_calls() { + let (a, b) = (corpus(), corpus()); + for (x, y) in a.artifacts.iter().zip(b.artifacts.iter()) { + assert_eq!(x.bytes, y.bytes, "{} is nondeterministic", x.name); + } + assert_eq!(a.gold, b.gold); + assert_eq!(a.manifest, b.manifest); + } + + #[test] + fn manifest_plants_two_scored_contradictions() { + assert_eq!(corpus().manifest.planted_contradictions(), 2); + } +} From f9173f53108ec58091d335fdcf31a5e3eab1e67b Mon Sep 17 00:00:00 2001 From: zuub-don Date: Thu, 23 Jul 2026 00:18:11 -0700 Subject: [PATCH 04/13] feat(world): smesh-refinery - Tier 1/2 experts live over OpenRouter (Phase 3) Oracle abstraction (OpenRouter prod, scripted test doubles), corpus packet builder, 4 extractor lenses on Kimi K3, confabulation firewall (verbatim quote -> Citation::grounded or attributed rejection), Grounding Auditor cross-family entailment verdicts, Contradiction Sentinel (deterministic supersession/uniqueness resolution + semantic pair scan), staged changeset with lanes + Meridian scorecard. cargo run -p smesh-refinery = live benchmark run. 15 tests. --- Cargo.toml | 1 + WORLD-MODEL.md | 2 +- smesh-refinery/Cargo.toml | 19 ++ smesh-refinery/src/extract.rs | 232 +++++++++++++++++++++++++ smesh-refinery/src/lib.rs | 110 ++++++++++++ smesh-refinery/src/main.rs | 43 +++++ smesh-refinery/src/packet.rs | 77 +++++++++ smesh-refinery/src/roster.rs | 89 ++++++++++ smesh-refinery/src/run.rs | 223 ++++++++++++++++++++++++ smesh-refinery/src/verify.rs | 315 ++++++++++++++++++++++++++++++++++ 10 files changed, 1110 insertions(+), 1 deletion(-) create mode 100644 smesh-refinery/Cargo.toml create mode 100644 smesh-refinery/src/extract.rs create mode 100644 smesh-refinery/src/lib.rs create mode 100644 smesh-refinery/src/main.rs create mode 100644 smesh-refinery/src/packet.rs create mode 100644 smesh-refinery/src/roster.rs create mode 100644 smesh-refinery/src/run.rs create mode 100644 smesh-refinery/src/verify.rs diff --git a/Cargo.toml b/Cargo.toml index 17813b9..ead0b04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "smesh-bounty", "smesh-cli", "smesh-world", + "smesh-refinery", ] [workspace.package] diff --git a/WORLD-MODEL.md b/WORLD-MODEL.md index 86a5033..f36f32e 100644 --- a/WORLD-MODEL.md +++ b/WORLD-MODEL.md @@ -210,6 +210,6 @@ config-as-policy, CSV org roster, wiki export, scanned-memo PDFs. - **Phase 0 — DONE:** types + state machine + gates green. - **Phase 1 — DONE:** Tier 0 intake adapters → CDM (markdown, `.eml`, PDF). MIME multipart + OCR deferred to the vendor-attachment / scanned-doc work. - **Phase 2 — DONE:** Meridian gold graph + deterministic renderer + planted defects; every artifact ingests through the Registrar, every evidence quote grounds a citation, negative space verified truly absent. -- **Phase 3:** Wire Tier 1/2 experts over `smesh-agent` (K3 extract, cross-model verify), run the field, emit first staged changeset. +- **Phase 3 — SHIPPED (`smesh-refinery`):** Tier 1/2 experts over `smesh-agent`: 4 extractor lenses on K3, grounding firewall (verbatim quote → `Citation::grounded` or attributed rejection), Grounding Auditor (cross-family entailment), Contradiction Sentinel (deterministic supersession/uniqueness resolution + cross-family semantic pairs), staged changeset + lanes + scorecard. `cargo run -p smesh-refinery` = live Meridian run. Field/reputation feedback into `smesh-core` is Phase 3b, deferred. - **Phase 4:** Ratification dashboard (Axum; prevue-api pattern) + Curator signing into AION. - **Phase 5:** Scorecard harness; Meridian becomes the regression gate. diff --git a/smesh-refinery/Cargo.toml b/smesh-refinery/Cargo.toml new file mode 100644 index 0000000..65be893 --- /dev/null +++ b/smesh-refinery/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "smesh-refinery" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +description = "The refinery: Tier 1/2 world-model experts over SMESH LLM backends, corpus in, staged changeset out" + +[dependencies] +smesh-world = { path = "../smesh-world" } +smesh-agent = { path = "../smesh-agent" } +tokio = { workspace = true } +async-trait = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/smesh-refinery/src/extract.rs b/smesh-refinery/src/extract.rs new file mode 100644 index 0000000..4ab1467 --- /dev/null +++ b/smesh-refinery/src/extract.rs @@ -0,0 +1,232 @@ +//! Tier 1 extraction and the confabulation firewall. +//! +//! An emission becomes a `CandidateEdge` only if its quote is found verbatim +//! in the cited document — `Citation::grounded` does the rest. Everything +//! else lands in `rejected`, counted and attributable, never silently gone. + +use crate::packet::NamedDoc; +use crate::roster; +use crate::{extract_json, Oracle, RefineryError}; +use serde::Deserialize; +use smesh_world::role::ModelPolicy; +use smesh_world::{CandidateEdge, CdmSpan, Citation, EdgeKind, WorldRole}; + +#[derive(Debug, Clone, Deserialize)] +pub struct Emission { + pub subject: String, + pub kind: String, + pub object: String, + pub doc: String, + pub quote: String, +} + +#[derive(Debug)] +pub struct RejectedEmission { + pub role: WorldRole, + pub emission: Emission, + pub reason: String, +} + +#[derive(Debug, Default)] +pub struct ExtractionOutcome { + pub candidates: Vec, + pub rejected: Vec, +} + +pub(crate) fn kind_from_str(s: &str) -> Option { + let normalized: String = s.chars().filter(|c| c.is_ascii_alphanumeric()).collect(); + EdgeKind::ALL + .into_iter() + .find(|k| format!("{k:?}").eq_ignore_ascii_case(&normalized)) +} + +fn extractor_model(role: WorldRole) -> &'static str { + match role.model_policy() { + ModelPolicy::Fixed(m) => m, + other => unreachable!("extractor {role:?} has policy {other:?}"), + } +} + +/// Run every Tier 1 extractor over the packet and ground each emission. +pub async fn run_extractors( + oracle: &dyn Oracle, + docs: &[NamedDoc], + doc_block: &str, +) -> Result { + let mut outcome = ExtractionOutcome::default(); + for role in roster::EXTRACTORS { + let response = oracle + .complete( + extractor_model(role), + &roster::extractor_system(role), + doc_block, + ) + .await?; + let emissions: Vec = serde_json::from_value(extract_json(&response)?) + .map_err(|e| RefineryError::Parse(format!("{role:?} emissions: {e}")))?; + for emission in emissions { + ground(role, emission, docs, &mut outcome); + } + } + Ok(outcome) +} + +/// The firewall. Grounding failures are recorded, not raised: one bad +/// emission must never abort a run. +pub fn ground( + role: WorldRole, + emission: Emission, + docs: &[NamedDoc], + outcome: &mut ExtractionOutcome, +) { + let reject = |reason: String, outcome: &mut ExtractionOutcome, emission: Emission| { + outcome.rejected.push(RejectedEmission { + role, + emission, + reason, + }); + }; + + let Some(kind) = kind_from_str(&emission.kind) else { + return reject( + format!("unknown kind {:?}", emission.kind), + outcome, + emission, + ); + }; + let Some(named) = docs.iter().find(|d| d.name == emission.doc) else { + return reject( + format!("unknown document {:?}", emission.doc), + outcome, + emission, + ); + }; + let Some(start) = named.doc.canonical_text.find(&emission.quote) else { + return reject( + "quote not found verbatim in document".into(), + outcome, + emission, + ); + }; + let span = CdmSpan::new(start, start + emission.quote.len()); + let citation = match Citation::grounded(&named.doc, span) { + Ok(c) => c, + Err(e) => return reject(format!("citation rejected: {e}"), outcome, emission), + }; + + if let Some(existing) = outcome + .candidates + .iter_mut() + .find(|c| c.subject == emission.subject && c.kind == kind && c.object == emission.object) + { + if let smesh_world::ProvenanceClass::CorpusDerived { citations } = &mut existing.provenance + { + if !citations.contains(&citation) { + citations.push(citation); + } + } + return; + } + + match CandidateEdge::emit( + role, + emission.subject.clone(), + kind, + emission.object.clone(), + vec![citation], + ) { + Ok(candidate) => outcome.candidates.push(candidate), + Err(e) => reject(format!("emit rejected: {e}"), outcome, emission), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::packet::ingest_all; + + fn docs() -> Vec { + ingest_all(&[( + "handbook.md".to_owned(), + b"# Handbook\n\nClaims must be filed within 30 days.\n".to_vec(), + )]) + .unwrap() + } + + fn emission(quote: &str) -> Emission { + Emission { + subject: "Claims".into(), + kind: "GovernedBy".into(), + object: "30-day filing window".into(), + doc: "handbook.md".into(), + quote: quote.into(), + } + } + + #[test] + fn verbatim_quotes_ground_into_candidates() { + let docs = docs(); + let mut outcome = ExtractionOutcome::default(); + ground( + WorldRole::Policy, + emission("Claims must be filed within 30 days."), + &docs, + &mut outcome, + ); + assert_eq!(outcome.candidates.len(), 1); + assert!(outcome.rejected.is_empty()); + } + + #[test] + fn hallucinated_quotes_are_rejected_and_counted() { + let docs = docs(); + let mut outcome = ExtractionOutcome::default(); + ground( + WorldRole::Policy, + emission("Claims must be filed within 45 days."), + &docs, + &mut outcome, + ); + assert!(outcome.candidates.is_empty()); + assert_eq!(outcome.rejected.len(), 1); + assert!(outcome.rejected[0].reason.contains("not found verbatim")); + } + + #[test] + fn duplicate_emissions_merge_citations_instead_of_duplicating() { + let docs = docs(); + let mut outcome = ExtractionOutcome::default(); + let quote = "Claims must be filed within 30 days."; + ground(WorldRole::Policy, emission(quote), &docs, &mut outcome); + ground(WorldRole::Lexicon, emission(quote), &docs, &mut outcome); + assert_eq!(outcome.candidates.len(), 1); + match &outcome.candidates[0].provenance { + smesh_world::ProvenanceClass::CorpusDerived { citations } => { + assert_eq!(citations.len(), 1); + } + other => panic!("unexpected provenance {other:?}"), + } + } + + #[test] + fn unknown_kind_and_unknown_doc_are_rejected() { + let docs = docs(); + let mut outcome = ExtractionOutcome::default(); + let mut bad_kind = emission("Claims must be filed within 30 days."); + bad_kind.kind = "Blesses".into(); + ground(WorldRole::Policy, bad_kind, &docs, &mut outcome); + let mut bad_doc = emission("Claims must be filed within 30 days."); + bad_doc.doc = "ghost.md".into(); + ground(WorldRole::Policy, bad_doc, &docs, &mut outcome); + assert!(outcome.candidates.is_empty()); + assert_eq!(outcome.rejected.len(), 2); + } + + #[test] + fn kind_parsing_accepts_case_and_snake_variants() { + assert_eq!(kind_from_str("GovernedBy"), Some(EdgeKind::GovernedBy)); + assert_eq!(kind_from_str("governed_by"), Some(EdgeKind::GovernedBy)); + assert_eq!(kind_from_str("DEFINES_TERM"), Some(EdgeKind::DefinesTerm)); + assert_eq!(kind_from_str("Blesses"), None); + } +} diff --git a/smesh-refinery/src/lib.rs b/smesh-refinery/src/lib.rs new file mode 100644 index 0000000..a46becb --- /dev/null +++ b/smesh-refinery/src/lib.rs @@ -0,0 +1,110 @@ +//! # SMESH Refinery +//! +//! Tier 1/2 of the world-model pipeline (`WORLD-MODEL.md`): expert roles run +//! over LLM backends, a corpus goes in, a staged changeset comes out — ready +//! for human ratification. The refinery never signs anything. +//! +//! The confabulation firewall lives in [`extract`]: an extractor's emission +//! only becomes a candidate if its quote is found verbatim in the cited +//! document and `Citation::grounded` succeeds. Everything else is rejected +//! and counted, never silently dropped. + +pub mod extract; +pub mod packet; +pub mod roster; +pub mod run; +pub mod verify; + +use async_trait::async_trait; +use smesh_agent::OpenRouterClient; +use smesh_world::WorldError; + +#[derive(Debug, thiserror::Error)] +pub enum RefineryError { + #[error("backend error: {0}")] + Backend(String), + #[error("unparseable expert response: {0}")] + Parse(String), + #[error(transparent)] + World(#[from] WorldError), +} + +/// Minimal completion interface the refinery needs from any LLM provider. +/// Production uses OpenRouter; tests use scripted oracles. +#[async_trait] +pub trait Oracle: Send + Sync { + async fn complete( + &self, + model: &str, + system: &str, + prompt: &str, + ) -> Result; +} + +#[async_trait] +impl Oracle for OpenRouterClient { + async fn complete( + &self, + model: &str, + system: &str, + prompt: &str, + ) -> Result { + self.generate_with_model(model, prompt, Some(system)) + .await + .map_err(|e| RefineryError::Backend(e.to_string())) + } +} + +/// Extract the first JSON value from an LLM response, tolerating code +/// fences and prose around it. +pub(crate) fn extract_json(text: &str) -> Result { + let start = text + .find(['[', '{']) + .ok_or_else(|| RefineryError::Parse(format!("no JSON in response: {text:.100}")))?; + let candidate = &text[start..]; + let mut depth = 0usize; + let mut in_string = false; + let mut escaped = false; + for (i, c) in candidate.char_indices() { + if escaped { + escaped = false; + continue; + } + match c { + '\\' if in_string => escaped = true, + '"' => in_string = !in_string, + '[' | '{' if !in_string => depth += 1, + ']' | '}' if !in_string => { + depth -= 1; + if depth == 0 { + return serde_json::from_str(&candidate[..=i]) + .map_err(|e| RefineryError::Parse(e.to_string())); + } + } + _ => {} + } + } + Err(RefineryError::Parse("unterminated JSON in response".into())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extract_json_tolerates_fences_and_prose() { + let v = extract_json("Here you go:\n```json\n[{\"a\": 1}]\n```\nDone.").unwrap(); + assert_eq!(v[0]["a"], 1); + } + + #[test] + fn extract_json_handles_brackets_inside_strings() { + let v = extract_json("{\"q\": \"a ] tricky [ one\"}").unwrap(); + assert_eq!(v["q"], "a ] tricky [ one"); + } + + #[test] + fn extract_json_rejects_json_free_prose() { + assert!(extract_json("I could not find any candidates.").is_err()); + } +} diff --git a/smesh-refinery/src/main.rs b/smesh-refinery/src/main.rs new file mode 100644 index 0000000..c42d640 --- /dev/null +++ b/smesh-refinery/src/main.rs @@ -0,0 +1,43 @@ +//! Live refinery run against the Meridian benchmark. +//! +//! ```sh +//! cargo run -p smesh-refinery +//! ``` +//! +//! Requires OpenRouter credentials (`OPENROUTER_API_KEY` or +//! `~/.creds/openrouter.env`). Extraction runs on Kimi K3, verification on a +//! distinct model family, per the role model policy. Temperature is pinned +//! to 0 — the refinery wants reproducibility, not creativity. + +use smesh_agent::openrouter::OpenRouterConfig; +use smesh_agent::OpenRouterClient; +use smesh_refinery::run::refine_meridian; + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + let Some(mut config) = OpenRouterConfig::from_env() else { + eprintln!("no OpenRouter credentials (OPENROUTER_API_KEY or ~/.creds/openrouter.env)"); + std::process::exit(2); + }; + config.temperature = 0.0; + config.max_tokens = 8192; + config.timeout_secs = 420; + let client = OpenRouterClient::new(config); + + match refine_meridian(&client).await { + Ok(report) => { + print!("{}", report.render()); + let gate_passed = report + .scorecard + .as_ref() + .is_some_and(smesh_world::corpus::Scorecard::passes_gate); + std::process::exit(i32::from(!gate_passed)); + } + Err(e) => { + eprintln!("refinery run failed: {e}"); + std::process::exit(2); + } + } +} diff --git a/smesh-refinery/src/packet.rs b/smesh-refinery/src/packet.rs new file mode 100644 index 0000000..9f6ee61 --- /dev/null +++ b/smesh-refinery/src/packet.rs @@ -0,0 +1,77 @@ +//! Corpus packet: named CDM documents plus the prompt block extractors read. +//! Tier 1 sees canonical text and intake metadata only — never raw formats. + +use smesh_world::{CdmDocument, Registrar, WorldError}; + +pub struct NamedDoc { + pub name: String, + pub doc: CdmDocument, +} + +/// Ingest raw artifacts through the Registrar into a packet, failing loudly +/// on any artifact Tier 0 cannot handle. +pub fn ingest_all(artifacts: &[(String, Vec)]) -> Result, WorldError> { + artifacts + .iter() + .map(|(name, bytes)| { + Registrar::ingest(bytes).map(|doc| NamedDoc { + name: name.clone(), + doc, + }) + }) + .collect() +} + +/// Render the packet as the document block shared by every extractor prompt. +pub fn prompt_block(docs: &[NamedDoc]) -> String { + let mut out = String::new(); + for named in docs { + out.push_str(&format!("=== document: {} ===\n", named.name)); + let m = &named.doc.metadata; + if let Some(title) = &m.title { + out.push_str(&format!("title: {title}\n")); + } + if let Some(author) = &m.author { + out.push_str(&format!("author: {author}\n")); + } + if let Some(date) = &m.date { + out.push_str(&format!("date: {date}\n")); + } + if let Some(department) = &m.department { + out.push_str(&format!("department: {department}\n")); + } + out.push_str("---\n"); + out.push_str(&named.doc.canonical_text); + out.push_str("\n\n"); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use smesh_world::meridian; + + fn meridian_packet() -> Vec { + let artifacts: Vec<(String, Vec)> = meridian::corpus() + .artifacts + .into_iter() + .map(|a| (a.name.to_owned(), a.bytes)) + .collect(); + ingest_all(&artifacts).unwrap() + } + + #[test] + fn meridian_ingests_fully_into_a_packet() { + assert_eq!(meridian_packet().len(), 9); + } + + #[test] + fn prompt_block_carries_names_metadata_and_text() { + let block = prompt_block(&meridian_packet()); + assert!(block.contains("=== document: memo-4417.eml ===")); + assert!(block.contains("department: Claims")); + assert!(block.contains("the claims filing window is 45 days")); + assert!(block.contains("=== document: clinical-policy-manual.pdf ===")); + } +} diff --git a/smesh-refinery/src/roster.rs b/smesh-refinery/src/roster.rs new file mode 100644 index 0000000..a229cb9 --- /dev/null +++ b/smesh-refinery/src/roster.rs @@ -0,0 +1,89 @@ +//! Prompts for the LLM-backed roles. One lens per expert; the JSON contract +//! is shared so grounding code stays role-agnostic. + +use smesh_world::WorldRole; + +/// Shared emission contract appended to every extractor system prompt. +const EMISSION_CONTRACT: &str = r#" +Respond with ONLY a JSON array. Each element: + {"subject": "...", "kind": "...", "object": "...", "doc": "", "quote": "..."} + +Hard rules: +- "quote" MUST be a verbatim, contiguous substring of the named document's + text, at most 200 characters, staying on a single line. Never paraphrase, + never join text across line breaks. Emissions whose quote is not found + verbatim are discarded. +- "doc" MUST be one of the document names exactly as given. +- Only assert what a quote directly supports. If the corpus does not state + something, do not emit it. An empty array [] is a valid, good answer. +- Use short canonical names for subject/object, not sentences. +"#; + +/// The Tier 1 extractors this refinery runs, in emission order. +pub const EXTRACTORS: [WorldRole; 4] = [ + WorldRole::Lexicon, + WorldRole::Policy, + WorldRole::Structure, + WorldRole::Process, +]; + +/// System prompt for a Tier 1 extractor role. +pub fn extractor_system(role: WorldRole) -> String { + let lens = match role { + WorldRole::Lexicon => { + "You are the Lexicon expert: you extract the organization's own \ + verbiage — defined terms, acronyms, canonical names. Allowed kinds: \ + \"DefinesTerm\", \"ScopedTo\". When the same term means different \ + things in different departments, emit one subject per department \ + using the form \"Term (Department)\", plus a ScopedTo edge from \ + that subject to the department." + } + WorldRole::Policy => { + "You are the Policy expert: you extract normative statements — what \ + must, shall, or may happen — plus effective dates and supersession. \ + Allowed kinds: \"GovernedBy\", \"Requires\", \"Triggers\", \ + \"Supersedes\". Subject of GovernedBy is the governed department or \ + function; object is a short rule name. When one rule replaces \ + another, also emit new-rule Supersedes old-rule." + } + WorldRole::Structure => { + "You are the Structure expert: you extract organizational structure — \ + departments, roles, systems, ownership. Allowed kinds: \ + \"ReportsTo\", \"Owns\", \"Operates\", \"MemberOf\"." + } + WorldRole::Process => { + "You are the Process expert: you extract workflows and ordering — \ + what precedes, requires, or triggers what. Allowed kinds: \ + \"Precedes\", \"Requires\", \"Triggers\"." + } + other => unreachable!("{other:?} is not an extractor"), + }; + format!("{lens}\n{EMISSION_CONTRACT}") +} + +/// System prompt for the Grounding Auditor (Tier 2, cross-model). +pub fn auditor_system() -> String { + r#"You are a skeptical grounding auditor. You are given a claimed edge +(subject, kind, object) and the verbatim quote cited as its evidence, with +surrounding context. Judge STRICTLY whether the quote entails the claim. +Default to refute when uncertain, when the quote merely relates to the topic, +or when the claim adds anything the quote does not state. + +Respond with ONLY a JSON object: {"verdict": "corroborate" | "refute", "rationale": "..."}"# + .to_owned() +} + +/// System prompt for the Contradiction Sentinel's semantic pass (Tier 2, +/// cross-model). Structural checks are code; this catches contradictions +/// mechanics cannot see (different subjects, conflicting substance). +pub fn sentinel_system() -> String { + r#"You are a contradiction sentinel reviewing a numbered list of candidate +edges extracted from one organization's documents. Identify pairs whose +claims cannot BOTH be true operational rules at the same time (e.g. two +different limits for the same activity). Ignore pairs where one explicitly +supersedes the other. Do not flag mere overlap or restatement. + +Respond with ONLY a JSON array of index pairs, e.g. [[0,3],[2,7]]. An empty +array [] means no contradictions."# + .to_owned() +} diff --git a/smesh-refinery/src/run.rs b/smesh-refinery/src/run.rs new file mode 100644 index 0000000..a34c187 --- /dev/null +++ b/smesh-refinery/src/run.rs @@ -0,0 +1,223 @@ +//! Orchestration: corpus in, staged changeset + report out. +//! +//! The refinery's output stops exactly where the human's authority begins — +//! a `Changeset` with lanes. Ratification and signing are Phase 4. + +use crate::extract::{run_extractors, RejectedEmission}; +use crate::packet::{ingest_all, prompt_block, NamedDoc}; +use crate::verify::{audit_grounding, sentinel_pass}; +use crate::{Oracle, RefineryError}; +use smesh_world::corpus::Scorecard; +use smesh_world::meridian; +use smesh_world::{CandidateEdge, Changeset, Lane, Staged}; + +pub struct RunReport { + pub docs: Vec, + pub staged: Option>, + pub rejected: Vec, + pub contradictions_caught: usize, + pub scorecard: Option, +} + +impl RunReport { + pub fn lane_counts(&self) -> (usize, usize, usize) { + let mut counts = (0, 0, 0); + if let Some(staged) = &self.staged { + for (lane, _) in staged.lanes() { + match lane { + Lane::Green => counts.0 += 1, + Lane::Amber => counts.1 += 1, + Lane::Red => counts.2 += 1, + } + } + } + counts + } + + /// Human-readable run summary for the CLI. + pub fn render(&self) -> String { + let mut out = String::new(); + let (green, amber, red) = self.lane_counts(); + out.push_str(&format!( + "documents ingested: {}\nrejected emissions (confabulation firewall): {}\ncontradiction pairs flagged: {}\n", + self.docs.len(), + self.rejected.len(), + self.contradictions_caught + )); + for r in &self.rejected { + out.push_str(&format!( + " rejected [{:?}] {} --[{}]--> {} : {}\n", + r.role, r.emission.subject, r.emission.kind, r.emission.object, r.reason + )); + } + match &self.staged { + None => out.push_str("staged changeset: EMPTY — nothing survived\n"), + Some(staged) => { + out.push_str(&format!( + "staged changeset vs {}: {} candidates (green {green} / amber {amber} / red {red})\n", + staged.base_rev, + staged.edges.len() + )); + for (lane, edge) in staged.lanes() { + out.push_str(&format!( + " [{lane:?}] \"{}\" --[{:?}]--> \"{}\" (+{}/-{})\n", + edge.subject, + edge.kind, + edge.object, + edge.corroborations(), + edge.refutations() + )); + } + } + } + if let Some(card) = &self.scorecard { + out.push_str(&format!( + "scorecard: precision {:.2} recall {:.2} confabulated {} contradictions {}/{} gate {}\n", + card.precision(), + card.recall(), + card.confabulated, + card.contradictions_caught, + card.contradictions_planted, + if card.passes_gate() { "PASS" } else { "FAIL" } + )); + } + out + } +} + +/// Refine an arbitrary corpus: ingest, extract, audit, sentinel, stage. +pub async fn refine( + oracle: &dyn Oracle, + artifacts: &[(String, Vec)], + base_rev: &str, +) -> Result { + let docs = ingest_all(artifacts)?; + let block = prompt_block(&docs); + + let mut outcome = run_extractors(oracle, &docs, &block).await?; + audit_grounding(oracle, &docs, &mut outcome.candidates).await?; + let contradictions_caught = sentinel_pass(oracle, &mut outcome.candidates).await?; + + let staged = if outcome.candidates.is_empty() { + None + } else { + Some(Changeset::stage( + base_rev.to_owned(), + outcome.candidates.clone(), + )?) + }; + + Ok(RunReport { + docs, + staged, + rejected: outcome.rejected, + contradictions_caught, + scorecard: None, + }) +} + +/// Refine the Meridian benchmark and score the run against its gold graph. +pub async fn refine_meridian(oracle: &dyn Oracle) -> Result { + let corpus = meridian::corpus(); + let artifacts: Vec<(String, Vec)> = corpus + .artifacts + .iter() + .map(|a| (a.name.to_owned(), a.bytes.clone())) + .collect(); + let mut report = refine(oracle, &artifacts, "meridian-rev0").await?; + + let observed: &[CandidateEdge] = report + .staged + .as_ref() + .map(|s| s.edges.as_slice()) + .unwrap_or(&[]); + report.scorecard = Some(Scorecard::evaluate( + &corpus.gold, + observed, + &corpus.manifest, + report.contradictions_caught, + )); + Ok(report) +} + +#[cfg(test)] +mod tests { + use super::*; + use async_trait::async_trait; + + /// Routes by system-prompt content: extractor roles get scripted + /// emissions, the auditor corroborates, the sentinel finds nothing new. + struct MeridianScript; + + #[async_trait] + impl Oracle for MeridianScript { + async fn complete( + &self, + _model: &str, + system: &str, + _prompt: &str, + ) -> Result { + if system.contains("Policy expert") { + Ok(r#"[ + {"subject": "Claims", "kind": "GovernedBy", "object": "45-day filing window", + "doc": "memo-4417.eml", "quote": "the claims filing window is 45 days"}, + {"subject": "Claims", "kind": "GovernedBy", "object": "30-day filing window", + "doc": "employee-handbook.md", "quote": "Claims must be filed within 30 days of the date of service."}, + {"subject": "Claims", "kind": "GovernedBy", "object": "wormhole intake", + "doc": "employee-handbook.md", "quote": "claims teleport through the wormhole"} + ]"# + .to_owned()) + } else if system.contains("Lexicon expert") { + Ok(r#"[ + {"subject": "Claim (Legal)", "kind": "DefinesTerm", + "object": "a formal demand for coverage under a policy", + "doc": "legal-definitions.md", + "quote": "a Claim means a formal demand for coverage under a policy"} + ]"# + .to_owned()) + } else if system.contains("Structure expert") || system.contains("Process expert") { + Ok("[]".to_owned()) + } else if system.contains("grounding auditor") { + Ok(r#"{"verdict": "corroborate", "rationale": "entailed"}"#.to_owned()) + } else if system.contains("contradiction sentinel") { + Ok("[]".to_owned()) + } else { + Err(RefineryError::Parse(format!( + "unscripted system: {system:.60}" + ))) + } + } + } + + #[tokio::test] + async fn scripted_meridian_run_stages_scores_and_firewalls() { + let report = refine_meridian(&MeridianScript).await.unwrap(); + + assert_eq!( + report.rejected.len(), + 1, + "wormhole quote must be firewalled" + ); + assert!(report.rejected[0].reason.contains("not found verbatim")); + + let staged = report.staged.as_ref().unwrap(); + assert_eq!(staged.edges.len(), 3); + + assert_eq!( + report.contradictions_caught, 1, + "30 vs 45 with no supersedes edge is an unresolved conflict" + ); + let contested = staged.edges.iter().filter(|e| e.is_contested()).count(); + assert_eq!(contested, 2, "both filing rules reach the human contested"); + let (_, _, red) = report.lane_counts(); + assert_eq!(red, 2); + + let card = report.scorecard.as_ref().unwrap(); + assert_eq!(card.confabulated, 0, "firewalled emissions never score"); + assert!(card.precision() > 0.6, "2 of 3 staged edges are gold"); + assert!( + card.recall() < 0.5, + "most of the gold graph was not extracted" + ); + } +} diff --git a/smesh-refinery/src/verify.rs b/smesh-refinery/src/verify.rs new file mode 100644 index 0000000..1ca6de9 --- /dev/null +++ b/smesh-refinery/src/verify.rs @@ -0,0 +1,315 @@ +//! Tier 2 verification. +//! +//! The Grounding Auditor re-verifies every citation mechanically, then asks +//! a cross-family model whether the quote entails the claim. The +//! Contradiction Sentinel is deterministic code where it can be (same +//! subject + kind, supersession resolution) and a cross-family model where +//! it must be (contradictions across different subjects). Verifiers cannot +//! emit facts — only verdicts. + +use crate::packet::NamedDoc; +use crate::roster; +use crate::{extract_json, Oracle, RefineryError}; +use serde::Deserialize; +use smesh_world::role::ModelPolicy; +use smesh_world::{ + CandidateEdge, EdgeKind, Judgment, ProvenanceClass, StructuralConstraint, Verdict, WorldRole, +}; +use std::collections::BTreeMap; + +fn verifier_model(role: WorldRole) -> &'static str { + match role.model_policy() { + ModelPolicy::DistinctFamilyFromExtraction(m) => m, + other => unreachable!("verifier {role:?} has policy {other:?}"), + } +} + +#[derive(Debug, Deserialize)] +struct AuditorResponse { + verdict: String, + #[serde(default)] + rationale: String, +} + +fn context_window(text: &str, start: usize, end: usize) -> &str { + let mut lo = start.saturating_sub(200); + while lo > 0 && !text.is_char_boundary(lo) { + lo -= 1; + } + let mut hi = (end + 200).min(text.len()); + while hi < text.len() && !text.is_char_boundary(hi) { + hi += 1; + } + &text[lo..hi] +} + +/// Run the Grounding Auditor over every corpus-derived candidate. +pub async fn audit_grounding( + oracle: &dyn Oracle, + docs: &[NamedDoc], + candidates: &mut [CandidateEdge], +) -> Result<(), RefineryError> { + let model = verifier_model(WorldRole::GroundingAuditor); + let system = roster::auditor_system(); + for candidate in candidates.iter_mut() { + let ProvenanceClass::CorpusDerived { citations } = &candidate.provenance else { + continue; + }; + let citation = citations[0].clone(); + let named = docs + .iter() + .find(|d| d.doc.id == citation.doc) + .ok_or_else(|| RefineryError::Parse("citation references unknown doc".into()))?; + citation.verify_against(&named.doc)?; + + let context = context_window( + &named.doc.canonical_text, + citation.span.start, + citation.span.end, + ); + let prompt = format!( + "Claim: \"{}\" --[{:?}]--> \"{}\"\nCited quote: \"{}\"\nContext:\n...{}...", + candidate.subject, candidate.kind, candidate.object, citation.quote, context + ); + let response = oracle.complete(model, &system, &prompt).await?; + let parsed: AuditorResponse = serde_json::from_value(extract_json(&response)?) + .map_err(|e| RefineryError::Parse(format!("auditor verdict: {e}")))?; + let judgment = if parsed.verdict.eq_ignore_ascii_case("corroborate") { + Judgment::Corroborate + } else { + Judgment::Refute + }; + candidate.record(Verdict::new( + WorldRole::GroundingAuditor, + judgment, + parsed.rationale, + )?); + } + Ok(()) +} + +/// Run the Contradiction Sentinel. Returns the number of conflict pairs +/// flagged — the `contradictions_caught` input to the scorecard. +pub async fn sentinel_pass( + oracle: &dyn Oracle, + candidates: &mut [CandidateEdge], +) -> Result { + let supersedes: Vec<(String, String)> = candidates + .iter() + .filter(|c| c.kind == EdgeKind::Supersedes) + .map(|c| (c.subject.clone(), c.object.clone())) + .collect(); + let superseded_by = |loser: &str| -> Option<&str> { + supersedes + .iter() + .find(|(_, l)| l == loser) + .map(|(w, _)| w.as_str()) + }; + let related = |a: &str, b: &str| { + supersedes + .iter() + .any(|(w, l)| (w == a && l == b) || (w == b && l == a)) + }; + + let mut refutations: BTreeMap = BTreeMap::new(); + let mut conflicts = 0usize; + + let mut groups: BTreeMap<(String, String), Vec> = BTreeMap::new(); + for (i, c) in candidates.iter().enumerate() { + let conflict_prone = c.kind == EdgeKind::GovernedBy + || c.kind.structural_constraint() + == Some(StructuralConstraint::UniquePerSubjectPerTimeslice); + if conflict_prone { + groups + .entry((c.subject.clone(), format!("{:?}", c.kind))) + .or_default() + .push(i); + } + } + for indices in groups.values() { + for (a_pos, &i) in indices.iter().enumerate() { + for &j in &indices[a_pos + 1..] { + let (oi, oj) = (&candidates[i].object, &candidates[j].object); + if oi == oj { + continue; + } + if let Some(winner) = superseded_by(oi) { + refutations + .entry(i) + .or_insert_with(|| format!("superseded by {winner}")); + } else if let Some(winner) = superseded_by(oj) { + refutations + .entry(j) + .or_insert_with(|| format!("superseded by {winner}")); + } else { + conflicts += 1; + refutations + .entry(i) + .or_insert_with(|| format!("conflicts with {oj:?} for the same subject")); + refutations + .entry(j) + .or_insert_with(|| format!("conflicts with {oi:?} for the same subject")); + } + } + } + } + + if candidates.len() > 1 { + let listing: String = candidates + .iter() + .enumerate() + .map(|(i, c)| { + format!( + "{i}. \"{}\" --[{:?}]--> \"{}\"\n", + c.subject, c.kind, c.object + ) + }) + .collect(); + let response = oracle + .complete( + verifier_model(WorldRole::ContradictionSentinel), + &roster::sentinel_system(), + &listing, + ) + .await?; + let pairs: Vec> = serde_json::from_value(extract_json(&response)?) + .map_err(|e| RefineryError::Parse(format!("sentinel pairs: {e}")))?; + for pair in pairs { + let [i, j] = pair.as_slice() else { continue }; + let (i, j) = (*i, *j); + if i >= candidates.len() || j >= candidates.len() || i == j { + continue; + } + if related(&candidates[i].object, &candidates[j].object) { + continue; + } + conflicts += 1; + let (oi, oj) = (candidates[i].object.clone(), candidates[j].object.clone()); + refutations + .entry(i) + .or_insert_with(|| format!("contradicts candidate asserting {oj:?}")); + refutations + .entry(j) + .or_insert_with(|| format!("contradicts candidate asserting {oi:?}")); + } + } + + for (i, candidate) in candidates.iter_mut().enumerate() { + let verdict = match refutations.get(&i) { + Some(rationale) => Verdict::new( + WorldRole::ContradictionSentinel, + Judgment::Refute, + rationale.clone(), + )?, + None => Verdict::new( + WorldRole::ContradictionSentinel, + Judgment::Corroborate, + "no conflicts within the candidate set".into(), + )?, + }; + candidate.record(verdict); + } + Ok(conflicts) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::packet::ingest_all; + use async_trait::async_trait; + use smesh_world::{CdmSpan, Citation}; + + struct Scripted(&'static str); + + #[async_trait] + impl Oracle for Scripted { + async fn complete(&self, _: &str, _: &str, _: &str) -> Result { + Ok(self.0.to_owned()) + } + } + + fn candidate(subject: &str, kind: EdgeKind, object: &str) -> CandidateEdge { + let docs = ingest_all(&[( + "doc.md".to_owned(), + b"# Doc\n\nClaims must be filed within 30 days. The window is 45 days now.\n".to_vec(), + )]) + .unwrap(); + let citation = Citation::grounded(&docs[0].doc, CdmSpan::new(8, 14)).unwrap(); + CandidateEdge::emit( + WorldRole::Policy, + subject.into(), + kind, + object.into(), + vec![citation], + ) + .unwrap() + } + + #[tokio::test] + async fn unresolved_same_subject_conflict_contests_both() { + let mut candidates = vec![ + candidate("Claims", EdgeKind::GovernedBy, "30-day filing window"), + candidate("Claims", EdgeKind::GovernedBy, "45-day filing window"), + ]; + let conflicts = sentinel_pass(&Scripted("[]"), &mut candidates) + .await + .unwrap(); + assert_eq!(conflicts, 1); + assert_eq!(candidates[0].refutations(), 1); + assert_eq!(candidates[1].refutations(), 1); + } + + #[tokio::test] + async fn supersession_resolves_the_conflict_deterministically() { + let mut candidates = vec![ + candidate("Claims", EdgeKind::GovernedBy, "30-day filing window"), + candidate("Claims", EdgeKind::GovernedBy, "45-day filing window"), + candidate( + "45-day filing window", + EdgeKind::Supersedes, + "30-day filing window", + ), + ]; + let conflicts = sentinel_pass(&Scripted("[]"), &mut candidates) + .await + .unwrap(); + assert_eq!(conflicts, 0); + assert_eq!(candidates[0].refutations(), 1, "superseded rule is refuted"); + assert_eq!(candidates[1].refutations(), 0, "superseding rule survives"); + } + + #[tokio::test] + async fn semantic_pairs_from_the_model_contest_both_sides() { + let mut candidates = vec![ + candidate("HR", EdgeKind::GovernedBy, "equipment back in 14 days"), + candidate( + "IT-Security", + EdgeKind::GovernedBy, + "equipment back in 7 days", + ), + ]; + let conflicts = sentinel_pass(&Scripted("[[0,1]]"), &mut candidates) + .await + .unwrap(); + assert_eq!(conflicts, 1); + assert!(candidates[0].refutations() == 1 && candidates[1].refutations() == 1); + } + + #[tokio::test] + async fn auditor_records_cross_model_verdicts() { + let docs = ingest_all(&[( + "doc.md".to_owned(), + b"# Doc\n\nClaims must be filed within 30 days. The window is 45 days now.\n".to_vec(), + )]) + .unwrap(); + let mut candidates = vec![candidate("Claims", EdgeKind::GovernedBy, "30-day window")]; + audit_grounding( + &Scripted(r#"{"verdict": "corroborate", "rationale": "entailed"}"#), + &docs, + &mut candidates, + ) + .await + .unwrap(); + assert_eq!(candidates[0].corroborations(), 1); + } +} From a7c9e63e275afad69f0159057f48879efb4032b1 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Thu, 23 Jul 2026 00:35:42 -0700 Subject: [PATCH 05/13] feat(world): smesh-ratify - the ratification bench (Phase 4) The human merge authority, made real: - StagedRun handoff format (smesh-world): candidates + reviewer-ready evidence views (quote, native anchor, context); refinery writes refinery-staged.json after each run - Ed25519 reviewer identity: persistent keypair, ratification record signed over base_rev + reviewer + full decision map, offline verification (tamper/wrong-rev tests) - Session state: crash-safe decision ledger, total-coverage gate, green-lane batch approval, ratify -> kernel type-state path -> SignedChangeset written to disk; AlreadySigned is terminal - Axum bench UI (impeccable, Zuub navy/teal, institutional-ledger aesthetic): Red/Amber/Green attention lanes, per-claim evidence drill-in with highlighted verbatim quote + native anchor + verdict rationales, amend/reject/defer flows, confabulation-firewall transparency panel, signing ceremony with revision receipt; hand-drawn SVG icon sprite, Familjen Grotesk / Atkinson Hyperlegible / Red Hat Mono - --demo mode: offline Meridian session (all lanes populated); full flow verified in-browser: 8 decisions -> signed revision, 8 staged - 1 rejected - 1 deferred = 6 edges in the revision Workspace: 197 tests, 0 failures, clippy clean on world/refinery/ratify. --- .impeccable.md | 38 +++ Cargo.toml | 5 + smesh-ratify/.gitignore | 2 + smesh-ratify/Cargo.toml | 20 ++ smesh-ratify/assets/index.html | 489 +++++++++++++++++++++++++++++++++ smesh-ratify/src/demo.rs | 259 +++++++++++++++++ smesh-ratify/src/lib.rs | 33 +++ smesh-ratify/src/main.rs | 102 +++++++ smesh-ratify/src/signer.rs | 137 +++++++++ smesh-ratify/src/state.rs | 246 +++++++++++++++++ smesh-ratify/src/web.rs | 327 ++++++++++++++++++++++ smesh-refinery/src/main.rs | 10 + smesh-refinery/src/run.rs | 77 +++++- smesh-world/src/cdm.rs | 19 +- smesh-world/src/delta.rs | 31 +++ smesh-world/src/lib.rs | 4 +- 16 files changed, 1795 insertions(+), 4 deletions(-) create mode 100644 .impeccable.md create mode 100644 smesh-ratify/.gitignore create mode 100644 smesh-ratify/Cargo.toml create mode 100644 smesh-ratify/assets/index.html create mode 100644 smesh-ratify/src/demo.rs create mode 100644 smesh-ratify/src/lib.rs create mode 100644 smesh-ratify/src/main.rs create mode 100644 smesh-ratify/src/signer.rs create mode 100644 smesh-ratify/src/state.rs create mode 100644 smesh-ratify/src/web.rs diff --git a/.impeccable.md b/.impeccable.md new file mode 100644 index 0000000..cbbe3aa --- /dev/null +++ b/.impeccable.md @@ -0,0 +1,38 @@ +# Impeccable Design Context — smesh-rust + +## Design Context + +### Users +Internal domain experts with tribal knowledge — compliance-minded operators +(Zuub world: dental RCM / payer operations) acting as the merge authority +over machine-extracted world models. Desk work, careful evidence reading, +daytime. They are legally accountable for what they sign. + +### Brand Personality +Evidentiary, institutional, calm. Zuub brand family: deep navy `#05375E` +dominant, teal `#6CDCD8` reserved for action, mid-blue `#1870C9`, pale +navy-tinted surfaces (`#E9F1FB` family), indigo `#2B3C6A`. Three words: +**careful, evidentiary, institutional.** + +### Aesthetic Direction +"Institutional ledger": a court docket crossed with a well-set legal brief. +Light theme (evidence reading, daytime desk work). Paper-like surfaces +tinted toward navy; teal appears only on actions and progress. Rich inline +SVG iconography — a consistent hand-drawn 20px/1.5px-stroke set (seal, +scales, quote, per-format document glyphs, verdict marks) — never emoji, +never icon fonts. Anti-references: SaaS metric dashboards, cards-in-cards, +dark-mode-with-glow, gradient text. + +### Design Principles +1. **The mesh proposes; the human disposes.** Attention ranks by lane: + Red (mandatory judgment) before Amber before Green (batch, spot-check). +2. **Evidence is the interface.** Every claim renders with its verbatim + quote highlighted in context plus its native anchor (page / heading + path / message-id). Nothing asks for trust without showing its source. +3. **Signing is ceremony.** The ratify action is visually terminal and + weighty — disabled until coverage is total, followed by a receipt state + showing the new revision hash. +4. **Transparency is credibility.** Firewall rejections and verdict + rationales are visible, not hidden. +5. **Typography:** display Familjen Grotesk, body Atkinson Hyperlegible + (misread-proof evidence), Red Hat Mono strictly for hashes/anchors. diff --git a/Cargo.toml b/Cargo.toml index ead0b04..528741b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "smesh-cli", "smesh-world", "smesh-refinery", + "smesh-ratify", ] [workspace.package] @@ -42,6 +43,10 @@ blake3 = "1.5" # global thread pool, which is nondeterministic under test parallelism. lopdf = { version = "0.36", default-features = false } pdf-extract = "0.10" + +# Ratification dashboard +axum = "0.8" +ed25519-dalek = { version = "2.2", features = ["rand_core"] } rand = "0.8" uuid = { version = "1.6", features = ["v4", "serde"] } diff --git a/smesh-ratify/.gitignore b/smesh-ratify/.gitignore new file mode 100644 index 0000000..141785f --- /dev/null +++ b/smesh-ratify/.gitignore @@ -0,0 +1,2 @@ +demo-session.* +.ratify/ diff --git a/smesh-ratify/Cargo.toml b/smesh-ratify/Cargo.toml new file mode 100644 index 0000000..498fa6a --- /dev/null +++ b/smesh-ratify/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "smesh-ratify" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +description = "Ratification dashboard: the human merge authority over staged world-model changesets, with Ed25519 sign-off" + +[dependencies] +smesh-world = { path = "../smesh-world" } +axum = { workspace = true } +tokio = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +ed25519-dalek = { workspace = true } +rand = { workspace = true } diff --git a/smesh-ratify/assets/index.html b/smesh-ratify/assets/index.html new file mode 100644 index 0000000..2249f6d --- /dev/null +++ b/smesh-ratify/assets/index.html @@ -0,0 +1,489 @@ + + + + + +Ratification Bench — SMESH World Model + + + + + + + + + + + + + + +
+ + +
+
+ staged changeset · human review +

The mesh proposes. You dispose.

+

Every candidate below was extracted from your corpus, grounded to a verbatim quote, and cross-examined by a second model family. Nothing becomes part of the signed world without your judgment.

+
+
+
+
+
+ +
+ + + + diff --git a/smesh-ratify/src/demo.rs b/smesh-ratify/src/demo.rs new file mode 100644 index 0000000..a42364a --- /dev/null +++ b/smesh-ratify/src/demo.rs @@ -0,0 +1,259 @@ +//! Offline demo session: a realistic staged run assembled from the Meridian +//! corpus with no network and no refinery — grounded citations, mixed +//! verdicts, all three lanes populated. Drives UI work and tests. + +use smesh_world::meridian; +use smesh_world::{ + CandidateEdge, CdmDocument, CdmSpan, Citation, EdgeKind, EvidenceView, Judgment, Registrar, + RejectedView, StagedRun, Verdict, WorldRole, +}; +use std::collections::BTreeMap; + +struct Bench { + docs: Vec<(String, CdmDocument)>, +} + +impl Bench { + fn new() -> Self { + let docs = meridian::corpus() + .artifacts + .into_iter() + .map(|a| { + let doc = Registrar::ingest(&a.bytes).expect("meridian ingests"); + (a.name.to_owned(), doc) + }) + .collect(); + Self { docs } + } + + fn doc(&self, name: &str) -> &CdmDocument { + &self + .docs + .iter() + .find(|(n, _)| n == name) + .expect("known doc") + .1 + } + + fn grounded( + &self, + role: WorldRole, + subject: &str, + kind: EdgeKind, + object: &str, + doc_name: &str, + quote: &str, + ) -> (CandidateEdge, EvidenceView) { + let doc = self.doc(doc_name); + let start = doc + .canonical_text + .find(quote) + .unwrap_or_else(|| panic!("{quote:?} not in {doc_name}")); + let span = CdmSpan::new(start, start + quote.len()); + let citation = Citation::grounded(doc, span).expect("demo quotes ground"); + let lo = start.saturating_sub(120); + let hi = (span.end + 120).min(doc.canonical_text.len()); + let view = EvidenceView { + doc_name: doc_name.to_owned(), + quote: quote.to_owned(), + anchor: doc + .native_anchor(span) + .map(ToString::to_string) + .unwrap_or_default(), + context: doc.canonical_text[lo..hi].to_owned(), + }; + let edge = CandidateEdge::emit(role, subject.into(), kind, object.into(), vec![citation]) + .expect("demo emissions are valid"); + (edge, view) + } +} + +fn corroborate(by: WorldRole, why: &str) -> Verdict { + Verdict::new(by, Judgment::Corroborate, why.into()).expect("verifier role") +} + +fn refute(by: WorldRole, why: &str) -> Verdict { + Verdict::new(by, Judgment::Refute, why.into()).expect("verifier role") +} + +/// A staged run with 2 green, 2 amber, and 4 red (two contested pairs). +pub fn demo_staged_run() -> StagedRun { + let bench = Bench::new(); + let mut candidates = Vec::new(); + let mut evidence: BTreeMap> = BTreeMap::new(); + + let mut push = |edge: CandidateEdge, view: EvidenceView| { + evidence.insert(edge.key(), vec![view]); + candidates.push(edge); + }; + + let (mut legal_term, v) = bench.grounded( + WorldRole::Lexicon, + "Claim (Legal)", + EdgeKind::DefinesTerm, + "a formal demand for coverage under a policy", + "legal-definitions.md", + "a Claim means a formal demand for coverage under a policy", + ); + legal_term.record(corroborate( + WorldRole::GroundingAuditor, + "quote entails the definition", + )); + legal_term.record(corroborate( + WorldRole::ContradictionSentinel, + "no conflicts", + )); + push(legal_term, v); + + let (mut ledger, v) = bench.grounded( + WorldRole::Structure, + "Finance", + EdgeKind::Owns, + "general ledger", + "finance-glossary.md", + "Finance owns the general ledger", + ); + ledger.record(corroborate( + WorldRole::GroundingAuditor, + "direct statement of ownership", + )); + ledger.record(corroborate( + WorldRole::ContradictionSentinel, + "no conflicts", + )); + push(ledger, v); + + let (mut tokens, v) = bench.grounded( + WorldRole::Policy, + "IT-Security", + EdgeKind::GovernedBy, + "90-day token rotation", + "it-security-policy.md", + "All access tokens rotate every 90 days", + ); + tokens.record(corroborate( + WorldRole::GroundingAuditor, + "quote states the rotation rule", + )); + push(tokens, v); + + let (mut pto, v) = bench.grounded( + WorldRole::Policy, + "HR", + EdgeKind::GovernedBy, + "PTO fortnight accrual capped at 26 days", + "employee-handbook.md", + "PTO accrues at 1 day per fortnight worked, capped at 26 days", + ); + pto.record(corroborate( + WorldRole::GroundingAuditor, + "quote states the accrual rule", + )); + push(pto, v); + + let (mut filing_45, v) = bench.grounded( + WorldRole::Policy, + "Claims", + EdgeKind::GovernedBy, + "45-day filing window", + "memo-4417.eml", + "the claims filing window is 45 days", + ); + filing_45.record(corroborate( + WorldRole::GroundingAuditor, + "memo states the new window", + )); + filing_45.record(refute( + WorldRole::ContradictionSentinel, + "conflicts with \"30-day filing window\" for the same subject", + )); + push(filing_45, v); + + let (mut filing_30, v) = bench.grounded( + WorldRole::Policy, + "Claims", + EdgeKind::GovernedBy, + "30-day filing window", + "employee-handbook.md", + "Claims must be filed within 30 days of the date of service", + ); + filing_30.record(corroborate( + WorldRole::GroundingAuditor, + "handbook states the old window", + )); + filing_30.record(refute( + WorldRole::ContradictionSentinel, + "conflicts with \"45-day filing window\" for the same subject", + )); + push(filing_30, v); + + let (mut hr_equipment, v) = bench.grounded( + WorldRole::Policy, + "HR", + EdgeKind::GovernedBy, + "equipment returned within 14 days of departure", + "employee-handbook.md", + "Departing employees must return company equipment within 14 days", + ); + hr_equipment.record(corroborate( + WorldRole::GroundingAuditor, + "quote states the deadline", + )); + hr_equipment.record(refute( + WorldRole::ContradictionSentinel, + "contradicts candidate asserting \"equipment returned within 7 days of departure\"", + )); + push(hr_equipment, v); + + let (mut it_equipment, v) = bench.grounded( + WorldRole::Policy, + "IT-Security", + EdgeKind::GovernedBy, + "equipment returned within 7 days of departure", + "it-security-policy.md", + "Departing employees must return company equipment within 7 days", + ); + it_equipment.record(corroborate( + WorldRole::GroundingAuditor, + "quote states the deadline", + )); + it_equipment.record(refute( + WorldRole::ContradictionSentinel, + "contradicts candidate asserting \"equipment returned within 14 days of departure\"", + )); + push(it_equipment, v); + + StagedRun { + base_rev: "meridian-rev0".into(), + candidates, + evidence, + rejected: vec![RejectedView { + role: "Policy".into(), + summary: "Claims --[GovernedBy]--> wormhole intake".into(), + reason: "quote not found verbatim in document".into(), + }], + contradictions_caught: 2, + scorecard: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use smesh_world::Lane; + + #[test] + fn demo_populates_all_three_lanes_with_grounded_evidence() { + let run = demo_staged_run(); + let lanes: Vec = run.candidates.iter().map(Lane::assign).collect(); + assert_eq!(lanes.iter().filter(|l| **l == Lane::Green).count(), 2); + assert_eq!(lanes.iter().filter(|l| **l == Lane::Amber).count(), 2); + assert_eq!(lanes.iter().filter(|l| **l == Lane::Red).count(), 4); + for c in &run.candidates { + let views = &run.evidence[&c.key()]; + assert!(!views.is_empty()); + assert!(!views[0].anchor.is_empty()); + assert!(views[0].context.contains(&views[0].quote)); + } + } +} diff --git a/smesh-ratify/src/lib.rs b/smesh-ratify/src/lib.rs new file mode 100644 index 0000000..1e57eb0 --- /dev/null +++ b/smesh-ratify/src/lib.rs @@ -0,0 +1,33 @@ +//! # SMESH Ratify +//! +//! The human's bench. Serves a staged world-model changeset for review, +//! collects total-coverage decisions, and turns human sign-off into a real +//! Ed25519 signature over the ratification record — which the kernel's +//! type-state machinery then converts into a signed revision. +//! +//! The dashboard never edits the graph; it edits *decisions*. The kernel's +//! `Changeset → ratify → Ratified → sign` path remains the only +//! road to a new revision. + +pub mod demo; +pub mod signer; +pub mod state; +pub mod web; + +#[derive(Debug, thiserror::Error)] +pub enum RatifyError { + #[error("io error: {0}")] + Io(#[from] std::io::Error), + #[error("serialization error: {0}")] + Json(#[from] serde_json::Error), + #[error(transparent)] + World(#[from] smesh_world::WorldError), + #[error("unknown candidate key {0:?}")] + UnknownCandidate(String), + #[error("cannot ratify: {undecided} of {total} candidates still undecided")] + IncompleteCoverage { undecided: usize, total: usize }, + #[error("changeset already ratified and signed")] + AlreadySigned, + #[error("signing key error: {0}")] + Key(String), +} diff --git a/smesh-ratify/src/main.rs b/smesh-ratify/src/main.rs new file mode 100644 index 0000000..34aa121 --- /dev/null +++ b/smesh-ratify/src/main.rs @@ -0,0 +1,102 @@ +//! Serve the ratification bench. +//! +//! ```sh +//! cargo run -p smesh-ratify -- refinery-staged.json # review a real run +//! cargo run -p smesh-ratify -- --demo # offline demo session +//! ``` +//! +//! Reviewer identity comes from `SMESH_REVIEWER` (falling back to `USER`); +//! the Ed25519 keypair persists at `.ratify/reviewer.ed25519`. Decisions and +//! the signed revision are written next to the staged file. + +use smesh_ratify::demo::demo_staged_run; +use smesh_ratify::signer::ReviewerKey; +use smesh_ratify::state::Session; +use smesh_ratify::web::{router, App}; +use smesh_world::ReviewerId; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + let mut demo = false; + let mut port: u16 = 8093; + let mut staged_path = "refinery-staged.json".to_owned(); + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "--demo" => demo = true, + "--port" => { + port = args.next().and_then(|p| p.parse().ok()).unwrap_or_else(|| { + eprintln!("--port requires a number"); + std::process::exit(2); + }) + } + other => staged_path = other.to_owned(), + } + } + + let (staged, stem) = if demo { + (demo_staged_run(), PathBuf::from("demo-session")) + } else { + let path = staged_path; + let bytes = match std::fs::read(&path) { + Ok(b) => b, + Err(e) => { + eprintln!("cannot read staged run {path}: {e}"); + eprintln!("produce one with `cargo run -p smesh-refinery`, or use --demo"); + std::process::exit(2); + } + }; + let staged = match serde_json::from_slice(&bytes) { + Ok(s) => s, + Err(e) => { + eprintln!("{path} is not a staged run: {e}"); + std::process::exit(2); + } + }; + (staged, PathBuf::from(path)) + }; + + let reviewer_name = std::env::var("SMESH_REVIEWER") + .or_else(|_| std::env::var("USER")) + .unwrap_or_else(|_| "reviewer".to_owned()); + let reviewer = match ReviewerKey::load_or_generate( + &PathBuf::from(".ratify/reviewer.ed25519"), + ReviewerId(reviewer_name), + ) { + Ok(k) => k, + Err(e) => { + eprintln!("reviewer key error: {e}"); + std::process::exit(2); + } + }; + + let session = match Session::open( + staged, + stem.with_extension("decisions.json"), + stem.with_extension("revision.json"), + ) { + Ok(s) => s, + Err(e) => { + eprintln!("cannot open session: {e}"); + std::process::exit(2); + } + }; + + let app = App { + session: Arc::new(Mutex::new(session)), + reviewer: Arc::new(reviewer), + }; + + let addr = format!("127.0.0.1:{port}"); + println!("ratification bench: http://{addr}"); + let listener = tokio::net::TcpListener::bind(&addr) + .await + .expect("bind ratification bench port"); + axum::serve(listener, router(app)) + .await + .expect("serve ratification bench"); +} diff --git a/smesh-ratify/src/signer.rs b/smesh-ratify/src/signer.rs new file mode 100644 index 0000000..0c5293a --- /dev/null +++ b/smesh-ratify/src/signer.rs @@ -0,0 +1,137 @@ +//! Reviewer identity: a persistent Ed25519 keypair. Ratification is a +//! signed act — "which human approved this and when" must verify offline. + +use crate::RatifyError; +use ed25519_dalek::{Signer, SigningKey, Verifier, VerifyingKey}; +use smesh_world::{RatificationRecord, ReviewerId, Signature}; +use std::path::Path; + +pub struct ReviewerKey { + pub reviewer: ReviewerId, + signing: SigningKey, +} + +impl ReviewerKey { + /// Load the keypair at `path`, or generate and persist one (0600). + pub fn load_or_generate(path: &Path, reviewer: ReviewerId) -> Result { + let signing = if path.exists() { + let bytes = std::fs::read(path)?; + let key: [u8; 32] = bytes.as_slice().try_into().map_err(|_| { + RatifyError::Key(format!("{} is not a 32-byte seed", path.display())) + })?; + SigningKey::from_bytes(&key) + } else { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let key = SigningKey::generate(&mut rand::rngs::OsRng); + std::fs::write(path, key.to_bytes())?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; + } + key + }; + Ok(Self { reviewer, signing }) + } + + pub fn verifying_key(&self) -> VerifyingKey { + self.signing.verifying_key() + } + + /// Canonical bytes the signature covers: base rev + reviewer + the full + /// decision map, serialized deterministically (BTreeMap ordering). + pub fn ratification_message( + base_rev: &str, + reviewer: &ReviewerId, + decisions: &std::collections::BTreeMap, + ) -> Vec { + let payload = serde_json::json!({ + "base_rev": base_rev, + "reviewer": reviewer.0, + "decisions": decisions, + }); + serde_json::to_vec(&payload).expect("canonical ratification payload serializes") + } + + /// Produce a signed `RatificationRecord` over the decision map. + pub fn sign_ratification( + &self, + base_rev: &str, + decisions: std::collections::BTreeMap, + ) -> RatificationRecord { + let message = Self::ratification_message(base_rev, &self.reviewer, &decisions); + let signature = self.signing.sign(&message); + RatificationRecord { + reviewer: self.reviewer.clone(), + decisions, + signature: Signature(signature.to_bytes().to_vec()), + } + } + + /// Offline verification: does this record carry a valid signature from + /// `key` over `base_rev`? + pub fn verify_ratification( + key: &VerifyingKey, + base_rev: &str, + record: &RatificationRecord, + ) -> bool { + let message = Self::ratification_message(base_rev, &record.reviewer, &record.decisions); + ed25519_dalek::Signature::from_slice(&record.signature.0) + .map(|sig| key.verify(&message, &sig).is_ok()) + .unwrap_or(false) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use smesh_world::ReviewDecision; + use std::collections::BTreeMap; + + fn temp_key_path(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("smesh-ratify-test-{name}-{}", std::process::id())) + } + + fn decisions() -> BTreeMap { + [("edge-a".to_owned(), ReviewDecision::Approve)] + .into_iter() + .collect() + } + + #[test] + fn keypair_persists_across_loads() { + let path = temp_key_path("persist"); + let a = ReviewerKey::load_or_generate(&path, ReviewerId("dj".into())).unwrap(); + let b = ReviewerKey::load_or_generate(&path, ReviewerId("dj".into())).unwrap(); + assert_eq!(a.verifying_key(), b.verifying_key()); + std::fs::remove_file(&path).unwrap(); + } + + #[test] + fn signed_ratification_verifies_and_tampering_fails() { + let path = temp_key_path("verify"); + let key = ReviewerKey::load_or_generate(&path, ReviewerId("dj".into())).unwrap(); + let record = key.sign_ratification("rev0", decisions()); + assert!(ReviewerKey::verify_ratification( + &key.verifying_key(), + "rev0", + &record + )); + assert!( + !ReviewerKey::verify_ratification(&key.verifying_key(), "rev1", &record), + "signature must bind the base revision" + ); + let mut tampered = record.clone(); + tampered + .decisions + .insert("edge-b".into(), ReviewDecision::Approve); + assert!(!ReviewerKey::verify_ratification( + &key.verifying_key(), + "rev0", + &tampered + )); + std::fs::remove_file(&path).unwrap(); + } +} diff --git a/smesh-ratify/src/state.rs b/smesh-ratify/src/state.rs new file mode 100644 index 0000000..658f6d1 --- /dev/null +++ b/smesh-ratify/src/state.rs @@ -0,0 +1,246 @@ +//! Review-session state: the staged run, the decision ledger, and the +//! transition into a signed revision. Decisions persist to disk after every +//! change — a browser crash must never cost the reviewer their work. + +use crate::signer::ReviewerKey; +use crate::RatifyError; +use serde::Serialize; +use smesh_world::{ + CandidateEdge, Changeset, Lane, ReviewDecision, SignedChangeset, StagedRun, WorldError, +}; +use std::collections::BTreeMap; +use std::path::PathBuf; + +pub struct Session { + pub staged: StagedRun, + pub decisions: BTreeMap, + pub signed: Option, + pub decisions_path: PathBuf, + pub revision_path: PathBuf, +} + +#[derive(Debug, Serialize)] +pub struct Progress { + pub total: usize, + pub decided: usize, + pub green: usize, + pub amber: usize, + pub red: usize, +} + +impl Session { + /// Open a session over a staged run, restoring any persisted decisions. + pub fn open( + staged: StagedRun, + decisions_path: PathBuf, + revision_path: PathBuf, + ) -> Result { + let decisions = match std::fs::read(&decisions_path) { + Ok(bytes) => serde_json::from_slice(&bytes)?, + Err(_) => BTreeMap::new(), + }; + let signed = match std::fs::read(&revision_path) { + Ok(bytes) => Some(serde_json::from_slice(&bytes)?), + Err(_) => None, + }; + Ok(Self { + staged, + decisions, + signed, + decisions_path, + revision_path, + }) + } + + pub fn candidate(&self, key: &str) -> Option<&CandidateEdge> { + self.staged.candidates.iter().find(|c| c.key() == key) + } + + pub fn lane_of(&self, candidate: &CandidateEdge) -> Lane { + Lane::assign(candidate) + } + + pub fn progress(&self) -> Progress { + let mut p = Progress { + total: self.staged.candidates.len(), + decided: 0, + green: 0, + amber: 0, + red: 0, + }; + for c in &self.staged.candidates { + match Lane::assign(c) { + Lane::Green => p.green += 1, + Lane::Amber => p.amber += 1, + Lane::Red => p.red += 1, + } + if self.decisions.contains_key(&c.key()) { + p.decided += 1; + } + } + p + } + + /// Record one decision and persist the ledger. + pub fn decide(&mut self, key: String, decision: ReviewDecision) -> Result<(), RatifyError> { + if self.signed.is_some() { + return Err(RatifyError::AlreadySigned); + } + if self.candidate(&key).is_none() { + return Err(RatifyError::UnknownCandidate(key)); + } + self.decisions.insert(key, decision); + self.persist_decisions() + } + + /// Approve every green-lane candidate that has no decision yet — the + /// batch gesture; ratification stays total, attention stays tiered. + pub fn approve_green_lane(&mut self) -> Result { + if self.signed.is_some() { + return Err(RatifyError::AlreadySigned); + } + let mut approved = 0; + let keys: Vec = self + .staged + .candidates + .iter() + .filter(|c| Lane::assign(c) == Lane::Green) + .map(CandidateEdge::key) + .collect(); + for key in keys { + self.decisions.entry(key).or_insert_with(|| { + approved += 1; + ReviewDecision::Approve + }); + } + self.persist_decisions()?; + Ok(approved) + } + + /// Total-coverage gate, then the kernel path: stage → ratify → sign. + /// Writes the signed revision to disk and returns it. + pub fn ratify_and_sign(&mut self, key: &ReviewerKey) -> Result { + if self.signed.is_some() { + return Err(RatifyError::AlreadySigned); + } + let undecided = self + .staged + .candidates + .iter() + .filter(|c| !self.decisions.contains_key(&c.key())) + .count(); + if undecided > 0 { + return Err(RatifyError::IncompleteCoverage { + undecided, + total: self.staged.candidates.len(), + }); + } + + let record = key.sign_ratification(&self.staged.base_rev, self.decisions.clone()); + let changeset = + Changeset::stage(self.staged.base_rev.clone(), self.staged.candidates.clone())?; + let outcome = changeset.ratify(record)?; + let signed = outcome.changeset.sign(outcome.ratification); + + std::fs::write(&self.revision_path, serde_json::to_vec_pretty(&signed)?)?; + self.signed = Some(signed.clone()); + Ok(signed) + } + + fn persist_decisions(&self) -> Result<(), RatifyError> { + std::fs::write( + &self.decisions_path, + serde_json::to_vec_pretty(&self.decisions)?, + )?; + Ok(()) + } +} + +/// Fully-rejected changesets surface the kernel's refusal as a first-class +/// outcome the UI can explain. +pub fn is_empty_changeset_error(err: &RatifyError) -> bool { + matches!(err, RatifyError::World(WorldError::EmptyChangeset)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::demo::demo_staged_run; + use smesh_world::ReviewerId; + + fn temp(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("smesh-ratify-{name}-{}", std::process::id())) + } + + fn session(tag: &str) -> Session { + let d = temp(&format!("{tag}-decisions.json")); + let r = temp(&format!("{tag}-revision.json")); + let _ = std::fs::remove_file(&d); + let _ = std::fs::remove_file(&r); + Session::open(demo_staged_run(), d, r).unwrap() + } + + fn reviewer(tag: &str) -> ReviewerKey { + ReviewerKey::load_or_generate(&temp(&format!("{tag}.key")), ReviewerId("dj".into())) + .unwrap() + } + + #[test] + fn ratify_refuses_partial_coverage() { + let mut s = session("partial"); + let key = reviewer("partial"); + let first = s.staged.candidates[0].key(); + s.decide(first, ReviewDecision::Approve).unwrap(); + match s.ratify_and_sign(&key) { + Err(RatifyError::IncompleteCoverage { undecided, total }) => { + assert!(undecided > 0 && undecided < total); + } + other => panic!("expected coverage error, got {other:?}"), + } + } + + #[test] + fn full_coverage_signs_and_persists_a_revision() { + let mut s = session("signs"); + let key = reviewer("signs"); + s.approve_green_lane().unwrap(); + let keys: Vec = s.staged.candidates.iter().map(CandidateEdge::key).collect(); + for k in keys { + if !s.decisions.contains_key(&k) { + s.decide(k, ReviewDecision::Approve).unwrap(); + } + } + let signed = s.ratify_and_sign(&key).unwrap(); + assert_eq!(signed.base_rev, s.staged.base_rev); + assert_eq!(signed.new_rev.len(), 64); + assert!(s.revision_path.exists()); + assert!(matches!( + s.decide("anything".into(), ReviewDecision::Approve), + Err(RatifyError::AlreadySigned) + )); + } + + #[test] + fn decisions_survive_a_session_restart() { + let d = temp("restart-decisions.json"); + let r = temp("restart-revision.json"); + let _ = std::fs::remove_file(&d); + let _ = std::fs::remove_file(&r); + let mut s = Session::open(demo_staged_run(), d.clone(), r.clone()).unwrap(); + let first = s.staged.candidates[0].key(); + s.decide(first.clone(), ReviewDecision::Defer).unwrap(); + drop(s); + let restored = Session::open(demo_staged_run(), d, r).unwrap(); + assert_eq!(restored.decisions.get(&first), Some(&ReviewDecision::Defer)); + } + + #[test] + fn green_lane_batch_approval_skips_decided_and_non_green() { + let mut s = session("batch"); + let approved = s.approve_green_lane().unwrap(); + let p = s.progress(); + assert_eq!(approved, p.green); + assert_eq!(p.decided, p.green, "only green candidates were decided"); + assert!(p.red > 0, "demo data must include contested red-lane rows"); + } +} diff --git a/smesh-ratify/src/web.rs b/smesh-ratify/src/web.rs new file mode 100644 index 0000000..c9128c8 --- /dev/null +++ b/smesh-ratify/src/web.rs @@ -0,0 +1,327 @@ +//! HTTP surface: one page, four JSON endpoints. The browser edits +//! decisions; only `POST /api/ratify` reaches the kernel's signing path. + +use crate::signer::ReviewerKey; +use crate::state::Session; +use crate::RatifyError; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::Html; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use smesh_world::{CandidateEdge, EdgeKind, Lane, ProvenanceClass, ReviewDecision}; +use std::sync::{Arc, Mutex}; + +#[derive(Clone)] +pub struct App { + pub session: Arc>, + pub reviewer: Arc, +} + +pub fn router(app: App) -> Router { + Router::new() + .route("/", get(page)) + .route("/api/state", get(api_state)) + .route("/api/decision", post(api_decision)) + .route("/api/approve-green", post(api_approve_green)) + .route("/api/ratify", post(api_ratify)) + .with_state(app) +} + +async fn page() -> Html<&'static str> { + Html(include_str!("../assets/index.html")) +} + +#[derive(Serialize)] +struct VerdictView { + by: String, + judgment: String, + rationale: String, +} + +#[derive(Serialize)] +struct CandidateView { + key: String, + subject: String, + kind: String, + object: String, + lane: String, + emitted_by: String, + corroborations: u32, + refutations: u32, + citations: usize, + verdicts: Vec, + evidence: Vec, + decision: Option, +} + +fn candidate_view(session: &Session, c: &CandidateEdge) -> CandidateView { + let key = c.key(); + CandidateView { + lane: format!("{:?}", Lane::assign(c)), + subject: c.subject.clone(), + kind: format!("{:?}", c.kind), + object: c.object.clone(), + emitted_by: format!("{:?}", c.emitted_by), + corroborations: c.corroborations(), + refutations: c.refutations(), + citations: match &c.provenance { + ProvenanceClass::CorpusDerived { citations } => citations.len(), + ProvenanceClass::HumanAttested { .. } => 0, + }, + verdicts: c + .verdicts + .iter() + .map(|v| VerdictView { + by: format!("{:?}", v.by), + judgment: format!("{:?}", v.judgment), + rationale: v.rationale.clone(), + }) + .collect(), + evidence: session + .staged + .evidence + .get(&key) + .cloned() + .unwrap_or_default(), + decision: session + .decisions + .get(&key) + .map(|d| serde_json::to_value(d).expect("decision serializes")), + key, + } +} + +async fn api_state(State(app): State) -> Json { + let session = app.session.lock().expect("session lock"); + let candidates: Vec = session + .staged + .candidates + .iter() + .map(|c| candidate_view(&session, c)) + .collect(); + Json(json!({ + "base_rev": session.staged.base_rev, + "reviewer": app.reviewer.reviewer.0, + "verifying_key": hex(&app.reviewer.verifying_key().to_bytes()), + "progress": session.progress(), + "candidates": candidates, + "rejected": session.staged.rejected, + "contradictions_caught": session.staged.contradictions_caught, + "scorecard": session.staged.scorecard, + "signed": session.signed.as_ref().map(|s| json!({ + "base_rev": s.base_rev, + "new_rev": s.new_rev, + "edges": s.edges.len(), + "reviewer": s.ratification.reviewer.0, + })), + })) +} + +#[derive(Deserialize)] +struct DecisionBody { + key: String, + action: String, + #[serde(default)] + reason: String, + subject: Option, + kind: Option, + object: Option, +} + +fn build_decision( + session: &Session, + body: &DecisionBody, +) -> Result { + match body.action.as_str() { + "approve" => Ok(ReviewDecision::Approve), + "defer" => Ok(ReviewDecision::Defer), + "reject" => Ok(ReviewDecision::Reject { + reason: if body.reason.trim().is_empty() { + "rejected by reviewer".to_owned() + } else { + body.reason.clone() + }, + }), + "edit" => { + let original = session + .candidate(&body.key) + .ok_or((StatusCode::NOT_FOUND, "unknown candidate".to_owned()))?; + let kind: EdgeKind = match &body.kind { + Some(k) => serde_json::from_value(json!(k)) + .map_err(|_| (StatusCode::BAD_REQUEST, format!("unknown kind {k:?}")))?, + None => original.kind, + }; + let mut amended = original.clone(); + amended.subject = body + .subject + .clone() + .unwrap_or_else(|| original.subject.clone()); + amended.kind = kind; + amended.object = body + .object + .clone() + .unwrap_or_else(|| original.object.clone()); + Ok(ReviewDecision::Edit { amended }) + } + other => Err((StatusCode::BAD_REQUEST, format!("unknown action {other:?}"))), + } +} + +async fn api_decision( + State(app): State, + Json(body): Json, +) -> Result, (StatusCode, String)> { + let mut session = app.session.lock().expect("session lock"); + let decision = build_decision(&session, &body)?; + session + .decide(body.key.clone(), decision) + .map_err(ratify_status)?; + Ok(Json(json!({ "progress": session.progress() }))) +} + +async fn api_approve_green( + State(app): State, +) -> Result, (StatusCode, String)> { + let mut session = app.session.lock().expect("session lock"); + let approved = session.approve_green_lane().map_err(ratify_status)?; + Ok(Json( + json!({ "approved": approved, "progress": session.progress() }), + )) +} + +async fn api_ratify( + State(app): State, +) -> Result, (StatusCode, String)> { + let mut session = app.session.lock().expect("session lock"); + let signed = session + .ratify_and_sign(&app.reviewer) + .map_err(ratify_status)?; + Ok(Json(json!({ + "base_rev": signed.base_rev, + "new_rev": signed.new_rev, + "edges": signed.edges.len(), + "reviewer": signed.ratification.reviewer.0, + }))) +} + +fn ratify_status(err: RatifyError) -> (StatusCode, String) { + let code = match &err { + RatifyError::UnknownCandidate(_) => StatusCode::NOT_FOUND, + RatifyError::IncompleteCoverage { .. } | RatifyError::AlreadySigned => StatusCode::CONFLICT, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }; + (code, err.to_string()) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::demo::demo_staged_run; + use smesh_world::ReviewerId; + + fn app(tag: &str) -> App { + let base = + std::env::temp_dir().join(format!("smesh-ratify-web-{tag}-{}", std::process::id())); + let _ = std::fs::remove_file(base.with_extension("decisions.json")); + let _ = std::fs::remove_file(base.with_extension("revision.json")); + App { + session: Arc::new(Mutex::new( + Session::open( + demo_staged_run(), + base.with_extension("decisions.json"), + base.with_extension("revision.json"), + ) + .unwrap(), + )), + reviewer: Arc::new( + ReviewerKey::load_or_generate(&base.with_extension("key"), ReviewerId("dj".into())) + .unwrap(), + ), + } + } + + #[tokio::test] + async fn state_endpoint_reports_lanes_and_progress() { + let state = api_state(State(app("state"))).await; + let v = state.0; + assert_eq!(v["base_rev"], "meridian-rev0"); + assert_eq!(v["progress"]["total"], 8); + assert_eq!(v["candidates"].as_array().unwrap().len(), 8); + assert!(v["signed"].is_null()); + } + + #[tokio::test] + async fn edit_decision_amends_and_ratify_gates_on_coverage() { + let app = app("edit"); + let key = { + let s = app.session.lock().unwrap(); + s.staged.candidates[0].key() + }; + let _ = api_decision( + State(app.clone()), + Json(DecisionBody { + key: key.clone(), + action: "edit".into(), + reason: String::new(), + subject: None, + kind: None, + object: Some("a formal demand for coverage".into()), + }), + ) + .await + .unwrap(); + { + let s = app.session.lock().unwrap(); + match s.decisions.get(&key).unwrap() { + ReviewDecision::Edit { amended } => { + assert_eq!(amended.object, "a formal demand for coverage"); + } + other => panic!("expected edit, got {other:?}"), + } + } + let err = api_ratify(State(app)).await.unwrap_err(); + assert_eq!(err.0, StatusCode::CONFLICT); + } + + #[tokio::test] + async fn full_flow_signs_through_the_kernel() { + let app = app("flow"); + let _ = api_approve_green(State(app.clone())).await.unwrap(); + let keys: Vec = { + let s = app.session.lock().unwrap(); + s.staged + .candidates + .iter() + .map(CandidateEdge::key) + .filter(|k| !s.decisions.contains_key(k)) + .collect() + }; + for key in keys { + let _ = api_decision( + State(app.clone()), + Json(DecisionBody { + key, + action: "approve".into(), + reason: String::new(), + subject: None, + kind: None, + object: None, + }), + ) + .await + .unwrap(); + } + let receipt = api_ratify(State(app.clone())).await.unwrap().0; + assert_eq!(receipt["edges"], 8); + assert_eq!(receipt["new_rev"].as_str().unwrap().len(), 64); + let state = api_state(State(app)).await.0; + assert_eq!(state["signed"]["new_rev"], receipt["new_rev"]); + } +} diff --git a/smesh-refinery/src/main.rs b/smesh-refinery/src/main.rs index c42d640..f587ad9 100644 --- a/smesh-refinery/src/main.rs +++ b/smesh-refinery/src/main.rs @@ -29,6 +29,16 @@ async fn main() { match refine_meridian(&client).await { Ok(report) => { print!("{}", report.render()); + if let Some(staged) = report.staged_run() { + let path = "refinery-staged.json"; + match serde_json::to_string_pretty(&staged) + .map_err(|e| e.to_string()) + .and_then(|json| std::fs::write(path, json).map_err(|e| e.to_string())) + { + Ok(()) => println!("staged run written to {path} (serve with smesh-ratify)"), + Err(e) => eprintln!("failed to write {path}: {e}"), + } + } let gate_passed = report .scorecard .as_ref() diff --git a/smesh-refinery/src/run.rs b/smesh-refinery/src/run.rs index a34c187..5b8c56c 100644 --- a/smesh-refinery/src/run.rs +++ b/smesh-refinery/src/run.rs @@ -9,7 +9,9 @@ use crate::verify::{audit_grounding, sentinel_pass}; use crate::{Oracle, RefineryError}; use smesh_world::corpus::Scorecard; use smesh_world::meridian; -use smesh_world::{CandidateEdge, Changeset, Lane, Staged}; +use smesh_world::{ + CandidateEdge, Changeset, EvidenceView, Lane, ProvenanceClass, RejectedView, Staged, StagedRun, +}; pub struct RunReport { pub docs: Vec, @@ -34,6 +36,64 @@ impl RunReport { counts } + /// Package the staged changeset for the ratification dashboard: + /// candidates plus reviewer-ready evidence (quote, native anchor, + /// surrounding context) resolved back through the CDM. + pub fn staged_run(&self) -> Option { + let staged = self.staged.as_ref()?; + let mut evidence = std::collections::BTreeMap::new(); + for edge in &staged.edges { + let ProvenanceClass::CorpusDerived { citations } = &edge.provenance else { + continue; + }; + let views: Vec = citations + .iter() + .filter_map(|citation| { + let named = self.docs.iter().find(|d| d.doc.id == citation.doc)?; + let text = &named.doc.canonical_text; + let mut lo = citation.span.start.saturating_sub(160); + while lo > 0 && !text.is_char_boundary(lo) { + lo -= 1; + } + let mut hi = (citation.span.end + 160).min(text.len()); + while hi < text.len() && !text.is_char_boundary(hi) { + hi += 1; + } + Some(EvidenceView { + doc_name: named.name.clone(), + quote: citation.quote.clone(), + anchor: named + .doc + .native_anchor(citation.span) + .map(ToString::to_string) + .unwrap_or_else(|| format!("offset {}", citation.span.start)), + context: text[lo..hi].to_owned(), + }) + }) + .collect(); + evidence.insert(edge.key(), views); + } + Some(StagedRun { + base_rev: staged.base_rev.clone(), + candidates: staged.edges.clone(), + evidence, + rejected: self + .rejected + .iter() + .map(|r| RejectedView { + role: format!("{:?}", r.role), + summary: format!( + "{} --[{}]--> {}", + r.emission.subject, r.emission.kind, r.emission.object + ), + reason: r.reason.clone(), + }) + .collect(), + contradictions_caught: self.contradictions_caught, + scorecard: self.scorecard.clone(), + }) + } + /// Human-readable run summary for the CLI. pub fn render(&self) -> String { let mut out = String::new(); @@ -212,6 +272,21 @@ mod tests { let (_, _, red) = report.lane_counts(); assert_eq!(red, 2); + let staged_run = report.staged_run().unwrap(); + assert_eq!(staged_run.candidates.len(), 3); + let memo_key = staged + .edges + .iter() + .find(|e| e.object == "45-day filing window") + .unwrap() + .key(); + let memo_evidence = &staged_run.evidence[&memo_key]; + assert_eq!(memo_evidence[0].doc_name, "memo-4417.eml"); + assert!(memo_evidence[0] + .anchor + .contains("memo-4417@meridianmutual.example")); + assert!(memo_evidence[0].context.contains("45 days")); + let card = report.scorecard.as_ref().unwrap(); assert_eq!(card.confabulated, 0, "firewalled emissions never score"); assert!(card.precision() > 0.6, "2 of 3 staged edges are gold"); diff --git a/smesh-world/src/cdm.rs b/smesh-world/src/cdm.rs index 7cdc8c4..64e4fee 100644 --- a/smesh-world/src/cdm.rs +++ b/smesh-world/src/cdm.rs @@ -42,7 +42,8 @@ pub enum SourceFormat { Unknown, } -/// Native coordinates in the original artifact, per format. +/// Native coordinates in the original artifact, per format. `Display` +/// renders the human-facing anchor label shown next to citations. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum NativeAnchor { PdfPage { @@ -64,6 +65,22 @@ pub enum NativeAnchor { }, } +impl fmt::Display for NativeAnchor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + NativeAnchor::PdfPage { page } => write!(f, "page {page}"), + NativeAnchor::EmailLine { message_id, line } => { + write!(f, "{message_id}, line {line}") + } + NativeAnchor::MarkdownHeading { heading_path, line } => { + write!(f, "{}, line {line}", heading_path.join(" › ")) + } + NativeAnchor::Line { line } => write!(f, "line {line}"), + NativeAnchor::ByteOffset { offset } => write!(f, "byte {offset}"), + } + } +} + /// A half-open byte range `start..end` into a document's canonical text. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct CdmSpan { diff --git a/smesh-world/src/delta.rs b/smesh-world/src/delta.rs index 4c7a904..65e418b 100644 --- a/smesh-world/src/delta.rs +++ b/smesh-world/src/delta.rs @@ -143,6 +143,37 @@ impl Changeset { } } +/// One citation prepared for human eyes: the quote, where it lives in the +/// original artifact, and enough surrounding context to judge it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EvidenceView { + pub doc_name: String, + pub quote: String, + pub anchor: String, + pub context: String, +} + +/// A firewalled emission, summarized for the transparency panel. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RejectedView { + pub role: String, + pub summary: String, + pub reason: String, +} + +/// The refinery → dashboard handoff: everything a reviewer needs to ratify, +/// serialized to disk between the mesh run and the human session. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StagedRun { + pub base_rev: String, + pub candidates: Vec, + /// Evidence per candidate `key()`, in citation order. + pub evidence: std::collections::BTreeMap>, + pub rejected: Vec, + pub contradictions_caught: usize, + pub scorecard: Option, +} + /// The Curator's output: a new revision whose identity commits to the base /// revision, the surviving edges, and the human ratification that authorized /// them. diff --git a/smesh-world/src/lib.rs b/smesh-world/src/lib.rs index d7ab8fd..71926e5 100644 --- a/smesh-world/src/lib.rs +++ b/smesh-world/src/lib.rs @@ -24,8 +24,8 @@ pub use candidate::{CandidateEdge, Citation, Judgment, ProvenanceClass, Verdict} pub use cdm::{CdmDocument, CdmSpan, DocId, NativeAnchor, SourceFormat}; pub use corpus::{DefectManifest, GoldEdge, GoldGraph, PlantedDefect, Scorecard}; pub use delta::{ - Changeset, Lane, RatificationRecord, Ratified, ReviewDecision, ReviewerId, Signature, Signed, - Staged, + Changeset, EvidenceView, Lane, RatificationRecord, Ratified, RejectedView, ReviewDecision, + ReviewerId, Signature, Signed, SignedChangeset, Staged, StagedRun, }; pub use intake::Registrar; pub use ontology::{ConsensusPolicy, EdgeKind, StructuralConstraint, TransportKind}; From f02b73f4a799d31c852d34e40ef761a794b2abd0 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Thu, 23 Jul 2026 00:36:17 -0700 Subject: [PATCH 06/13] docs(world): mark Phase 4 shipped in roadmap --- WORLD-MODEL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORLD-MODEL.md b/WORLD-MODEL.md index f36f32e..2eb5cf9 100644 --- a/WORLD-MODEL.md +++ b/WORLD-MODEL.md @@ -211,5 +211,5 @@ config-as-policy, CSV org roster, wiki export, scanned-memo PDFs. - **Phase 1 — DONE:** Tier 0 intake adapters → CDM (markdown, `.eml`, PDF). MIME multipart + OCR deferred to the vendor-attachment / scanned-doc work. - **Phase 2 — DONE:** Meridian gold graph + deterministic renderer + planted defects; every artifact ingests through the Registrar, every evidence quote grounds a citation, negative space verified truly absent. - **Phase 3 — SHIPPED (`smesh-refinery`):** Tier 1/2 experts over `smesh-agent`: 4 extractor lenses on K3, grounding firewall (verbatim quote → `Citation::grounded` or attributed rejection), Grounding Auditor (cross-family entailment), Contradiction Sentinel (deterministic supersession/uniqueness resolution + cross-family semantic pairs), staged changeset + lanes + scorecard. `cargo run -p smesh-refinery` = live Meridian run. Field/reputation feedback into `smesh-core` is Phase 3b, deferred. -- **Phase 4:** Ratification dashboard (Axum; prevue-api pattern) + Curator signing into AION. +- **Phase 4 — SHIPPED (`smesh-ratify`):** the ratification bench. Ed25519 reviewer identity, crash-safe decision ledger, total-coverage gate, lanes with evidence drill-in, firewall transparency, kernel type-state signing with revision receipt. `cargo run -p smesh-ratify -- --demo` or point it at `refinery-staged.json`. Curator → AION write is Phase 4b. - **Phase 5:** Scorecard harness; Meridian becomes the regression gate. From 50fe8368d61cb77156e752f2f7f7e7c8e57b6906 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Thu, 23 Jul 2026 00:39:57 -0700 Subject: [PATCH 07/13] fix(world): sentinel evidence quotes + scorecard canonicalization Sentinel semantic listing now carries each candidate verbatim quote - two edges can share a bland name ("PTO accrual") while their quoted substance conflicts; the first live run missed the boilerplate trap this way. Scorecard now matches on kind classes (GovernedBy/Requires/ Triggers assert the same fact class), alias-aware subjects (HR = Human Resources), canonicalized text, and object containment - exact-key equality was scoring naming disagreement as both FP and FN. Candidate dedup is canonicalization-aware (case/punctuation variants merge). --- smesh-refinery/src/extract.rs | 17 +++- smesh-refinery/src/roster.rs | 11 ++- smesh-refinery/src/verify.rs | 12 ++- smesh-world/src/corpus.rs | 143 ++++++++++++++++++++++++++++++++-- smesh-world/src/meridian.rs | 7 ++ 5 files changed, 176 insertions(+), 14 deletions(-) diff --git a/smesh-refinery/src/extract.rs b/smesh-refinery/src/extract.rs index 4ab1467..bbfdaff 100644 --- a/smesh-refinery/src/extract.rs +++ b/smesh-refinery/src/extract.rs @@ -8,6 +8,7 @@ use crate::packet::NamedDoc; use crate::roster; use crate::{extract_json, Oracle, RefineryError}; use serde::Deserialize; +use smesh_world::corpus::canon; use smesh_world::role::ModelPolicy; use smesh_world::{CandidateEdge, CdmSpan, Citation, EdgeKind, WorldRole}; @@ -114,10 +115,11 @@ pub fn ground( Err(e) => return reject(format!("citation rejected: {e}"), outcome, emission), }; + let (subj, obj) = (canon(&emission.subject), canon(&emission.object)); if let Some(existing) = outcome .candidates .iter_mut() - .find(|c| c.subject == emission.subject && c.kind == kind && c.object == emission.object) + .find(|c| c.kind == kind && canon(&c.subject) == subj && canon(&c.object) == obj) { if let smesh_world::ProvenanceClass::CorpusDerived { citations } = &mut existing.provenance { @@ -222,6 +224,19 @@ mod tests { assert_eq!(outcome.rejected.len(), 2); } + #[test] + fn case_variant_emissions_merge_into_one_candidate() { + let docs = docs(); + let mut outcome = ExtractionOutcome::default(); + let quote = "Claims must be filed within 30 days."; + ground(WorldRole::Policy, emission(quote), &docs, &mut outcome); + let mut variant = emission(quote); + variant.subject = "CLAIMS".into(); + variant.object = "30-Day Filing Window".into(); + ground(WorldRole::Policy, variant, &docs, &mut outcome); + assert_eq!(outcome.candidates.len(), 1); + } + #[test] fn kind_parsing_accepts_case_and_snake_variants() { assert_eq!(kind_from_str("GovernedBy"), Some(EdgeKind::GovernedBy)); diff --git a/smesh-refinery/src/roster.rs b/smesh-refinery/src/roster.rs index a229cb9..b6f7436 100644 --- a/smesh-refinery/src/roster.rs +++ b/smesh-refinery/src/roster.rs @@ -78,10 +78,13 @@ Respond with ONLY a JSON object: {"verdict": "corroborate" | "refute", "rational /// mechanics cannot see (different subjects, conflicting substance). pub fn sentinel_system() -> String { r#"You are a contradiction sentinel reviewing a numbered list of candidate -edges extracted from one organization's documents. Identify pairs whose -claims cannot BOTH be true operational rules at the same time (e.g. two -different limits for the same activity). Ignore pairs where one explicitly -supersedes the other. Do not flag mere overlap or restatement. +edges extracted from one organization's documents. Each line carries the +verbatim evidence quote behind the edge — judge the QUOTED SUBSTANCE, not +just the edge names: two edges with similar names can still conflict in +substance (different limits, rates, or deadlines for the same activity). +Identify pairs whose claims cannot BOTH be true operational rules at the +same time. Ignore pairs where one explicitly supersedes the other. Do not +flag mere overlap or restatement. Respond with ONLY a JSON array of index pairs, e.g. [[0,3],[2,7]]. An empty array [] means no contradictions."# diff --git a/smesh-refinery/src/verify.rs b/smesh-refinery/src/verify.rs index 1ca6de9..b06f2e0 100644 --- a/smesh-refinery/src/verify.rs +++ b/smesh-refinery/src/verify.rs @@ -155,12 +155,22 @@ pub async fn sentinel_pass( } if candidates.len() > 1 { + // Evidence quotes ride along: two edges can share a bland name + // ("PTO accrual") while their quoted substance conflicts — the + // Meridian boilerplate trap taught us names alone are not enough. let listing: String = candidates .iter() .enumerate() .map(|(i, c)| { + let quote = match &c.provenance { + ProvenanceClass::CorpusDerived { citations } => citations + .first() + .map(|ci| ci.quote.chars().take(160).collect::()) + .unwrap_or_default(), + ProvenanceClass::HumanAttested { .. } => String::new(), + }; format!( - "{i}. \"{}\" --[{:?}]--> \"{}\"\n", + "{i}. \"{}\" --[{:?}]--> \"{}\" evidence: \"{quote}\"\n", c.subject, c.kind, c.object ) }) diff --git a/smesh-world/src/corpus.rs b/smesh-world/src/corpus.rs index 5f7276f..7dc1cf6 100644 --- a/smesh-world/src/corpus.rs +++ b/smesh-world/src/corpus.rs @@ -20,6 +20,80 @@ impl GoldEdge { pub fn key(&self) -> String { format!("{}|{:?}|{}", self.subject, self.kind, self.object) } + + /// Semantic match against an observed candidate: same kind class, + /// alias-aware subject, canonicalized object with containment tolerance. + /// Exact-key equality measured naming agreement, not truth — the first + /// live run scored `Claims Requires 45-day filing window` as both a + /// false positive and a false negative against `Claims GovernedBy + /// 45-day filing window`. + pub fn matches(&self, observed: &CandidateEdge, aliases: &AliasTable) -> bool { + kind_class(self.kind) == kind_class(observed.kind) + && aliases.same_subject(&self.subject, &observed.subject) + && objects_match(&self.object, &observed.object) + } +} + +/// Groups of edge kinds that assert the same class of fact. Extractors +/// legitimately disagree about `GovernedBy` vs `Requires`; the scorecard +/// should not. +fn kind_class(kind: EdgeKind) -> u8 { + match kind { + EdgeKind::GovernedBy | EdgeKind::Requires | EdgeKind::Triggers => 0, + EdgeKind::DefinesTerm => 1, + EdgeKind::ScopedTo | EdgeKind::MemberOf => 2, + EdgeKind::Owns | EdgeKind::Operates => 3, + EdgeKind::ReportsTo => 4, + EdgeKind::Precedes => 5, + EdgeKind::Supersedes => 6, + } +} + +/// Lowercased, punctuation-free, whitespace-collapsed comparison form. +pub fn canon(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut pending_space = false; + for c in s.chars() { + if c.is_alphanumeric() { + if pending_space && !out.is_empty() { + out.push(' '); + } + pending_space = false; + out.extend(c.to_lowercase()); + } else { + pending_space = true; + } + } + out +} + +fn objects_match(gold: &str, observed: &str) -> bool { + let (g, o) = (canon(gold), canon(observed)); + if g == o { + return true; + } + let shorter = g.len().min(o.len()); + shorter >= 8 && (g.contains(&o) || o.contains(&g)) +} + +/// Declared name equivalences (e.g. `HR` ↔ `Human Resources`) so the gold +/// graph is not hostage to one arbitrary spelling. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AliasTable { + pub groups: Vec>, +} + +impl AliasTable { + pub fn same_subject(&self, a: &str, b: &str) -> bool { + let (ca, cb) = (canon(a), canon(b)); + if ca == cb { + return true; + } + self.groups.iter().any(|group| { + let canon_group: Vec = group.iter().map(|g| canon(g)).collect(); + canon_group.contains(&ca) && canon_group.contains(&cb) + }) + } } /// Each planted defect targets a specific subsystem of the mesh. @@ -72,6 +146,8 @@ impl DefectManifest { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct GoldGraph { pub edges: Vec, + #[serde(default)] + pub aliases: AliasTable, } /// The headline numbers. Confabulation rate targets zero — an edge asserted @@ -94,16 +170,28 @@ impl Scorecard { manifest: &DefectManifest, contradictions_caught: usize, ) -> Self { - let gold_keys: BTreeSet = gold.edges.iter().map(GoldEdge::key).collect(); - let observed_keys: BTreeSet = observed.iter().map(CandidateEdge::key).collect(); + let mut deduped: Vec<&CandidateEdge> = Vec::new(); + let mut seen: BTreeSet<(u8, String, String)> = BTreeSet::new(); + for e in observed { + if seen.insert((kind_class(e.kind), canon(&e.subject), canon(&e.object))) { + deduped.push(e); + } + } - let true_positives = observed_keys.intersection(&gold_keys).count(); - let false_positives = observed_keys.difference(&gold_keys).count(); - let false_negatives = gold_keys.difference(&observed_keys).count(); - let confabulated = observed + let true_positives = gold + .edges + .iter() + .filter(|g| deduped.iter().any(|o| g.matches(o, &gold.aliases))) + .count(); + let false_negatives = gold.edges.len() - true_positives; + let false_positives = deduped + .iter() + .filter(|o| !gold.edges.iter().any(|g| g.matches(o, &gold.aliases))) + .count(); + let confabulated = deduped .iter() .filter(|e| { - !gold_keys.contains(&e.key()) + !gold.edges.iter().any(|g| g.matches(e, &gold.aliases)) && match &e.provenance { ProvenanceClass::CorpusDerived { citations } => citations.is_empty(), ProvenanceClass::HumanAttested { .. } => false, @@ -170,9 +258,12 @@ mod tests { GoldEdge { subject: "Finance".into(), kind: EdgeKind::Owns, - object: "Ledger".into(), + object: "General Ledger".into(), }, ], + aliases: AliasTable { + groups: vec![vec!["Claims".into(), "Claims department".into()]], + }, } } @@ -243,4 +334,40 @@ mod tests { fn manifest_counts_planted_contradictions() { assert_eq!(manifest().planted_contradictions(), 1); } + + #[test] + fn canonical_matching_forgives_naming_noise_not_substance() { + let hits = vec![ + observed("Claims department", EdgeKind::ReportsTo, "Ops"), + observed("finance", EdgeKind::Operates, "the General Ledger"), + ]; + let card = Scorecard::evaluate(&gold(), &hits, &manifest(), 1); + assert_eq!( + card.true_positives, 2, + "alias subject, kind-class sibling, and object containment all match" + ); + assert_eq!(card.false_positives, 0); + + let miss = vec![observed("Claims", EdgeKind::ReportsTo, "Legal")]; + let card = Scorecard::evaluate(&gold(), &miss, &manifest(), 1); + assert_eq!(card.true_positives, 0, "different substance stays a miss"); + } + + #[test] + fn case_variant_duplicates_collapse_before_scoring() { + let hits = vec![ + observed("Claims", EdgeKind::ReportsTo, "Ops"), + observed("claims", EdgeKind::ReportsTo, "OPS"), + ]; + let card = Scorecard::evaluate(&gold(), &hits, &manifest(), 1); + assert_eq!(card.true_positives, 1); + assert_eq!(card.false_positives, 0); + } + + #[test] + fn canon_normalizes_case_punctuation_and_whitespace() { + assert_eq!(canon("IT-Security"), "it security"); + assert_eq!(canon(" Prior Authorization. "), "prior authorization"); + assert_eq!(canon("45-day filing window"), "45 day filing window"); + } } diff --git a/smesh-world/src/meridian.rs b/smesh-world/src/meridian.rs index b6266cc..f7c7840 100644 --- a/smesh-world/src/meridian.rs +++ b/smesh-world/src/meridian.rs @@ -97,6 +97,13 @@ pub fn corpus() -> MeridianCorpus { prior_auth.clone(), token_rotation.clone(), ], + aliases: crate::corpus::AliasTable { + groups: vec![ + vec!["HR".into(), "Human Resources".into()], + vec!["IT-Security".into(), "IT Security".into()], + vec!["Claims".into(), "Claims department".into()], + ], + }, }; let manifest = DefectManifest { From bf2bb2d8b1a4b5fcc4fd57a25f47f6de6ce388e2 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Thu, 23 Jul 2026 00:40:06 -0700 Subject: [PATCH 08/13] chore: gitignore refinery run artifacts --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 81b4229..3e039e8 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,6 @@ film/src/node_modules/ verify/tla/tla2tools.jar verify/tla/states/ verify/tla/*.old + +# Refinery run artifacts +refinery-staged.json* From bf265d52740ab97d473a57f835f384485ab3c5cd Mon Sep 17 00:00:00 2001 From: zuub-don Date: Wed, 19 Aug 2026 00:43:33 -0700 Subject: [PATCH 09/13] chore: reconcile ed25519-dalek after the rebase Both sides of the rebase declared it: main for signing attestations, with serde and pkcs8; this branch for its own use, at a 2.2 floor. One entry with the union of both satisfies each, and cargo rejects the duplicate outright. --- Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 528741b..cfcc807 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ async-trait = "0.1" # Crypto sha2 = "0.10" -ed25519-dalek = { version = "2.1", features = ["rand_core", "serde", "pkcs8"] } +ed25519-dalek = { version = "2.2", features = ["rand_core", "serde", "pkcs8"] } blake3 = "1.5" # PDF (Tier 0 intake + Meridian renderer) @@ -46,7 +46,6 @@ pdf-extract = "0.10" # Ratification dashboard axum = "0.8" -ed25519-dalek = { version = "2.2", features = ["rand_core"] } rand = "0.8" uuid = { version = "1.6", features = ["v4", "serde"] } From 2c4676380701ff3542bcd3c97852c3ec52549e06 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Wed, 19 Aug 2026 08:12:49 -0700 Subject: [PATCH 10/13] fix(world): close the holes review found in the signed path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six findings, all in the part of this design whose value is that it can be verified. **A ratification could be moved to a changeset it never approved.** The reviewer's signature covers the base revision, their identity and the decision map — not the edges. The type-state stops an unratified changeset being signed, but nothing stopped swapping records between two ratified ones with the same base. The result verified, having been approved by nobody, which is the single thing this design promises cannot happen. `sign` now refuses a record that does not correspond exactly to the edges in hand. The decision map is signed, so requiring that correspondence binds the signature to the edge set without changing what is signed. The existing test was called `signing_commits_to_base_edges_and_ratification` and only checked that changing the signature changed the revision hash. It was named for a property it did not enforce. There is now a test that performs the swap and expects refusal. **The curator's signing key was briefly world-readable.** It was written with `fs::write`, which uses the process umask, and chmod'd afterwards. A key that was momentarily readable is a key that may already be gone, and this one authorises every ratification. It is created private now, and a key file others can read is refused rather than loaded. **An empty quote passed the confabulation firewall.** `text.find("")` returns `Some(0)` for every document, so an empty quote satisfied "appears verbatim" and produced a citation pointing at nothing — precisely the claim-without- evidence the firewall exists to catch. Empty and trivially short quotes are rejected; below a few characters a match says more about the document than about the claim. **A permissions error looked like a clean slate.** Loading decisions and the signed revision treated every I/O error as "nothing recorded yet", so an unreadable file became an empty one and the next ratification would overwrite signed state that was still there. Only `NotFound` means absent now. **`citations[0]` panicked** on a candidate that arrived with none. The producer is a language model, so "cannot happen" was a claim about a model's output rather than about this code. **One malformed response aborted the whole run**, discarding every other expert's work. A model returning unparseable JSON is ordinary. It is recorded as that expert failing — in its own field, since there is no emission to attribute and inventing one would put words in the model's mouth — and the run continues. Co-Authored-By: Claude Opus 5 (1M context) --- smesh-ratify/src/signer.rs | 56 ++++++++++++++++++++--- smesh-ratify/src/state.rs | 12 +++-- smesh-refinery/src/extract.rs | 58 ++++++++++++++++++++++-- smesh-refinery/src/verify.rs | 9 +++- smesh-world/src/delta.rs | 84 ++++++++++++++++++++++++++++++++--- smesh-world/src/lib.rs | 2 + 6 files changed, 201 insertions(+), 20 deletions(-) diff --git a/smesh-ratify/src/signer.rs b/smesh-ratify/src/signer.rs index 0c5293a..8230380 100644 --- a/smesh-ratify/src/signer.rs +++ b/smesh-ratify/src/signer.rs @@ -15,6 +15,7 @@ impl ReviewerKey { /// Load the keypair at `path`, or generate and persist one (0600). pub fn load_or_generate(path: &Path, reviewer: ReviewerId) -> Result { let signing = if path.exists() { + Self::reject_if_exposed(path)?; let bytes = std::fs::read(path)?; let key: [u8; 32] = bytes.as_slice().try_into().map_err(|_| { RatifyError::Key(format!("{} is not a 32-byte seed", path.display())) @@ -25,17 +26,35 @@ impl ReviewerKey { std::fs::create_dir_all(parent)?; } let key = SigningKey::generate(&mut rand::rngs::OsRng); - std::fs::write(path, key.to_bytes())?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; - } + write_private(path, &key.to_bytes())?; key }; Ok(Self { reviewer, signing }) } + /// Refuse a key file anyone else on the box can read. + /// + /// A private key that was briefly world-readable is a private key that may + /// already have been copied, and this one authorises every ratification. + #[cfg(unix)] + fn reject_if_exposed(path: &Path) -> Result<(), RatifyError> { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(path)?.permissions().mode() & 0o077; + if mode != 0 { + return Err(RatifyError::Key(format!( + "{} is readable by others (mode {:o}); refusing to load a signing key", + path.display(), + mode + ))); + } + Ok(()) + } + + #[cfg(not(unix))] + fn reject_if_exposed(_path: &Path) -> Result<(), RatifyError> { + Ok(()) + } + pub fn verifying_key(&self) -> VerifyingKey { self.signing.verifying_key() } @@ -135,3 +154,28 @@ mod tests { std::fs::remove_file(&path).unwrap(); } } + +/// Create secret material already private, rather than fixing it afterwards. +/// +/// `fs::write` creates with the process umask — usually world-readable — and +/// relaxing the mode afterwards leaves a window in which the signing key is +/// readable by anyone on the machine. `create_new` with the mode set up front +/// closes it, and refuses to clobber an existing key. +#[cfg(unix)] +fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path)?; + file.write_all(bytes) +} + +/// Permissions are left to the platform here. +#[cfg(not(unix))] +fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + std::fs::write(path, bytes) +} diff --git a/smesh-ratify/src/state.rs b/smesh-ratify/src/state.rs index 658f6d1..aff291a 100644 --- a/smesh-ratify/src/state.rs +++ b/smesh-ratify/src/state.rs @@ -35,13 +35,19 @@ impl Session { decisions_path: PathBuf, revision_path: PathBuf, ) -> Result { + // Only a missing file means "nothing recorded yet". Treating every + // error that way turned a permissions problem or a bad disk into a + // clean slate, and the next ratification would overwrite signed state + // that was still there and merely unreadable. let decisions = match std::fs::read(&decisions_path) { Ok(bytes) => serde_json::from_slice(&bytes)?, - Err(_) => BTreeMap::new(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(), + Err(e) => return Err(RatifyError::Io(e)), }; let signed = match std::fs::read(&revision_path) { Ok(bytes) => Some(serde_json::from_slice(&bytes)?), - Err(_) => None, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(RatifyError::Io(e)), }; Ok(Self { staged, @@ -140,7 +146,7 @@ impl Session { let changeset = Changeset::stage(self.staged.base_rev.clone(), self.staged.candidates.clone())?; let outcome = changeset.ratify(record)?; - let signed = outcome.changeset.sign(outcome.ratification); + let signed = outcome.changeset.sign(outcome.ratification)?; std::fs::write(&self.revision_path, serde_json::to_vec_pretty(&signed)?)?; self.signed = Some(signed.clone()); diff --git a/smesh-refinery/src/extract.rs b/smesh-refinery/src/extract.rs index bbfdaff..eeec786 100644 --- a/smesh-refinery/src/extract.rs +++ b/smesh-refinery/src/extract.rs @@ -12,6 +12,12 @@ use smesh_world::corpus::canon; use smesh_world::role::ModelPolicy; use smesh_world::{CandidateEdge, CdmSpan, Citation, EdgeKind, WorldRole}; +/// Shortest quote that can count as grounding a claim. +/// +/// Not a magic number so much as a floor: below this, a match against the +/// document says more about the document than about the claim. +const MIN_QUOTE_CHARS: usize = 8; + #[derive(Debug, Clone, Deserialize)] pub struct Emission { pub subject: String, @@ -28,10 +34,23 @@ pub struct RejectedEmission { pub reason: String, } +/// An expert that produced nothing usable at all. +/// +/// Distinct from a rejected emission: there is no emission to attribute, so +/// recording it as one would put words in the model's mouth. Kept because this +/// module's contract is that nothing is silently gone. +#[derive(Debug, Clone)] +pub struct ExpertFailure { + pub role: WorldRole, + pub reason: String, +} + #[derive(Debug, Default)] pub struct ExtractionOutcome { pub candidates: Vec, pub rejected: Vec, + /// Experts whose response could not be parsed at all. + pub failures: Vec, } pub(crate) fn kind_from_str(s: &str) -> Option { @@ -63,10 +82,24 @@ pub async fn run_extractors( doc_block, ) .await?; - let emissions: Vec = serde_json::from_value(extract_json(&response)?) - .map_err(|e| RefineryError::Parse(format!("{role:?} emissions: {e}")))?; - for emission in emissions { - ground(role, emission, docs, &mut outcome); + // One expert returning malformed JSON used to abort the entire run and + // discard every other expert's work. A model producing something + // unparseable is ordinary, not exceptional: record it as that expert + // failing and carry on with the rest. + let emissions: Result, _> = extract_json(&response) + .map_err(|e| e.to_string()) + .and_then(|v| serde_json::from_value::>(v).map_err(|e| e.to_string())); + + match emissions { + Ok(emissions) => { + for emission in emissions { + ground(role, emission, docs, &mut outcome); + } + } + Err(e) => outcome.failures.push(ExpertFailure { + role, + reason: format!("unusable output: {e}"), + }), } } Ok(outcome) @@ -102,6 +135,23 @@ pub fn ground( emission, ); }; + // An empty quote is found at offset zero in every document, so it passes + // "appears verbatim" trivially and yields a citation that points at nothing. + // The firewall exists to stop exactly that: a claim carrying evidence that + // is not evidence. A floor on length is applied for the same reason — a + // one-character quote matches by accident, not by grounding. + let quote = emission.quote.trim(); + if quote.chars().count() < MIN_QUOTE_CHARS { + return reject( + format!( + "quote is {} characters; needs at least {MIN_QUOTE_CHARS} to ground anything", + quote.chars().count() + ), + outcome, + emission, + ); + } + let Some(start) = named.doc.canonical_text.find(&emission.quote) else { return reject( "quote not found verbatim in document".into(), diff --git a/smesh-refinery/src/verify.rs b/smesh-refinery/src/verify.rs index b06f2e0..674b843 100644 --- a/smesh-refinery/src/verify.rs +++ b/smesh-refinery/src/verify.rs @@ -55,7 +55,14 @@ pub async fn audit_grounding( let ProvenanceClass::CorpusDerived { citations } = &candidate.provenance else { continue; }; - let citation = citations[0].clone(); + // Indexing panicked on a candidate that reached here with no citations. + // The extractor is a language model, so "cannot happen" is a claim about + // a model's output rather than about this code. + let Some(citation) = citations.first().cloned() else { + return Err(RefineryError::Parse( + "candidate reached verification with no citation to audit".into(), + )); + }; let named = docs .iter() .find(|d| d.doc.id == citation.doc) diff --git a/smesh-world/src/delta.rs b/smesh-world/src/delta.rs index 65e418b..994d5ca 100644 --- a/smesh-world/src/delta.rs +++ b/smesh-world/src/delta.rs @@ -9,7 +9,7 @@ use crate::candidate::CandidateEdge; use crate::ontology::EdgeKind; use crate::WorldError; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct ReviewerId(pub String); @@ -187,19 +187,42 @@ pub struct SignedChangeset { impl Changeset { /// Only reachable through `ratify` — the type system is the protocol. - pub fn sign(self, ratification: RatificationRecord) -> SignedChangeset { + /// Attach a ratification, refusing one that does not cover these edges. + /// + /// The reviewer's signature covers the base revision, their identity and + /// the decision map — it does not cover the edges. So a record signed for + /// one changeset could previously be stapled onto a different one with the + /// same base revision, and the result verified, having been approved by + /// nobody. The premise of this whole design is that nothing reaches the + /// signed graph without a human ratification record; that only holds if the + /// record demonstrably concerns *these* edges. + /// + /// The decision map is signed, so requiring an exact correspondence between + /// it and the edges binds the signature to the edge set without changing + /// what is signed. + pub fn sign(self, ratification: RatificationRecord) -> Result { + let edge_keys: BTreeSet = self.edges.iter().map(|e| e.key()).collect(); + let decided: BTreeSet = ratification.decisions.keys().cloned().collect(); + + if let Some(missing) = edge_keys.difference(&decided).next() { + return Err(WorldError::UnreviewedCandidate(missing.clone())); + } + if let Some(extra) = decided.difference(&edge_keys).next() { + return Err(WorldError::ForeignRatification(extra.clone())); + } + let mut hasher = blake3::Hasher::new(); hasher.update(self.base_rev.as_bytes()); hasher.update(&ratification.signature.0); for edge in &self.edges { hasher.update(edge.key().as_bytes()); } - SignedChangeset { + Ok(SignedChangeset { base_rev: self.base_rev, new_rev: hasher.finalize().to_hex().to_string(), edges: self.edges, ratification, - } + }) } } @@ -347,13 +370,62 @@ mod tests { let outcome = staged .ratify(record("dj", vec![(a.key(), ReviewDecision::Approve)])) .unwrap(); - let signed = outcome.changeset.clone().sign(outcome.ratification.clone()); + let signed = outcome + .changeset + .clone() + .sign(outcome.ratification.clone()) + .unwrap(); assert_eq!(signed.base_rev, "rev0"); assert_eq!(signed.new_rev.len(), 64); let mut other_sig = outcome.ratification.clone(); other_sig.signature = Signature(vec![9; 64]); - let resigned = outcome.changeset.sign(other_sig); + let resigned = outcome.changeset.sign(other_sig).unwrap(); assert_ne!(signed.new_rev, resigned.new_rev); } + + #[test] + fn a_ratification_cannot_be_moved_to_a_changeset_it_did_not_approve() { + // The type-state already stops an unratified changeset being signed. + // What it does not stop is swapping records between two ratified ones: + // the reviewer's signature covers the base revision, their name and the + // decisions, not the edges. So a record approving one changeset could + // be stapled onto another with the same base, and the result would + // verify while having been approved by nobody. That is the single thing + // this design promises cannot happen. + let d = doc(); + let approved = edge(&d, "Claims", EdgeKind::ReportsTo, "Ops"); + let smuggled = edge(&d, "Claims", EdgeKind::ReportsTo, "Finance"); + + let honest = Changeset::stage("rev0".into(), vec![approved.clone()]) + .unwrap() + .ratify(record( + "dj", + vec![(approved.key(), ReviewDecision::Approve)], + )) + .unwrap(); + + // A separate changeset, properly ratified on its own terms. + let other = Changeset::stage("rev0".into(), vec![smuggled.clone()]) + .unwrap() + .ratify(record( + "dj", + vec![(smuggled.key(), ReviewDecision::Approve)], + )) + .unwrap(); + + // Now swap: sign the second changeset with the first one's record. + assert_eq!( + other + .changeset + .sign(honest.ratification.clone()) + .unwrap_err(), + WorldError::UnreviewedCandidate(smuggled.key()), + "an edge nobody approved must not ride in on someone else's signature" + ); + + // Its own record still works, so the check rejects the swap and not + // every signature. + assert!(honest.changeset.sign(honest.ratification).is_ok()); + } } diff --git a/smesh-world/src/lib.rs b/smesh-world/src/lib.rs index 71926e5..7379a59 100644 --- a/smesh-world/src/lib.rs +++ b/smesh-world/src/lib.rs @@ -55,4 +55,6 @@ pub enum WorldError { Malformed(String), #[error("ratification record does not cover candidate {0}")] UnreviewedCandidate(String), + #[error("ratification rules on {0}, which is not in this changeset")] + ForeignRatification(String), } From 09b93caffeb2eddee2a0cd3358c85d2fb505affe Mon Sep 17 00:00:00 2001 From: zuub-don Date: Wed, 19 Aug 2026 09:25:36 -0700 Subject: [PATCH 11/13] fix(refinery): report the expert failures instead of dropping them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-through gap in the previous commit. `ExpertFailure` was added so a model returning unusable output would not vanish, and then `refine` dropped the vector on the way to the run report — recorded and lost one level up, which is the same silence in a different place. Failures now reach `RunReport` and are printed as their own line, not folded in with rejected emissions. The distinction matters: a rejected emission is a judgement the firewall made, whereas a failed expert made no judgement at all. A run that looks clean because half the roster failed silently is a worse outcome than one that looks bad, and without this the two are indistinguishable. Co-Authored-By: Claude Opus 5 (1M context) --- smesh-refinery/src/run.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/smesh-refinery/src/run.rs b/smesh-refinery/src/run.rs index 5b8c56c..41125f9 100644 --- a/smesh-refinery/src/run.rs +++ b/smesh-refinery/src/run.rs @@ -3,7 +3,7 @@ //! The refinery's output stops exactly where the human's authority begins — //! a `Changeset` with lanes. Ratification and signing are Phase 4. -use crate::extract::{run_extractors, RejectedEmission}; +use crate::extract::{run_extractors, ExpertFailure, RejectedEmission}; use crate::packet::{ingest_all, prompt_block, NamedDoc}; use crate::verify::{audit_grounding, sentinel_pass}; use crate::{Oracle, RefineryError}; @@ -17,6 +17,12 @@ pub struct RunReport { pub docs: Vec, pub staged: Option>, pub rejected: Vec, + /// Experts that returned nothing usable at all. + /// + /// Separate from `rejected`, which is per-emission. An empty run with three + /// silent failures and an empty run with three experts that genuinely found + /// nothing are very different results, and without this they look identical. + pub failures: Vec, pub contradictions_caught: usize, pub scorecard: Option, } @@ -110,6 +116,20 @@ impl RunReport { r.role, r.emission.subject, r.emission.kind, r.emission.object, r.reason )); } + + // Surfaced rather than counted alongside rejections. An expert that + // returned nothing usable did not contribute a judgement at all, and a + // run that looks clean because half the roster failed silently is worse + // than one that looks bad. + if !self.failures.is_empty() { + out.push_str(&format!( + "experts that returned nothing usable: {}\n", + self.failures.len() + )); + for f in &self.failures { + out.push_str(&format!(" failed [{:?}] {}\n", f.role, f.reason)); + } + } match &self.staged { None => out.push_str("staged changeset: EMPTY — nothing survived\n"), Some(staged) => { @@ -171,6 +191,7 @@ pub async fn refine( docs, staged, rejected: outcome.rejected, + failures: outcome.failures, contradictions_caught, scorecard: None, }) From e54cddb7b722f86c2e295d1afcd1ae6e856231f2 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Wed, 19 Aug 2026 09:36:24 -0700 Subject: [PATCH 12/13] fix(world): stop showing a reviewer text the source does not contain Three bugs that all end the same way: a human approves a claim while looking at something the document never said. **Evidence quotes were corrupted by `String.replace`.** The replacement string expands `$&`, ``$` ``, `$'` and `$1`, so any quote containing a dollar sign rendered as something else entirely. This is the text a reviewer reads immediately before deciding, which makes it the one string in the system that must be reproduced exactly. Spliced by index now, marking the first occurrence only. **Headings ending in a hash lost it.** `trim_end_matches('#')` removed trailing hashes unconditionally, so "## Sprint #" became "Sprint" and "### C#" became "C". CommonMark only treats a *closing sequence* as decoration, and requires a space before it; anything else is part of the heading. An anchor naming the wrong section is a citation pointing somewhere the reader cannot check. **Encoded and multipart email bodies were stored verbatim.** A quoted-printable body cited directly quotes the transfer encoding, showing "=E2=80=99" where an apostrophe belongs; a multipart body buries the prose among MIME boundaries. Both are now refused with a reason. Decoding them is a real adapter feature rather than a line of code, and until it exists refusing the format is honest where storing a mangled reading of it is not. Co-Authored-By: Claude Opus 5 (1M context) --- smesh-ratify/assets/index.html | 10 ++++- smesh-world/src/intake/eml.rs | 61 ++++++++++++++++++++++++++++++ smesh-world/src/intake/markdown.rs | 13 ++++++- 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/smesh-ratify/assets/index.html b/smesh-ratify/assets/index.html index 2249f6d..1e951b4 100644 --- a/smesh-ratify/assets/index.html +++ b/smesh-ratify/assets/index.html @@ -313,7 +313,15 @@

The mesh proposes. You dispose.

function evidenceHtml(ev){ const ctx = esc(ev.context); const q = esc(ev.quote); - const marked = ctx.includes(q) ? ctx.replace(q, `${q}`) : `${ctx}
${q}`; + // String.replace expands $& $` $' and $1 in the replacement, so a quote + // containing a dollar sign is rendered as something the document does not + // say. This is the text a reviewer reads before approving, so a mangled + // quote means approving something they were never shown. Splice by index + // instead, and only mark the first occurrence. + const at = ctx.indexOf(q); + const marked = at === -1 + ? `${ctx}
${q}` + : ctx.slice(0, at) + `${q}` + ctx.slice(at + q.length); return `
diff --git a/smesh-world/src/intake/eml.rs b/smesh-world/src/intake/eml.rs index 71cbbbf..de8415f 100644 --- a/smesh-world/src/intake/eml.rs +++ b/smesh-world/src/intake/eml.rs @@ -24,6 +24,34 @@ pub fn ingest(bytes: &[u8]) -> Result { .map(|v| v.trim_matches(['<', '>']).to_owned()) .ok_or_else(|| WorldError::Malformed("missing Message-ID header".into()))?; + // Refuse what we cannot faithfully represent. + // + // A quoted-printable or base64 body stored verbatim is not the text the + // author wrote: citations would quote transfer encoding, and a reviewer + // would be shown "=E2=80=99" where an apostrophe belongs. A multipart + // message stored verbatim buries the prose in MIME boundaries. Better to + // reject the format outright than to ground a claim in a mangled reading of + // it — decoding these is a real job for the adapter, not a one-liner here. + if let Some(encoding) = headers.get("content-transfer-encoding") { + let encoding = encoding.trim().to_ascii_lowercase(); + if !matches!(encoding.as_str(), "7bit" | "8bit" | "binary" | "") { + return Err(WorldError::Malformed(format!( + "content-transfer-encoding {encoding} is not decoded yet; \ + storing it verbatim would cite the encoding rather than the text" + ))); + } + } + if let Some(content_type) = headers.get("content-type") { + let content_type = content_type.trim().to_ascii_lowercase(); + if content_type.starts_with("multipart/") { + return Err(WorldError::Malformed( + "multipart messages are not split yet; storing one verbatim \ + would bury the prose in MIME boundaries" + .into(), + )); + } + } + let canonical = body.to_owned(); let mut anchors = Vec::new(); let mut offset = 0usize; @@ -149,3 +177,36 @@ mod tests { ); } } + +#[cfg(test)] +mod encoding_tests { + use super::*; + + #[test] + fn an_encoded_body_is_refused_rather_than_stored_raw() { + // Storing this verbatim would let a citation quote "=E2=80=99" and + // present it to a reviewer as the author's words. + let raw = b"Message-ID: \nContent-Transfer-Encoding: quoted-printable\n\nit=E2=80=99s fine\n"; + let err = ingest(raw).unwrap_err(); + assert!( + matches!(err, WorldError::Malformed(ref m) if m.contains("quoted-printable")), + "got {err:?}" + ); + } + + #[test] + fn a_multipart_body_is_refused() { + let raw = b"Message-ID: \nContent-Type: multipart/mixed; boundary=xyz\n\n--xyz\nhello\n--xyz--\n"; + let err = ingest(raw).unwrap_err(); + assert!( + matches!(err, WorldError::Malformed(ref m) if m.contains("multipart")), + "got {err:?}" + ); + } + + #[test] + fn a_plain_body_still_ingests() { + let raw = b"Message-ID: \nContent-Type: text/plain\n\nthe handbook says two weeks\n"; + assert!(ingest(raw).is_ok()); + } +} diff --git a/smesh-world/src/intake/markdown.rs b/smesh-world/src/intake/markdown.rs index 3446106..233b8c9 100644 --- a/smesh-world/src/intake/markdown.rs +++ b/smesh-world/src/intake/markdown.rs @@ -56,7 +56,18 @@ fn parse_heading(line: &str) -> Option<(u8, String)> { return None; } let rest = line[hashes..].strip_prefix(' ')?; - let heading = rest.trim().trim_end_matches('#').trim(); + + // Only a *closing sequence* is decoration, and CommonMark requires it to be + // preceded by a space. `trim_end_matches` removed hashes unconditionally, + // so "## Sprint #" became "Sprint" and "### C#" became "C" — the heading + // silently lost the character that identified it. + let trimmed = rest.trim(); + let heading = match trimmed.rsplit_once(' ') { + Some((before, tail)) if !tail.is_empty() && tail.bytes().all(|b| b == b'#') => { + before.trim() + } + _ => trimmed, + }; if heading.is_empty() { return None; } From e42349b6b98de4df05fafbda74ff05949fa2520e Mon Sep 17 00:00:00 2001 From: zuub-don Date: Wed, 19 Aug 2026 09:45:09 -0700 Subject: [PATCH 13/13] fix(world): the remaining review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight, none in the signed path, but several with consequences worth naming. **Candidate keys could collide across the separator.** `subject|kind|object` made `("a|b", X, "c")` and `("a", X, "b|c")` identical. That key decides which ratification decision applies to which edge, so a collision is two different claims sharing one human approval. Length-prefixed now, with a test. **Contradictions were counted once per conflicting partner** rather than once per pair, so a single disputed claim inflated both the conflict total and the detection rate derived from it. Now counted per pair — and the first attempt at this nested the guard inside itself, which suppressed the count entirely and was caught by two existing tests doing their job. **A poisoned mutex took the whole server down.** One panicking handler made every later request fail. The session is plain data with no invariant a panic could leave half-applied, so recovering keeps a reviewer's in-progress work reachable. **Two copies of the context window, correct in one of them.** The refinery clipped to character boundaries; the demo sliced by byte offset and would panic on any multi-byte character in a document. One copy now, in the crate both already depend on. **The UI failed silently.** A failed `/api/state` left the previous render on screen, so a reviewer could go on approving against state the server had moved past; a failed bulk approval looked identical to a deliberate no-op. Both say so now. Bulk approval is exactly the action a reviewer must not be wrong about having taken. **No visible focus ring.** Ratification is a keyboard-driven review, and a control you cannot see yourself on is one you can approve by accident. Reviewer keys and session files are ignored at the repository root rather than only inside the crate. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 5 +++++ smesh-ratify/assets/index.html | 40 +++++++++++++++++++++++++++++++--- smesh-ratify/src/demo.rs | 5 ++--- smesh-ratify/src/web.rs | 27 +++++++++++++++++++---- smesh-refinery/src/verify.rs | 24 ++++++++++---------- smesh-world/src/candidate.rs | 29 +++++++++++++++++++++++- smesh-world/src/lib.rs | 31 ++++++++++++++++++++++++++ 7 files changed, 139 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 3e039e8..f76bfe9 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,8 @@ verify/tla/*.old # Refinery run artifacts refinery-staged.json* + +# Reviewer signing keys and ratification sessions +.ratify/ +demo-session.* +*.reviewer.key diff --git a/smesh-ratify/assets/index.html b/smesh-ratify/assets/index.html index 1e951b4..43c1e12 100644 --- a/smesh-ratify/assets/index.html +++ b/smesh-ratify/assets/index.html @@ -213,9 +213,17 @@ @media (prefers-reduced-motion:reduce){ *,*::before,*::after{transition-duration:.01ms!important;animation-duration:.01ms!important} } + +/* A visible focus ring. Ratification is a keyboard-driven review, and a + control you cannot see yourself on is one you can approve by accident. */ +#errorbar{background:#7f1d1d;color:#fee2e2;padding:.6rem 1rem;font-size:.9rem} +a:focus-visible, button:focus-visible, [tabindex]:focus-visible, summary:focus-visible{ + outline:3px solid #60a5fa; outline-offset:2px; border-radius:2px; +} +

The mesh proposes. You dispose.

let STATE = null; const open = new Set(); +function showError(msg){ + const bar = document.getElementById("errorbar"); + if(!bar) return; + bar.textContent = msg; + bar.hidden = false; +} + async function load(){ - STATE = await (await fetch("/api/state")).json(); - render(); + // A failed fetch used to reject silently and leave the last render on + // screen, so a reviewer could go on approving against state the server had + // already moved past. Say so instead. + try { + const res = await fetch("/api/state"); + if(!res.ok) throw new Error(`server returned ${res.status}`); + STATE = await res.json(); + const bar = document.getElementById("errorbar"); + if(bar) bar.hidden = true; + render(); + } catch (e) { + showError(`Could not load the review session: ${e.message}. What you see may be out of date.`); + } } function decisionChip(d){ @@ -477,8 +503,16 @@

Revision signed

}); const batch = document.getElementById("batch-green"); if(batch) batch.onclick = async () => { + // A failure here silently did nothing, and the reload made it look like a + // deliberate no-op. Bulk approval is exactly the action a reviewer must not + // be wrong about having taken. const res = await fetch("/api/approve-green", {method:"POST"}); - if(res.ok){ const j = await res.json(); toast(`${j.approved} green candidates approved`); } + if(res.ok){ + const j = await res.json(); + toast(`${j.approved} green candidates approved`); + } else { + showError(`Bulk approval failed (${res.status}). Nothing was approved.`); + } await load(); }; const signBtn = document.getElementById("sign-btn"); diff --git a/smesh-ratify/src/demo.rs b/smesh-ratify/src/demo.rs index a42364a..634b375 100644 --- a/smesh-ratify/src/demo.rs +++ b/smesh-ratify/src/demo.rs @@ -51,8 +51,6 @@ impl Bench { .unwrap_or_else(|| panic!("{quote:?} not in {doc_name}")); let span = CdmSpan::new(start, start + quote.len()); let citation = Citation::grounded(doc, span).expect("demo quotes ground"); - let lo = start.saturating_sub(120); - let hi = (span.end + 120).min(doc.canonical_text.len()); let view = EvidenceView { doc_name: doc_name.to_owned(), quote: quote.to_owned(), @@ -60,7 +58,8 @@ impl Bench { .native_anchor(span) .map(ToString::to_string) .unwrap_or_default(), - context: doc.canonical_text[lo..hi].to_owned(), + context: smesh_world::context_window(&doc.canonical_text, span.start, span.end, 120) + .to_owned(), }; let edge = CandidateEdge::emit(role, subject.into(), kind, object.into(), vec![citation]) .expect("demo emissions are valid"); diff --git a/smesh-ratify/src/web.rs b/smesh-ratify/src/web.rs index c9128c8..b7b08ef 100644 --- a/smesh-ratify/src/web.rs +++ b/smesh-ratify/src/web.rs @@ -94,8 +94,18 @@ fn candidate_view(session: &Session, c: &CandidateEdge) -> CandidateView { } } +/// Take the session lock, tolerating a poisoned one. +/// +/// A panic elsewhere in a handler poisons the mutex, and propagating that +/// turns one failed request into a server that refuses every subsequent one. +/// The session is a plain data structure — there is no invariant a panic could +/// have left half-applied — so recovering is safe and keeps a reviewer's +/// in-progress work reachable. async fn api_state(State(app): State) -> Json { - let session = app.session.lock().expect("session lock"); + let session = app + .session + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); let candidates: Vec = session .staged .candidates @@ -174,7 +184,10 @@ async fn api_decision( State(app): State, Json(body): Json, ) -> Result, (StatusCode, String)> { - let mut session = app.session.lock().expect("session lock"); + let mut session = app + .session + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); let decision = build_decision(&session, &body)?; session .decide(body.key.clone(), decision) @@ -185,7 +198,10 @@ async fn api_decision( async fn api_approve_green( State(app): State, ) -> Result, (StatusCode, String)> { - let mut session = app.session.lock().expect("session lock"); + let mut session = app + .session + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); let approved = session.approve_green_lane().map_err(ratify_status)?; Ok(Json( json!({ "approved": approved, "progress": session.progress() }), @@ -195,7 +211,10 @@ async fn api_approve_green( async fn api_ratify( State(app): State, ) -> Result, (StatusCode, String)> { - let mut session = app.session.lock().expect("session lock"); + let mut session = app + .session + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); let signed = session .ratify_and_sign(&app.reviewer) .map_err(ratify_status)?; diff --git a/smesh-refinery/src/verify.rs b/smesh-refinery/src/verify.rs index 674b843..200c152 100644 --- a/smesh-refinery/src/verify.rs +++ b/smesh-refinery/src/verify.rs @@ -32,15 +32,7 @@ struct AuditorResponse { } fn context_window(text: &str, start: usize, end: usize) -> &str { - let mut lo = start.saturating_sub(200); - while lo > 0 && !text.is_char_boundary(lo) { - lo -= 1; - } - let mut hi = (end + 200).min(text.len()); - while hi < text.len() && !text.is_char_boundary(hi) { - hi += 1; - } - &text[lo..hi] + smesh_world::context_window(text, start, end, 200) } /// Run the Grounding Auditor over every corpus-derived candidate. @@ -120,6 +112,12 @@ pub async fn sentinel_pass( let mut refutations: BTreeMap = BTreeMap::new(); let mut conflicts = 0usize; + // Contradictions are counted per pair. A candidate that conflicts with + // several others added one per conflict, so a single disputed claim + // inflated the total and the detection rate computed from it. + let mut counted_pairs: std::collections::HashSet<(usize, usize)> = + std::collections::HashSet::new(); + let mut count_pair = move |i: usize, j: usize| counted_pairs.insert((i.min(j), i.max(j))); let mut groups: BTreeMap<(String, String), Vec> = BTreeMap::new(); for (i, c) in candidates.iter().enumerate() { @@ -149,7 +147,9 @@ pub async fn sentinel_pass( .entry(j) .or_insert_with(|| format!("superseded by {winner}")); } else { - conflicts += 1; + if count_pair(i, j) { + conflicts += 1; + } refutations .entry(i) .or_insert_with(|| format!("conflicts with {oj:?} for the same subject")); @@ -200,7 +200,9 @@ pub async fn sentinel_pass( if related(&candidates[i].object, &candidates[j].object) { continue; } - conflicts += 1; + if count_pair(i, j) { + conflicts += 1; + } let (oi, oj) = (candidates[i].object.clone(), candidates[j].object.clone()); refutations .entry(i) diff --git a/smesh-world/src/candidate.rs b/smesh-world/src/candidate.rs index a0a177d..f0acd3c 100644 --- a/smesh-world/src/candidate.rs +++ b/smesh-world/src/candidate.rs @@ -115,8 +115,26 @@ impl CandidateEdge { } /// Stable identity used by ratification records and dedup. + /// + /// Length-prefixed rather than delimited. A bare separator means a subject + /// containing it can impersonate another candidate — `("a|b", X, "c")` and + /// `("a", X, "b|c")` produced the same key. That key decides which + /// ratification decision applies to which edge, so a collision is two + /// different claims sharing one human approval. pub fn key(&self) -> String { - format!("{}|{:?}|{}", self.subject, self.kind, self.object) + Self::key_parts(&self.subject, self.kind, &self.object) + } + + /// The key computation, over its parts. + pub fn key_parts(subject: &str, kind: EdgeKind, object: &str) -> String { + format!( + "{}:{}|{:?}|{}:{}", + subject.len(), + subject, + kind, + object.len(), + object + ) } pub fn record(&mut self, verdict: Verdict) { @@ -156,6 +174,15 @@ impl CandidateEdge { #[cfg(test)] mod tests { + #[test] + fn keys_cannot_collide_across_a_separator() { + // The key decides which ratification decision applies to which edge, so + // two different claims sharing one is two claims sharing one approval. + let a = CandidateEdge::key_parts("a|b", EdgeKind::ReportsTo, "c"); + let b = CandidateEdge::key_parts("a", EdgeKind::ReportsTo, "b|c"); + assert_ne!(a, b); + } + use super::*; use crate::cdm::{DocMetadata, SourceFormat}; diff --git a/smesh-world/src/lib.rs b/smesh-world/src/lib.rs index 7379a59..ea22045 100644 --- a/smesh-world/src/lib.rs +++ b/smesh-world/src/lib.rs @@ -58,3 +58,34 @@ pub enum WorldError { #[error("ratification rules on {0}, which is not in this changeset")] ForeignRatification(String), } + +/// A window of surrounding text, clipped to character boundaries. +/// +/// Slicing a `str` by byte offset panics when the cut lands inside a multi-byte +/// character, and the text here comes from arbitrary documents. Lived in two +/// places, correct in one of them; keeping a single copy is the point. +pub fn context_window(text: &str, start: usize, end: usize, radius: usize) -> &str { + let mut lo = start.saturating_sub(radius); + while lo > 0 && !text.is_char_boundary(lo) { + lo -= 1; + } + let mut hi = (end + radius).min(text.len()); + while hi < text.len() && !text.is_char_boundary(hi) { + hi += 1; + } + &text[lo..hi] +} + +#[cfg(test)] +mod context_window_tests { + #[test] + fn a_window_never_splits_a_character() { + let text = "policy: it’s ninety days — confirmed"; + for start in 0..text.len() { + for radius in [0usize, 1, 3, 200] { + // Must not panic for any offset or radius. + let _ = super::context_window(text, start, start, radius); + } + } + } +}