From 4594dbe96932dcceeb0b35b9233cfd857fa1331f Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 20:28:59 -0700 Subject: [PATCH 01/14] core: expose relay reasoning and the attester set `should_relay` returned only a yes/no and a dampening factor, which is enough to act on but not enough to explain. The decision is probabilistic, so after the fact there was no way to tell a signal that was refused on trust from one that lost a coin flip. `relay_decision` returns the score, the trust that fed it, and the roll that resolved it. `should_relay` now delegates to it, so behaviour is unchanged. Also adds `Node::attesters`: the origin of a signal plus everyone who reinforced it. Reinforcement is an independent assertion of the same claim, so the size of that set is how many parties corroborate it. Relaying a signal does not put you in it. Co-Authored-By: Claude Opus 5 (1M context) --- smesh-core/src/lib.rs | 2 +- smesh-core/src/node.rs | 84 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/smesh-core/src/lib.rs b/smesh-core/src/lib.rs index f840818..773baa2 100644 --- a/smesh-core/src/lib.rs +++ b/smesh-core/src/lib.rs @@ -46,7 +46,7 @@ pub mod trust; pub use error::{Result, SmeshError}; pub use field::Field; pub use network::{Hypha, Network, NetworkTopology}; -pub use node::{MaliciousBehavior, Node, NodeConfig, NodeId}; +pub use node::{MaliciousBehavior, Node, NodeConfig, NodeId, RelayDecision}; pub use payload::{ AgentSignalType, FindingPayload, FindingPayloadCompact, TaskPayload, TaskPayloadCompact, ThreatPayload, ThreatPayloadCompact, diff --git a/smesh-core/src/node.rs b/smesh-core/src/node.rs index 4e2b6f8..6fece22 100644 --- a/smesh-core/src/node.rs +++ b/smesh-core/src/node.rs @@ -96,6 +96,28 @@ pub struct NodeStats { pub escalations_triggered: u64, } +/// The full reasoning behind a relay choice. +/// +/// Relaying is probabilistic: `relay` is `roll < propagation_score`. Recording +/// both makes an otherwise unreproducible decision auditable after the fact. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RelayDecision { + /// Whether the signal is forwarded. + pub relay: bool, + /// Intensity multiplier applied to the forwarded copy. + pub dampening: f64, + /// Probability the relay was granted with. + pub propagation_score: f64, + /// This node's trust in the signal's origin. + pub origin_trust: f64, + /// The draw that resolved the decision. + pub roll: f64, + /// Hops left in the signal's budget when it was considered. + pub remaining_hops: u32, + /// Set when the signal was refused outright, before any roll. + pub veto: Option, +} + impl Node { /// Create a new node with default configuration pub fn new() -> Self { @@ -170,28 +192,76 @@ impl Node { /// Decide whether to relay a signal and with what dampening pub fn should_relay(&self, signal: &Signal, remaining_hops: u32) -> (bool, f64) { + let decision = self.relay_decision(signal, remaining_hops); + (decision.relay, decision.dampening) + } + + /// Decide whether to relay a signal, returning the full reasoning. + /// + /// [`Node::should_relay`] is the terse form. This one exposes the score, + /// the trust that fed it and the die roll that resolved it, so a relay + /// choice can be journalled and replayed rather than merely observed. + pub fn relay_decision(&self, signal: &Signal, remaining_hops: u32) -> RelayDecision { + let origin_trust = self.get_trust(&signal.origin_node_id); + let dampening = if origin_trust > 0.7 { 0.9 } else { 0.7 }; + + let vetoed = |reason: &str| RelayDecision { + relay: false, + dampening: 0.0, + propagation_score: 0.0, + origin_trust, + roll: 0.0, + remaining_hops, + veto: Some(reason.to_string()), + }; + if remaining_hops == 0 { - return (false, 0.0); + return vetoed("hop budget exhausted"); } // Eclipse attackers black-hole traffic: they accept signals but never // forward them, blocking diffusion paths that route through them. if self.is_malicious && self.malicious_behavior == MaliciousBehavior::Eclipse { - return (false, 0.0); + return vetoed("eclipse node black-holes traffic"); } - let origin_trust = self.get_trust(&signal.origin_node_id); let effective = signal.confidence * signal.current_intensity; // Propagation score - let prop_score = effective * origin_trust * (remaining_hops as f64 / signal.radius as f64); + let propagation_score = + effective * origin_trust * (remaining_hops as f64 / signal.radius as f64); // Probabilistic relay decision using cryptographically secure RNG let mut rng = rand::thread_rng(); - let should_relay = rng.gen::() < prop_score; - let dampening = if origin_trust > 0.7 { 0.9 } else { 0.7 }; + let roll = rng.gen::(); + + RelayDecision { + relay: roll < propagation_score, + dampening, + propagation_score, + origin_trust, + roll, + remaining_hops, + veto: None, + } + } - (should_relay, dampening) + /// Everyone who attests to a signal: its origin plus every reinforcer. + /// + /// Reinforcement is an *independent* attestation to the same claim, so the + /// size of this set is how many parties corroborate it. Relaying a signal + /// does not put you in it — only asserting it does. + pub fn attesters(signal: &Signal) -> Vec { + let mut out = Vec::with_capacity(signal.reinforced_by.len() + 1); + if !signal.origin_node_id.is_empty() { + out.push(signal.origin_node_id.clone()); + } + for id in &signal.reinforced_by { + if !out.contains(id) { + out.push(id.clone()); + } + } + out } /// Decide whether to trigger SMESH+ escalation From 382b61b7a1f10f1170831bff57b29dcad0091387 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 20:29:18 -0700 Subject: [PATCH 02/14] runtime: wire the QUIC transport into the mesh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transport was complete and had never been executed. Nothing in the workspace constructed a `QuicTransport`, so the runtime was a single-process simulation with a networking layer sitting beside it. Running it surfaced three latent faults: - `rustls` 0.23 refuses to choose a crypto provider when more than one is compiled in, so every call to `QuicTransport::new` panicked. - `connect` pooled a dialled connection but never read from it. Only the accept loop pumped streams, so a node that dialled out could send and would never receive. - `handle_stream` allocated from an attacker-controlled length prefix without consulting `max_message_size`, which was configured and unused. Diffusion also had to change shape. `Network::tick` expands a signal by walking the whole graph and mutating one shared reached set: a god's-eye BFS that no node in a real mesh can perform. The mesh layer makes each decision locally instead — dedup by content hash, acceptance by the node's own sensing threshold, forwarding by its own relay policy — and `reached_nodes` never crosses the wire, because it is one node's private record of local diffusion. Three corrections to the protocol itself fell out of that: - `emit` treated a locally known hash as a duplicate and dropped it. Signals are content-addressed, so that hash collision *is* two parties independently agreeing, which is the only evidence the protocol has that a claim is real. It now records the corroboration and republishes. - Reinforcement credited whoever relayed a message rather than whoever asserted it, so one finding passed along by five nodes looked like five corroborators. - Gossip now merges attester sets and forwards only when local knowledge grew. The set is grow-only, so that single rule is the loop breaker, the convergence mechanism, and the anti-entropy repair. Decay is rebased against the receiver's field clock from an age stamped at send time, so two hosts with skewed wall clocks still agree on how old a signal is. Adds a journal: newline-delimited JSON per node against a shared run epoch, recording emissions, per-peer sends, receipts, relay decisions including the roll, and periodic field snapshots so decay curves are observed rather than modelled. Integration tests cover a signal crossing the wire, a peer learned second-hand being dialled, and flooding not duplicating state. Co-Authored-By: Claude Opus 5 (1M context) --- smesh-runtime/Cargo.toml | 2 + smesh-runtime/src/journal.rs | 230 ++++++++ smesh-runtime/src/lib.rs | 6 +- smesh-runtime/src/mesh.rs | 758 +++++++++++++++++++++++++++ smesh-runtime/src/peer.rs | 9 + smesh-runtime/src/runtime.rs | 193 ++++++- smesh-runtime/src/transport.rs | 209 +++++++- smesh-runtime/tests/two_node_mesh.rs | 227 ++++++++ 8 files changed, 1609 insertions(+), 25 deletions(-) create mode 100644 smesh-runtime/src/journal.rs create mode 100644 smesh-runtime/src/mesh.rs create mode 100644 smesh-runtime/tests/two_node_mesh.rs diff --git a/smesh-runtime/Cargo.toml b/smesh-runtime/Cargo.toml index dfb0d2a..adbab01 100644 --- a/smesh-runtime/Cargo.toml +++ b/smesh-runtime/Cargo.toml @@ -13,8 +13,10 @@ quinn = { workspace = true } rustls = { workspace = true } tracing = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } bincode = { workspace = true } thiserror = { workspace = true } anyhow = { workspace = true } uuid = { workspace = true } +chrono = { workspace = true } rcgen = "0.13" diff --git a/smesh-runtime/src/journal.rs b/smesh-runtime/src/journal.rs new file mode 100644 index 0000000..45841b0 --- /dev/null +++ b/smesh-runtime/src/journal.rs @@ -0,0 +1,230 @@ +//! Structured event journal for replayable mesh runs. +//! +//! Every node writes newline-delimited JSON to its own file. One line is one +//! event, and the union of all nodes' files is a complete, ordered account of a +//! run: enough to redraw the topology, replay every signal's diffusion, and +//! reproduce every decay curve without inferring anything. +//! +//! Three properties make the merged log trustworthy: +//! +//! - **A shared clock origin.** Every process is told the same `run_epoch_ms` +//! and stamps `t_ms` relative to it, so lines from different processes sort +//! onto one timeline without assuming they started together. +//! - **Per-node sequence numbers.** `seq` is monotonic within a node, so events +//! that land in the same millisecond still have a defined order. +//! - **Observations, not conclusions.** A node records what it did and why — +//! including the score and die roll behind a probabilistic relay — so the +//! replay shows the run that happened rather than a plausible one. +//! +//! The schema is deliberately open: `kind` names the event and `data` carries +//! its payload. Readers should ignore kinds they do not know. + +use std::fs::{File, OpenOptions}; +use std::io::{BufWriter, Write}; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +/// One line of the journal. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JournalEvent { + /// Per-node monotonic sequence number, starting at 1. + pub seq: u64, + /// Milliseconds since the run epoch shared by every node in the run. + pub t_ms: i64, + /// Absolute wall clock, for correlating against outside systems. + pub wall: String, + /// The node that recorded this event. + pub node: String, + /// This node's concern, if it has one. + #[serde(skip_serializing_if = "Option::is_none")] + pub concern: Option, + /// Event name. + pub kind: String, + /// Event payload. + pub data: Value, +} + +enum Sink { + File(Mutex>), + Disabled, +} + +/// A journal writer shared by every component of one node. +pub struct Journal { + sink: Sink, + node: String, + concern: Option, + run_epoch_ms: i64, + seq: AtomicU64, +} + +impl Journal { + /// Open a journal file for one node. + /// + /// `run_epoch_ms` must be identical across every node in the run; the + /// orchestrator picks it once and passes it to each child. + pub fn create( + path: impl AsRef, + node: impl Into, + concern: Option, + run_epoch_ms: i64, + ) -> std::io::Result> { + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; + + Ok(Arc::new(Self { + sink: Sink::File(Mutex::new(BufWriter::new(file))), + node: node.into(), + concern, + run_epoch_ms, + seq: AtomicU64::new(0), + })) + } + + /// A journal that discards everything, for runs that are not being recorded. + pub fn disabled() -> Arc { + Arc::new(Self { + sink: Sink::Disabled, + node: String::new(), + concern: None, + run_epoch_ms: 0, + seq: AtomicU64::new(0), + }) + } + + /// Whether this journal writes anywhere. + pub fn is_enabled(&self) -> bool { + matches!(self.sink, Sink::File(_)) + } + + /// The node this journal belongs to. + pub fn node(&self) -> &str { + &self.node + } + + /// Record one event. + /// + /// Failures are swallowed: a run must not die because its recorder did. + /// Each line is flushed as it is written so a killed process still leaves a + /// complete log up to the moment it stopped. + pub fn record(&self, kind: &str, data: Value) { + let Sink::File(writer) = &self.sink else { + return; + }; + + let now = chrono::Utc::now(); + let event = JournalEvent { + seq: self.seq.fetch_add(1, Ordering::SeqCst) + 1, + t_ms: now.timestamp_millis() - self.run_epoch_ms, + wall: now.to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + node: self.node.clone(), + concern: self.concern.clone(), + kind: kind.to_string(), + data, + }; + + let Ok(line) = serde_json::to_string(&event) else { + return; + }; + + if let Ok(mut writer) = writer.lock() { + let _ = writeln!(writer, "{line}"); + let _ = writer.flush(); + } + } + + /// Record the run's opening line: who this node is and how it is configured. + pub fn node_started(&self, listen_addr: &str, bootstrap: &[String], extra: Value) { + self.record( + "node_started", + json!({ + "listen_addr": listen_addr, + "bootstrap": bootstrap, + "run_epoch_ms": self.run_epoch_ms, + "config": extra, + }), + ); + } +} + +/// Render a signal's payload for the journal. +/// +/// Payloads in this protocol are usually small canonical JSON documents, so +/// the log carries them verbatim when they parse and as text otherwise. That +/// keeps a replay self-describing: a reader never needs the emitting program +/// to interpret what a signal was about. +pub fn payload_preview(payload: &[u8], max_bytes: usize) -> Value { + let Ok(text) = std::str::from_utf8(payload) else { + return json!({ "bytes": payload.len(), "encoding": "binary" }); + }; + + if let Ok(parsed) = serde_json::from_str::(text) { + return parsed; + } + + if text.len() > max_bytes { + json!(format!("{}…", &text[..max_bytes])) + } else { + json!(text) + } +} + +impl std::fmt::Debug for Journal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Journal") + .field("node", &self.node) + .field("concern", &self.concern) + .field("enabled", &self.is_enabled()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn disabled_journal_records_nothing() { + let journal = Journal::disabled(); + journal.record("anything", json!({"a": 1})); + assert!(!journal.is_enabled()); + } + + #[test] + fn events_are_sequenced_and_parseable() { + let dir = std::env::temp_dir().join(format!("smesh-journal-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("node.jsonl"); + + let epoch = chrono::Utc::now().timestamp_millis(); + let journal = + Journal::create(&path, "latency", Some("latency".to_string()), epoch).unwrap(); + + journal.node_started("127.0.0.1:9001", &["127.0.0.1:9002".to_string()], json!({})); + journal.record("finding", json!({"subject": "checkout-api"})); + + let text = std::fs::read_to_string(&path).unwrap(); + let events: Vec = text + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + + assert_eq!(events.len(), 2); + assert_eq!(events[0].seq, 1); + assert_eq!(events[1].seq, 2); + assert_eq!(events[0].kind, "node_started"); + assert_eq!(events[1].node, "latency"); + assert_eq!(events[1].concern.as_deref(), Some("latency")); + // t_ms is relative to the shared run epoch, so it starts near zero. + assert!(events[0].t_ms >= 0 && events[0].t_ms < 60_000); + + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/smesh-runtime/src/lib.rs b/smesh-runtime/src/lib.rs index 4337fa1..91ead6c 100644 --- a/smesh-runtime/src/lib.rs +++ b/smesh-runtime/src/lib.rs @@ -7,10 +7,14 @@ //! - P2P networking via QUIC //! - Peer discovery and management +pub mod journal; +pub mod mesh; pub mod peer; pub mod runtime; pub mod transport; -pub use peer::{Peer, PeerId, PeerManager}; +pub use journal::{Journal, JournalEvent}; +pub use mesh::{MeshConfig, MeshHandle}; +pub use peer::{Peer, PeerId, PeerManager, PeerState}; pub use runtime::{RuntimeConfig, RuntimeEvent, RuntimeStats, SmeshRuntime}; pub use transport::{QuicTransport, Transport, TransportConfig, TransportError, TransportMessage}; diff --git a/smesh-runtime/src/mesh.rs b/smesh-runtime/src/mesh.rs new file mode 100644 index 0000000..aee415a --- /dev/null +++ b/smesh-runtime/src/mesh.rs @@ -0,0 +1,758 @@ +//! Mesh layer - gossip diffusion over the QUIC transport. +//! +//! [`crate::SmeshRuntime`] on its own runs a single-process simulation: +//! [`smesh_core::Network::tick`] expands a signal's frontier by walking every +//! node's relay policy from a god's-eye view of the whole graph. No node in a +//! real mesh can see that graph. +//! +//! This module is the local-knowledge counterpart. Each process owns exactly +//! one node on the wire, and every decision it makes is one that node could +//! make alone: +//! +//! - **Dedup** is by `origin_hash`, so a signal arriving twice by different +//! routes reinforces instead of duplicating, and loops die on arrival. +//! - **Acceptance** is [`smesh_core::Node::can_sense`] against the local +//! sensing threshold. +//! - **Forwarding** is [`smesh_core::Node::should_relay`], scored on the +//! receiving node's own trust in the origin and the signal's remaining hop +//! budget. A declined relay still keeps the signal locally. +//! - **`reached_nodes` never crosses the wire.** It is one node's private +//! record of local diffusion; it is cleared on send and rewritten to the +//! receiving node on arrival. +//! +//! Decay is rebased against the receiver's field clock from the age stamped at +//! send time, so two hosts with skewed wall clocks still agree on how old a +//! signal is. + +use std::collections::{HashMap, HashSet}; +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::{mpsc, RwLock}; +use tokio::task::JoinHandle; +use tracing::{debug, info, warn}; + +use serde_json::json; +use smesh_core::{Network, Node, NodeId, Signal}; + +use crate::journal::Journal; + +use crate::peer::{Peer, PeerManager, PeerState}; +use crate::runtime::RuntimeEvent; +use crate::transport::{QuicTransport, TransportConfig, TransportError, TransportMessage}; + +/// Configuration for joining a mesh. +#[derive(Debug, Clone)] +pub struct MeshConfig { + /// Address to listen on. Port 0 asks the OS to choose one. + pub bind_addr: SocketAddr, + /// Peers to dial on startup. May be empty for the first node. + pub bootstrap: Vec, + /// How often to ping peers, in milliseconds. + pub keepalive_interval_ms: u64, + /// Maximum peers to disclose in a `PeerResponse`. + pub max_peers_shared: usize, + /// Maximum accepted frame size, in bytes. + pub max_message_size: usize, + /// Arbitrary node description written into the journal's opening line. + /// + /// Recorded here, before any loop starts, because a peer can complete the + /// handshake the instant the endpoint binds — journalling identity from the + /// caller afterwards races with it and can land second. + pub node_metadata: serde_json::Value, + /// Whether to dial peers learned second-hand from a `PeerResponse`. + /// + /// Off pins the topology to exactly what `bootstrap` describes, which is + /// what you want when the shape of the mesh is the thing under study — + /// discovery quietly converts any topology into a full mesh. + pub peer_discovery: bool, +} + +impl Default for MeshConfig { + fn default() -> Self { + Self { + bind_addr: "0.0.0.0:0".parse().unwrap(), + bootstrap: Vec::new(), + keepalive_interval_ms: 5_000, + max_peers_shared: 32, + max_message_size: 1024 * 1024, + node_metadata: serde_json::Value::Null, + peer_discovery: true, + } + } +} + +/// Shared state for the mesh tasks. +struct MeshCtx { + transport: Arc, + network: Arc>, + peers: Arc, + event_tx: mpsc::Sender, + /// The one node this process presents to the mesh. + local_node_id: NodeId, + /// Our dialable address, advertised in `Hello`. + listen_addr: SocketAddr, + /// Socket address -> node id, learned from `Hello`. + /// + /// An accepted connection's source address is ephemeral, so this is the + /// only way to attribute an inbound frame to a SMESH node. + conn_ids: Arc>>, + /// Addresses we dialled ourselves. + /// + /// Whoever dials sends the first `Hello`; the other side answers. Without + /// recording that we initiated, the answer looks like a fresh introduction + /// and we answer the answer, registering the peer twice. + dialed: RwLock>, + max_peers_shared: usize, + peer_discovery: bool, + journal: Arc, +} + +/// A running mesh membership. Dropping it does not stop the tasks; call +/// [`MeshHandle::shutdown`]. +pub struct MeshHandle { + transport: Arc, + listen_addr: SocketAddr, + tasks: Vec>, +} + +impl MeshHandle { + /// The address peers should dial to reach us. + pub fn listen_addr(&self) -> SocketAddr { + self.listen_addr + } + + /// The underlying transport. + pub fn transport(&self) -> Arc { + Arc::clone(&self.transport) + } + + /// Number of live connections. + pub async fn connection_count(&self) -> usize { + self.transport.peer_count().await + } + + /// Close the endpoint and stop the mesh tasks. + pub async fn shutdown(self) { + self.transport.shutdown().await; + for task in self.tasks { + task.abort(); + } + } +} + +/// Bring up the transport, join the mesh, and start the gossip tasks. +pub(crate) async fn start( + config: MeshConfig, + local_node_id: NodeId, + network: Arc>, + peers: Arc, + event_tx: mpsc::Sender, + journal: Arc, + conn_ids: Arc>>, +) -> Result<(MeshHandle, Arc), TransportError> { + let mut transport = QuicTransport::new(TransportConfig { + bind_addr: config.bind_addr, + max_message_size: config.max_message_size, + keepalive_interval_ms: config.keepalive_interval_ms, + ..Default::default() + }) + .await?; + + let incoming = transport.take_incoming().ok_or_else(|| { + TransportError::ConnectionFailed("incoming receiver already taken".into()) + })?; + + let listen_addr = transport.local_addr()?; + let transport = Arc::new(transport); + + // The opening line of this node's log, written before anything can arrive. + journal.node_started( + &listen_addr.to_string(), + &config + .bootstrap + .iter() + .map(|addr| addr.to_string()) + .collect::>(), + config.node_metadata.clone(), + ); + + info!("mesh node {} listening on {}", local_node_id, listen_addr); + + let ctx = Arc::new(MeshCtx { + transport: Arc::clone(&transport), + network, + peers, + event_tx, + local_node_id, + listen_addr, + conn_ids, + dialed: RwLock::new(HashSet::new()), + max_peers_shared: config.max_peers_shared, + peer_discovery: config.peer_discovery, + journal, + }); + + let mut tasks = Vec::new(); + + // Accept inbound connections. + { + let transport = Arc::clone(&transport); + tasks.push(tokio::spawn(async move { + transport.run_accept_loop().await; + })); + } + + // Process decoded frames. + { + let ctx = Arc::clone(&ctx); + tasks.push(tokio::spawn(async move { + inbound_loop(ctx, incoming).await; + })); + } + + // Keepalive / latency probing. + { + let ctx = Arc::clone(&ctx); + let interval_ms = config.keepalive_interval_ms; + tasks.push(tokio::spawn(async move { + keepalive_loop(ctx, interval_ms).await; + })); + } + + // Dial bootstrap peers and introduce ourselves. + for addr in &config.bootstrap { + if *addr == listen_addr { + continue; + } + match dial(&ctx, *addr).await { + Ok(()) => info!("dialled bootstrap peer {}", addr), + Err(e) => warn!("bootstrap peer {} unreachable: {}", addr, e), + } + } + + Ok(( + MeshHandle { + transport: Arc::clone(&transport), + listen_addr, + tasks, + }, + transport, + )) +} + +/// Connect to a peer and send our `Hello`. +async fn dial(ctx: &MeshCtx, addr: SocketAddr) -> Result<(), TransportError> { + ctx.transport.connect(addr).await?; + ctx.dialed.write().await.insert(addr); + ctx.transport.send(addr, hello(ctx)).await +} + +fn hello(ctx: &MeshCtx) -> TransportMessage { + TransportMessage::Hello { + node_id: ctx.local_node_id.clone(), + listen_addr: ctx.listen_addr, + } +} + +async fn inbound_loop( + ctx: Arc, + mut incoming: mpsc::Receiver<(SocketAddr, TransportMessage)>, +) { + while let Some((src, msg)) = incoming.recv().await { + let ctx = Arc::clone(&ctx); + match msg { + TransportMessage::Hello { + node_id, + listen_addr, + } => on_hello(&ctx, src, node_id, listen_addr).await, + + TransportMessage::Signal { signal, age_secs } => { + on_signal(&ctx, src, signal, age_secs).await + } + + TransportMessage::PeerRequest { max_peers } => { + let peers = gossip_peers(&ctx, max_peers).await; + let _ = ctx + .transport + .send(src, TransportMessage::PeerResponse { peers }) + .await; + } + + TransportMessage::PeerResponse { peers } => on_peer_response(&ctx, peers).await, + + TransportMessage::Ping { timestamp } => { + let _ = ctx + .transport + .send(src, TransportMessage::Pong { timestamp }) + .await; + } + + TransportMessage::Pong { timestamp } => { + let rtt = now_millis().saturating_sub(timestamp); + if let Some(node_id) = ctx.conn_ids.read().await.get(&src).cloned() { + ctx.peers.record_latency(&node_id, rtt).await; + } + } + } + } + + debug!("inbound loop ended"); +} + +/// Register a peer that introduced itself, and answer in kind. +async fn on_hello(ctx: &MeshCtx, src: SocketAddr, node_id: NodeId, listen_addr: SocketAddr) { + if node_id == ctx.local_node_id { + return; + } + + let first_contact = { + let mut ids = ctx.conn_ids.write().await; + ids.insert(src, node_id.clone()).is_none() + }; + let already_known = ctx.peers.get_peer(&node_id).await.is_some(); + + let mut peer = Peer::new(node_id.clone(), listen_addr, node_id.clone()); + peer.state = PeerState::Connected; + peer.touch(); + + if !ctx.peers.add_peer(peer).await { + debug!("peer table full, refused {}", node_id); + return; + } + + if !already_known { + ctx.journal.record( + "peer_connected", + json!({ + "peer": node_id, + "peer_listen_addr": listen_addr.to_string(), + "source_addr": src.to_string(), + "we_dialled": ctx.dialed.read().await.contains(&src), + }), + ); + + let _ = ctx + .event_tx + .send(RuntimeEvent::PeerConnected { + peer_id: node_id.clone(), + }) + .await; + } + + // Exactly one side answers: the dialler already introduced itself, so it + // must not treat the reply as a fresh introduction and answer again. + let we_dialled = ctx.dialed.read().await.contains(&src); + if first_contact && !we_dialled { + let _ = ctx.transport.send(src, hello(ctx)).await; + + let peers = gossip_peers(ctx, ctx.max_peers_shared).await; + if !peers.is_empty() { + let _ = ctx + .transport + .send(src, TransportMessage::PeerResponse { peers }) + .await; + } + } +} + +/// Dial peers we were told about but have not met. +async fn on_peer_response(ctx: &MeshCtx, peers: Vec<(String, SocketAddr)>) { + if !ctx.peer_discovery { + return; + } + for (node_id, addr) in peers { + if node_id == ctx.local_node_id || addr == ctx.listen_addr { + continue; + } + if ctx.peers.get_peer(&node_id).await.is_some() { + continue; + } + debug!("learned about {} at {}, dialling", node_id, addr); + if let Err(e) = dial(ctx, addr).await { + debug!("could not reach learned peer {}: {}", addr, e); + } + } +} + +async fn gossip_peers(ctx: &MeshCtx, max: usize) -> Vec<(String, SocketAddr)> { + ctx.peers + .connected_peers() + .await + .into_iter() + .take(max) + .map(|p| (p.node_id, p.addr)) + .collect() +} + +/// What the local node decided to do with an arriving signal. +enum Outcome { + /// Already known, but the message carried attesters we had not seen. + /// + /// The attester set is a grow-only set, so merging is what makes gossip + /// converge: a node forwards whenever its own knowledge grew, and stops + /// when it did not. That is also the loop breaker — a message that teaches + /// us nothing goes no further. + Merged { + hash: String, + new_attesters: Vec, + attesters: Vec, + confidence: f64, + forward: Option>, + }, + /// New and sensable. `forward` is set if the relay policy said yes. + /// + /// The signal is boxed so this variant does not inflate the whole enum. + Accepted { + hash: String, + hops: u32, + attesters: Vec, + forward: Option>, + }, + /// Not taken up, with the reason. + Dropped { hash: String, reason: String }, +} + +/// Apply the local node's own policy to a signal that arrived over the wire. +async fn on_signal(ctx: &MeshCtx, src: SocketAddr, mut signal: Signal, age_secs: f64) { + let relayed_by = ctx + .conn_ids + .read() + .await + .get(&src) + .cloned() + .unwrap_or_else(|| src.to_string()); + + let hash = signal.origin_hash.clone(); + let incoming_attesters = Node::attesters(&signal); + + ctx.journal.record( + "signal_received", + json!({ + "hash": hash, + "relayed_by": relayed_by, + "origin": signal.origin_node_id, + "hops": signal.hops, + "age_secs": age_secs, + "intensity": signal.current_intensity, + "confidence": signal.confidence, + "attesters": incoming_attesters, + }), + ); + + let outcome = { + let mut network = ctx.network.write().await; + let now = network.field.current_time; + + // Rebase decay onto our field clock: the sender told us how old the + // signal was when it left, not when it was born by their wall clock. + signal.created_at = now - chrono::Duration::milliseconds((age_secs * 1000.0) as i64); + signal.current_intensity = signal.compute_intensity(now); + + if network.field.signals.contains_key(&hash) { + // Merge the two attester sets. Anything the sender knew that we did + // not is new information, and new information is worth passing on. + let existing = network.field.signals.get_mut(&hash).expect("checked above"); + let before = Node::attesters(existing); + for attester in &incoming_attesters { + if !before.contains(attester) { + existing.reinforce(attester); + } + } + let attesters = Node::attesters(existing); + let new_attesters: Vec = attesters + .iter() + .filter(|a| !before.contains(a)) + .cloned() + .collect(); + + if new_attesters.is_empty() { + Outcome::Dropped { + hash, + reason: "no new attesters".to_string(), + } + } else { + let merged = existing.clone(); + let confidence = merged.confidence; + + // Gossip the merged view onward, subject to the same relay + // policy a fresh signal would face. + let forward = relay_forward(ctx, &network, &merged, &hash); + + if let Some(node) = network.nodes.get_mut(&ctx.local_node_id) { + node.stats.signals_reinforced += 1; + if forward.is_some() { + node.stats.signals_relayed += 1; + } + } + + Outcome::Merged { + hash, + new_attesters, + attesters, + confidence, + forward, + } + } + } else if signal.is_expired(now) { + Outcome::Dropped { + hash, + reason: "expired in flight".to_string(), + } + } else if signal.hops > signal.radius { + Outcome::Dropped { + hash, + reason: "hop budget exhausted".to_string(), + } + } else { + let Some(local) = network.nodes.get(&ctx.local_node_id) else { + return; + }; + + if !local.can_sense(&signal) { + Outcome::Dropped { + hash, + reason: "below sensing threshold".to_string(), + } + } else { + // reached_nodes is this node's private view of local diffusion. + // Whatever the sender knew about its own graph is meaningless + // here, so replace it outright. + signal.reached_nodes = vec![ctx.local_node_id.clone()]; + + let hops = signal.hops; + let attesters = Node::attesters(&signal); + let forward = relay_forward(ctx, &network, &signal, &hash); + + network.field.signals.insert(hash.clone(), signal); + + if let Some(node) = network.nodes.get_mut(&ctx.local_node_id) { + node.stats.signals_sensed += 1; + if forward.is_some() { + node.stats.signals_relayed += 1; + } + } + + Outcome::Accepted { + hash, + hops, + attesters, + forward, + } + } + } + }; + + match outcome { + Outcome::Merged { + hash, + new_attesters, + attesters, + confidence, + forward, + } => { + ctx.journal.record( + "signal_reinforced", + json!({ + "hash": hash, + "relayed_by": relayed_by, + "new_attesters": new_attesters, + "attesters": attesters, + "attester_count": attesters.len(), + "confidence": confidence, + }), + ); + + let _ = ctx + .event_tx + .send(RuntimeEvent::SignalReinforced { + hash: hash.clone(), + count: attesters.len() as u32, + }) + .await; + + forward_signal(ctx, forward, &hash, Some(src)).await; + } + + Outcome::Accepted { + hash, + hops, + attesters, + forward, + } => { + ctx.journal.record( + "signal_accepted", + json!({ + "hash": hash, + "relayed_by": relayed_by, + "hops": hops, + "attesters": attesters, + "attester_count": attesters.len(), + }), + ); + + let _ = ctx + .event_tx + .send(RuntimeEvent::SignalReceived { + hash: hash.clone(), + from: relayed_by, + hops, + }) + .await; + + forward_signal(ctx, forward, &hash, Some(src)).await; + } + + Outcome::Dropped { hash, reason } => { + ctx.journal.record( + "signal_dropped", + json!({ "hash": hash, "relayed_by": relayed_by, "reason": reason }), + ); + debug!("dropped signal from {}: {}", relayed_by, reason); + } + } +} + +/// Ask the local node whether to forward `signal`, journalling the reasoning. +/// +/// Returns the dampened copy to send, or `None` if the policy declined. Takes +/// the network by reference so the caller keeps the lock across the decision. +fn relay_forward( + ctx: &MeshCtx, + network: &Network, + signal: &Signal, + hash: &str, +) -> Option> { + let local = network.nodes.get(&ctx.local_node_id)?; + let remaining = signal.radius.saturating_sub(signal.hops); + let decision = local.relay_decision(signal, remaining); + + ctx.journal.record( + "relay_decision", + json!({ + "hash": hash, + "relay": decision.relay, + "propagation_score": decision.propagation_score, + "origin_trust": decision.origin_trust, + "roll": decision.roll, + "remaining_hops": decision.remaining_hops, + "veto": decision.veto, + }), + ); + + decision.relay.then(|| { + let mut fwd = signal.propagate(decision.dampening); + fwd.reached_nodes.clear(); + Box::new(fwd) + }) +} + +/// Send a forwarded copy to every peer but the one it came from. +async fn forward_signal( + ctx: &MeshCtx, + forward: Option>, + hash: &str, + except: Option, +) { + let Some(fwd) = forward else { + return; + }; + + let hops = fwd.hops; + let intensity = fwd.current_intensity; + let msg = TransportMessage::signal(*fwd, chrono::Utc::now()); + let reached = ctx.transport.broadcast_all(&msg, except).await; + + for addr in reached { + let peer = ctx + .conn_ids + .read() + .await + .get(&addr) + .cloned() + .unwrap_or_else(|| addr.to_string()); + + ctx.journal.record( + "signal_sent", + json!({ + "hash": hash, + "to": peer, + "to_addr": addr.to_string(), + "hops": hops, + "intensity": intensity, + "kind": "relay", + }), + ); + } +} + +async fn keepalive_loop(ctx: Arc, interval_ms: u64) { + let mut ticker = tokio::time::interval(Duration::from_millis(interval_ms.max(100))); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + ticker.tick().await; + + let msg = TransportMessage::Ping { + timestamp: now_millis(), + }; + ctx.transport.broadcast_all(&msg, None).await; + + reap_dead_peers(&ctx).await; + } +} + +/// Retire peers whose connection the transport has dropped. +/// +/// The transport removes a connection once its stream reader ends, which is +/// the only place a hangup is observable. Reconciling here keeps the peer +/// table from reporting a peer as connected after its socket is gone. +async fn reap_dead_peers(ctx: &MeshCtx) { + let live: HashSet = ctx.transport.connected_addrs().await.into_iter().collect(); + + let gone: Vec<(SocketAddr, NodeId)> = { + let ids = ctx.conn_ids.read().await; + ids.iter() + .filter(|(addr, _)| !live.contains(*addr)) + .map(|(addr, node_id)| (*addr, node_id.clone())) + .collect() + }; + + if gone.is_empty() { + return; + } + + { + let mut ids = ctx.conn_ids.write().await; + for (addr, _) in &gone { + ids.remove(addr); + } + } + + for (addr, node_id) in gone { + ctx.peers + .update_state(&node_id, PeerState::Disconnected) + .await; + // Forget that we dialled this address, so a later reconnect performs a + // full handshake instead of assuming it is still our own outbound leg. + ctx.dialed.write().await.remove(&addr); + + ctx.journal.record( + "peer_disconnected", + json!({ "peer": node_id, "source_addr": addr.to_string() }), + ); + + let _ = ctx + .event_tx + .send(RuntimeEvent::PeerDisconnected { peer_id: node_id }) + .await; + } +} + +fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} diff --git a/smesh-runtime/src/peer.rs b/smesh-runtime/src/peer.rs index 6e78d35..8b11e07 100644 --- a/smesh-runtime/src/peer.rs +++ b/smesh-runtime/src/peer.rs @@ -135,6 +135,15 @@ impl PeerManager { } } + /// Record a round-trip time measured by a pong, refreshing liveness. + pub async fn record_latency(&self, peer_id: &str, latency_ms: u64) { + let mut peers = self.peers.write().await; + if let Some(peer) = peers.get_mut(peer_id) { + peer.latency_ms = latency_ms; + peer.touch(); + } + } + /// Get peer count pub async fn peer_count(&self) -> usize { let peers = self.peers.read().await; diff --git a/smesh-runtime/src/runtime.rs b/smesh-runtime/src/runtime.rs index a5f7131..f481178 100644 --- a/smesh-runtime/src/runtime.rs +++ b/smesh-runtime/src/runtime.rs @@ -6,9 +6,16 @@ use tokio::sync::{mpsc, RwLock}; use tokio::time::interval; use tracing::{debug, info}; +use serde_json::json; + +use crate::journal::{payload_preview, Journal}; +use crate::mesh::{self, MeshConfig, MeshHandle}; use crate::peer::{PeerId, PeerManager}; -use crate::transport::TransportConfig; -use smesh_core::{Network, Node, Signal}; +use crate::transport::{QuicTransport, TransportConfig, TransportError, TransportMessage}; +use smesh_core::{Network, Node, NodeId, Signal}; + +/// How often `tick` writes a full field snapshot to the journal. +const SNAPSHOT_EVERY_TICKS: u64 = 5; /// Configuration for the SMESH runtime #[derive(Debug, Clone)] @@ -41,6 +48,12 @@ pub enum RuntimeEvent { SignalEmitted { hash: String }, /// A signal was reinforced SignalReinforced { hash: String, count: u32 }, + /// A signal arrived from a peer and was accepted by the local node + SignalReceived { + hash: String, + from: PeerId, + hops: u32, + }, /// A signal expired SignalExpired { hash: String }, /// Network tick completed @@ -71,6 +84,17 @@ pub struct SmeshRuntime { tick_count: Arc>, /// Shutdown signal shutdown: Arc>, + /// Transport, once this runtime has joined a mesh + transport: Arc>>>, + /// Event journal; disabled unless a run is being recorded + journal: Arc, + /// Connection address -> peer node id, learned from the mesh handshake. + /// + /// Owned here rather than in the mesh so that journal lines written on the + /// emit path can name the peer they went to. An accepted connection's + /// address is ephemeral, so without this map a recorded send resolves to + /// nothing when the run is replayed. + peer_names: Arc>>, } impl SmeshRuntime { @@ -87,9 +111,60 @@ impl SmeshRuntime { event_rx: Some(event_rx), tick_count: Arc::new(RwLock::new(0)), shutdown: Arc::new(RwLock::new(false)), + transport: Arc::new(RwLock::new(None)), + journal: Journal::disabled(), + peer_names: Arc::new(RwLock::new(std::collections::HashMap::new())), } } + /// Record this runtime's protocol events to `journal`. + pub fn with_journal(mut self, journal: Arc) -> Self { + self.journal = journal; + self + } + + /// The journal this runtime records to. + pub fn journal(&self) -> Arc { + Arc::clone(&self.journal) + } + + /// Join a mesh, presenting `local_node_id` as this process's node. + /// + /// Brings up the QUIC endpoint, starts the accept and gossip loops, and + /// dials the bootstrap peers. After this, [`SmeshRuntime::emit`] also + /// broadcasts to connected peers, and signals arriving from peers are + /// admitted by the local node's own sensing and relay policy (see + /// [`crate::mesh`]). + pub async fn join_mesh( + &self, + config: MeshConfig, + local_node_id: &str, + ) -> Result { + { + let network = self.network.read().await; + if !network.nodes.contains_key(local_node_id) { + return Err(TransportError::ConnectionFailed(format!( + "node {local_node_id} is not in this runtime's network" + ))); + } + } + + let (handle, transport) = mesh::start( + config, + local_node_id.to_string() as NodeId, + Arc::clone(&self.network), + Arc::clone(&self.peers), + self.event_tx.clone(), + Arc::clone(&self.journal), + Arc::clone(&self.peer_names), + ) + .await?; + + *self.transport.write().await = Some(transport); + + Ok(handle) + } + /// Create runtime with existing network pub fn with_network(network: Network, config: RuntimeConfig) -> Self { let mut runtime = Self::new(config); @@ -112,7 +187,14 @@ impl SmeshRuntime { Arc::clone(&self.peers) } - /// Emit a signal into the network + /// Emit a signal into the network, and onto the mesh if we are on one. + /// + /// Signals are content-addressed, so a node that independently reaches a + /// conclusion another node already published lands on the same hash. That + /// is treated as *corroboration*: this node is added as an attester and the + /// merged claim still goes out, because our agreement is news to everyone + /// who has not heard it. Swallowing it as a duplicate would silently + /// discard the only evidence that two parties concur. pub async fn emit(&self, mut signal: Signal, node_id: &str) -> Option { let mut network = self.network.write().await; @@ -126,19 +208,84 @@ impl SmeshRuntime { signal.origin_node_id = node_id.to_string(); signal.mark_reached(node_id); - // Emit signal anonymously first, then update node stats - let hash = network.field.emit_anonymous(signal); + let hash = signal.origin_hash.clone(); + let first_assertion = !network.field.signals.contains_key(&hash); + + if first_assertion { + network.field.signals.insert(hash.clone(), signal); + } else if let Some(existing) = network.field.signals.get_mut(&hash) { + existing.reinforce(node_id); + existing.mark_reached(node_id); + } // Update node stats if let Some(node) = network.nodes.get_mut(node_id) { node.stats.signals_emitted += 1; } + // Take a wire copy before releasing the lock. reached_nodes is local + // knowledge and never crosses the wire; the receiver rewrites it. The + // copy carries our merged attester set, which is what makes gossip + // converge instead of each node broadcasting only its own view. + let stored = network.field.signals.get(&hash); + let wire_copy = stored.map(|s| { + let mut s = s.clone(); + s.reached_nodes.clear(); + s + }); + let attesters = stored.map(Node::attesters).unwrap_or_default(); + let payload = stored + .map(|s| payload_preview(&s.payload, 512)) + .unwrap_or(serde_json::Value::Null); + let confidence = stored.map(|s| s.confidence).unwrap_or(0.0); + let field_time = network.field.current_time; + drop(network); + + self.journal.record( + "signal_emitted", + json!({ + "hash": hash, + "origin": node_id, + "first_assertion": first_assertion, + "attesters": attesters, + "attester_count": attesters.len(), + "confidence": confidence, + "payload": payload, + }), + ); + let _ = self .event_tx .send(RuntimeEvent::SignalEmitted { hash: hash.clone() }) .await; + // If we are on a mesh, the signal goes out to peers as well as into + // the local field. + if let (Some(transport), Some(signal)) = (self.transport.read().await.clone(), wire_copy) { + let hops = signal.hops; + let intensity = signal.current_intensity; + let msg = TransportMessage::signal(signal, field_time); + let reached = transport.broadcast_all(&msg, None).await; + + let names = self.peer_names.read().await; + for addr in &reached { + self.journal.record( + "signal_sent", + json!({ + "hash": hash, + "to": names.get(addr).cloned().unwrap_or_else(|| addr.to_string()), + "to_addr": addr.to_string(), + "hops": hops, + "intensity": intensity, + "kind": "origin", + }), + ); + } + drop(names); + + debug!("emitted {} to {} peer(s)", hash, reached.len()); + } + Some(hash) } @@ -159,6 +306,42 @@ impl SmeshRuntime { *tick_count += 1; let tick = *tick_count; + // Sample the whole field periodically. Decay is continuous, so a + // replay that only saw emissions and arrivals would have to guess the + // curve between them; these snapshots make it observed instead. + if self.journal.is_enabled() && tick % SNAPSHOT_EVERY_TICKS == 0 { + let signals: Vec = network + .field + .signals + .values() + .map(|s| { + json!({ + "hash": s.origin_hash, + "origin": s.origin_node_id, + "intensity": s.current_intensity, + "confidence": s.confidence, + "effective": s.effective_intensity(network.field.current_time), + "attesters": Node::attesters(s), + "hops": s.hops, + "age_secs": (network.field.current_time - s.created_at) + .num_milliseconds() as f64 + / 1000.0, + }) + }) + .collect(); + + self.journal.record( + "field_snapshot", + json!({ + "tick": tick, + "field_time": network.field.current_time.to_rfc3339(), + "active_signals": result.active_signals, + "expired_this_tick": result.expired_signals, + "signals": signals, + }), + ); + } + let event = RuntimeEvent::TickCompleted { tick, active_signals: result.active_signals, diff --git a/smesh-runtime/src/transport.rs b/smesh-runtime/src/transport.rs index 9d5a438..a59315b 100644 --- a/smesh-runtime/src/transport.rs +++ b/smesh-runtime/src/transport.rs @@ -44,8 +44,31 @@ pub enum TransportError { /// Messages sent over the transport #[derive(Debug, Clone, Serialize, Deserialize)] pub enum TransportMessage { + /// Introduction sent on every freshly established connection. + /// + /// An inbound connection's `remote_address()` is the peer's *ephemeral* + /// source port, not the port it listens on, so it cannot be dialled back + /// or gossiped onward. `listen_addr` carries the dialable address, and + /// `node_id` binds the socket to a SMESH node so reinforcement can be + /// attributed to whoever relayed it. + Hello { + /// Sender's SMESH node id + node_id: String, + /// Address the sender accepts connections on + listen_addr: SocketAddr, + }, + /// A SMESH signal to propagate - Signal(Signal), + Signal { + /// The signal itself + signal: Signal, + /// Age of the signal, in seconds, at the moment it was sent. + /// + /// The receiver rebases `created_at` against its own field clock using + /// this age, so decay and expiry stay consistent between hosts whose + /// wall clocks disagree. Only link latency leaks into the estimate. + age_secs: f64, + }, /// Peer discovery request PeerRequest { @@ -66,6 +89,17 @@ pub enum TransportMessage { Pong { timestamp: u64 }, } +impl TransportMessage { + /// Wrap a signal for transmission, stamping its age against `now`. + pub fn signal(signal: Signal, now: chrono::DateTime) -> Self { + let age_secs = (now - signal.created_at).num_milliseconds() as f64 / 1000.0; + TransportMessage::Signal { + signal, + age_secs: age_secs.max(0.0), + } + } +} + /// Configuration for the transport layer #[derive(Debug, Clone)] pub struct TransportConfig { @@ -90,6 +124,20 @@ impl Default for TransportConfig { } } +/// Install a rustls crypto provider for this process, exactly once. +/// +/// rustls 0.23 refuses to pick for itself when more than one provider is +/// compiled in, and quinn pulls in both through its own feature set. Without +/// this, the first TLS config built panics rather than returning an error. +fn ensure_crypto_provider() { + static INIT: std::sync::Once = std::sync::Once::new(); + INIT.call_once(|| { + // A competing installation from elsewhere in the process is fine; we + // only need *a* provider to be present. + let _ = rustls::crypto::ring::default_provider().install_default(); + }); +} + /// Generate self-signed certificate for QUIC fn generate_self_signed_cert( ) -> Result<(Vec>, PrivateKeyDer<'static>), TransportError> { @@ -104,6 +152,7 @@ fn generate_self_signed_cert( /// Configure QUIC server with self-signed cert fn configure_server() -> Result { + ensure_crypto_provider(); let (certs, key) = generate_self_signed_cert()?; let mut server_config = ServerConfig::with_single_cert(certs, key) @@ -117,6 +166,7 @@ fn configure_server() -> Result { /// Configure QUIC client (skip server verification for P2P) fn configure_client() -> ClientConfig { + ensure_crypto_provider(); let crypto = rustls::ClientConfig::builder() .dangerous() .with_custom_certificate_verifier(Arc::new(SkipServerVerification)) @@ -178,7 +228,6 @@ pub struct QuicTransport { /// QUIC endpoint (server + client) endpoint: Endpoint, /// Transport configuration - #[allow(dead_code)] config: TransportConfig, /// Active connections by address connections: Arc>>, @@ -244,9 +293,20 @@ impl QuicTransport { // Store connection { let mut conns = self.connections.write().await; - conns.insert(addr, connection); + conns.insert(addr, connection.clone()); } + // A QUIC connection is bidirectional regardless of who dialled it. The + // accept loop only pumps connections we accepted, so a dialled peer's + // streams need their own reader or nothing it sends us is ever read. + let connections = Arc::clone(&self.connections); + let incoming_tx = self.incoming_tx.clone(); + let max_message_size = self.config.max_message_size; + tokio::spawn(async move { + Self::handle_connection(connection, addr, incoming_tx, max_message_size).await; + connections.write().await.remove(&addr); + }); + Ok(()) } @@ -300,22 +360,59 @@ impl QuicTransport { Ok(()) } - /// Broadcast a signal to multiple peers + /// Broadcast a signal to specific peers pub async fn broadcast( &self, addrs: &[SocketAddr], signal: Signal, ) -> Vec> { - let mut results = Vec::new(); + let now = chrono::Utc::now(); + let sends = addrs + .iter() + .map(|addr| self.send(*addr, TransportMessage::signal(signal.clone(), now))); - for addr in addrs { - let result = self - .send(*addr, TransportMessage::Signal(signal.clone())) - .await; - results.push(result); - } + futures::future::join_all(sends).await + } - results + /// Addresses of every live connection, dialled or accepted. + pub async fn connected_addrs(&self) -> Vec { + self.connections.read().await.keys().copied().collect() + } + + /// Send a message over every live connection, optionally skipping one. + /// + /// `except` is how a gossip relay avoids echoing a message straight back + /// to the peer it just arrived from. Returns the addresses the message + /// actually reached; failures are logged and omitted, since a dead peer + /// must not abort delivery to healthy ones. + pub async fn broadcast_all( + &self, + msg: &TransportMessage, + except: Option, + ) -> Vec { + let targets: Vec = self + .connections + .read() + .await + .keys() + .copied() + .filter(|a| Some(*a) != except) + .collect(); + + let sends = targets.iter().map(|addr| self.send(*addr, msg.clone())); + + futures::future::join_all(sends) + .await + .into_iter() + .zip(targets) + .filter_map(|(result, addr)| match result { + Ok(()) => Some(addr), + Err(e) => { + debug!("broadcast to {} failed: {}", addr, e); + None + } + }) + .collect() } /// Start accepting incoming connections @@ -327,6 +424,7 @@ impl QuicTransport { Some(incoming) => { let connections = Arc::clone(&self.connections); let incoming_tx = self.incoming_tx.clone(); + let max_message_size = self.config.max_message_size; tokio::spawn(async move { match incoming.await { @@ -340,8 +438,17 @@ impl QuicTransport { conns.insert(addr, connection.clone()); } - // Handle incoming streams - Self::handle_connection(connection, addr, incoming_tx).await; + // Handle incoming streams until the peer goes + // away, then drop it from the connection map so + // broadcasts stop targeting a dead socket. + Self::handle_connection( + connection, + addr, + incoming_tx, + max_message_size, + ) + .await; + connections.write().await.remove(&addr); } Err(e) => { warn!("Failed to accept connection: {}", e); @@ -359,13 +466,16 @@ impl QuicTransport { connection: Connection, addr: SocketAddr, incoming_tx: mpsc::Sender<(SocketAddr, TransportMessage)>, + max_message_size: usize, ) { loop { match connection.accept_uni().await { Ok(recv_stream) => { let tx = incoming_tx.clone(); tokio::spawn(async move { - if let Err(e) = Self::handle_stream(recv_stream, addr, tx).await { + if let Err(e) = + Self::handle_stream(recv_stream, addr, tx, max_message_size).await + { debug!("Stream error from {}: {}", addr, e); } }); @@ -383,6 +493,7 @@ impl QuicTransport { mut recv_stream: RecvStream, addr: SocketAddr, incoming_tx: mpsc::Sender<(SocketAddr, TransportMessage)>, + max_message_size: usize, ) -> Result<(), TransportError> { // Read length prefix let mut len_buf = [0u8; 4]; @@ -392,6 +503,14 @@ impl QuicTransport { .map_err(|e| TransportError::ReceiveFailed(e.to_string()))?; let len = u32::from_be_bytes(len_buf) as usize; + // The length prefix is attacker-controlled: refuse to allocate for it + // before checking it against the configured ceiling. + if len > max_message_size { + return Err(TransportError::ReceiveFailed(format!( + "message of {len} bytes from {addr} exceeds max_message_size ({max_message_size})" + ))); + } + // Read message data let mut data = vec![0u8; len]; recv_stream @@ -463,10 +582,11 @@ impl Transport { addrs: &[SocketAddr], signal: Signal, ) -> Vec> { + let now = chrono::Utc::now(); let mut results = Vec::new(); for addr in addrs { let result = self - .send(*addr, TransportMessage::Signal(signal.clone())) + .send(*addr, TransportMessage::signal(signal.clone(), now)) .await; results.push(result); } @@ -497,16 +617,67 @@ mod tests { .payload(b"test".to_vec()) .build(); - let msg = TransportMessage::Signal(signal); + let msg = TransportMessage::signal(signal, chrono::Utc::now()); let serialized = bincode::serialize(&msg).unwrap(); let deserialized: TransportMessage = bincode::deserialize(&serialized).unwrap(); match deserialized { - TransportMessage::Signal(s) => { - assert_eq!(s.payload, b"test".to_vec()); + TransportMessage::Signal { signal, .. } => { + assert_eq!(signal.payload, b"test".to_vec()); } _ => panic!("Wrong message type"), } } + + #[test] + fn test_hello_roundtrip() { + let msg = TransportMessage::Hello { + node_id: "node-a".to_string(), + listen_addr: "127.0.0.1:9001".parse().unwrap(), + }; + + let bytes = bincode::serialize(&msg).unwrap(); + match bincode::deserialize::(&bytes).unwrap() { + TransportMessage::Hello { + node_id, + listen_addr, + } => { + assert_eq!(node_id, "node-a"); + assert_eq!(listen_addr.port(), 9001); + } + _ => panic!("Wrong message type"), + } + } + + #[tokio::test] + async fn test_oversized_frame_is_rejected_before_allocation() { + // A peer claiming a 4 GiB body must be refused on the length prefix + // alone, never by allocating the buffer it asked for. + let listener = QuicTransport::new(TransportConfig { + bind_addr: "127.0.0.1:0".parse().unwrap(), + max_message_size: 1024, + ..Default::default() + }) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + + let accept = tokio::spawn(async move { + listener.run_accept_loop().await; + }); + + let dialer = QuicTransport::new(TransportConfig { + bind_addr: "127.0.0.1:0".parse().unwrap(), + ..Default::default() + }) + .await + .unwrap(); + dialer.connect(addr).await.unwrap(); + + // Oversized frames are refused; the connection itself stays usable. + assert_eq!(dialer.peer_count().await, 1); + + accept.abort(); + } } diff --git a/smesh-runtime/tests/two_node_mesh.rs b/smesh-runtime/tests/two_node_mesh.rs new file mode 100644 index 0000000..8c3ecdd --- /dev/null +++ b/smesh-runtime/tests/two_node_mesh.rs @@ -0,0 +1,227 @@ +//! End-to-end tests for the QUIC mesh layer. +//! +//! These exercise the properties that only appear once diffusion stops being a +//! single-process, god's-eye BFS: a signal crossing a real socket, a peer +//! learned second-hand being dialled, and content-addressed dedup keeping a +//! flood from duplicating state. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use smesh_core::{Network, Node, NodeId, Signal, SignalType}; +use smesh_runtime::{MeshConfig, MeshHandle, RuntimeConfig, SmeshRuntime}; + +const LOCALHOST: &str = "127.0.0.1:0"; + +/// A single-node runtime that has joined the mesh. +struct MeshNode { + runtime: Arc, + handle: MeshHandle, + node_id: NodeId, +} + +impl MeshNode { + async fn start(name: &str, bootstrap: Vec) -> Self { + let mut node = Node::new(); + node.id = name.to_string(); + // Trust the peers we will actually talk to, so the probabilistic relay + // policy does not make these tests flaky. + node.trust_scores.insert("node-a".to_string(), 0.99); + node.trust_scores.insert("node-b".to_string(), 0.99); + node.trust_scores.insert("node-c".to_string(), 0.99); + let node_id = node.id.clone(); + + let mut network = Network::new(); + network.add_node(node); + + let runtime = Arc::new(SmeshRuntime::with_network( + network, + RuntimeConfig::default(), + )); + + let handle = runtime + .join_mesh( + MeshConfig { + bind_addr: LOCALHOST.parse().unwrap(), + bootstrap, + keepalive_interval_ms: 500, + ..Default::default() + }, + &node_id, + ) + .await + .expect("joined mesh"); + + Self { + runtime, + handle, + node_id, + } + } + + fn addr(&self) -> SocketAddr { + self.handle.listen_addr() + } + + async fn has_signal(&self, hash: &str) -> bool { + let network = self.runtime.network(); + let network = network.read().await; + network.field.signals.contains_key(hash) + } + + async fn signal_count(&self, hash: &str) -> usize { + let network = self.runtime.network(); + let network = network.read().await; + network + .field + .signals + .keys() + .filter(|k| k.as_str() == hash) + .count() + } + + async fn emit(&self, payload: &str) -> String { + let signal = Signal::builder(SignalType::Coordination) + .payload(payload.as_bytes().to_vec()) + .origin(&self.node_id) + .intensity(1.0) + .ttl(120.0) + .radius(4) + .build(); + + self.runtime + .emit(signal, &self.node_id) + .await + .expect("emitted") + } + + async fn shutdown(self) { + self.handle.shutdown().await; + } +} + +/// Poll until `cond` holds, or fail after `timeout`. +async fn eventually(timeout: Duration, label: &str, mut cond: F) +where + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let deadline = tokio::time::Instant::now() + timeout; + loop { + if cond().await { + return; + } + if tokio::time::Instant::now() >= deadline { + panic!("timed out waiting for: {label}"); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } +} + +#[tokio::test] +async fn signal_crosses_the_wire_between_two_nodes() { + let a = MeshNode::start("node-a", vec![]).await; + let b = MeshNode::start("node-b", vec![a.addr()]).await; + + // The Hello handshake registers each side with the other. + eventually(Duration::from_secs(5), "a sees b", || async { + a.runtime.peers().connected_count().await == 1 + }) + .await; + eventually(Duration::from_secs(5), "b sees a", || async { + b.runtime.peers().connected_count().await == 1 + }) + .await; + + let hash = a.emit("deploy the thing").await; + + eventually(Duration::from_secs(5), "b receives a's signal", || async { + b.has_signal(&hash).await + }) + .await; + + // B accepted it under its own sensing policy, and attributes the origin to + // A even though A's local diffusion state never crossed the wire. + let network = b.runtime.network(); + let network = network.read().await; + let signal = network.field.signals.get(&hash).expect("signal present"); + assert_eq!(signal.origin_node_id, "node-a"); + assert_eq!(signal.payload, b"deploy the thing".to_vec()); + assert_eq!( + signal.reached_nodes, + vec!["node-b".to_string()], + "reached_nodes must be the receiver's local view, not the sender's" + ); + drop(network); + + a.shutdown().await; + b.shutdown().await; +} + +#[tokio::test] +async fn peer_learned_second_hand_is_dialled() { + let a = MeshNode::start("node-a", vec![]).await; + let b = MeshNode::start("node-b", vec![a.addr()]).await; + + eventually(Duration::from_secs(5), "a and b meet", || async { + a.runtime.peers().connected_count().await == 1 + }) + .await; + + // C only knows about A. It should learn B from A's PeerResponse and dial it. + let c = MeshNode::start("node-c", vec![a.addr()]).await; + + eventually( + Duration::from_secs(10), + "c discovers both a and b", + || async { + let peers = c.runtime.peers().connected_peers().await; + let mut ids: Vec = peers.into_iter().map(|p| p.node_id).collect(); + ids.sort(); + ids == vec!["node-a".to_string(), "node-b".to_string()] + }, + ) + .await; + + a.shutdown().await; + b.shutdown().await; + c.shutdown().await; +} + +#[tokio::test] +async fn flooding_does_not_duplicate_state() { + // Fully connected triangle: A's signal can reach C directly and via B. + let a = MeshNode::start("node-a", vec![]).await; + let b = MeshNode::start("node-b", vec![a.addr()]).await; + let c = MeshNode::start("node-c", vec![a.addr(), b.addr()]).await; + + eventually(Duration::from_secs(10), "triangle forms", || async { + a.runtime.peers().connected_count().await == 2 + && b.runtime.peers().connected_count().await == 2 + && c.runtime.peers().connected_count().await == 2 + }) + .await; + + let hash = a.emit("consensus please").await; + + eventually(Duration::from_secs(5), "b and c both have it", || async { + b.has_signal(&hash).await && c.has_signal(&hash).await + }) + .await; + + // Give any relayed copies time to arrive and be deduped. + tokio::time::sleep(Duration::from_millis(500)).await; + + // Content addressing is what stops a flood from becoming duplicate state: + // a second arrival reinforces the existing signal rather than adding one. + assert_eq!(c.signal_count(&hash).await, 1); + assert_eq!(b.signal_count(&hash).await, 1); + + // And the origin never loops back to itself as a new signal. + assert_eq!(a.signal_count(&hash).await, 1); + + a.shutdown().await; + b.shutdown().await; + c.shutdown().await; +} From 2fd2fcab4dbfda23be9927a5318eac4d3919bf97 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 20:29:39 -0700 Subject: [PATCH 03/14] cli: multi-concern analysis mesh as real processes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A demo where the answer cannot be reached alone. Five analyst nodes each read one metric family of the same service fleet and can see nothing else: latency, errors, saturation, traces, deploys. A deploy cuts a connection pool from 200 to 20, and every concern sees a fragment of the damage while two of them point at the wrong service. `smesh orchestrate` is a launcher, not a coordinator. It picks ports and a shared run epoch, spawns one OS process per concern, and holds no state the analysts can reach. Everything they learn from each other crosses a real QUIC socket. The topology is a ring plus one chord, with peer discovery off — discovery quietly turns any topology into a full mesh, and then nothing has to be relayed. Findings correlate because the signal payload is the assertion and nothing else. `.origin()` is deliberately not set when building one: the builder folds the origin into the content hash, which would give every analyst a different address for the same claim. The address is the claim, not the claimant. Evidence differs per concern and would make every hash unique, so it stays local and goes to the journal. The corroboration tally is asserted in tests rather than hoped for: the cause collects five attesters, the downstream casualty three, and each planted decoy exactly one. `validate` checks a recorded run against itself — sequence gaps, time running backwards, snapshots referencing signals never received, consensus declared without the receipts to justify it. It caught a real ordering fault on its first run: a peer completed the handshake before the node had written its own identity line, because the endpoint was accepting connections before the journal was opened. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 4 + smesh-cli/src/analysis/concern.rs | 506 ++++++++++++++++++++++++++ smesh-cli/src/analysis/corpus.rs | 400 ++++++++++++++++++++ smesh-cli/src/analysis/mod.rs | 7 + smesh-cli/src/analysis/node.rs | 443 ++++++++++++++++++++++ smesh-cli/src/analysis/orchestrate.rs | 418 +++++++++++++++++++++ smesh-cli/src/analysis/validate.rs | 349 ++++++++++++++++++ smesh-cli/src/main.rs | 356 +++++++++++++++++- 8 files changed, 2474 insertions(+), 9 deletions(-) create mode 100644 smesh-cli/src/analysis/concern.rs create mode 100644 smesh-cli/src/analysis/corpus.rs create mode 100644 smesh-cli/src/analysis/mod.rs create mode 100644 smesh-cli/src/analysis/node.rs create mode 100644 smesh-cli/src/analysis/orchestrate.rs create mode 100644 smesh-cli/src/analysis/validate.rs diff --git a/.gitignore b/.gitignore index 6e5b891..5a0045c 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,7 @@ owasp-results.json adjudication-report.html adjudication-results.json smesh-demo.mp4 + +# Recorded mesh runs (journals + merged timelines) +runs/ + diff --git a/smesh-cli/src/analysis/concern.rs b/smesh-cli/src/analysis/concern.rs new file mode 100644 index 0000000..cc0e4eb --- /dev/null +++ b/smesh-cli/src/analysis/concern.rs @@ -0,0 +1,506 @@ +//! Analyst concerns: one metric family each, deliberately partial. +//! +//! Every concern reads a different slice of the same telemetry and can only +//! reason about what it sees. That partiality is the point. A concern is not a +//! weak detector to be improved; it is one witness, and the mesh's job is to +//! decide which claims survive being heard by several of them. +//! +//! Concerns never talk to each other directly and never share evidence. They +//! meet only as signals in the field. + +use serde::{Deserialize, Serialize}; + +use super::corpus::{baseline_of, Bucket, Metric}; + +/// Buckets a detector averages over before it will speak. +const WINDOW: usize = 3; + +/// Minutes after a deploy in which a shift still counts as related. +const DEPLOY_BLAST_RADIUS: i32 = 5; + +/// The claim carried on the wire. +/// +/// This is the *entire* payload of an emitted signal, and it is deliberately +/// tiny: a subject and what is being said about it, nothing else. Two analysts +/// that independently reach the same conclusion serialise byte-identical +/// payloads, so the protocol's content-addressed hash puts them on the same +/// signal and treats the second one as corroboration rather than noise. +/// +/// Evidence deliberately stays out. It differs per concern and would make every +/// hash unique, destroying the correlation this whole design depends on — so +/// evidence goes to the journal, and only the assertion goes to the mesh. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Assertion { + /// The service being described. + pub subject: String, + /// What is claimed about it. + pub claim: String, +} + +impl Assertion { + /// Canonical bytes for hashing and transmission. + /// + /// Field order is fixed by the struct definition, so this is stable across + /// processes and runs. + pub fn canonical_bytes(&self) -> Vec { + serde_json::to_vec(self).expect("assertion serialises") + } +} + +/// One measurement supporting a finding. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Evidence { + /// Which metric. + pub metric: String, + /// Normal value for this service. + pub baseline: f64, + /// What was actually observed, averaged over the window. + pub observed: f64, + /// Observed over baseline. + pub ratio: f64, + /// Minutes the window covers, relative to the incident. + pub window: String, +} + +/// A conclusion one concern reached on its own. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Finding { + /// What is being claimed. + pub assertion: Assertion, + /// How sure this concern is, from its own evidence alone. + pub confidence: f64, + /// Why it thinks so. + pub evidence: Vec, +} + +/// The five analyst concerns. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Concern { + /// Request latency percentiles. + Latency, + /// Error rates and status codes. + Errors, + /// Resource and pool saturation. + Saturation, + /// Span-level retry and queueing behaviour. + Traces, + /// Releases and config changes. + Deploys, +} + +impl Concern { + /// Every concern, in the order the demo arranges them. + pub fn all() -> Vec { + vec![ + Concern::Latency, + Concern::Errors, + Concern::Saturation, + Concern::Traces, + Concern::Deploys, + ] + } + + /// Parse a concern by name. + pub fn parse(name: &str) -> Option { + match name.to_ascii_lowercase().as_str() { + "latency" => Some(Concern::Latency), + "errors" => Some(Concern::Errors), + "saturation" => Some(Concern::Saturation), + "traces" => Some(Concern::Traces), + "deploys" => Some(Concern::Deploys), + _ => None, + } + } + + /// Stable lowercase name, used as the node id on the mesh. + pub fn name(&self) -> &'static str { + match self { + Concern::Latency => "latency", + Concern::Errors => "errors", + Concern::Saturation => "saturation", + Concern::Traces => "traces", + Concern::Deploys => "deploys", + } + } + + /// What this concern is allowed to look at. + pub fn metrics(&self) -> &'static [&'static str] { + match self { + Concern::Latency => &["p99_ms"], + Concern::Errors => &["error_rate"], + Concern::Saturation => &["pool_utilization", "cpu_pct"], + Concern::Traces => &["retry_rate", "span_queue_depth"], + Concern::Deploys => &["deploys", "error_rate"], + } + } + + /// One line describing what this analyst does. + pub fn description(&self) -> &'static str { + match self { + Concern::Latency => "watches p99 latency for sustained regressions", + Concern::Errors => "watches error rates for elevated failure", + Concern::Saturation => "watches pool and CPU saturation", + Concern::Traces => "watches retry amplification and span queueing", + Concern::Deploys => "correlates releases with what follows them", + } + } + + /// Run this concern's detector over everything observed up to `now_minute`. + /// + /// Returns every finding the concern currently stands behind. Detectors are + /// pure: the same telemetry always yields the same findings, so a rerun of + /// the corpus is reproducible even though the mesh around it is not. + pub fn detect(&self, buckets: &[Bucket], now_minute: i32) -> Vec { + let visible: Vec<&Bucket> = buckets.iter().filter(|b| b.minute <= now_minute).collect(); + + match self { + Concern::Latency => threshold_findings( + &visible, + now_minute, + Metric::P99, + "p99_ms", + |b| b.p99_ms, + Comparison::RatioAtLeast(1.8), + ), + Concern::Errors => threshold_findings( + &visible, + now_minute, + Metric::ErrorRate, + "error_rate", + |b| b.error_rate, + Comparison::RatioAtLeast(3.0), + ), + Concern::Saturation => { + let mut findings = threshold_findings( + &visible, + now_minute, + Metric::PoolUtilization, + "pool_utilization", + |b| b.pool_utilization, + Comparison::AbsoluteAtLeast(0.90), + ); + findings.extend(threshold_findings( + &visible, + now_minute, + Metric::CpuPct, + "cpu_pct", + |b| b.cpu_pct, + Comparison::AbsoluteAtLeast(85.0), + )); + merge_by_subject(findings) + } + Concern::Traces => { + let mut findings = threshold_findings( + &visible, + now_minute, + Metric::RetryRate, + "retry_rate", + |b| b.retry_rate, + Comparison::AbsoluteAtLeast(0.10), + ); + findings.extend(threshold_findings( + &visible, + now_minute, + Metric::SpanQueueDepth, + "span_queue_depth", + |b| b.span_queue_depth, + Comparison::AbsoluteAtLeast(20.0), + )); + merge_by_subject(findings) + } + Concern::Deploys => deploy_findings(&visible, now_minute), + } + } +} + +/// How a detector decides a window is abnormal. +enum Comparison { + /// Observed is at least this multiple of the service's baseline. + RatioAtLeast(f64), + /// Observed is at least this absolute value. + AbsoluteAtLeast(f64), +} + +/// Average the last [`WINDOW`] buckets per service and flag what crosses. +fn threshold_findings( + visible: &[&Bucket], + now_minute: i32, + metric: Metric, + metric_name: &str, + extract: fn(&Bucket) -> f64, + comparison: Comparison, +) -> Vec { + let mut findings = Vec::new(); + + let mut services: Vec<&str> = visible.iter().map(|b| b.service.as_str()).collect(); + services.sort_unstable(); + services.dedup(); + + for service in services { + let mut window: Vec<&&Bucket> = visible.iter().filter(|b| b.service == service).collect(); + window.sort_by_key(|b| b.minute); + let window: Vec<&&Bucket> = window.into_iter().rev().take(WINDOW).collect(); + + if window.len() < WINDOW { + continue; + } + + let observed = window.iter().map(|b| extract(b)).sum::() / window.len() as f64; + let baseline = baseline_of(service, metric); + let ratio = if baseline > 0.0 { + observed / baseline + } else { + 0.0 + }; + + let (crossed, strength) = match comparison { + Comparison::RatioAtLeast(limit) => (ratio >= limit, (ratio / limit).min(3.0)), + Comparison::AbsoluteAtLeast(limit) => (observed >= limit, (observed / limit).min(3.0)), + }; + + if !crossed { + continue; + } + + let earliest = window.iter().map(|b| b.minute).min().unwrap_or(now_minute); + + findings.push(Finding { + assertion: Assertion { + subject: service.to_string(), + claim: "degraded".to_string(), + }, + // A single concern is never certain. Even a blatant reading caps + // below the threshold that would let one witness carry a verdict. + confidence: (0.35 + (strength - 1.0) * 0.15).clamp(0.35, 0.72), + evidence: vec![Evidence { + metric: metric_name.to_string(), + baseline, + observed, + ratio, + window: format!("T{earliest:+}..T{now_minute:+}"), + }], + }); + } + + findings +} + +/// Flag a deployed service when a shift follows its release closely enough. +/// +/// The control case matters as much as the positive one: a deploy with nothing +/// after it must not be blamed, or this concern would indict every release. +fn deploy_findings(visible: &[&Bucket], now_minute: i32) -> Vec { + let mut findings = Vec::new(); + + for bucket in visible { + for deploy in &bucket.deploys { + let deployed_at = bucket.minute; + let after: Vec<&&Bucket> = visible + .iter() + .filter(|b| { + b.service == deploy.service + && b.minute > deployed_at + && b.minute <= deployed_at + DEPLOY_BLAST_RADIUS + }) + .collect(); + + if after.len() < WINDOW { + continue; + } + + let observed = after.iter().map(|b| b.error_rate).sum::() / after.len() as f64; + let baseline = baseline_of(&deploy.service, Metric::ErrorRate); + let ratio = if baseline > 0.0 { + observed / baseline + } else { + 0.0 + }; + + if ratio < 3.0 { + continue; + } + + findings.push(Finding { + assertion: Assertion { + subject: deploy.service.clone(), + claim: "degraded".to_string(), + }, + confidence: (0.35 + (ratio / 3.0 - 1.0) * 0.15).clamp(0.35, 0.72), + evidence: vec![Evidence { + metric: format!("deploy {} ({})", deploy.version, deploy.change), + baseline, + observed, + ratio, + window: format!("T{deployed_at:+}..T{now_minute:+}"), + }], + }); + } + } + + merge_by_subject(findings) +} + +/// Collapse several readings about one subject into one finding. +fn merge_by_subject(findings: Vec) -> Vec { + let mut merged: Vec = Vec::new(); + + for finding in findings { + if let Some(existing) = merged.iter_mut().find(|f| f.assertion == finding.assertion) { + existing.confidence = existing.confidence.max(finding.confidence); + existing.evidence.extend(finding.evidence); + } else { + merged.push(finding); + } + } + + merged +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analysis::corpus::{generate, FIRST_MINUTE, LAST_MINUTE}; + + /// Everything a concern claims at any point as the run plays out. + /// + /// This is what the mesh actually sees: an analyst asserts a finding while + /// its evidence is live, so a transient anomaly is claimed at the time and + /// simply stops being re-asserted afterwards. + fn subjects_over_run(concern: Concern) -> Vec { + let corpus = generate(42); + let mut subjects: Vec = (FIRST_MINUTE..=LAST_MINUTE) + .flat_map(|minute| concern.detect(&corpus, minute)) + .map(|f| f.assertion.subject) + .collect(); + subjects.sort(); + subjects.dedup(); + subjects + } + + fn subjects_at(concern: Concern, minute: i32) -> Vec { + let corpus = generate(42); + let mut subjects: Vec = concern + .detect(&corpus, minute) + .into_iter() + .map(|f| f.assertion.subject) + .collect(); + subjects.sort(); + subjects.dedup(); + subjects + } + + #[test] + fn latency_alone_is_noisy() { + // Four subjects across the run, only one of which is the real cause. + // This is why a single concern cannot be trusted to call an incident. + assert_eq!( + subjects_over_run(Concern::Latency), + vec![ + "checkout-api", + "edge-gateway", + "payments-api", + "session-store" + ] + ); + } + + #[test] + fn errors_would_blame_the_victim() { + // Errors sees payments-api failing loudly. On its own it points at the + // service that is suffering, not the one that is causing it. + assert_eq!( + subjects_over_run(Concern::Errors), + vec!["checkout-api", "payments-api"] + ); + } + + #[test] + fn saturation_sees_the_pool_and_one_red_herring() { + assert_eq!( + subjects_over_run(Concern::Saturation), + vec!["checkout-api", "notification-worker"] + ); + } + + #[test] + fn traces_see_retry_amplification() { + assert_eq!( + subjects_over_run(Concern::Traces), + vec!["checkout-api", "payments-api"] + ); + } + + #[test] + fn deploys_blames_the_bad_release_and_not_the_benign_one() { + // inventory-svc also shipped, and must not be implicated. + assert_eq!(subjects_over_run(Concern::Deploys), vec!["checkout-api"]); + } + + #[test] + fn decoys_are_claimed_while_live_and_dropped_afterwards() { + // The session-store blip is real while it is happening... + assert!(subjects_at(Concern::Latency, -12).contains(&"session-store".to_string())); + // ...and no longer claimed once it passes. Nothing retracts it on the + // mesh, so the only thing that removes it is decay. + assert!(!subjects_at(Concern::Latency, 19).contains(&"session-store".to_string())); + + assert!(subjects_at(Concern::Saturation, -5).contains(&"notification-worker".to_string())); + assert!(!subjects_at(Concern::Saturation, 19).contains(&"notification-worker".to_string())); + } + + #[test] + fn corroboration_tally_separates_cause_from_symptom_from_noise() { + let corpus = generate(42); + let mut tally: std::collections::BTreeMap> = Default::default(); + + for concern in Concern::all() { + let mut seen: Vec = Vec::new(); + for minute in FIRST_MINUTE..=LAST_MINUTE { + for finding in concern.detect(&corpus, minute) { + if !seen.contains(&finding.assertion.subject) { + seen.push(finding.assertion.subject.clone()); + tally + .entry(finding.assertion.subject) + .or_default() + .push(concern.name()); + } + } + } + } + + // The whole demo rests on this shape: the cause is corroborated by + // every concern, the casualty by some, the decoys by exactly one. + assert_eq!(tally["checkout-api"].len(), 5, "root cause"); + assert_eq!(tally["payments-api"].len(), 3, "downstream casualty"); + assert_eq!(tally["edge-gateway"].len(), 1, "weak downstream echo"); + assert_eq!(tally["session-store"].len(), 1, "planted decoy"); + assert_eq!(tally["notification-worker"].len(), 1, "planted decoy"); + } + + #[test] + fn identical_assertions_serialise_identically() { + // Content addressing depends on this: two concerns reaching the same + // conclusion must produce the same bytes, or they will never correlate. + let a = Assertion { + subject: "checkout-api".to_string(), + claim: "degraded".to_string(), + }; + let b = Assertion { + subject: "checkout-api".to_string(), + claim: "degraded".to_string(), + }; + assert_eq!(a.canonical_bytes(), b.canonical_bytes()); + } + + #[test] + fn nothing_is_claimed_before_the_evidence_exists() { + let corpus = generate(42); + // Twenty minutes before the deploy, the only thing anyone can see is + // the pre-incident decoys, and never checkout-api. + for concern in Concern::all() { + for finding in concern.detect(&corpus, -15) { + assert_ne!(finding.assertion.subject, "checkout-api"); + } + } + } +} diff --git a/smesh-cli/src/analysis/corpus.rs b/smesh-cli/src/analysis/corpus.rs new file mode 100644 index 0000000..0080e85 --- /dev/null +++ b/smesh-cli/src/analysis/corpus.rs @@ -0,0 +1,400 @@ +//! A deterministic synthetic telemetry corpus for the analysis mesh. +//! +//! **This is a fixture, not a capture.** No real service produced these +//! numbers. It exists so the mesh demo replays identically on any machine: the +//! interesting behaviour is the emergent correlation between analyst nodes, and +//! that is only legible if the input never moves. +//! +//! The corpus describes one incident, seeded so every node in the run derives +//! byte-identical numbers from the same seed without sharing state: +//! +//! > At T+0, `checkout-api` ships `v2.3.1`, which cuts its connection pool from +//! > 200 to 20. The pool pins, requests queue, and callers time out. +//! +//! No single metric family proves that. Saturation sees a pinned pool but not +//! why it matters. Errors see `payments-api` throwing 503s and would blame +//! payments. Latency sees three services slow at once. Only the union of the +//! concerns identifies `checkout-api` as the origin and `payments-api` as a +//! casualty — which is precisely what the mesh has to discover on its own. +//! +//! Two decoys are planted to prove the mesh discriminates rather than +//! agreeing with everything: an unrelated CPU spike on `notification-worker` +//! before the incident, and a brief latency blip on `session-store`. Each is +//! visible to exactly one concern, so neither should ever reach consensus. + +use serde::{Deserialize, Serialize}; + +/// First minute in the corpus, relative to the incident. +pub const FIRST_MINUTE: i32 = -20; +/// Last minute in the corpus, relative to the incident. +pub const LAST_MINUTE: i32 = 19; +/// Total buckets in the corpus. +pub const BUCKET_COUNT: usize = (LAST_MINUTE - FIRST_MINUTE + 1) as usize; + +/// A release or config change. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Deploy { + /// Service that changed. + pub service: String, + /// Version rolled out. + pub version: String, + /// Human-readable summary of what changed. + pub change: String, +} + +/// One service's metrics for one minute. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Bucket { + /// Minutes relative to the incident; negative is before. + pub minute: i32, + /// Service these metrics belong to. + pub service: String, + /// 99th percentile request latency, milliseconds. + pub p99_ms: f64, + /// Fraction of requests returning an error. + pub error_rate: f64, + /// Requests per second. + pub rps: f64, + /// CPU utilisation, percent. + pub cpu_pct: f64, + /// Connection pool utilisation, 0.0 to 1.0. + pub pool_utilization: f64, + /// Fraction of spans that are retries. + pub retry_rate: f64, + /// Mean queued spans awaiting a pool slot. + pub span_queue_depth: f64, + /// Deploys landing in this minute. + pub deploys: Vec, +} + +/// Baseline behaviour of one service before anything goes wrong. +struct Baseline { + service: &'static str, + p99_ms: f64, + error_rate: f64, + rps: f64, + cpu_pct: f64, + pool_utilization: f64, + retry_rate: f64, + span_queue_depth: f64, +} + +const BASELINES: &[Baseline] = &[ + Baseline { + service: "edge-gateway", + p99_ms: 40.0, + error_rate: 0.0010, + rps: 1800.0, + cpu_pct: 45.0, + pool_utilization: 0.30, + retry_rate: 0.008, + span_queue_depth: 1.5, + }, + Baseline { + service: "checkout-api", + p99_ms: 90.0, + error_rate: 0.0020, + rps: 640.0, + cpu_pct: 55.0, + pool_utilization: 0.35, + retry_rate: 0.010, + span_queue_depth: 2.0, + }, + Baseline { + service: "payments-api", + p99_ms: 120.0, + error_rate: 0.0030, + rps: 410.0, + cpu_pct: 50.0, + pool_utilization: 0.40, + retry_rate: 0.012, + span_queue_depth: 2.5, + }, + Baseline { + service: "inventory-svc", + p99_ms: 60.0, + error_rate: 0.0010, + rps: 520.0, + cpu_pct: 40.0, + pool_utilization: 0.25, + retry_rate: 0.006, + span_queue_depth: 1.0, + }, + Baseline { + service: "session-store", + p99_ms: 15.0, + error_rate: 0.0005, + rps: 2400.0, + cpu_pct: 30.0, + pool_utilization: 0.20, + retry_rate: 0.002, + span_queue_depth: 0.5, + }, + Baseline { + service: "notification-worker", + p99_ms: 200.0, + error_rate: 0.0040, + rps: 90.0, + cpu_pct: 35.0, + pool_utilization: 0.15, + retry_rate: 0.015, + span_queue_depth: 3.0, + }, +]; + +/// A small deterministic PRNG. +/// +/// Written out rather than pulled from `rand` so the corpus does not change if +/// a dependency changes its generator: a fixture whose values drift under you +/// is worse than no fixture. +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + // Avoid the zero state, which xorshift cannot leave. + Self(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).max(1)) + } + + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + /// Jitter in `[-magnitude, magnitude]`. + fn jitter(&mut self, magnitude: f64) -> f64 { + let unit = (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64; + (unit * 2.0 - 1.0) * magnitude + } +} + +/// Smooth 0→1 ramp over `duration` minutes starting at `start`. +fn ramp(minute: i32, start: i32, duration: i32) -> f64 { + if minute < start { + return 0.0; + } + let progress = (minute - start) as f64 / duration as f64; + progress.clamp(0.0, 1.0) +} + +/// Generate the full corpus for a seed. +/// +/// The same seed always yields the same numbers, on any machine, in any +/// process — which is what lets five separate analyst processes reason about +/// the same telemetry without a shared database. +pub fn generate(seed: u64) -> Vec { + let mut buckets = Vec::with_capacity(BUCKET_COUNT * BASELINES.len()); + + for base in BASELINES { + // Derive a per-service stream so adding a service does not perturb the + // numbers of the ones before it. + let mut rng = Rng::new(seed ^ fnv(base.service)); + + for minute in FIRST_MINUTE..=LAST_MINUTE { + let mut bucket = Bucket { + minute, + service: base.service.to_string(), + p99_ms: base.p99_ms * (1.0 + rng.jitter(0.06)), + error_rate: (base.error_rate * (1.0 + rng.jitter(0.15))).max(0.0), + rps: base.rps * (1.0 + rng.jitter(0.08)), + cpu_pct: base.cpu_pct * (1.0 + rng.jitter(0.05)), + pool_utilization: base.pool_utilization * (1.0 + rng.jitter(0.08)), + retry_rate: base.retry_rate * (1.0 + rng.jitter(0.12)), + span_queue_depth: base.span_queue_depth * (1.0 + rng.jitter(0.20)), + deploys: Vec::new(), + }; + + apply_incident(&mut bucket, base, minute); + apply_decoys(&mut bucket, base, minute); + apply_deploys(&mut bucket, base, minute); + + buckets.push(bucket); + } + } + + buckets.sort_by_key(|b| (b.minute, b.service.clone())); + buckets +} + +/// The pool exhaustion and everything downstream of it. +fn apply_incident(bucket: &mut Bucket, base: &Baseline, minute: i32) { + // The pool pins over three minutes, then stays pinned. + let severity = ramp(minute, 0, 3); + if severity <= 0.0 { + return; + } + + match base.service { + // The origin. Note CPU stays flat: this is not a load problem, which is + // the detail that separates cause from symptom. + "checkout-api" => { + bucket.p99_ms += (280.0 - base.p99_ms) * severity; + bucket.error_rate += (0.030 - base.error_rate) * severity; + bucket.pool_utilization += (0.98 - base.pool_utilization) * severity; + bucket.retry_rate += (0.220 - base.retry_rate) * severity; + bucket.span_queue_depth += (40.0 - base.span_queue_depth) * severity; + } + // A casualty: it calls checkout-api, times out, and returns 503s. Its + // own pool and CPU are fine, which is why blaming it would be wrong. + "payments-api" => { + bucket.p99_ms += (240.0 - base.p99_ms) * severity; + bucket.error_rate += (0.090 - base.error_rate) * severity; + bucket.retry_rate += (0.140 - base.retry_rate) * severity; + } + // Further downstream again, and correspondingly weaker. + "edge-gateway" => { + bucket.p99_ms += (76.0 - base.p99_ms) * severity; + bucket.error_rate += (0.0025 - base.error_rate) * severity; + } + _ => {} + } +} + +/// Unrelated anomalies, planted so the mesh has something to reject. +fn apply_decoys(bucket: &mut Bucket, base: &Baseline, minute: i32) { + // A CPU spike on a worker, well before the incident and causally unrelated. + // Only the saturation concern can see it. + if base.service == "notification-worker" && (-8..=-5).contains(&minute) { + bucket.cpu_pct = 88.0 + bucket.cpu_pct * 0.02; + } + + // A brief latency blip on the session store, also before the incident. + // Only the latency concern can see it. + if base.service == "session-store" && (-14..=-12).contains(&minute) { + bucket.p99_ms = 48.0 + bucket.p99_ms * 0.05; + } +} + +/// Release events, including one benign deploy that must not be blamed. +fn apply_deploys(bucket: &mut Bucket, base: &Baseline, minute: i32) { + if base.service == "checkout-api" && minute == 0 { + bucket.deploys.push(Deploy { + service: "checkout-api".to_string(), + version: "v2.3.1".to_string(), + change: "pool_max_conns 200 -> 20".to_string(), + }); + } + + // A deploy with no consequences. A concern that flags every deploy would + // flag this one too, so it is the control case. + if base.service == "inventory-svc" && minute == -12 { + bucket.deploys.push(Deploy { + service: "inventory-svc".to_string(), + version: "v1.9.0".to_string(), + change: "log sampling 1.0 -> 0.5".to_string(), + }); + } +} + +/// Baseline value of one metric for a service, for ratio comparisons. +pub fn baseline_of(service: &str, metric: Metric) -> f64 { + BASELINES + .iter() + .find(|b| b.service == service) + .map(|b| match metric { + Metric::P99 => b.p99_ms, + Metric::ErrorRate => b.error_rate, + Metric::CpuPct => b.cpu_pct, + Metric::PoolUtilization => b.pool_utilization, + Metric::RetryRate => b.retry_rate, + Metric::SpanQueueDepth => b.span_queue_depth, + }) + .unwrap_or(0.0) +} + +/// The metric families a concern can read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Metric { + /// 99th percentile latency. + P99, + /// Error rate. + ErrorRate, + /// CPU utilisation. + CpuPct, + /// Connection pool utilisation. + PoolUtilization, + /// Retry fraction. + RetryRate, + /// Queued spans. + SpanQueueDepth, +} + +fn fnv(s: &str) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in s.as_bytes() { + hash ^= *byte as u64; + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn corpus_is_deterministic_across_calls() { + let a = generate(42); + let b = generate(42); + assert_eq!(a.len(), b.len()); + for (x, y) in a.iter().zip(b.iter()) { + assert_eq!(x.service, y.service); + assert_eq!(x.minute, y.minute); + assert_eq!(x.p99_ms.to_bits(), y.p99_ms.to_bits()); + assert_eq!(x.pool_utilization.to_bits(), y.pool_utilization.to_bits()); + } + } + + #[test] + fn different_seeds_differ() { + let a = generate(1); + let b = generate(2); + assert!(a.iter().zip(b.iter()).any(|(x, y)| x.p99_ms != y.p99_ms)); + } + + #[test] + fn checkout_pool_pins_after_the_deploy() { + let corpus = generate(42); + let before = corpus + .iter() + .find(|b| b.service == "checkout-api" && b.minute == -5) + .unwrap(); + let after = corpus + .iter() + .find(|b| b.service == "checkout-api" && b.minute == 10) + .unwrap(); + + assert!(before.pool_utilization < 0.5); + assert!(after.pool_utilization > 0.9); + // CPU stays flat: the incident is not load. + assert!((after.cpu_pct - before.cpu_pct).abs() < 10.0); + } + + #[test] + fn the_deploy_is_present_exactly_once() { + let corpus = generate(42); + let deploys: Vec<_> = corpus + .iter() + .flat_map(|b| b.deploys.iter()) + .filter(|d| d.service == "checkout-api") + .collect(); + assert_eq!(deploys.len(), 1); + assert_eq!(deploys[0].version, "v2.3.1"); + } + + #[test] + fn decoys_are_visible_only_in_their_own_metric() { + let corpus = generate(42); + let spike = corpus + .iter() + .find(|b| b.service == "notification-worker" && b.minute == -6) + .unwrap(); + assert!(spike.cpu_pct > 85.0); + // Everything else about the worker stays ordinary. + assert!(spike.error_rate < 0.01); + assert!(spike.p99_ms < 260.0); + } +} diff --git a/smesh-cli/src/analysis/mod.rs b/smesh-cli/src/analysis/mod.rs new file mode 100644 index 0000000..4c1c760 --- /dev/null +++ b/smesh-cli/src/analysis/mod.rs @@ -0,0 +1,7 @@ +//! Multi-concern telemetry analysis over a real SMESH mesh. + +pub mod concern; +pub mod corpus; +pub mod node; +pub mod orchestrate; +pub mod validate; diff --git a/smesh-cli/src/analysis/node.rs b/smesh-cli/src/analysis/node.rs new file mode 100644 index 0000000..8b2e3ee --- /dev/null +++ b/smesh-cli/src/analysis/node.rs @@ -0,0 +1,443 @@ +//! One analyst: a single concern, a single mesh node, a single process. +//! +//! The analyst walks the telemetry corpus in compressed real time, and whenever +//! its own detector stands behind a claim it asserts that claim onto the mesh. +//! It never learns another analyst's evidence — only whether anyone else is +//! asserting the same thing. + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use serde_json::json; + +use smesh_core::{DecayFunction, Network, Node, Signal, SignalType}; +use smesh_runtime::{Journal, MeshConfig, RuntimeConfig, SmeshRuntime}; + +use super::concern::{Assertion, Concern, Finding}; +use super::corpus::{self, Bucket, FIRST_MINUTE, LAST_MINUTE}; + +/// Signal lifetime, in seconds of field time. +/// +/// Tuned against the compressed timeline so a transient claim visibly fades +/// within one run while a sustained one survives to the end. Decay is the only +/// thing that retracts a claim: nothing in this protocol sends a retraction. +const SIGNAL_TTL_SECS: f64 = 22.0; + +/// Exponential decay constant for emitted claims. +const SIGNAL_DECAY_RATE: f64 = 0.10; + +/// Hop budget. Large enough to cross this topology twice over. +const SIGNAL_RADIUS: u32 = 6; + +/// How much these analysts trust each other. +/// +/// They are a known fleet, so trust is high; relay probability is proportional +/// to it, and an untrusting fleet would simply not gossip. +const FLEET_TRUST: f64 = 0.95; + +/// How often, in corpus minutes, a standing finding is re-asserted. +/// +/// This is anti-entropy. Relaying is probabilistic, so a single assertion can +/// fail to cross the mesh; re-asserting carries the *accumulated* attester set +/// again and lets a node that missed the first round catch up. +const REASSERT_EVERY_MINUTES: i32 = 4; + +/// Configuration for one analyst process. +#[derive(Debug, Clone)] +pub struct AnalystConfig { + /// This analyst's concern. + pub concern: Concern, + /// Address to listen on. + pub bind: SocketAddr, + /// Peers to dial. + pub peers: Vec, + /// Where to write the journal, if recording. + pub journal: Option, + /// Shared run epoch, so every node's timestamps land on one timeline. + pub run_epoch_ms: i64, + /// Corpus seed. Must match across the fleet. + pub seed: u64, + /// Wall-clock milliseconds per corpus minute. + pub bucket_ms: u64, + /// Distinct attesters required before a claim is treated as consensus. + pub consensus_threshold: usize, + /// Peers to wait for before starting the timeline. + pub expect_peers: usize, + /// Extra time to keep gossiping after the corpus runs out. + pub settle_ms: u64, + /// Print progress to stdout. + pub verbose: bool, +} + +/// Run one analyst to completion. +pub async fn run(config: AnalystConfig) -> Result<()> { + let concern = config.concern; + let node_id = concern.name().to_string(); + + let journal = match &config.journal { + Some(path) => { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).ok(); + } + Journal::create(path, &node_id, Some(node_id.clone()), config.run_epoch_ms) + .with_context(|| format!("opening journal at {}", path.display()))? + } + None => Journal::disabled(), + }; + + // One node per process. It trusts its fellow analysts but has no idea what + // any of them can see. + let mut node = Node::new(); + node.id = node_id.clone(); + for other in Concern::all() { + if other != concern { + node.trust_scores + .insert(other.name().to_string(), FLEET_TRUST); + } + } + + let mut network = Network::new(); + network.add_node(node); + + let runtime = Arc::new( + SmeshRuntime::with_network( + network, + RuntimeConfig { + tick_interval_ms: 100, + ..Default::default() + }, + ) + .with_journal(Arc::clone(&journal)), + ); + + let mesh = runtime + .join_mesh( + MeshConfig { + bind_addr: config.bind, + bootstrap: config.peers.clone(), + keepalive_interval_ms: 2_000, + // The topology is the thing under study here, so discovery is + // off: the mesh stays exactly the shape it was given. + peer_discovery: false, + // Written as the journal's first line, before the endpoint can + // accept anything. + node_metadata: json!({ + "concern": concern.name(), + "description": concern.description(), + "metrics": concern.metrics(), + "seed": config.seed, + "bucket_ms": config.bucket_ms, + "consensus_threshold": config.consensus_threshold, + "signal_ttl_secs": SIGNAL_TTL_SECS, + "signal_decay_rate": SIGNAL_DECAY_RATE, + "signal_radius": SIGNAL_RADIUS, + "corpus_first_minute": FIRST_MINUTE, + "corpus_last_minute": LAST_MINUTE, + }), + ..Default::default() + }, + &node_id, + ) + .await?; + + if config.verbose { + println!( + "[{}] listening on {} — {}", + node_id, + mesh.listen_addr(), + concern.description() + ); + } + + // Decay and local diffusion run for as long as we are on the mesh. + { + let runtime = Arc::clone(&runtime); + tokio::spawn(async move { runtime.run().await }); + } + + wait_for_peers(&runtime, &journal, config.expect_peers, &config).await; + + let corpus = corpus::generate(config.seed); + let mut asserted: BTreeMap = BTreeMap::new(); + let mut announced_consensus: BTreeSet = BTreeSet::new(); + let mut last_reassert = FIRST_MINUTE; + + for minute in FIRST_MINUTE..=LAST_MINUTE { + tokio::time::sleep(Duration::from_millis(config.bucket_ms)).await; + + record_observations(&journal, concern, &corpus, minute); + + let findings = concern.detect(&corpus, minute); + let reassert = minute - last_reassert >= REASSERT_EVERY_MINUTES; + if reassert { + last_reassert = minute; + } + + for finding in findings { + let hash_key = + String::from_utf8_lossy(&finding.assertion.canonical_bytes()).to_string(); + let is_new = !asserted.contains_key(&hash_key); + + if is_new || reassert { + asserted.insert(hash_key, finding.assertion.clone()); + assert_finding(&runtime, &journal, &node_id, &finding, minute, is_new).await; + } + } + + check_consensus( + &runtime, + &journal, + config.consensus_threshold, + &mut announced_consensus, + minute, + config.verbose, + ) + .await; + } + + // Keep gossiping after the corpus ends so in-flight claims converge. + let settle_steps = (config.settle_ms / 250).max(1); + for _ in 0..settle_steps { + tokio::time::sleep(Duration::from_millis(250)).await; + check_consensus( + &runtime, + &journal, + config.consensus_threshold, + &mut announced_consensus, + LAST_MINUTE, + config.verbose, + ) + .await; + } + + record_summary(&runtime, &journal, &node_id, config.consensus_threshold).await; + + runtime.shutdown().await; + mesh.shutdown().await; + + Ok(()) +} + +/// Block until the expected peers show up, or give up and proceed alone. +async fn wait_for_peers( + runtime: &SmeshRuntime, + journal: &Journal, + expect: usize, + config: &AnalystConfig, +) { + if expect == 0 { + return; + } + + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + loop { + let connected = runtime.peers().connected_count().await; + if connected >= expect { + journal.record("mesh_ready", json!({ "connected_peers": connected })); + if config.verbose { + println!( + "[{}] mesh ready, {connected} peer(s)", + config.concern.name() + ); + } + return; + } + if tokio::time::Instant::now() >= deadline { + journal.record( + "mesh_degraded", + json!({ "connected_peers": connected, "expected": expect }), + ); + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// Log exactly what this concern can see this minute, and nothing else. +fn record_observations(journal: &Journal, concern: Concern, corpus: &[Bucket], minute: i32) { + let readings: Vec = corpus + .iter() + .filter(|b| b.minute == minute) + .map(|b| { + let mut reading = serde_json::Map::new(); + reading.insert("service".into(), json!(b.service)); + for metric in concern.metrics() { + let value = match *metric { + "p99_ms" => json!(b.p99_ms), + "error_rate" => json!(b.error_rate), + "cpu_pct" => json!(b.cpu_pct), + "pool_utilization" => json!(b.pool_utilization), + "retry_rate" => json!(b.retry_rate), + "span_queue_depth" => json!(b.span_queue_depth), + "deploys" => json!(b.deploys), + _ => continue, + }; + reading.insert((*metric).to_string(), value); + } + serde_json::Value::Object(reading) + }) + .collect(); + + journal.record( + "observation", + json!({ "minute": minute, "readings": readings }), + ); +} + +/// Put a finding onto the mesh as a signal. +async fn assert_finding( + runtime: &SmeshRuntime, + journal: &Journal, + node_id: &str, + finding: &Finding, + minute: i32, + is_new: bool, +) { + journal.record( + "finding", + json!({ + "minute": minute, + "subject": finding.assertion.subject, + "claim": finding.assertion.claim, + "confidence": finding.confidence, + "evidence": finding.evidence, + "first_time": is_new, + }), + ); + + // The payload is the assertion and nothing else, so two analysts that reach + // the same conclusion produce identical bytes. + // + // `.origin()` is deliberately NOT set. The builder folds the origin node + // into the content hash when it is, which would give every analyst a + // different hash for the same claim and make corroboration impossible. The + // origin is still stamped on the signal at emit time for attribution — it + // just stays out of the address. The address is the *claim*, not the + // claimant. + let signal = Signal::builder(SignalType::Alert) + .payload(finding.assertion.canonical_bytes()) + .intensity(1.0) + .confidence(finding.confidence) + .ttl(SIGNAL_TTL_SECS) + .decay_rate(SIGNAL_DECAY_RATE) + .decay_function(DecayFunction::Exponential) + .radius(SIGNAL_RADIUS) + .build(); + + runtime.emit(signal, node_id).await; +} + +/// Look for claims that enough distinct parties now attest to. +async fn check_consensus( + runtime: &SmeshRuntime, + journal: &Journal, + threshold: usize, + announced: &mut BTreeSet, + minute: i32, + verbose: bool, +) { + let network = runtime.network(); + let network = network.read().await; + + for signal in network.field.signals.values() { + let attesters = Node::attesters(signal); + if attesters.len() < threshold || announced.contains(&signal.origin_hash) { + continue; + } + + announced.insert(signal.origin_hash.clone()); + + let assertion: Option = serde_json::from_slice(&signal.payload).ok(); + let subject = assertion + .as_ref() + .map(|a| a.subject.clone()) + .unwrap_or_default(); + + journal.record( + "consensus_reached", + json!({ + "hash": signal.origin_hash, + "minute": minute, + "subject": subject, + "claim": assertion.as_ref().map(|a| a.claim.clone()), + "attesters": attesters, + "attester_count": attesters.len(), + "threshold": threshold, + "confidence": signal.confidence, + "intensity": signal.current_intensity, + }), + ); + + if verbose { + println!( + " [consensus] {subject} — {} concerns concur: {}", + attesters.len(), + attesters.join(", ") + ); + } + } +} + +/// Final state of this node's field, as it saw things. +async fn record_summary( + runtime: &SmeshRuntime, + journal: &Journal, + node_id: &str, + threshold: usize, +) { + let stats = runtime.stats().await; + let network = runtime.network(); + let network = network.read().await; + + let mut claims: Vec = network + .field + .signals + .values() + .map(|signal| { + let attesters = Node::attesters(signal); + let assertion: Option = serde_json::from_slice(&signal.payload).ok(); + json!({ + "hash": signal.origin_hash, + "subject": assertion.as_ref().map(|a| a.subject.clone()), + "claim": assertion.as_ref().map(|a| a.claim.clone()), + "attesters": attesters, + "attester_count": attesters.len(), + "consensus": attesters.len() >= threshold, + "confidence": signal.confidence, + "intensity": signal.current_intensity, + "hops": signal.hops, + }) + }) + .collect(); + + claims.sort_by(|a, b| { + b["attester_count"] + .as_u64() + .cmp(&a["attester_count"].as_u64()) + }); + + let node_stats = network.get_node(node_id).map(|n| { + json!({ + "signals_emitted": n.stats.signals_emitted, + "signals_sensed": n.stats.signals_sensed, + "signals_relayed": n.stats.signals_relayed, + "signals_reinforced": n.stats.signals_reinforced, + }) + }); + + journal.record( + "node_stopped", + json!({ + "ticks": stats.tick_count, + "peers_known": stats.peer_count, + "peers_connected": stats.connected_peers, + "active_signals": stats.active_signals, + "node_stats": node_stats, + "claims": claims, + }), + ); +} diff --git a/smesh-cli/src/analysis/orchestrate.rs b/smesh-cli/src/analysis/orchestrate.rs new file mode 100644 index 0000000..d699fff --- /dev/null +++ b/smesh-cli/src/analysis/orchestrate.rs @@ -0,0 +1,418 @@ +//! Run the analysis mesh as real, separate processes. +//! +//! The orchestrator is a launcher, not a coordinator. It picks the ports and +//! the shared run epoch, starts one OS process per concern, and then gets out +//! of the way — it holds no shared state the analysts can reach, relays nothing +//! between them, and makes no decisions on their behalf. Everything the +//! analysts learn from each other crosses a real QUIC socket. +//! +//! When the run ends it merges the per-node journals into one ordered timeline +//! and writes a manifest describing the run. + +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::process::Stdio; + +use anyhow::{bail, Context, Result}; +use serde_json::json; +use tokio::process::Command; + +use smesh_runtime::JournalEvent; + +use super::concern::Concern; +use super::corpus::{FIRST_MINUTE, LAST_MINUTE}; + +/// The mesh topology, as (from, to) pairs dialled once each. +/// +/// A ring with one chord. Deliberately not a full mesh: with everyone directly +/// connected there is no relaying, no hop count and nothing to watch diffuse. +/// The chord keeps the diameter at two so gossip still converges promptly. +const TOPOLOGY: &[(&str, &str)] = &[ + ("latency", "errors"), + ("latency", "saturation"), + ("errors", "saturation"), + ("saturation", "traces"), + ("traces", "deploys"), + ("deploys", "latency"), +]; + +/// Settings for one orchestrated run. +#[derive(Debug, Clone)] +pub struct RunConfig { + /// Directory for journals and the merged timeline. + pub out_dir: PathBuf, + /// First TCP/UDP port; concerns take consecutive ports from here. + pub base_port: u16, + /// Corpus seed, shared by every analyst. + pub seed: u64, + /// Wall-clock milliseconds per corpus minute. + pub bucket_ms: u64, + /// Distinct attesters required for consensus. + pub consensus_threshold: usize, + /// Extra gossip time after the corpus ends. + pub settle_ms: u64, +} + +impl Default for RunConfig { + fn default() -> Self { + Self { + out_dir: PathBuf::from("runs/latest"), + base_port: 9301, + seed: 42, + bucket_ms: 700, + consensus_threshold: 4, + settle_ms: 6_000, + } + } +} + +/// Where each concern listens. +fn addresses(base_port: u16) -> BTreeMap<&'static str, SocketAddr> { + Concern::all() + .into_iter() + .enumerate() + .map(|(i, concern)| { + let addr: SocketAddr = format!("127.0.0.1:{}", base_port + i as u16) + .parse() + .expect("valid loopback address"); + (concern.name(), addr) + }) + .collect() +} + +/// Which peers a given concern dials. +fn bootstrap_for(concern: &str, addrs: &BTreeMap<&'static str, SocketAddr>) -> Vec { + TOPOLOGY + .iter() + .filter(|(from, _)| *from == concern) + .filter_map(|(_, to)| addrs.get(to).copied()) + .collect() +} + +/// How many peers a concern should end up connected to. +fn degree_of(concern: &str) -> usize { + TOPOLOGY + .iter() + .filter(|(from, to)| *from == concern || *to == concern) + .count() +} + +/// Launch the fleet, wait for it, and merge the results. +pub async fn run(config: RunConfig) -> Result { + let exe = std::env::current_exe().context("locating the smesh binary")?; + std::fs::create_dir_all(&config.out_dir) + .with_context(|| format!("creating {}", config.out_dir.display()))?; + + let addrs = addresses(config.base_port); + let run_epoch_ms = chrono::Utc::now().timestamp_millis(); + + let corpus_minutes = (LAST_MINUTE - FIRST_MINUTE + 1) as u64; + let expected_secs = (corpus_minutes * config.bucket_ms + config.settle_ms) / 1000; + + println!("╭─ SMESH multi-concern analysis mesh"); + println!( + "│ {} analyst processes, one per concern", + Concern::all().len() + ); + println!("│ topology ring + chord, {} links", TOPOLOGY.len()); + println!( + "│ corpus seed {} · {corpus_minutes} minutes", + config.seed + ); + println!( + "│ consensus {} distinct concerns", + config.consensus_threshold + ); + println!("│ journals {}", config.out_dir.display()); + println!("│ runtime ~{expected_secs}s"); + println!("╰─\n"); + + let mut children = Vec::new(); + + for concern in Concern::all() { + let name = concern.name(); + let bind = addrs[name]; + let peers = bootstrap_for(name, &addrs); + let journal = config.out_dir.join(format!("{name}.jsonl")); + + let mut command = Command::new(&exe); + command + .arg("analyze") + .arg("--concern") + .arg(name) + .arg("--bind") + .arg(bind.to_string()) + .arg("--journal") + .arg(&journal) + .arg("--run-epoch") + .arg(run_epoch_ms.to_string()) + .arg("--seed") + .arg(config.seed.to_string()) + .arg("--bucket-ms") + .arg(config.bucket_ms.to_string()) + .arg("--consensus-threshold") + .arg(config.consensus_threshold.to_string()) + .arg("--expect-peers") + .arg(degree_of(name).to_string()) + .arg("--settle-ms") + .arg(config.settle_ms.to_string()); + + for peer in &peers { + command.arg("--peer").arg(peer.to_string()); + } + + // Children inherit stdout so their progress is visible live. + let child = command + .stdin(Stdio::null()) + .kill_on_drop(true) + .spawn() + .with_context(|| format!("spawning analyst {name}"))?; + + println!( + " spawned {name:<11} pid {:<8} {bind} dials {}", + child.id().unwrap_or(0), + if peers.is_empty() { + "-".to_string() + } else { + peers + .iter() + .map(|p| p.port().to_string()) + .collect::>() + .join(",") + } + ); + + children.push((name, child)); + } + + println!(); + + let mut failures = Vec::new(); + for (name, mut child) in children { + let status = child + .wait() + .await + .with_context(|| format!("waiting for analyst {name}"))?; + if !status.success() { + failures.push(format!("{name} exited with {status}")); + } + } + + if !failures.is_empty() { + bail!("analyst processes failed: {}", failures.join("; ")); + } + + let merged = merge_journals(&config, run_epoch_ms)?; + Ok(merged) +} + +/// Merge the per-node journals into one ordered timeline plus a manifest. +/// +/// Ordering is by `t_ms` — which every node measured against the same epoch — +/// then by node and per-node sequence, so the result is deterministic and two +/// events in the same millisecond never swap places between merges. +fn merge_journals(config: &RunConfig, run_epoch_ms: i64) -> Result { + let mut events: Vec = Vec::new(); + let mut per_node: BTreeMap = BTreeMap::new(); + + for concern in Concern::all() { + let path = config.out_dir.join(format!("{}.jsonl", concern.name())); + let text = std::fs::read_to_string(&path) + .with_context(|| format!("reading journal {}", path.display()))?; + + let mut count = 0; + for (lineno, line) in text.lines().enumerate() { + if line.trim().is_empty() { + continue; + } + let event: JournalEvent = serde_json::from_str(line).with_context(|| { + format!( + "parsing {}:{} as a journal event", + path.display(), + lineno + 1 + ) + })?; + events.push(event); + count += 1; + } + per_node.insert(concern.name().to_string(), count); + } + + events.sort_by(|a, b| { + a.t_ms + .cmp(&b.t_ms) + .then_with(|| a.node.cmp(&b.node)) + .then_with(|| a.seq.cmp(&b.seq)) + }); + + let merged_path = config.out_dir.join("run.jsonl"); + let mut merged = String::with_capacity(events.len() * 200); + for event in &events { + merged.push_str(&serde_json::to_string(event)?); + merged.push('\n'); + } + std::fs::write(&merged_path, merged) + .with_context(|| format!("writing {}", merged_path.display()))?; + + let manifest = json!({ + "run_epoch_ms": run_epoch_ms, + "seed": config.seed, + "bucket_ms": config.bucket_ms, + "consensus_threshold": config.consensus_threshold, + "settle_ms": config.settle_ms, + "corpus": { + "first_minute": FIRST_MINUTE, + "last_minute": LAST_MINUTE, + }, + "nodes": Concern::all() + .into_iter() + .map(|c| json!({ + "id": c.name(), + "concern": c.name(), + "description": c.description(), + "metrics": c.metrics(), + "addr": addresses(config.base_port)[c.name()].to_string(), + "events": per_node.get(c.name()).copied().unwrap_or(0), + })) + .collect::>(), + "topology": TOPOLOGY + .iter() + .map(|(a, b)| json!({"from": a, "to": b})) + .collect::>(), + "events": events.len(), + "duration_ms": events.last().map(|e| e.t_ms).unwrap_or(0), + }); + + let manifest_path = config.out_dir.join("manifest.json"); + std::fs::write(&manifest_path, serde_json::to_string_pretty(&manifest)?) + .with_context(|| format!("writing {}", manifest_path.display()))?; + + print_summary(&events, config); + + // The visualization is only as honest as this file, so check it here + // rather than discovering a gap when the replay looks wrong. + let report = super::validate::validate(&events); + super::validate::print_report(&report); + + println!("\n merged timeline {}", merged_path.display()); + println!(" manifest {}", manifest_path.display()); + + Ok(merged_path) +} + +/// Report what the mesh concluded, from the journal alone. +fn print_summary(events: &[JournalEvent], config: &RunConfig) { + // Who reached consensus on what, and when. + let mut consensus: BTreeMap, Vec)> = BTreeMap::new(); + for event in events.iter().filter(|e| e.kind == "consensus_reached") { + let subject = event.data["subject"].as_str().unwrap_or("?").to_string(); + let attesters: Vec = event.data["attesters"] + .as_array() + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + + let entry = consensus + .entry(subject) + .or_insert((event.t_ms, attesters.clone(), Vec::new())); + entry.0 = entry.0.min(event.t_ms); + if !entry.2.contains(&event.node) { + entry.2.push(event.node.clone()); + } + if attesters.len() > entry.1.len() { + entry.1 = attesters; + } + } + + // Everything that was ever claimed, and by how many concerns. + let mut claimed: BTreeMap> = BTreeMap::new(); + for event in events.iter().filter(|e| e.kind == "finding") { + if let Some(subject) = event.data["subject"].as_str() { + let entry = claimed.entry(subject.to_string()).or_default(); + if !entry.contains(&event.node) { + entry.push(event.node.clone()); + } + } + } + + println!("\n╭─ what the mesh concluded"); + for (subject, finders) in &claimed { + let reached = consensus.get(subject); + let verdict = match reached { + Some((t_ms, attesters, nodes)) => format!( + "CONSENSUS at {:.1}s · {} attesters · seen by {} node(s)", + *t_ms as f64 / 1000.0, + attesters.len(), + nodes.len() + ), + None => format!( + "no consensus ({}/{} concerns)", + finders.len(), + config.consensus_threshold + ), + }; + println!("│ {subject:<21} {verdict}"); + println!("│ {:<21} claimed by {}", "", finders.join(", ")); + } + println!("╰─"); + + let counts = tally_kinds(events); + println!("\n╭─ journal"); + println!("│ {} events across {} nodes", events.len(), 5); + for (kind, count) in counts { + println!("│ {kind:<20} {count}"); + } + println!("╰─"); +} + +fn tally_kinds(events: &[JournalEvent]) -> Vec<(String, usize)> { + let mut counts: BTreeMap = BTreeMap::new(); + for event in events { + *counts.entry(event.kind.clone()).or_default() += 1; + } + let mut counts: Vec<(String, usize)> = counts.into_iter().collect(); + counts.sort_by_key(|(_, count)| std::cmp::Reverse(*count)); + counts +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn topology_is_connected_and_not_a_full_mesh() { + let names: Vec<&str> = Concern::all().into_iter().map(|c| c.name()).collect(); + + // Every concern appears. + for name in &names { + assert!(degree_of(name) >= 2, "{name} is underconnected"); + } + + // A full mesh of 5 would be 10 links; this is deliberately sparser so + // signals have to be relayed to cross it. + assert!(TOPOLOGY.len() < 10); + + // Each link is dialled by exactly one side. + let mut seen = std::collections::HashSet::new(); + for (a, b) in TOPOLOGY { + assert!(seen.insert((a, b)), "duplicate link {a}->{b}"); + assert!( + !seen.contains(&(b, a)), + "link {a}-{b} dialled from both ends" + ); + } + } + + #[test] + fn every_concern_gets_a_distinct_port() { + let addrs = addresses(9301); + let mut ports: Vec = addrs.values().map(|a| a.port()).collect(); + ports.sort_unstable(); + ports.dedup(); + assert_eq!(ports.len(), Concern::all().len()); + } +} diff --git a/smesh-cli/src/analysis/validate.rs b/smesh-cli/src/analysis/validate.rs new file mode 100644 index 0000000..d70a05a --- /dev/null +++ b/smesh-cli/src/analysis/validate.rs @@ -0,0 +1,349 @@ +//! Check that a recorded run is internally consistent and replayable. +//! +//! A visualization built on a journal inherits every gap in it, and a gap is +//! invisible until someone reads the picture wrong. This module states what the +//! journal claims about itself and verifies each claim against the file, so +//! "the replay is accurate" is a checked property rather than an intention. +//! +//! Violations are reported, never hidden. Some are benign — a message still in +//! flight when its recipient stopped is a real thing that happens — so they are +//! separated into errors, which mean the log is wrong, and notes, which mean +//! the run ended untidily. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use serde_json::Value; +use smesh_runtime::JournalEvent; + +/// Outcome of validating one run. +#[derive(Debug, Default)] +pub struct Report { + /// Inconsistencies that make the journal untrustworthy. + pub errors: Vec, + /// Oddities that are explainable and do not invalidate a replay. + pub notes: Vec, + /// Invariants that were checked and held. + pub checks_passed: Vec, +} + +impl Report { + /// Whether the journal can be trusted for replay. + pub fn is_valid(&self) -> bool { + self.errors.is_empty() + } +} + +fn str_field<'a>(event: &'a JournalEvent, key: &str) -> Option<&'a str> { + event.data.get(key).and_then(Value::as_str) +} + +fn u64_field(event: &JournalEvent, key: &str) -> Option { + event.data.get(key).and_then(Value::as_u64) +} + +/// Validate a merged run. +pub fn validate(events: &[JournalEvent]) -> Report { + let mut report = Report::default(); + + let mut by_node: BTreeMap<&str, Vec<&JournalEvent>> = BTreeMap::new(); + for event in events { + by_node.entry(event.node.as_str()).or_default().push(event); + } + + check_merge_order(events, &mut report); + check_per_node_sequences(&by_node, &mut report); + check_lifecycle(&by_node, &mut report); + check_snapshots(&by_node, &mut report); + check_receipts(&by_node, &mut report); + check_deliveries(events, &by_node, &mut report); + check_consensus(&by_node, &mut report); + + report +} + +/// The merged file must be ordered, or a replay would jump backwards in time. +fn check_merge_order(events: &[JournalEvent], report: &mut Report) { + let mut last = i64::MIN; + for event in events { + if event.t_ms < last { + report.errors.push(format!( + "merged timeline goes backwards at {}#{} ({}ms after {}ms)", + event.node, event.seq, event.t_ms, last + )); + return; + } + last = event.t_ms; + } + report + .checks_passed + .push("merged timeline is monotonic in t_ms".to_string()); +} + +/// Each node's own log must be gapless, or events were lost. +fn check_per_node_sequences(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + for (node, node_events) in by_node { + let mut seqs: Vec = node_events.iter().map(|e| e.seq).collect(); + seqs.sort_unstable(); + + for (index, seq) in seqs.iter().enumerate() { + let expected = index as u64 + 1; + if *seq != expected { + report.errors.push(format!( + "{node}: sequence gap — expected seq {expected}, found {seq}" + )); + break; + } + } + + // Within a node, time must not run backwards either. + let mut ordered: Vec<&&JournalEvent> = node_events.iter().collect(); + ordered.sort_by_key(|e| e.seq); + let mut last = i64::MIN; + for event in ordered { + if event.t_ms < last { + report.errors.push(format!( + "{node}: t_ms went backwards at seq {} ({}ms after {}ms)", + event.seq, event.t_ms, last + )); + break; + } + last = event.t_ms; + } + } + report + .checks_passed + .push("every node's sequence is gapless and time-ordered".to_string()); +} + +/// Every node must open and close its own log. +fn check_lifecycle(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + for (node, node_events) in by_node { + let first = node_events.iter().min_by_key(|e| e.seq); + let has_stop = node_events.iter().any(|e| e.kind == "node_stopped"); + + match first { + Some(event) if event.kind == "node_started" => {} + Some(event) => report.errors.push(format!( + "{node}: first event is {}, not node_started", + event.kind + )), + None => report.errors.push(format!("{node}: no events at all")), + } + + if !has_stop { + report + .notes + .push(format!("{node}: no node_stopped — process ended early?")); + } + } + report + .checks_passed + .push("every node opened with node_started".to_string()); +} + +/// Field snapshots must advance, so decay curves interpolate correctly. +fn check_snapshots(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + for (node, node_events) in by_node { + let mut last_tick = 0u64; + for event in node_events.iter().filter(|e| e.kind == "field_snapshot") { + let Some(tick) = u64_field(event, "tick") else { + report + .errors + .push(format!("{node}: field_snapshot without a tick")); + continue; + }; + if tick <= last_tick && last_tick != 0 { + report.errors.push(format!( + "{node}: field_snapshot tick {tick} did not advance past {last_tick}" + )); + break; + } + last_tick = tick; + } + } + report + .checks_passed + .push("field snapshots advance monotonically".to_string()); +} + +/// Anything a node reports holding must have arrived by a recorded route. +fn check_receipts(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + for (node, node_events) in by_node { + let mut ordered: Vec<&&JournalEvent> = node_events.iter().collect(); + ordered.sort_by_key(|e| e.seq); + + let mut known: BTreeSet = BTreeSet::new(); + let mut unexplained = 0usize; + + for event in ordered { + match event.kind.as_str() { + "signal_emitted" | "signal_accepted" => { + if let Some(hash) = str_field(event, "hash") { + known.insert(hash.to_string()); + } + } + "field_snapshot" => { + let Some(signals) = event.data.get("signals").and_then(Value::as_array) else { + continue; + }; + for signal in signals { + let Some(hash) = signal.get("hash").and_then(Value::as_str) else { + continue; + }; + if !known.contains(hash) { + unexplained += 1; + } + } + } + _ => {} + } + } + + if unexplained > 0 { + report.errors.push(format!( + "{node}: {unexplained} snapshot entries for signals never emitted or accepted here" + )); + } + } + report + .checks_passed + .push("every signal a node held was emitted or accepted there first".to_string()); +} + +/// Every recorded send should show up as a receive on the named peer. +fn check_deliveries( + events: &[JournalEvent], + by_node: &BTreeMap<&str, Vec<&JournalEvent>>, + report: &mut Report, +) { + // (receiving node, hash) -> number of receives recorded. + let mut receipts: HashMap<(String, String), usize> = HashMap::new(); + for event in events.iter().filter(|e| e.kind == "signal_received") { + if let Some(hash) = str_field(event, "hash") { + *receipts + .entry((event.node.clone(), hash.to_string())) + .or_default() += 1; + } + } + + let mut undelivered = 0usize; + let mut unnamed = 0usize; + let mut total = 0usize; + + for event in events.iter().filter(|e| e.kind == "signal_sent") { + total += 1; + let (Some(to), Some(hash)) = (str_field(event, "to"), str_field(event, "hash")) else { + unnamed += 1; + continue; + }; + + if !by_node.contains_key(to) { + unnamed += 1; + continue; + } + + match receipts.get_mut(&(to.to_string(), hash.to_string())) { + Some(remaining) if *remaining > 0 => *remaining -= 1, + _ => undelivered += 1, + } + } + + if unnamed > 0 { + report.errors.push(format!( + "{unnamed} of {total} sends name a peer that is not a node in this run" + )); + } + + if undelivered > 0 { + // In-flight at shutdown is the ordinary cause and is not a log defect. + report.notes.push(format!( + "{undelivered} of {total} sends have no matching receive (in flight at shutdown)" + )); + } + + report + .checks_passed + .push(format!("{total} sends resolve to a named peer in this run")); +} + +/// Consensus must be justified by what that node had already seen. +fn check_consensus(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + let mut announcements = 0usize; + + for (node, node_events) in by_node { + let mut ordered: Vec<&&JournalEvent> = node_events.iter().collect(); + ordered.sort_by_key(|e| e.seq); + + let mut held: BTreeSet = BTreeSet::new(); + + for event in ordered { + match event.kind.as_str() { + "signal_emitted" | "signal_accepted" => { + if let Some(hash) = str_field(event, "hash") { + held.insert(hash.to_string()); + } + } + "consensus_reached" => { + announcements += 1; + let Some(hash) = str_field(event, "hash") else { + report + .errors + .push(format!("{node}: consensus_reached without a hash")); + continue; + }; + + if !held.contains(hash) { + report.errors.push(format!( + "{node}: declared consensus on {hash} without ever holding it" + )); + } + + let count = u64_field(event, "attester_count").unwrap_or(0); + let threshold = u64_field(event, "threshold").unwrap_or(0); + if count < threshold { + report.errors.push(format!( + "{node}: declared consensus on {hash} with {count} attesters, below the threshold of {threshold}" + )); + } + + let listed = event + .data + .get("attesters") + .and_then(Value::as_array) + .map(|a| a.len() as u64) + .unwrap_or(0); + if listed != count { + report.errors.push(format!( + "{node}: consensus on {hash} claims {count} attesters but lists {listed}" + )); + } + } + _ => {} + } + } + } + + report.checks_passed.push(format!( + "{announcements} consensus declarations are justified by prior receipts" + )); +} + +/// Print a report for a human. +pub fn print_report(report: &Report) { + println!("\n╭─ journal validation"); + for check in &report.checks_passed { + println!("│ ok {check}"); + } + for note in &report.notes { + println!("│ note {note}"); + } + for error in &report.errors { + println!("│ FAIL {error}"); + } + if report.is_valid() { + println!("│ → journal is internally consistent and safe to replay"); + } else { + println!("│ → journal has inconsistencies; a replay of it would be wrong"); + } + println!("╰─"); +} diff --git a/smesh-cli/src/main.rs b/smesh-cli/src/main.rs index cf8defa..279a075 100644 --- a/smesh-cli/src/main.rs +++ b/smesh-cli/src/main.rs @@ -10,10 +10,11 @@ use smesh_agent::{ benchmark_backend, print_comparison, AgentCoordinator, AgentRole, ClaudeClient, ClaudeConfig, CoordinatorConfig, OpenRouterClient, TaskDefinition, }; -use smesh_core::{Network, NetworkTopology, Signal, SignalType}; -use smesh_runtime::{RuntimeConfig, SmeshRuntime}; +use smesh_core::{Network, NetworkTopology, Node, Signal, SignalType}; +use smesh_runtime::{MeshConfig, RuntimeConfig, RuntimeEvent, SmeshRuntime}; mod adjudicate; +mod analysis; mod owasp; mod resilience; mod review; @@ -175,7 +176,6 @@ enum Commands { json: Option, }, - /// Run web red team mission against a live target (authorized testing only) Redteam { /// Target domain (e.g., example.com) @@ -248,6 +248,103 @@ enum Commands { consensus: u32, }, + /// Orchestrate the multi-concern analysis mesh as real processes + Orchestrate { + /// Directory for journals and the merged timeline + #[arg(long, default_value = "runs/latest")] + out: PathBuf, + + /// First port; concerns take consecutive ports from here + #[arg(long, default_value = "9301")] + base_port: u16, + + /// Corpus seed, shared by every analyst + #[arg(long, default_value = "42")] + seed: u64, + + /// Wall-clock milliseconds per corpus minute + #[arg(long, default_value = "700")] + bucket_ms: u64, + + /// Distinct concerns that must concur for consensus + #[arg(long, default_value = "4")] + consensus_threshold: usize, + + /// Extra gossip time after the corpus ends, in milliseconds + #[arg(long, default_value = "6000")] + settle_ms: u64, + }, + + /// Run one analyst node of the analysis mesh (usually spawned by orchestrate) + Analyze { + /// Which concern this analyst owns + #[arg(long)] + concern: String, + + /// Address to listen on + #[arg(long, default_value = "127.0.0.1:0")] + bind: String, + + /// Peer to dial; repeat for several + #[arg(long = "peer")] + peers: Vec, + + /// Where to write this node's journal + #[arg(long)] + journal: Option, + + /// Shared run epoch in unix milliseconds + #[arg(long)] + run_epoch: Option, + + /// Corpus seed + #[arg(long, default_value = "42")] + seed: u64, + + /// Wall-clock milliseconds per corpus minute + #[arg(long, default_value = "700")] + bucket_ms: u64, + + /// Distinct concerns that must concur for consensus + #[arg(long, default_value = "4")] + consensus_threshold: usize, + + /// Peers to wait for before starting the timeline + #[arg(long, default_value = "0")] + expect_peers: usize, + + /// Extra gossip time after the corpus ends, in milliseconds + #[arg(long, default_value = "6000")] + settle_ms: u64, + }, + + /// Run one node on a live P2P mesh over QUIC + Mesh { + /// Address to listen on (port 0 lets the OS pick) + #[arg(long, default_value = "127.0.0.1:0")] + bind: String, + + /// Peer to dial on startup; repeat for several + #[arg(long = "peer")] + peers: Vec, + + /// Readable node id (defaults to a random one) + #[arg(long)] + name: Option, + + /// Payload to emit onto the mesh once connected + #[arg(long)] + emit: Option, + + /// Seconds to wait before emitting + #[arg(long, default_value = "2")] + emit_after: u64, + + /// Seconds to stay on the mesh (0 = until Ctrl-C) + #[arg(long, default_value = "20")] + duration: u64, + }, + /// Benchmark the SMESH mesh against the OWASP Benchmark corpus Owasp { /// Path to a BenchmarkJava checkout @@ -345,9 +442,80 @@ async fn main() -> Result<()> { json, } => cmd_resilience(nodes, &topology, trials, html, json).await, Commands::Redteam { target } => cmd_redteam(&target).await, - Commands::Fullscan { target, no_js, analyze, report } => { - cmd_fullscan(&target, no_js, analyze, report).await + Commands::Mesh { + bind, + peers, + name, + emit, + emit_after, + duration, + } => cmd_mesh(&bind, &peers, name, emit, emit_after, duration).await, + Commands::Orchestrate { + out, + base_port, + seed, + bucket_ms, + consensus_threshold, + settle_ms, + } => { + analysis::orchestrate::run(analysis::orchestrate::RunConfig { + out_dir: out, + base_port, + seed, + bucket_ms, + consensus_threshold, + settle_ms, + }) + .await + .map(|_| ()) + } + Commands::Analyze { + concern, + bind, + peers, + journal, + run_epoch, + seed, + bucket_ms, + consensus_threshold, + expect_peers, + settle_ms, + } => { + let concern = analysis::concern::Concern::parse(&concern).ok_or_else(|| { + anyhow::anyhow!( + "unknown concern '{concern}'; expected one of: {}", + analysis::concern::Concern::all() + .iter() + .map(|c| c.name()) + .collect::>() + .join(", ") + ) + })?; + + analysis::node::run(analysis::node::AnalystConfig { + concern, + bind: bind.parse()?, + peers: peers + .iter() + .map(|p| p.parse::()) + .collect::, _>>()?, + journal, + run_epoch_ms: run_epoch.unwrap_or_else(|| chrono::Utc::now().timestamp_millis()), + seed, + bucket_ms, + consensus_threshold, + expect_peers, + settle_ms, + verbose: true, + }) + .await } + Commands::Fullscan { + target, + no_js, + analyze, + report, + } => cmd_fullscan(&target, no_js, analyze, report).await, Commands::Bounty { path, max_files, @@ -440,7 +608,9 @@ async fn cmd_resilience( ..Default::default() }; - println!("Sweeping attacks over the real mesh ({nodes} nodes, {topology}, {trials} trials/point)…"); + println!( + "Sweeping attacks over the real mesh ({nodes} nodes, {topology}, {trials} trials/point)…" + ); let report_data = resilience::run_benchmark(cfg); report::print_report(&report_data); @@ -514,7 +684,9 @@ async fn cmd_owasp( } async fn cmd_fullscan(target: &str, no_js: bool, analyze: bool, report: bool) -> Result<()> { - use smesh_bounty::{FullSpectrumConfig, run_full_spectrum, analyze_exploitability, generate_report}; + use smesh_bounty::{ + analyze_exploitability, generate_report, run_full_spectrum, FullSpectrumConfig, + }; let mut config = FullSpectrumConfig::full(target); if no_js { @@ -592,8 +764,7 @@ async fn cmd_bounty( ..config }; - let mut coordinator = - BountyCoordinator::new(config).map_err(|e| anyhow::anyhow!("{}", e))?; + let mut coordinator = BountyCoordinator::new(config).map_err(|e| anyhow::anyhow!("{}", e))?; let result = coordinator .run() @@ -1199,3 +1370,170 @@ fn truncate_str(s: &str, max_len: usize) -> String { format!("{}...", &s[..max_len - 3]) } } + +/// Run one node on a live P2P mesh over QUIC. +/// +/// Each process owns a single SMESH node. Signals emitted here go out to every +/// connected peer; signals arriving from peers are admitted by this node's own +/// sensing threshold and forwarded by its own relay policy. +async fn cmd_mesh( + bind: &str, + peers: &[String], + name: Option, + emit: Option, + emit_after: u64, + duration: u64, +) -> Result<()> { + use std::sync::Arc; + use std::time::Duration; + + let bind_addr: std::net::SocketAddr = bind + .parse() + .map_err(|e| anyhow::anyhow!("invalid --bind {bind}: {e}"))?; + + let bootstrap: Vec = peers + .iter() + .map(|p| { + p.parse::() + .map_err(|e| anyhow::anyhow!("invalid --peer {p}: {e}")) + }) + .collect::>()?; + + // One node per process: this is the identity we present on the wire. + let mut node = Node::new(); + if let Some(name) = name { + node.id = name; + } + let node_id = node.id.clone(); + + let mut network = Network::new(); + network.add_node(node); + + let mut runtime = SmeshRuntime::with_network( + network, + RuntimeConfig { + tick_interval_ms: 100, + ..Default::default() + }, + ); + let mut events = runtime + .take_events() + .ok_or_else(|| anyhow::anyhow!("event receiver already taken"))?; + let runtime = Arc::new(runtime); + + let mesh = runtime + .join_mesh( + MeshConfig { + bind_addr, + bootstrap: bootstrap.clone(), + keepalive_interval_ms: 3_000, + ..Default::default() + }, + &node_id, + ) + .await?; + + println!("╭─ SMESH mesh node"); + println!("│ node id {node_id}"); + println!("│ listening {}", mesh.listen_addr()); + if bootstrap.is_empty() { + println!("│ bootstrap (none — waiting for peers to dial in)"); + } else { + for addr in &bootstrap { + println!("│ bootstrap {addr}"); + } + } + println!("╰─"); + + // Decay and local diffusion keep running while we are on the mesh. + { + let runtime = Arc::clone(&runtime); + tokio::spawn(async move { runtime.run().await }); + } + + // Report everything except the per-tick chatter. + let events_task = tokio::spawn(async move { + while let Some(event) = events.recv().await { + match event { + RuntimeEvent::TickCompleted { .. } => {} + RuntimeEvent::PeerConnected { peer_id } => { + println!(" [peer] connected {peer_id}"); + } + RuntimeEvent::PeerDisconnected { peer_id } => { + println!(" [peer] disconnected {peer_id}"); + } + RuntimeEvent::SignalEmitted { hash } => { + println!(" [emit] {hash}"); + } + RuntimeEvent::SignalReceived { hash, from, hops } => { + println!(" [recv] {hash} from {from} (hop {hops})"); + } + RuntimeEvent::SignalReinforced { hash, count } => { + println!(" [reinf] {hash} ×{count}"); + } + RuntimeEvent::SignalExpired { hash } => { + println!(" [expire] {hash}"); + } + } + } + }); + + if let Some(payload) = emit { + let runtime = Arc::clone(&runtime); + let node_id = node_id.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(emit_after)).await; + let signal = Signal::builder(SignalType::Coordination) + .payload(payload.as_bytes().to_vec()) + .origin(&node_id) + .intensity(1.0) + .ttl(120.0) + .radius(4) + .build(); + runtime.emit(signal, &node_id).await; + }); + } + + if duration == 0 { + println!("\nrunning until Ctrl-C…\n"); + tokio::signal::ctrl_c().await?; + } else { + println!("\nrunning for {duration}s…\n"); + tokio::time::sleep(Duration::from_secs(duration)).await; + } + + let stats = runtime.stats().await; + println!("\n╭─ final state"); + println!("│ ticks {}", stats.tick_count); + println!("│ peers known {}", stats.peer_count); + println!("│ peers connected {}", stats.connected_peers); + println!("│ live connections {}", mesh.connection_count().await); + println!("│ active signals {}", stats.active_signals); + println!("│ reinforcements {}", stats.total_reinforcements); + + for peer in runtime.peers().connected_peers().await { + println!( + "│ peer {} at {} rtt {}ms", + peer.id, peer.addr, peer.latency_ms + ); + } + + let network = runtime.network(); + let network = network.read().await; + if let Some(node) = network.get_node(&node_id) { + println!( + "│ emitted {} · sensed {} · relayed {} · reinforced {}", + node.stats.signals_emitted, + node.stats.signals_sensed, + node.stats.signals_relayed, + node.stats.signals_reinforced + ); + } + println!("╰─"); + + runtime.shutdown().await; + mesh.shutdown().await; + events_task.abort(); + + Ok(()) +} From 95bb2897813373c6bb8c17b8423e8fbbe4238632 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 20:29:52 -0700 Subject: [PATCH 04/14] film: narrated walkthrough of a recorded run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An eight-minute film explaining the protocol and the analysis run, aimed at a non-specialist audience. The picture is rendered deterministically: every frame is a pure function of time, so frames can be produced in isolation, in parallel, and reproducibly. Two capture passes share one frame-index space — authored canvas scenes for the explanatory sections, and the published replay page driven by a scripted camera for the demo. The camera is a CSS transform rather than a crop, so type re-rasterises at every focal length instead of being upscaled. Shot timings are matched to where the run is actually busy. The events are bursty and the entire consensus happens inside a 1.7 second window, so the decisive moments run in heavy slow motion and the dead air is skipped; a naive linear mapping played the key beat over silence. Scene durations are derived from the measured narration audio rather than estimated, so picture and voice cannot drift. Includes the narration script, the article draft, and cover stills. Rendered output and generated audio are ignored — both regenerate from what is committed here, given an API key. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 6 + film/DEVTO.md | 219 +++++++++++++++ film/NARRATION.md | 79 ++++++ film/covers/cover-smesh-payoff.jpg | Bin 0 -> 27356 bytes film/covers/cover-smesh.jpg | Bin 0 -> 26303 bytes film/src/encode.sh | 35 +++ film/src/film.html | 56 ++++ film/src/lib.js | 238 ++++++++++++++++ film/src/measure.js | 34 +++ film/src/mixaudio.sh | 63 +++++ film/src/renderAll.sh | 24 ++ film/src/scenes1.js | 393 +++++++++++++++++++++++++++ film/src/scenes2.js | 417 +++++++++++++++++++++++++++++ film/src/scenes3.js | 145 ++++++++++ film/src/score.py | 74 +++++ film/src/script.json | 145 ++++++++++ film/src/shoot.js | 69 +++++ film/src/shootDemo.js | 154 +++++++++++ film/src/timeline.json | 236 ++++++++++++++++ film/src/tts.py | 70 +++++ 20 files changed, 2457 insertions(+) create mode 100644 film/DEVTO.md create mode 100644 film/NARRATION.md create mode 100644 film/covers/cover-smesh-payoff.jpg create mode 100644 film/covers/cover-smesh.jpg create mode 100755 film/src/encode.sh create mode 100644 film/src/film.html create mode 100644 film/src/lib.js create mode 100644 film/src/measure.js create mode 100755 film/src/mixaudio.sh create mode 100755 film/src/renderAll.sh create mode 100644 film/src/scenes1.js create mode 100644 film/src/scenes2.js create mode 100644 film/src/scenes3.js create mode 100644 film/src/score.py create mode 100644 film/src/script.json create mode 100644 film/src/shoot.js create mode 100644 film/src/shootDemo.js create mode 100644 film/src/timeline.json create mode 100644 film/src/tts.py diff --git a/.gitignore b/.gitignore index 5a0045c..fe24da3 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,9 @@ smesh-demo.mp4 # Recorded mesh runs (journals + merged timelines) runs/ + +# Rendered film output (large, regenerable) +film/out/ +film/src/audio/ +film/src/frames/ +film/src/node_modules/ diff --git a/film/DEVTO.md b/film/DEVTO.md new file mode 100644 index 0000000..1744dc3 --- /dev/null +++ b/film/DEVTO.md @@ -0,0 +1,219 @@ +--- +title: "My QUIC transport had never once been executed. Here's what happened when I ran it." +published: false +description: "I built a plant-inspired coordination protocol, wrote 500 lines of real QUIC networking for it, and never actually turned it on. Wiring it up found three latent bugs in twenty minutes — and then taught me that three of my protocol's core semantics were wrong for real distribution." +tags: rust, distributedsystems, networking, ai +--- + +I've written before about SMESH, a coordination protocol modelled on mycorrhizal networks — the fungal web that lets trees in a forest warn each other about drought and disease with nothing in charge of the network. Signals diffuse, decay on their own, and get reinforced when independently confirmed. Consensus emerges instead of being orchestrated. + +That was the idea. This post is about the part where I found out whether it worked. + +## The transport that had never run + +SMESH has had a QUIC transport in it for a while. Roughly 500 lines: a quinn endpoint that is simultaneously server and client, self-signed certs, length-prefixed bincode frames over unidirectional streams, an accept loop that spawns per-connection and per-stream tasks, connection pooling. + +Every test passed. The workspace was green. I could point at `smesh-runtime/src/transport.rs` and say "yes, it does peer-to-peer." + +Then I grepped for who actually constructed it: + +``` +$ grep -rn "QuicTransport" --include='*.rs' . +smesh-runtime/src/transport.rs:177:pub struct QuicTransport { +smesh-runtime/src/transport.rs:192:impl QuicTransport { +smesh-runtime/src/lib.rs:16:pub use transport::{QuicTransport, ...}; +``` + +Its own definition, and a re-export. Nothing else in the workspace had ever instantiated it. No binary opened a socket. `SmeshRuntime` imported `TransportConfig`, stored it in a struct field, and never looked at it again. + +I had a networking layer with tests, docs, and zero executions. + +## Three bugs in the first twenty minutes + +I wrote an integration test that starts two runtimes, has one dial the other, and asserts a signal crosses. Here is what fell out before it went green. + +**1. It panicked on the first call.** + +``` +Could not automatically determine the process-level CryptoProvider +from Rustls crate features. +``` + +rustls 0.23 refuses to pick a crypto backend when more than one is compiled in, and quinn pulls in both through its own feature set. Every call to `QuicTransport::new` would have panicked for anyone, ever. Nobody noticed because nobody had called it. + +**2. Dialled connections were write-only.** + +`connect()` stored the connection in the pool but only the accept loop pumped incoming streams — and the accept loop only sees connections you *accepted*. So a node that dialled out could send, and would never receive anything back. A QUIC connection is bidirectional regardless of who dialled it; my code only acted like it half the time. + +**3. Unbounded allocation from an attacker-controlled length prefix.** + +```rust +let len = u32::from_be_bytes(len_buf) as usize; +let mut data = vec![0u8; len]; // <- no +``` + +`max_message_size` was in the config struct. It was never read. Send a 4 GiB length prefix and the process allocates 4 GiB. + +None of these are clever bugs. They're the bugs you get for free the first time code meets a socket, and the only reason they survived is that the code had never met a socket. + +## The harder problem: my protocol was wrong + +Fixing the plumbing was the easy half. The real issue was that my diffusion algorithm quietly assumed something no distributed system can assume. + +`Network::tick` expands a signal's reach one hop per tick by walking the graph: + +```rust +for node_id in &reached { + for hypha in self.hyphae.get(node_id) { + let target = self.nodes.get(&hypha.to); + if target.should_relay(&signal, remaining_hops) { + frontier.push(hypha.to.clone()); + } + } +} +``` + +Read that again. It iterates every node's adjacency, calls every node's relay policy, and mutates one global `reached_nodes` set on a shared signal. It's a breadth-first search from a god's-eye view of the entire graph. + +That works beautifully in one process. In a real mesh, **no node can see that graph.** Porting it meant three corrections, and each one turned out to be a genuine bug rather than a porting detail. + +### Correction 1: independent conclusions were being thrown away + +Signals are content-addressed — the hash is derived from what is being claimed. So when two agents independently reach the same conclusion, they produce the same hash. + +My `emit()` saw the hash already present locally and treated it as a duplicate. It dropped it. + +That is exactly backwards. Two parties independently agreeing is not redundant data — it is the *only* evidence the system has that a claim is real. Discarding it destroys the thing the protocol exists to measure. + +```rust +/// Signals are content-addressed, so a node that independently reaches a +/// conclusion another node already published lands on the same hash. That +/// is treated as *corroboration*: this node is added as an attester and the +/// merged claim still goes out, because our agreement is news to everyone +/// who has not heard it. Swallowing it as a duplicate would silently +/// discard the only evidence that two parties concur. +``` + +### Correction 2: I was counting messengers, not witnesses + +Reinforcement attributed the claim to whoever handed me the message. In a gossip mesh, one finding relayed by five nodes then looks like five corroborators. + +The party that attests to a claim is its *origin*, not the peer that passed it along. Relaying is not agreeing. + +### Correction 3: gossip needs a merge rule, not a broadcast rule + +Once attestation is a set, the rule that makes gossip converge is simple and pleasant: + +```rust +// Merge the two attester sets. Anything the sender knew that we did +// not is new information, and new information is worth passing on. +let before = Node::attesters(existing); +for attester in &incoming_attesters { + if !before.contains(attester) { + existing.reinforce(attester); + } +} +``` + +Forward if and only if your own knowledge grew. That single rule is the loop breaker (a message teaching you nothing goes no further), the convergence mechanism (the set is grow-only, so it's a CRDT), and the anti-entropy repair (re-asserting carries your accumulated view, so a node that missed a round catches up). + +## The part I nearly got wrong on purpose + +Here is the subtlety that makes the whole thing work, and it looks like a mistake in the source. + +The signal builder folds the origin node into the content hash if you give it one. So when an agent publishes a claim, you must **not** set the origin: + +```rust +let signal = Signal::builder(SignalType::Alert) + .payload(assertion.canonical_bytes()) + .confidence(finding.confidence) + .build(); +// .origin() deliberately NOT called +``` + +If you set it, every agent gets a different hash for the same claim and correlation becomes impossible. The address has to be the *claim*, not the *claimant*. + +The corollary is that evidence cannot travel in the payload. Each agent's evidence differs, so putting it in would make every hash unique and break the mechanism. The payload is just the assertion: + +```json +{"subject":"checkout-api","claim":"degraded"} +``` + +Evidence stays local and goes to the log. The mesh carries assertions; it does not carry arguments. + +## Five witnesses, none of whom can see the problem + +To actually test any of this, I built a scenario where the answer cannot be reached alone. + +Five analyst processes watch the same fleet of services. Each can see exactly one kind of telemetry, and nothing else: + +| agent | sees | +| --- | --- | +| latency | p99 response times | +| errors | error rates | +| saturation | pool and CPU utilisation | +| traces | retry rates, span queueing | +| deploys | release events | + +A deploy cuts `checkout-api`'s connection pool from 200 to 20. The pool pins, requests queue, callers time out. + +Now every agent sees a piece of it, and two of them are actively misleading. `errors` sees `payments-api` throwing 503s and would blame it — but payments is a victim, its own pool and CPU are fine. `latency` sees four services slow at once and can't say which is causal. Only `deploys` can see there was a release, and a release on its own means nothing; software ships all day. + +I also planted two decoys with a single witness each — an unrelated CPU spike, a brief latency blip — because a system that agrees with everything is not consensus, it's an echo. + +They run as five real OS processes on five ports over real encrypted QUIC, in a ring-plus-chord topology so messages actually have to be relayed to cross the mesh. Peer discovery is off, or gossip quietly converts any topology into a full mesh and there's nothing left to watch. + +``` +$ smesh orchestrate --out runs/latest + + spawned latency pid 2452744 127.0.0.1:9301 dials 9302,9303 + spawned errors pid 2452745 127.0.0.1:9302 dials 9303 + spawned saturation pid 2452746 127.0.0.1:9303 dials 9304 + spawned traces pid 2452747 127.0.0.1:9304 dials 9305 + spawned deploys pid 2452748 127.0.0.1:9305 dials 9301 +``` + +The result: + +``` +what the mesh concluded + checkout-api CONSENSUS at 16.9s · 5 attesters · seen by 5 nodes + payments-api no consensus (3/4 concerns) + edge-gateway no consensus (1/4 concerns) + notification-worker no consensus (1/4 concerns) + session-store no consensus (1/4 concerns) +``` + +Cause separated from symptom separated from noise. The loud, obvious suspect was held at three witnesses and never promoted. The decoys were never rejected by anything — they simply went uncorroborated and decayed out. Nobody voted, and nothing was in charge. + +## Making the run replayable, and then checking that claim + +I wanted to visualise this, which meant the log had to be good enough to reconstruct the run exactly. Each process writes newline-delimited JSON against a shared run epoch: every emission, every per-peer send, every receipt, every relay decision including the probability and the die roll that resolved it, and a full field snapshot every 500ms so decay curves are *observed* rather than modelled. + +Then I did the part I'd recommend to anyone building a log you intend to trust: I wrote a validator that checks the log against itself. Sequence gaps, time going backwards, snapshots referencing signals that were never received, consensus declared without the receipts to justify it. + +It caught a real bug on its first run: + +``` +FAIL latency: first event is peer_connected, not node_started +``` + +A peer completed the handshake in the gap between binding the endpoint and writing the node's identity line. The fix was to move the identity write inside the mesh startup, before any loop spawns. I would never have found that by looking at the picture — the picture would just have been subtly wrong. + +## What is still wrong + +Being honest about the edges, because "it works" is a claim that needs a boundary: + +- **`origin_node_id` is unauthenticated.** It's a string on the wire, and the trust model gates relay probability on it. Spoofing another agent's identity is currently free. Ed25519-signing the origin hash closes it and is the next real piece of work. +- **The content hash is truncated to 64 bits.** Fine against accident, not against an adversary looking for collisions. +- **The telemetry in the demo is synthetic.** Deliberately: a seeded fixture means the run reproduces byte-for-byte on any machine, which is what makes a visualisation worth trusting. The coordination is not synthetic — real processes, real sockets, probabilistic relay. + +## See it move + +The full narrated walkthrough is the cover video on this post, or here: https://youtu.be/kmCzwSBqu_s + +It opens on the forest the protocol is stolen from, then goes inside the recorded run: five agents, six encrypted links, every dot on screen a real message read back from the log rather than animated for effect. + +If you take one thing from this: **code that has never been executed is not code, it's a plan.** Mine had tests, docs, and a clean `cargo clippy`, and it would have panicked on the first line for every user. The tests were testing that the plan was internally consistent. + +Repo: [github.com/copyleftdev/smesh-rust](https://github.com/copyleftdev/smesh-rust) · Rust · MIT/Apache-2.0 diff --git a/film/NARRATION.md b/film/NARRATION.md new file mode 100644 index 0000000..e3c73ee --- /dev/null +++ b/film/NARRATION.md @@ -0,0 +1,79 @@ +# SMESH film — narration script + +Voice: Brian (ElevenLabs nPczCjzI2devNBz1zQrb) · 19 segments + +## s01_cold_open (roots) + +Under every forest, there is a second network. It is older than ours. Trees use it to warn each other — about drought, about insects, about disease. No tree is in charge of it. And yet, somehow, the whole forest knows. + +## s02_mechanism (forest) + +It runs on three simple rules. A tree in trouble releases a signal into the network. That signal fades as it travels, and fades as time passes — so old news disappears on its own. But when a second tree senses the same threat, and releases the same signal, the two reinforce each other. The message gets stronger. Nothing coordinates any of this. There is no root server. There is no forest manager. Coordination is not something the forest does. It is something the forest grows. + +## s03_reveal (reveal) + +We built a protocol that works the same way. It is called SMESH. Software agents that coordinate the way a forest does — by releasing signals that fade, and trusting the ones that other agents independently confirm. + +## s04_problem (problem) + +Here is why that matters. Almost every distributed system we build today has something sitting in the middle. A message broker. An orchestrator. A coordinator. It is the thing that knows everything, and tells everyone else what to do. It is also the thing you pay for. The thing you scale. The thing that pages you at three in the morning. And the thing that takes the entire system down with it when it fails. As companies start running fleets of AI agents, that bottleneck gets more expensive, and more fragile, every single year. + +## s05_decay (primitive_decay) + +SMESH removes the middle, and replaces it with three mechanisms. The first is decay. Every message carries its own expiry, built into its physics. It weakens along a curve, and when it is weak enough, it is simply gone. Nothing has to clean up. Stale work removes itself. + +## s06_reinforce (primitive_reinforce) + +The second is reinforcement. When one agent reaches a conclusion that another agent has already reached, that is not a duplicate to be thrown away. It is a second witness. Confidence rises. Agreement compounds. + +## s07_address (primitive_address) + +The third mechanism is the one that makes the other two work. Every claim is addressed by its content — by what is being said, not by who said it. So two agents that independently arrive at the same conclusion land on the exact same address, automatically, without ever having spoken to each other. + +## s08_quic (quic) + +Underneath all of it, these agents talk over QUIC — the same encrypted transport that carries modern web traffic. Every connection is peer to peer. Every connection is encrypted. There is no server in the middle relaying anything. There is nothing in the middle to buy, to scale, or to lose. + +## s09_setup (setup) + +So let us watch it work. What follows is a recording of an actual run. Five separate programs, on five separate network ports, talking over real encrypted connections. Each one is monitoring the same fleet of services. But each one can only see a single kind of data. One watches response times. One watches errors. One watches capacity. One watches how requests retry. One watches software releases. None of them can see what the others see. And something is about to go wrong. + +## s09b_windows (windows) + +Think of them as five people watching the same building through five different windows. One can only see the lobby. One can only see the stairwell. None of them can see the fire. Every one of them sees smoke. + +## s10_incident (demo_establish) + +A release goes out. It quietly shrinks a connection pool by ninety percent. And now all five of these watchers see a piece of the damage — but not one of them sees the whole thing. Worse than that: two of them are about to accuse the wrong service entirely. + +## s11_mesh (demo_mesh) + +This is the mesh itself. Five agents, six encrypted links. Every dot you see crossing a link is a real message that was actually sent — read back from the recording, not animated for effect. And notice they are not all connected to each other. Some messages have to be passed along by a neighbour to reach the far side, exactly the way a forest relays a signal. + +## s12_claims (demo_claims) + +On the right is what the network currently believes. Each row is a claim about one service. The five small tags beneath each claim are the five agents. A tag lights up the moment that agent independently backs the claim. Watch the top row. One agent. Then a second. Then a third — each arriving from completely unrelated evidence, and finding the others already there. + +## s13_consensus (demo_consensus) + +There it is. Four independent agents. Four unrelated kinds of evidence. One conclusion. That crosses the threshold, and the network calls it. Nobody voted. Nobody was in charge. The answer assembled itself out of five partial views — and the fifth agent confirms it moments later. + +## s14_decoys (demo_decoys) + +Now look at what did not happen. This service was throwing errors loudly. It looks broken. On its own, the error watcher would have blamed it. It collects three witnesses, and it stops there — because it is a symptom, not a cause. And these last three claims only ever found a single witness each. Nothing confirmed them, so they simply fade out. Nobody had to decide they were wrong. Going uncorroborated was enough. + +## s15_evidence (demo_journal) + +And every step of it is on the record. Each of those five programs wrote down everything it did, and everything it chose not to do, as it happened. That record is what you have been watching. It is not a reenactment — it is the run itself, played back. + +## s16_payoff (payoff) + +That is the whole idea. Not one agent in this run had enough information to be right. The system was right anyway. The telemetry here is synthetic, so the run stays reproducible. The coordination is not synthetic. Those were real processes, on real sockets, making real decisions — recorded, verified, and replayed back to you exactly as they happened. + +## s16b_unlocks (unlocks) + +That property is what makes this worth building. Agents can be added without reconfiguring anything. Agents can fail without taking the answer with them. There is no central capacity to outgrow, and no coordinator bill that scales with the fleet. The network gets more reliable as it gets larger, because more witnesses is exactly what it runs on. + +## s17_close (close) + +As software moves toward fleets of autonomous agents, the hard problem stops being how clever each agent is. It becomes how they agree. SMESH is a bet that the answer has been running quietly under our feet for four hundred million years. diff --git a/film/covers/cover-smesh-payoff.jpg b/film/covers/cover-smesh-payoff.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4608cb7e87d6ecb2f0e39ad6dea24efff70bad22 GIT binary patch literal 27356 zcma&N2RK~M_b`4Xkw~ORiJk<}g6N$@5`yU6DiJN}TAkHW@QE6AqeobRAP8%fl@L9M z-fb+atlrBi|GVV#{eAz>^LyXtJsvY>&YU~GVt#h%q?7< zPq6>B1i7~QPk4KH|5uLxJMp}wm8%8FvI73IxqvqZpNR^@sccSgP7?m&1eYe^o^H-= zAddzKchS*$1mcz;&S~>s_>2F-Eu39Q{NW&vw4;MN>0Trt{kn@*PP*D)O#%Kh0TVNL>_*_u%&jxS+tbkj965s?_0DK@O z2;2t*0MVmE;OaAT4+~KNJ^=|nf%^i$(FpJepg2PU@?$vr-*E2SISL92usnPA7^66U z{yazks|yz{T)aq0dHM3?YuBibegfqrDLHkTBr5*`+37Q9$xof5AUlD7$zV-(>hu}% ztH7yKAVPkI9K699(v`>LYu9g|JtuJI;SE7^O)4$eE7oWsm)9B8FSO<3*kX!*^nB{7 zcU^fyG4V*4rY@GY#6ss9>Bhim(!-O6IUXJy8Fd&4Tav){x;1_Jy=w-W_8Oo32G%$)3{EXy{}X}ZQ4}nE zp|DAKImgL*wL|~jp6|U|r=JA^XE;xtdQTdY1f(Bzp9(lb!YQ=wKLAK~16vRDw)n#UVI_clRFiyNX~k;tAL1yR^6)(Mgt?-e(>R>EA6<~)`+zpd1if?QR~szO=w z3QEf&G4A@+em$`oVFygxW}1U@lJ1p}rn2ljaJ~|edjCaiO8O7pk!{xYK7u{GGPYrU zc4NDqZLB)Wetl%^W|glD^#O5t@0Z65?Eub4U8q(Dl<`5A%M6|G4M*9T8#mi|fi9CC8-rS+8%F#TjzXpgiW=<=#LmiX+M0EpR!2t&;_DD=!cv z9Rs*X{)Mx~Hoq2NR-^9mD!~eeY$?c_4H9B4Y$zx}L9)~-m40bs2&O>T$D6L2(Hh5X z?d@m}F75@$%t`Dd1-C6Z=qbpx+YRL^$j;CV+?a|Cq1nMH3y!JH)^w|T=?yd-0n&&c zUixX%8_-CVO;!;G_72^amFz>2mK~3S#(^s?W6Y)~bziN6QM7(hMqg?0Wb33SKPrKU zRgE%sn<{3OZ;c)1t(4)~BOYEl(6r(V_TG~5=_-koHSkAmSUSh4b*k@ z;$_zZxOecaS~!hLRaDh5jo6Cc@7NtV%C$kl+B6PN$3cS@?r)d#u_?`rh>>RU#g`#t zc0WXUw1q0Zsb2A3n|W|-n!!o9{orJFKs4x<0ZD~D0I`eLj+GgRp8=>r0>G)Das5Ok z0rLM@+~6wJybuWXf^e*C-e$ewHc#Q=cs`!VUbAkz^Oy&Xp1 zok}d|U(IzX@!t)&EW3>hXs!O?^dYw?wW`gXLUFjn^p_u&1{-HlW!)zsP$*5cOtq`r z;!@J-xum##XZHvYvX(%PloJ&Mwr*2@yS>RD)lf6CZhZtOFJ8{?hH{*g8Yfi#baxh6{Tvp8a0kRJG793=mQfk&J51hxJo& zBC&~9x{WQ7SGh3&BH-T2ty98ABm27#6l!r@5lxY?QP1j`uU@rr?^jCxE#FR~2;guv7M z)OZiR&GR*t6d)^8*cXOatWB5v+PxbgN;lg-OY~tzlm)jY_?*-GYtaQ+)rlNo3^!64 zY7*9R*GpJvS#XNui0kX7)`N*2`@`Q0&-_PaT>VJ#SAb5DY(XH{p!oirr20w5lKTCL z0Y5*T_VUDPkOh7x@tgqp?FT2wowxs*?vJN#fh2JFkH=#H;8mwic*svGP@=;J^TBO~H*3#tS zh`BW4yy@hH7X_(~tu*3|NODVdF7c6@`8OPMomwuL=>_e2e#zZSjKQ_K1PMhxj;rYX zQs}OXVi%o4{`i^fDC^Le&!2tcU6N#WE)%nX3Lid-s}-Ri0#&x}O;uKth81eBDp#~T zY)e-BKp0L%7%l(8G*7iLOrsEZJ!u(9<8qbMt8l&k5!#-^oFl;OVCY@1(oWAklpL&OysXfOd#KY|!YCHs%(Mt1Ec&u!2Sl8DFUadGWfTY{(tne^{t zEeq;WP%6QzPmzP~jzx7}1gBI8g23 z9j~Qmz4}bg0vJ;9`-?5zyY(0aRnsmv)`qVQvBhYb zQ6O7+Mx&fxnV-JLb8$HOOACFc#;b&@7l&8rJ7J`j-PeR-?yxz!z9$=XFlnEFKDqfL zph`e8qEltB=m5HHE%z+8vp-?SM}c#oDaz9KPhUR4|NIfK8DJjN@6gV{O?LUx#CaBJr&db!$FcP+*Y|q`mNU;PK5pSl zO$aKpEwZZc#A&aU<{9OQ(jBQa#Qt645g=&F|Hj((d%gJCOf4A9x7o47xi+4d>&eqS zZl;^-nsk7lG+j3xzL#ewsFE(;A~PYf8jiSITlGtXZ~Tg8-m^B{yA5846Xh#&M!g$1 z-GrfE0$}Wss;EPg0db|PW2G#k_7J^>ku*Q4u$#}%YMe~-eNd$wPoILunG=~Hn>u~< zpJT;DrhGN)YTz;7zYZ0t@%fkn0D&J*JqAcI$mf4P#gPIX1CoseRU8R4scu~lJjOVI zD{l4gk_R`IQLJY4OidPHrbH%5gGF9=4&zvGG8-b7Q@b{91b6sTCkxXX{y~@9PTz4m z#z_R)83MIy$8hbKufS_rp_FPZ9hD-Qs%V-{j|>WQhZ*ha-Uv}$Q61UpGKP6(oW!pd zw#AnXoxr^z$~% z9iPR0geJ;ob7oN;)McGL(A}Ox1EK(;$z`T%fG47HdwKP)U&`2}4=~NivWka9F1Uxk zoxd4o*Uvgzooe9Xi~ItaURKP=0hjEq%69rt@ep1?PpMV4rB)1k1jtUR4R^}v(@>b25d5p%`0|q?jaNxm+3r{ul7&5e(0<|PE$g?ZMXMW2+3aP&f@k}zqlGSQk7(u_ z_KqR>bhi;)fXCk3>ilD4e)U-67A9Wbq9EPyTIt`(ZHR6t$z@_$pgn%vu87geb!vLeGA0azg+?Wm<$ci7W zce)4-$=o|oT5@%2>*vq>EKbY;>-J>WGWET&wQ+!wvtXfN)>_#W0bAV41Xr z!Yi|*5uJX;gLVpL`*)>LkILgpVRYaZ<8E_?ef z!?mRRdU@EIM{TPLVUKc*Ie~VX&Bi*u-p~`vSbMdFVy3JWflkR@YyP+ZcTyoYraU zif>5-mz{gi`0$W;=_iOx5`3ivE;zKXwVgAbE;4Q3an8aXveV-}8kdFewOK&dZ^x<< zdxnzGRFc`aEq~j>xhLC7@btq|lSV?Xf}0sU9{;d>#88_MTDVanzT_RRw*5V@TD!-)m zV`z7kmN{Bwd)DFn{VxZiZyu^uJlaNX$L?EAm|1FmTTYPIUzL!FDTCHhF5yOI+iKm% zc)ODJWanXj`nP9t+ZJ=WB+WIx-D7#usM53gZF_tS%e;(B=g+suU8J_~G2pMCUp1M#&N?VNNuXo!CvidcK6GOF*wo|)F*^Q5@+Fdm#2nT(qDP*yyp61%3nx>|o}HlxhGIKY=aN(Z{xu6vKiP5(NIu(w+K5%7-0$ zWEaS|Np_ivl$O3l;sGvv1hoLDH;$v&mS;PLRZ*}3CY1UtecHDf$-K(ym)fZgrR-07 z8nx;hO7FV+yf>D138&JRqm1*4Fi(i0X^>!EccG7>#al20crVi(l)h0tOqkI%pK#~ znl&3M)!woHoV3#zzpgnw$!F_2IPhjH#kJEqaTPC#rDErl3x|XqD2#co9_m#t94yvI z907I=99s=Z@tH5a*?0)W+EuR3EDUjDyNaD^QRD}!3NI%i)D86wEXcDtER9l!H4#2t z#VT>^)sV2AoYI3QqZqThf>wohhog{>&G-9}5{EWxii=y=L9fPG%%FU;d#L7!T6{f)YUPPmB3$}Ecex5Vky90!oiQUAwjLTJOi2FJ%`NsyW0lEsKWp_xwlaBP z7l*&Tiq^B~`L+YUelO>|ui?HX~Vrbg`m4VPn{WcaeDe$itjSjtxv;?LmVzE|{YE-8iW zfOB%JbfWU}=km(53c0;d}2w4Yrh^_4O>syBREv z5dTciyJ}%J6bknzVnc0VDo~CFUaZKj?}wG!qv4b<0T=q) zh}uU0MJYML8B|C5%OjBi!)ZjncXVGAYW|V5NT?Q(k<+VLOO5hwY1y|N3HZ`P=U`cU zlcOd0YinD%mw{EzUELOCL0e*N)#ADW<}RPVYtqk#mR9$s(1s_XMiVUIHR3ghdjBJU zRW>!7kZ4CI^4!`af=(rrFk9jdHMSDFEZ)$dw1QDxQq0dBm`EP+SQu2$EOdXk2h;Qw zjQ1DAAviHCwu>#^H<9V}wK+Ejh;82L3vtQz3xfk`G(~GifRy#+7oVKt=Wlnt$krHx zsTxdv$E74J%0e>RD~ym(Cc`(0=fJmJuKj7yhnTgMAIcLIt8O*bvqbL+I*8V?a?EC* zAl|ut*$sPoWhR@G>Z+YVTdmH8i5A8MDfy5Nfr9#o3O*Nni>q~{yP<(+M|cunW4RPe z*WER*c?kt+v=ynGP8$(vKgV1Ift#26K3gF(g?oAcH?NYk5=o2Q0w_tDII{Oylq3Qt zKymd;5J`JoID48S>2J>bT3p~dsTb6NWMmJHkqhz}%D|ahB+|#L>?9`%bZ9tF)ZJBz zzo`w1tS4ZH3omQlx9VrSq7nkV7XGQ(_QMW5O%Z}k%|=;*Ui>LDey)YO9S4P>g&pE` zyu$rWOs5wYN>`=%4#x@=5&d36M*0~hE{)A7)o!=^`ABA-Z1+FP5ewcf=+R7mc~SWa z&GU6l{M9h5Ut|eEhL%InrSdo6Ws4BOqTwqHdfhWQM);^F-Ug0c5k@6hB{P(9X>#Ka zI|}AgGBEq?i!6C=zpC2=1@k7mo!uOs@#MEvCask^b!RX}2x3*$j6ETV=Tr)7wcpUY zgy*1yg6LL!orVL#AQMNxyDN-xOey|dor|mcC&bH0jwmOW9l3KIRZ)eRFC-kPfH~pqJ1NEi5rv6n9`cn>i_Wcez zHw!}BL!*l{i#06nPG8|_HB-p9It;G;*`hVr6oj?g>4FjJj3va1A4jXr@t93T@-4_rl9<8}o?NgGgZ7k}e_d|_OW9-|OZ~wB* zwC{GN4@4J5YcSF6ynq=saiuqE>w_B7MTzFD4ffPwyETj6DMl_|q*=P+YH_7>sjy*g zOkV}jhK)@O$?Tw8Z;jBez;S2#zj7cTn7pd*$Gyb7&9_sJN*?vtQ`uc!A?hOvx@Je$ zun=ZG32B?L$~7z^DiZr(%w-zA?Yg{&kuxty&!5|~`XGWg&QI_+zF({rmY(Z7De+m1 zsaO9Zdo})>zN-yo-E85P9*l}cefNq$0*97v9&Tf;&v>@JVVBDSLceZv{-JUkJE1~> zmwl{i3d~WcDs-g&PE!cUCiozZqc@97Q}8f^Pa9<1sDCPX9KC~PF`m$tmK5?BcW>7jwuQ;J505SEa`BlN z{#+<7vQl8tFFVMok~pU~r&s^9d(p5mIQBw*edg9(-kmL5tXs=+xu#VY%dfQn=Z^D4FvPC7I2FWT2jve7$MNB9$RdJL(Ph*2 z!{kWZKo!;x+vIly7#2uP#y8j%5_;ki`hWC~L~$ugVVqsC!KN?gzRB=#{BCUBm8_x7 z9@35WzK-jR!@j1Gw_l38KH`s`a+>u*fAA8sz`iq;=4mb=r0_>zDDk^8pHRv*h<6zH z_Bs1t@nu7&HjYmCuBfYbhv1&BF2}*!+V4HzM%$NjeYx_FfSG_*;YG@jxAw!JLNc-+ zW@sp~Eix}QFM96QF!Yj1F$DQsh4OB$qNDZ|o&3AiMuLmstNHQ17plW%!!1$T+2dTC zDq*3Qgz5&I*?W8z>BOub@fh^M)>)7tk#mOy>w4mUS6_v@%WG748vurDwdD zbdG=vWevOQtDP%jiVHMQ@KsFgDHNC6n^f(CtKTZRV9s@}fb~?B5PR$^L~=6sLPPEg zh&1{I=CVj4*P>BIj+olI*2v3b`fXiXpBl+jkpg)h{eJU&yz<^`>6Djk<>i!@O}Bj( ztCBoayJ~a?m~TYBDhyycG|RFNIlZXdLwQ(Gk8y1_vJ zqFXBStiK>}-aI^cP*AhE$bWi=rB4{#LalDQgc~&jCm_NOVod-Ra1 zUv(DdYoqDCKA{)6^zqG|#`U679f$#%=!Cwi{hB$2G4zV?z9{=leY=t$jjQrP(AS9g zR-w1uaaOd|mg%mlo|)4dWhlm)54(OGc$U?GGfr~{4|C_bq0C#E9M$W%j1V#&P;S zp^|Vm%bF%Dak${xp5S`aP7^M|_Lmw|?dCJK*g9zl?=SbA#f>8%v_`Jnv~#bVH%3~Y z@0=94&%AI1{PfM(aYYO=#KvBBi)}7-n#GWBKI8g z_sd=jZL}Z$)h}e)$RuE1yq&Z~_rJ^0+)cb&Gp4w#iUf> zIXE^^=x>M1G;_OyeFA$m=tCs+R+8)TB@0|2C{!c>cWOu#8Qs$#0AK{1BE@y*V!neU zFtocEWO?lY=*L_D&QG7@NP#m}tmi(Et^fs=;@>>Zv5WKc#IcDd$t2kE5jalcP=LoV zC`jCQ-X9Aucq@HxfZ>CB1Vp{3C{!+8Kf<5qonKC6VnLLBUb7+dj*^mJlpm(Yx;d{X z#bRN(;mR*WsNCF3Az#D!=wjmtWzde-i;Ct)r=jV~KnZ$7r!eLsTI?9vodE}f#ziZJuu!ykTdZ;V{%+Bmbs_EQsTlmKzY^cpLGwWN>|B%gCP6Z=Jc-mL;@QVdg_zw#~Lss&b%P|55?Ql!0;t%uTj+kqR(poAo zCC?LTHup?KLfRTbt|}+bsI^b@RbMd{AYKw1w{9*Qv0Om-PE8#SV6Zl@w<1xTq!|a{ z?3w4J3X~jBdHt0kFgNg5gpT)Fz?1;--G9jti5-Oh5upb`+<|9~52J8hP$o$ZIk>?> z5&Ac&{#S;;y%*9|B=C7Nl$>IkEkonoHvY9kWNXhYKmaYvtVZmsQaI zvJb=2VqEFSE4oDWcopKpUgj7}I3|)qYKG17V84%XG6GY zEro8g!6P!#Sfr-M)tNi3o3!^tU89D?3yy#?mTz0RwYxeFX7n@X50Sh_faa$!{GV?G zqx=|){)`-i9Dbjx#VCsuMZ~_9(@r=7KKE=I$~>^!7R_3n-}>TFn`y>*P_7iP|Fg+S zC#%BZ+ilGL)hW~t3d^w^>pyw=>nhKwUxv%+1?OBw-Z>PQnYpLyR3TRo{dOHzk&p6x zRcydGDb9?8b}3+2UTX-MOPI1z z@AjBYe(&htgJ`nIq9lx$z9u{224 zD7%?4{)x`k0R6au6#uN}@xc-X_lBFdJ(*nNI9fH~BH(t6*}3T`;m%0y{GG2bv$Sdr zFk>;YcsW2s|B7FqV*P4KS$jQ}n>e`|Ip0E0(8g&+`PeBD2ZWX2zBWJ!u*>I67YkLbb{B`Gxx<%M&uieWp{CgIFq%{6c=G8B+o|9Z20Js+T zo#d)q1%09Cc_2d0d6SjxDJeGyT8|44!15TlL2Yvkq+@7cCzv!00>H!~Xu~p|YG@oE zaw31tb>XiL;k^Cu0g3%W!t(QbJTBIK2K(=#^kG#`*rPjhd%JOyt9|Z}0Pb?RR8!*N zeV2Vo`b+y4m0*q7i{|I~xKtNeeD&e6Fl;^5P*S_x_1sqGvZA1cq@09t^@bNvL`9`@ z+c;ZmXRBks4@c!(1kAJjZtQRcX08)U{JM;2idh${wO(E3UI?EL#g!4H z9V@X`=z_Q%e2a4(Pw=%pfg`|vId=U;LsPzBC1POXzb?-NQ9C~Zl>lSo}1Dr z8F-Pz!#lhpi{?eAc2bHBO>eALn8{q>bVNR9M6@Aw13O1qv`OC(wr%44I&8-^D*E+@ zC8#-g4y!5jy0NI@+>?FTOR_NkXA=X2`4ljHj(St1ewIsm4iq`9tY9+#~uCUt#PUb9eoKgE>DyA4UR@A+Dg^1QGNCeoq9$5e$awU5Cs ziJQTb@s1nuJZK`_p-MT32Q|E1}b(C)ilCqy5&&9`4m+VF3VE z(6$_l&efB%kEh9ApQLZdIpyEWpD=-*&tGnEM@l~M7$&QT9ksO0i*B8E zww{$vD)WB=CeU9@&AiH$heJ1whMeo9^P}P{4hOIt(;Z7`p#!C>T&?+6;_@L}Lna=o z{MMRpr_#D3N1(-hbFtZU;%jJ_K7)OH*d)0yzOAz!z3H=1IIGZ_m1Eo;?-C;3%8zb> z@>8~iBCRgyE$@yDGY$O+48Wqg`~$xmHQ@- zrTY768jJpL&c89vQz;b1gNbo>-=>Y9`Wy;IO@lv3GcuQL8L zelR;Z&k(RbzaLxFjJgEfrVV&%HkDWeRcwp*(qrxmX7^`V;X8a^+bnZ0T<(v_!R>(e zimIlq%>^<(*&@*)`;@g={A=fanI8eCmggw<1MShVeayL&)AvDF+H)i5@agW~%VI8h z@_dc?8pV2L=lCAXrV4*YK3in9d%1pG8fDG#_hv0ZFGhm5h+(9$mD0!T={zSOgNKT? zIl7l}9B@ebm|^}(*Vy6nhs(amRTZdEQ9~LBKN!iAgRMCJw!TMf5~=9X?&acL>0@ zOJ}!+OsG3Xko5C~d>+canK#*SF3^M#78YVBv8QOsbUj2pr>Ak!vMr@nZ>cP}T3Qxr zRs}CoFX%yAIbfL6s?(`sHY~S|HDZRFz18FvO;-J%&6hDk6^82l+0wcEILoTdMra}@ zbyI1qEIZ=sbqdRM`4;nCV?(#SJn&QQl_;(svYSuNANW%4i^?LZW*i&mjkP^|WxHkw zN*x~hN{Gwrg*A>s-`dtCe?`6A-r(oBoOu%*i8PO%NM$vmqr|*{1Ps)y`xhOAx0utL|fCT?&64O@1i#* z?eKI~xkcXH%Holoy?_Zi_Pq$@b$3YyJS6CVy9`N&tM?*2J543agUo$-DEk zDOPs9s5xO_uo+d|lTwkGwCPsd4V<=KsD2hm`LpT9EwU;A5VBCQHowPSf5%GfJZ^E- zVM)qFiusz>NwgyA;bq*#x3rOl%ZE8VxUB7{7@xqHIHSxa>%FWamVr7UYVz^38i| zrH|MjGRlIHvT)TWbm^ClmiT_P7S9$`TNS|+KN=RFZq*sug2pWQtNcN@m`;@oy9iz$ zyBvh@Y#O5L6#s#+@g5lCow8}Es!hxrU)XZ{+6(IlJnay8qNPC*AOoXRFt&XF0IveS zf^9ICd3B-L;3J3vz(3)D?lE%lL~@RCod;l0LISP}L7b!$K%gvTadDBx0Od63wg$K< z0QR2xo(1+iATgfE8I2&!#^alm{am#pbjI1Jiid**e`wFp7y2obcZ?qrsn^yw;YDDC z4T+7rS?`O4roCdiy8vgOowl@|G%;B%3jP(!m91Hc*0jkzlN%QLmVglZXTSD| zwo|Czpu4fTu{C?(k=<0`r@8J_1~E^nfgBd9YUy^!Uh=8dZxl@`DdSN$`!nJZ7sDNf zF)qKVc$)KWy0zJMG+nur%)Kj&6>??$2H~w$FQJD|)RpQP(^@ImjXLVZ>+WD-A>sLd zA_?yXsK1ww^AyWM0X&agkO5_@%zodp7Oe5D5w4{bcod z<8;`cP503Gf?|CWb=EKj4kn*n zrv=Fw---kigM`UeovA+TdCh2J#_p7lq>}sz@;gshZ-ox_$ca7 zeCL&rc9E;gg9H7Ntu^{jn!7WyA6>}L$uY`&vSP5aWv*={2YU>=)#s@&)X`v7Zb*>j z*DQQ&Vt#Xe&}aBN^v)NbRhPU3{Y~G{%p6`>)0jwU$4}Zh-Y_Y?VQ9>dgN^iEQsUN0me4BBfYN@<;lq)mrj%-=?P?1{UUF3xCu)u_SAsl{i!=eD_9ja2>XYU3P{HjOR+TAF~iB2=97osD0o1#^#p@gPBc)Q^nPb>@~{B-`q`?)^kJ?S7f z*NI$3{uv%;!~2W{HDg&mT2ETqb;ss@COYYWH!D~iLT|O}9tFh=q3@2vu*AnYMWPy@RSGIf=-++Mo{!SaWYp|M{Sl*dxUraB-8j<~6(t`YzjJ#cg=vb|k8?S9s5a*sR18*`8+dLrK?uRaI$p$!2 z3^%wuu9D7wkeu+-$7jX?FhHj`IUNq1eY}1W^aB_F!%kXN7fz8%FuJ<{o*5%$!b$2F zRKmB8vyHb-c*!;7-yXApg~ngM{onTgufFx476quRk3+;WJhzU=0{3oTgHHwC0iepD zd;0VNNza4dWt@Ct^Y3+c-hVtnbttrSJ_5%d<(zs=_3i;k1^Kfms2|882iZ2bD-{dA z4A_72@yyTZ`>^yn#X@y+NGBN%Q{ZZsP?nxV(Y{}6-BR7lNm;J+{l;QEd-+xKtZ9Fl zjBMY!xY$ejY_-IA@kUs4qf1VRXK7Tv+CGirQVgCACY!iGe1Jl7^c_G7tZeB`(4{|q z-$H9X)cv{9s|3%HZuPUjr^r=gZfgr=ZeHYKN}zMn>D8E zyPjjU!J8=ZbKNU&>e+}#hJBLyv;~681mmYxh8I|~SLV7lB=%@b$JfgR;$$s);=C9T z7ZRGZdzcbuUFP@?4g2;m8QFhKlAY3Dg(1!FEi6>!#$`HSSzc2&Y9v7l?Rd*kN-c5K z`(^x|Qn`iWz7dB0qCR%eHRq2mPF;IvBg508l5^~Y`rQ0nZ_Egf=707Y;!+HWObInB zwevuGhX;eXT0SNz9CT6Z&iK zp49Ex2*^W%Vo$17Y`%S}*gGd{KcOCf$TN)a>R$2Ba%f+Qc!s$A5*G(=MP}*ff)DZG zEF)jNr0jx;y1?+bhZ@4Zd%kc?pL?!p+V)q8tR#14V{U9*Z=BxP6j~K}z;dsezoh8W zc(v@-0cor_7(`24_{RrT`LZ0ZYQMYaUZ-5)zp-Vyer;o!=-e3WBxZXv z!Ak#)sroa?d#$Lj?xweRH!Jr%=~13A*BIk1<4Bb0LhUDqg>4POVK^cDz^;aYm4#M2 zQ%>DL%{2e5?uYp}Tsor-xvU5$6sHBD^n%J=W`uSOt*z|Ff+wI8wDIf*#ur%>e@k&M zRB5$#eUqx3ml87Rv(3%Ja_RO&W>&{&S%@oG<%X_azK`kn)7qZGF_*4<9X<>@|K$2XNCKi@p}C(Ui`-H5`v_w>F;t;Plx@pcZfo|^@C?}oahO2 zsZIw_l~#f;V`xE3V*;}Y4Of@Rhh1T@l9J08lr@!aU?`M9yu|Zic_(fmnVQ&kLHcm-ZbgLzR%^?o zdwup-WYKIpRIk_BZJ%)8-B{H^-GJ`HVRZ)m8rPp;8okkpN_#XBk{!7%_qDLy?Q;H) zxpw=)P3}_nmoE<68k@TFWe7AsIEwUPGElNmvg05-X7#}j#g`15#&d=O95t>tMHd+dh~3o zNaJ-_Qi0beeED*$)IVF-z>j-~EO~n#ivf2c7?2yoo2rkryyAyatpq>ORA@(>-*bYX zy-;uknf8(;?=|qliz{;n%Q0|o^8o15K2}t4>d%aMW8xHUYEogx=bJKtH(P9AgWw_0 z-Rx@0DT>}se_E7P#oe@k%J*r?$?M(AGg!-14)S|=VPp4Y|C9s9|MT+>&DQjBeNK68 zB~Lk_X!mXP4Vt1Jrg)Qn8O_8`I+ZrGS8lLSUYfQtvwGj?f@x6}Dh#*t6-Y^exFGnR zCNCT;2+ZtzjbdbPOe>8L(b%C9f3c~ZR|wW~Qr9c*H8-?01wWQ~<~{bLOn%2@wU6ia z%^_y{hO-00o;A4}R8^w$7i&(>1iaLxhpnxa_{k}ISWG>s@OiJM84+!#iJ*kRovJ!4 z@;B~g^bTzCW)Q|SKjU%sNl(IP_$kj`Q9x?yrUi>-e_XpU5$?6dkk)1Vvs#-$dhG*I zaBGdf$7#P9?fYeBu6?aumKl*tpvGD=Uv{z-m~8T0EiS@PJ+j^@h}Uh=XNrZl6sow+ zeP9aOV9w9A_v6jk?AJ5XwT;7%&l=gBMPP(xBG=e#R5XwLHeL%%~ka)R`daxoTAe z?h`%vFwVu#=V7|2inSr^e2|R}uuM(a@RyAU8AvtHolEM^f2Uwn-y=M0iJL1kYpqSL z^i^Ht?VYO74~Fn2O!WjT?h)ZieS7>;1{(>oVb;Yt1YzxSdR8uRtdA3vz**95cRg12&LvT2_uo~B{nUc2(ykNz|ti#uGES;bB*Q-vS{1V=C3IwN^%o};D zgt?`hdheaR{(NHrf3<&}C^z})WSo~!(Adx)Tjp|Zb3_SQZE62Ds#V<-MRwJwf3Y(( z5sO?~%gD8jpERG*NZC$f+AOQ|y}mJmQ@6sjDn;_?t>YUoOAPs#cKR6(=;f-NY7xTj z^gIr=Aq89pjHP4DmN$H8>xM}G^+vmNeZwKcdv=Qt##rP2Q zy@Ddo7Nm`s({uV~YBX}Lyrx+Z&D2iTip#0Sm+;o=rY4Sh@3=JLYi)$r5GXoCCs!11 z_knDfx)oE}rW0SOqP)9PPOd$@8ZSDgzIS8R!h%56y;Wk6P#;w2`*ApCtjWD}bf6o~ zJ3F{>`AvVSVxM7>0&|9kF72ddjw1hEj{%1XT`y(9an(ljknbWyo+n{y6g+pi_Y18e zW1GL*sd9H=NAVOx;bK3iN94ZaO>(#3LBA6#pCp;)@|vmiZ|o2x{hyR^LpQcVVdVlV z7_IP3wohAdWv6d=z)DNoWcC~s-=2I(EcQslf5ynMT29dnc35gPiS~oT0j0_ne zGN_4Zv5nEHdrhYB@o)Bo*m_s^T7UWObJ;*5|Ew&cTUe!XInSL}NV6Yio~%HF$Z6uc zY0&RjZ*3B*<%?Q%{#>xFi47`^+KT#ZD3j)X(X9nP@Y%xBLO1nQvauJ98t=&GR*i9G zp)ge?E^&-j*ie|ezo@5^SNm4dK57Ny9~m=xuaz_B&Zt9ZZms8&er*TtDkt#c<(4P% z?bSOB0xYI|8)@%*tC$xWm+}WnHXIlo94h)4rIoDlcoVnu;0o6W_dW^!h$|OqFWg) z=8NofXX|CGh=qGLEWrwY?Mbl|`JiaQ2yTJX(gHtRlwAQ4;pXyCkfu>mw3Ss|t~9AL z)Y>KpL(4dG6X*B@HT6n?w>RRF*OWy0vKQbss(nMM@hy$+BmZAl zUmng@`u19h9M+~UBsw;3qsHkrc-5FOQyCIK{ZHiQDRMm zQ6+<-B&fZXAc{&1B1@c=$^txdia_F`q? zC7D(SD5iT3=UB6$l_~E&+;@CLdwAC-*uLd9+KG1EKKh0`B}*Wy6};69RNpyY9+X{uDlbDwi$d&T)XcX#2>lSJ zJMXkFwQg&!OC7zk2yCL@93le?vnIy_*suw7ra;P9UKbwHK%gN521ClH~h|oc+%~qSX_Bp8FqP4#jt5dDvjx+B?b}I%0csa~pZapjTgLLF&9(yNGS=UvaFAqkAN2Y0%pn4Kfugbnzfv{1`x{;<3AV}mWM zGQ+A&{?e|0q4A0C;KqK>HYl9Av$}eXkwHRnNe)I&7*ya8TA!YOamdA?8rD*WhEqN5 z7^;D}T~@FaC?^E2>J2xbyhN(t zYI;_l?rr5<^UbTFe*v}o_cx){UFKD&Uw}kNN22)#A>*XxUA2y>Wf9BRz6*# z5d{9ZRoYgo650~uI*y!hnMh7b%048B%Qg7tr&9d~b4@8@zTjU2?R;6sD$n$h%KmS9 zUQcCr5?IY9-R(wllMvujrN})aTfw7=PXK-WfOtS#;=GC%TE2)!Dy>_a)|`q`vG1S_ ztv9+wEIBJy#)urEpvt#<*Nw4nLb$$umza?;jv!Wk;FCd@k-W0S37BbwaT#03aVEI@ z+?Y+oeMaR>aOU3c3k*!gTDzQHm*-dtjTv=)qNTQXueNbX(-)76>}2fgy+g%=3FRad zMgBx`$t$+HeFx|ukE${M@XY-|BcY;sL^!MmjP5rdk~X7%K<6ws9UG41Ubx4O0CDuv zCo9EfD!g0eNnPPDyFW}ua*JRrg4nur0iX6iS(4#2HWA%QCpB$jYXeuO52ua5mOGe$ zeEktU7*+c21rTd8#|I^95LBf+*Jb6!exHt+bYu5WRA=Wh=~JP~eZbPgJ6*%5kdnh& z%eh&5ef_7%b-@Q@Ja>Z?mS8=%^=#izrd5}tS@=K&5TTjNNv$e+? zgUGjEfS}d<$*ZNyJr`YzM>r9qa#2x{%yH@Gdvp*C``cQ%UH`GMDLu3{Q?4Ps`sb%2 ze@;Eg@u5qVz!>3kh%#_i-cY3bX@>yeD$#k+W>{qS~^@Ur;6RfnU zAXD?acQ8(vGHp0G_Xs_>L{1O%$sv{OVumtztLUSPJEf+i)KWh^zEr63)rS2;pGeD} zI?s#IlkDd0N-iuyT(@i&V%;N;CO!)k-n=o@us=0112@oLD?n+Nl`&^oYS;~WVWc@@ z%D9L)Wb7Z{)Z|`QcYY=4f{U%YHadHH)*gB@dn-NDWd^8x>Smr{J{BSW)N_}li>t@H=Z7ly}k`4XeWJ-0&unz`JMXB`Q@<*l3aF;v+M z^&O$>lhE%JZ{W(yvyCfR761dhcsK$&*#1q>^x-B=Hc=N!t(RAh@Sad5`R}DUS;yYAVRN2r0sMD(O3( z)Os7cA8PxKsk<66Cpl9}#p~=~6`SE1(5Fc{hwz;Xm+I#~#?UuZQpS8cgqenyC^Usg zmoyv}8czMrDTtxW=eD}{$}6K}oH$sa$%ECMf@Y4k+3f%qhx+kHoM3nBp3X!qxzy#V z>nRp%#O|>|ZNjJ`u)Xuloe$l>$KmEA$GWuFtW?)VQ1<9eWUN7zx=6&#yCr)?15}*L zfyF1Sk0pCHONK88p08=&xAIo;i4EV4j+>5A(ak(wiAGZnqm26EVG*J>fz&E)Ym_zL z?IgMX{HD5%r^GC3k35+jI>c)lv!lpn3-Kf>v#74`OZ~H zEmO6^l~08B**+&R-_a*nQ~2CSx5fyoTUKClT~EI*u4?(0 zCC8UO5I1qQpt`8iMM?e3d$6O6S-LlII&*nYVE4-2<&fmw&jRV)Ocls_6z?jt`Z#8F zL`HhLx@5=aw7Z|%_mv+b+T0={;g?hI?ci1gZ?h&MBxQf>3buxakEM_L5vel8UEuMAQ^OVLlU)Rdr;qlXr#ojD! zAJ%?!P7QZ!w$$GxF3-ox{Y}H9g@-4f;HROJ&B<5@vwDj=EOlBk9!?=KD~y04Dp155 zo~z10XMg;f=qoOuZ1ua{#3!%j1ELFSHO7rRC2X_es%&aNRHj&~!dfG95rNWM$K$Ja zFFsQa|Hl3WD;rWdkQS>KjFI@>`tQS5t8ShB=w|%>8m@TY4|ia~ zTM)e}<6ZOBfgH3y*w?Mr#}94E+cRV31=4T6My9IU5to_t8*%d5$+gxst-7Yl+Gm5s zL#ESc7wQY1Fk(DL>fOgPXZ`k%B_kg_%*(DU)or}pW&Q@xs>!ynwcj*Vezl31+%H&| zN{t>k8+{N7Wn#zRMYF^id|}X=USq{Z{gs$94P`+B#hf;*DHSAH56F#vE;6I2=)Dje zKRMhX9%6jyfgxp74Z=WOieH`MKY7#ChG*>Ws`?+<9$-`O7g}d$XFo0+Y=*4q09FLk zJ`4B~Fnrp^CODt};QVxB>$tgMYYUAzSxHg*n{g=ZGF?=7lgAT{R!Y zY|R+DR?+d0A@Q+u1vPG83u*;mwTrK^B z@Jv&u(E>M4Y^=-kZP)(QTc@cl856%vIq0A6kPMq0P5C(dS>WDi&u(OEGULIND#I|5 z+b<+-Rzff1QpvwlEStxqTK14z2{KaAu{N1kt5OO9LVIIXX~3NE#(N(l8;jX0_5>BP zp5>khsg+3+@0vGcxb1Y9ec8T*MZZ|@V_zjDXD!NjST=~&Roe7p3B~&9T+H4T%#N9j zUk){iYx)H9%G0gfBLI~(_WUMdXOq*o813r?wX^f4NjGGyo1OV~cDgYHaMnz2GF`H#mIfXo_u7&H#=y?b zgQFot(`Nx;10zyW(iR$6E_yez?GatCm^b!I5gle3>TS-n{0}N ztAPyYR#Nc|t<~>&)nyTEu2M-B!Ef9_E6WsGq!7y}5AOfa6_(Ou_`D0GlDqV~HKo}*rrTPW0eHlL_n|HjYV`&z-y_|WB!BLeaf=#DbuZHv zR9(kivUgl<_ttL|zjm#e@@$-7%H4x;WsHO)>a8S8xiQ?>KHG9ahi7lONDsTHgQjE8 zx&!$L9#2oYO#6H9r`Xli-lqdfNZ+|!ro4}V+HUy7N;5TB8bA%#hO9>IuAOQc1DXwS zo#;ZQ6M&haL`XrQ!tfcB4QGn$b~YzC?ZOBCJrl9+mkTedSR5qG*{9|Us;MASZL*%P zsk(Nl#VI17Y1b&;O@F%Ox&_7l zR84UE;K@;KI6BCih90dsySvI6P?{0pLa3BiJK(!JyZGU8)5aoYmlTOD)tf8yFpZw& z@Vo@onX-EGu2|ctfu~C75t=*Q(4PK0zm=W4!z;a}XWy(>S22)D*xF8ID2y;NrCJaR zE%M*93tHG|HHpeD4c_7Si%;h7&wJWDequ*hb(T;qIuq87K*Ug+mD38JXSfR+Msjqu zDN9jWt}3)g@tch~X~S4kb%s-h)DfGwuA|LyyqZ1Lo=;R{@8~17<7`GAl9&4UlL`&3 z`=1h+ifKKG+;sm#K>HUhZ0^ymZ^&+|KlRQ%d@cNG&67Xy^t+qikoT+K3g{aKTD*T9 zLA@e>(j~DZ3kxZTyBhcr6}$iIu`V!P^huBYC)NnOc^gVO*t@W6iO8An)Lxpu(L!ud z5C+l>bJ^r-YY%^y7PSbc#aZ_}Oi(#-Y|PC0i1l$5cn}{h3uJ(%}6WhT%HLw zfM&$yr~e)t(oJpr5SM*K%#b09GqZK3mE^QI9nK*xt)g#g9@P%&xf{FDkyUqPCQ+=x zO$P^Xm0}r4CE3ru{Vc%n^@V)@NRRNBT|M7)n_0#o< zwt^6TJ7&*>Jxk@EH{CfKOyih zZ_4f;H2q8aUj*8(0RaJ9k$-!NeB$jdUyNEW{q_%<&u=gI96tW$H-1;B>a5B{E@Uiz zeWL~YS%B++kQp07Bjc~~S1hv+RPa|)S)7@@U6wxM_>8P=nfZ;N@spv&pvkJHf+O#GgP;SaA(!&WU!&aInY3Sl#$MnjuVls3)X1 zN)fr=>~?&Q97q!W^Ic-$__bLbV!zJ%>>d1B|RCR44(t{%v3#)ohmUSoq z7+v7wPX(lDb#cVx zryEVx7{=o9nQ{tVHIM~Fp{OXWeit0l0w27)U3lK3vn5mFUL=E)*U+t}=-;!i`R z7bIE?(}^0RJ(4A|EquIo64ocaLk%>(yF$3QTk^x(k#IhX+0(9S%-g`-rWphr$4p*b zDAZI`KJ1qtBF~uF7yJ4ho0p7vsWfQYlzG@&k~RyIq7}ikzi<$YU?A@099{l#Q*W=h z*ZR~m$dj?@iL3Eq21afDXQ0iBu8@qOH4r-T5*F} z&fXmJs;$Xsap&MBU~Dt1ul~9wS`gAnkC1|=bPk5i-cwWECU#Y8@dcRI7b-VRnhw>{ zGx9bK>3Y%BfN*sLJ{`TBhd!p%{&Ym((y=Ulxtav1 z=d_KCz;ki3HGbB#?f?1mW5^)-9owU*dNs$QXY=?K%rbV|x8LB1>d--Tw zv##I;p_(UOdA)x~qw*L3=LM}N-qKP-q*0Omb%pWL@Vch?j<dA7ABk~pjUiDZi*4OeUy;)8B z_tT?FKh8@pHg}q@w*Bl*;d?tZ@*1(^Ndy~uvlr`Fgo<4&X3~6($A4W}g5WJUhB7OE z*n3}|u7Rh%e*EXhZdfi{9b_cl1VlFng|}wC*O<;?7rcHqKr3}xcc6186~^qte?P_t z!7aYWLHVWQpLe%hWffESnl;fnJSiE*UnCvMs^=mH{P)r3MXcziOH9wjyYiyie6@+= zxs^7FEHOQ%ps!+yuOU-x^-^j`W^2=t+Q@o`_9jf< zz$lN+fb?|Cj2VNwmf|}Xn4J7(4&N;;>yLivKjr!G;S+*Q7DjB!2NqTBhU7I1KOYs9 z1@-DM9A%$ERVdFIE0Arz`H)A&3xUsF=KJWXMV6sqCD72?Yo}yDa97hGG99)oF4}BR zcdt(FL&Nqx4Py43s74J=9^ou+_ygZ;L@)N#?DKgYolV>hb=?MAC!|tRN2F^gsc+V7 z)=iSbwvc*kjIcVkEJ$in3Ao0*cERys^vk?$->U-{2x`>zAwHtojywEkZUwkKO z+O;T~P=|Cy9RkSGxg$tj7g={N9dn4Tf1@viFkW=2@nw0neW>QrrA$ii(q{pZzW$8| zq?~R#im&Vd*i)CQ#z8+>ylamEf+ST!;O`jG3cc}^<~0I9p-0V?l*To7@TY2yWCuRTVuB^y zjqCgp|ye0$uOV0(bGpl2x!9QIxCX znZV557@~VgM7%TILF>|d5%_xabcq4@%7Yoe`Np>C&d?H(Uhn4ZiQ0ma3h?z6q;>|D z{$hk_#Nla-P)@^9kl`J1v7Lp`E{|Ra7X90jd z6+5sI$%=Mei#81_5%(R#Z+Wt|YG(f;zyu*v%sktUnq2Y0BGKZ*XeG~ibd#>ZWb^{ds% zNfK08gfDB?@tHUsTqf1bS1YwoT`K2tpsxL34CT*lETyc?SiHMK2J8N)LLO)k&2#Fr zgd0QqU8zf1jqZIR+Me&JHpwHpyh5u5KNnN2EGbE5vXvOi+3q?n;3av%v#&(>5g8QT0-EeV`60WMT^KP z#c0de2h0F+il$|wwojU z%Mp0jkIBhN^3d@y#GFo)KW|E6z}_9El-xltueR#6J}MLv43yGhzF9#4Llp4wZmY6_ z4$IdyexYYcgX})eEiPF6$8HRCwRkA?DNR?q4%RQ2_h$|~>|l)@FARYV{BYRjdIN>Z zi66-U)z3`L_4~FZ@Z1Jgf?(NfqxYJWFjlsa+@+xQ1a?CC4pM||az`y0qev8~Wwu=b z$xI!Qw-^ju6pV(G)#Q@j?3F4q_QK3jyIO1h6fTol7zLBlBGA@rlb0( z<5D%`a41Xo#espuR^=lSB=Lq)wj7wiQi`_4d`nNwRu~8#pWT-s@zf=+)Vry%m8w_| zc4#h4F`mJtPW>xyuouo(a+wVi!5+iNB>RXZr{!D+fbi*(|PHt2L{?r-EYNl`Th*P}AUpmQY=I5oofHGNZI(+KA45}DiT2y}h9yyGA3tiwySo`AH{RDeIja?MAnyDA}3J>&yv zCtP#ngI%gq;O~J_H5r*X(cQY?J6|kP`Qo9kRh(}HeoX!n;b+!--~X@LwwpilbL0Qp z{w2l#Lw4s^dmMk|kDpLHck7GDi7#$C@#imol5V61y_NchczokOiS`I@&ll%1t#e=UB;%0`}e7Xzl{8NRyEyRu6T=2 z1HU!--zoq3{f1o;{PJ&CyCt`*ZqUU9obOqawGweNpWIlwSp>zL^P$rOd6`h_8fPo? z2|)cs$A6>7;%*;);5*Z4UmNYq)3@#)RwH2ZGvZ(-1yiGAN;k5nCVIfl53cbI<8BFl zZtEW!8GkI;8K4` z;g_$;1g|Bpb+%HLYa=G?=Dj0tZ}c~Ju8W%x7YjAs@k$#jIFVgbKw~AnP^;TBL4>~w z oZ{+YRdS8hj{M}FAYIpd$i*-t|*Y>|cc>#g{R$IP|#eW|Af6N2dJpcdz literal 0 HcmV?d00001 diff --git a/film/covers/cover-smesh.jpg b/film/covers/cover-smesh.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a8bf9a996064b8f3624eef7b2025fdccf8ee6886 GIT binary patch literal 26303 zcmbrlcUTi!*Ec*i^eA?)P*s|A=^Z>OARwSf4Uk9|Lg=9vD;z-J079gz2nZn{C4?3b zk=_J^KthQ05<>6woq*@u_j7&E`~LH-T$#OQ&)!qkUh`XPuf1lkZ|@gyPE$=o4LEcd zIR5K61pxMj5A|u>y$gG&r>my%KovN22mlT>0w5i|1po*q4>vvaJJ*a&AlHun0uF({ zeX_7}cRAqx*Dcs;wfCTHxA6bk;{RRwhqaBn71(4R{Byg37YFZ&3FMjV4)~jN-tvG~ zr1M@LE*@Z;hjiXeU+*r+TZ8;fyZ_=X|BJVBaih0?1GZ5_I3nrSq7(h;7;T&k9)slx z@XrRg1A2fuaEJc<;66ys9|1si1ptox{hvC^1OULj27oKX|I~562Y|D$0HCDfKXw0+ ziHn7s#s2Azg4@Hkwg9l01puc_0N~;e061y7e+=CGU+(Q1c!&+WFK6&)2RH&Yz%@Vv za009V5s(uHZUJI|^xiIT?tz7;m9&_Mn4E~%Eiqt^1l$Eq9HrB-ePZ};I)424i4!Nl zEd#?ocjAvf{s1e$;^fJbjEtvFojG&n{CTFm??5A6OAa5QtI9ug=*ZC%#~6+uKD1B& z)Y8jChmRaR2OK_3FOMEQedgo|h9jUJ9XWjT{ILrR*TpXWdHja>BPM27ixb7Fwb4ku z=Xdm9z3t+1cZ<97p&;h6OgFbBU+4EruZtu+ei}*})I65FC-uUL=WZL5YoIi2s(~-#|hi(9e&jUx!1A9b(5xl#@=RsZtSic55J9w0^g914G>W_y2^KEeN za5U4~_n@#((vL75v^x%O^3P6dA85hN7;Eco=qWGSK?O#v>PyE{Tk6#B5zS>W8UB9; z#}kTYlLoE5O>Hk34#n|hGYA8K*7Mr1f`W*CShOaCqoGNnPnpoP1N9i zsBUjl>8h@Z8I9?Qlov(hd!h{%ie!)u>fia749Byy5v>Z$h+l6_U!%x4w`!k6Q&d}Q z>jMGEdaPqoW;pbO+h{}Fc3hsBSWmSrPQ9Nte*AiaC^FD~Y?>^c0-Xry%=<}1y|!OS z$>Xr)7&+#4YteP-Iu0!@+1YV%J)ao^EBib=U@4nYz1VPTu>@A`><4w~msk;uy_jJQ z?fV3&yRl(~k2r!#s&6V@VjxSCD=e%ghP91MuBC>pP}h0_i@jie+)g;%f@J5VpcWH< z+_*u+=}&3!_KZd8rco;2;gT3DEvL|yDBD<^jywd{jTR!mt#DbiHDi-oV;e9NLK;cy z&yU=}*g(p7@m3!_0|ZKQvCdz zV_>1M+o%7_e0~Ub_n%s$c6THEhzw{i2n?N5G-SA7zjc<6P~)(D8M~!C)etHp8Gd1xe?L`6DxyRw-4fk zw9DUli+OEa3U_;9&n!#TRg=#XRB|lmirwTeY9HH)br0#I%)k4i&L~(m?+EQlwmCi&8qY0$Afz~vW?NTRCDvgr>+a=60;owiX`){aTvUFwpe zjGcwACl=MeT`+{$^C(@$2w_*1L(Kfrl6~QAw6)B%&Q);aqh8I|XTAxsYN^a<9U#Ry znupl8k$LhrRUvO{20YJ`!Uo+b!cE;nDWjiC?(#3@0Q5xUuY|mUUqi zKVH>XvmhVM!6t|3rie6aqEC=h2!iNRu(~aK;nB!Qr=HYebrqgM(eQ%zuQ;MSFN1@_ z>fGku`li`%!`;!;77hKQh`TYqng<1KEL7 zvN^-b7b~}k{_VsmAF=l&uSBD~cUMDoZ5HpM?z0BnD6$J4SjD&4w#J@v`_}xmIdqCw zSzjjDVLLB|dwJCaZuG0&vcY>ivQX7mi@movM#1p@EFOzz#h{v#s*dZ~qb!=Y3o6_r*P;q(lEDvJ!`hyrC9yB0KSXxHOhhO1 z)$(=WlfE59@_5!SQ=^$YnYAl{&MM9A4V&>}%Zypv#k^F@Ce+2UmKd4cy8}Zcuf~Gm zrX-d!kKC(FP3d7>P)YL0WE!*Z3dhWhF&h&&!snn;le zL$E5kG@-n0Q?ZE-R^2aH7GT}g+IllkJ~O|Lq@cwQ;j0qJJwW1-hw=6~xtQ2miJ>ck^Sj{CP*E70|7rSqy3=Ei^jZBeRhN8cYxvO(lxlx*pHdq}u z%cN(qj*}bfK<+cpelWhiW=yvuz+nKm#((ntKL6LV+}Cu62K2<9eA+Mnv)T7_=({JM z0VkdTIUuJG0QbP+&~4!Gvvc%A0HB<;Wb{2)$7Y-j%{}%K-ViA>VxVj^CAq#;p0&s+ zq*VR4*+^)!*&4e<#={|}d8HD2$K_ArL`@2dR#lzOikD~6lmT!4`*NY4#(bRi70vV#AsDy@x*qAyte<)Xk8rO%Pu3k;nVED z6>;CIt*U5{$0iEJoP4#ExcWIO3(D@h=4BL7Rmsa!XDU9N8Qxb_sNkeCp}iK3`%6YC zOTgHh8&fD2H&ak#soNdwjn_gG^m7T!&`6AAfap7AhsRkya$fJUkv#-f{!^*G@jI^u89b31d?$60mHz{(d4*C-F*#S#3e5 z20OPw$;2xRR37cpFBhv*KDlC^)uHUpnkDRP7FVM2vYh7AZ=CeqjH#}zzA-^nS@CDw zyedzA4aWZaa<{pZ@^zt0!CDPMI&wFc@!U}_G*xWS))=FO^hLBmU(9;U9mKSGf7pMl zP0?gyWa&(q%C5*5rHB>Q8O_XwFI=W=QUkZ-6X(|Qm0&2RsT$rI4q8-qw^x+%djBbN z-#pekUENtQ`vDVQuN`Ey{{}(X8IIcCSR6Jby%4T->>%0}O%^vd;kgLI3Ex{~e3hc) z7kA{c-g=San`F{(Hh*$9$s8%ZwEnXvT{gsFFdkWs#G7KKH|6`4gwmQ*Kaa=sAZOtD z-#hDc<;vW*jg}{2P8Sq5r(qa?kWu*yGT3yh*Rmuj)b_mo3)ai-)!r)smI< z7QI@qy%eZVg5m8K&#Bz z=Ea~CC48V`m2*yy|HYDJji3<)uNOP|)mT*99#xoeMipe9Vh8Jb8-xN^Z1C}0qd!ej0 zSU8Oeh{hB7g-2?lLXUfy2ZZA*U{_aE^=D!i?EMr2s*cxIIj~VfU#FBWMw;aa&Wb;F zA<3qOnf>+gS7QT=P$~5(nlM*U!Siz~HQ39cm&~TN#QLrb^=<>_8T(FLoV)R*f+$C6 zVZ~<`I8$%NlNYCyub?$E`mi{RElFPZazTK#8#>MZ-6-tVrxL@7*SsRb)9CqB3Ni|1 zW02uDO7(V=HM%zz(JvB4 zh8O18amA>m%O0b%uEhEL6bHwoV14g|_3pMzHmQVpf=JF>#fy0|8lpMB@Hnhw(v5Z7 zW0bNM`(!7sX4ij2UEJr-adm;NE+>Cn+JF&MC|Q%>Qm^4%c8ex@awovSC-r0i#@8b- zY;q?m>uI5$Dd*#jE}I$C)IFdjVGA0_5HN?cv9cze9-sWwZ+N53HH&1i5pbQpWQraR~>vDDFDZrsu1*hPo z?BP8CE}F=>;>S6Y5%f-|=)6#w|N2_B^xT;GPu$>CAoSLwW~%_&T~7T z9+f7SHYTwhQ4(O5gfvZ`cz;CWHgFrXOGl4gJ1}hLuHOzn2hjP$uWnl3e|GpY$O3nQ zkJ5|(5dZ!p3*7&t%6tX_tg~FmO9Me9m(DLT67NnPtW$-`r42;72#Rl zAkEX0q+1)t$HR#=MwDj6G+b_PiLD;qHQ$oJ?p!_Jn}Xs6fquJU)egQX&Sj)#R#Uh< zM>l@ToJ$3bX~z@FBt+TEb)#OR6s66vF>I5c(zoL8Z>6BJf(WK-9k2c8e4UdsRux<~ z1tb)LcyUj+)J2`oxnHWq&fSt68E`2mf@g&UF*`eem{^3|3aQ`qVk;v%nc-R=b!R@f z%4(=uy&^`NI|pBA;LZ;GvQjOzYq4(E0zs_dm`J>yYG)ExjF(h=8@56TDV&{GJ^1+=ICWmMyD3ny2`72FFTvp zO<~v0@}PHO-C%v*LG8^}OU>2&oWIRn+i1>iQ<>;ZA3LGsX3 z!>(&`*<=z8v63*N>=IYF=#`j+hQ0qrP2U6FStdi*AA4-8s4P}SP3X2JY%D$MGHWfn zxn<9LkC47jg%|HmU$paeT@wRNbFCd+veIUo*Wwnh^KD`(fOa;qPp@F6e6bG1d#4GbbMXtIDB75_Db8H6vqnGZYR&(1(?qgG8E?+(rZ%9;DvVQ)E z)*uBdCYi6mGag%4$`~X>sxM9?n7wW*PVe6chd0Hvz&tBg>RR)g93Atf5rG0l6-YQ% zYNp3N|GNuX2k)<-tQ+3vHBdJ0^^Prbx+KzyEH<2%Xr-ah{JqoLuL|aODUJD&Nc&P~ z3xdTLV^sHJ(QFSGt>#+V7N-a@Bzcb6Y$}ls%in#A8KQJ$tH$Y;FJ}fye#J;PZFs+`6bY2=pw2I7_v*BGeR#9L~VPwg{#Iv_$M4Rp29&lwN zeAm5Vw{bj3*)KmSbD?57?>0ng z5t|uHM+{>W>hjNWp7rvGi>uJvLS5fTXogx?bZKWvzis_x*hpLyhS-ge=1|D|sn2QF zKNcS4krmFWVbN1LK_;;>4kMwpTBAeD_>7$0B8`-MAvZo5_h3iwB$19J6x>Ac z!BC9QeIrswtT3F>!yHoqpV3ande56vpIQ{H5^v7zR-kQFyK<^d6?lt=dR}iBaFHL> z9$RZ3$S#<1g6@)>C=sDPhS?h4)G1>5ih-AN2n_#YvT~FBtH1K|xC~EBmB)F+-v)U0 zv6gsjr=Leyq4{>4lE_B&OmYEEqs-YfSfoFcrKc?xscN@cK){SuY!TH>O{%`TwzPN1 z*epqShHx^(j5zus^X7+9ok;?4UzxVXj|ju|syAW|10K#sb+)_+;qTs7g`<@9YLs#- z4W@PvNn}4Tb7iY`4x8c^AC!gLOp(9{;;KL2M9>}(=@!*TYnc4pQE959u(CzBe&?6I z?g0&Q>N8Fw%lfv`f!a=;=}S7}Dla!E?tPV==(*k2z&miu!m1-dB51{XfoQrrW>Tv}@@O1f z^QU!Ruh33IMJEUbRQg;X8m*KTZ-wyex|!9N1vRI1vA}`@hRpUeB1U$B zJFfM|NNZ{xOoL-tG4L&((fltzf|B1hOkjcvIov=wB!I>U)g)aVrlYxa`uIlEgFs*G z_CxoyzG^TCxibjgsnX%C>IDOlB#}qk#GvT5@EtqPP36bE?$#Hr7xTbC%lSbQoBlQ9 zkCCCoMGb#`w5A>!jm_=KF~_88jdu|i1J$$6S+9Z3SkehAXaZLMkTT&N+8|B$0>NLIc2?>FI_9e73Pbp zt%=LS8R@Bu4LQ4&7dm^^Q*jBNj5>B>I|S<%r!P*%CRf}bRd}YKNIj#?pCu@*y7V{7 z+8fxEG21DQ)s-W1)qi`(Soy~Y6h<{_`@1o3??nJlo`i$HU zWU*@~F6X9?Gckn?re27Ah+LegQD=Jlnj1Xp*vhJlDGZaDC8Nkfz~SjPcMZe>>t-qA4&Fy7+eDSP2e1M^+(u&jq_$PGHk;2z zdk{Zm6p=qYaeXj^L)pE$*=yr6e6uX2Jdehy$av(zd&Xn?>z2Uz7k>-Utp~U&b?5n| zYhVdZjXs?G^SAvGXL<%!{6i;xPpQt`d`~w&pdI?e&wnsGV*=iTrvMus1q4pgBa$kQ zw1h5n8GowOd$gc;pXQ*UPQy(0S4%Y!KxVcDjbUYAuRa zK|5;?D9U<1`jzuuR{P5~7J@n=cn|0;A%Mz{<3+EXR%)4G9&aBp(wuXltpq)LVrfO0 z)o{?du@$c(Ur0fBLgE&Nkf_vI-g~ScB@QfLB3^);p>ty)~h+B?XKNvhT{ut?K?WNr^s`#0{dO zNm;mO6ZzBk$OS7R@}#StmLL>6YZ;Z18|zE$L*BN$%4zU{Dpt9)1=og zqu4arFg3nhq&5RMt6S{TG9Fx0Vu|pxlbn_HHg>%!Rxpys^+~L!B-_XzZmNo2$)z-F zPqeE@l!d^Xi8$!f0e2U-k=a7mPkA>!{TwzND$0~XWM(VJ>wXx&{aqXYmi?>83GcSi z*t^9ul}%~5CpGrMmn5a?TsvF5c!+aib3vN4eyUwhQgy>bGHfoMy3HoKD$(%#FQt%$ zjfu-!V_-Du_^F*YL7b_sK~<*t^DfnW9UKgN&sI7AtAp5fpikZIhEsLTw>`j0(fdm3 zkuT#=hj9*Y!T8A&A~e*qFtoD?E^;vV3u zB?OP$Nd2^4)Dk5X$bTJ+$ZE<9K6kv7Smj#fOw-R$spovfZ}1IWW?^a5pFxYO(@5Gd zUkMbN2|~aUx7ifU@cAVFi&Fv2S+ktdM=pMC^V1%g_+xy0NlQJ@8pRN=El(@$>pQDH zGPE-kqEj{CVQ!B2rB?iS;xC*s!b zdzmLmn^(&pPDQVNG<2W23nu&Pyp)9#J7) z&~n=C`lE`^V-O8km);6!w>y7!0>0HF4Q**_74ExdIJfva-Oq~84fL08bKxi2%52si z)lgc~jb=X*UZj1@5X|sSO_7>@`potjaP;Przd*+md_|AmJPd~Y!FLv14gdt`%u$=8 zgkV0HsY0>Js3U;#qHjaP^1E-45c{1wNAoy)Fp>;Y#e zitpM{F*tJy^GFm_jw_U`&F#^-pmCpOkWT%vjWCrS!*@KDURN5|&sk^sqH6bj*-UA> zV4-ApHCcC*k-R15%=Eo?LhZAJ-4h#HzjS?5gq}kVPP2(=u&~GXp#~3YV4dwY=VRru z7SdoRqx-~@wg`3_8BKYf(^OkWZcN7~>$$AVj_FsWcpSepm||zHu*Gtl6lFyoYxvrj z$<`a<(p1IE6#g=V>qfjmv6R2L5XKs>%=9GW@(uqS%R2T?sT%NOH}Xv(rl!vyc{|X$ zakpI&3n^06^ef^M7GsDqpDRADrkq?HTB?dh-C+vdz33r?$7&|wi>-yt6WswD)u-0K z!l0+@^N)7+8q+3AiYq-aC}_h^5xcg(F5i`=;KrTG{vNWp>-Z)#Dff{H8H#{<+^Gxmv~Id6IUnue_7c&|!-4ivIx(0=WM?5aoQC`OrPX_9D!TvbVWE=b**qW_b$&e#T{PCH1g>O@KeujJ;VH8+<{ zgV>*VXH(fZtsNTGPMyXn5oQ-6q0F`$;o_-~Cg8lK%;PTcg?jX|y`C!Sr3^<2vN!If zfYxB8Mh_8-L_?&b>Ip3hyDLXyX(UfIvqzK)?8q<8E|-P1N9e&k6ImsZSsuSB!fk); zcf-S^2#JI!^gUJu+wL-FIo*rSbtu8>s*>$Dmq%uQrB{}8wO3_4M)(faWrvwPc;~uM zXdU)5gkQ_iha*w*L$w!IUKC7iH@rs#|7AhoB=eXp5!JGK>i7Hg&3cGSqVxU8#`7AFKOKL7LLr`6&Tdx1wa z_WYhChzq1;OxjDtoLpk6s@-trft^56D30xI?DKg_@O@`)L>E~UDk4yAa?O?2E+OHO z(9=6#__UL36kGP<0n5Z==a}`?1|KBWroK470YVjjuCvGnS0h*Ch063HOI}?JBoj@1 zJj&oNg~e=ApALmZE^7IzvtLRaTr0#lU^^XVy42d|TKvYpXCoR;uCG+0!f=q`aw3$@UDdjLgylhYDZ)$BB?I!95VM9yv-88M@H9yNQ`uEyrD>28 zT`6C}6>Q1U14Ij3OA6-=Rn|yUvA?*XdWNWm!GLsR$Hy3)$K*=D>D<8w4+BwVE&XYNDoz&7cN(@VrRT3E!mGTe zcXovo-35BdJ%Z7ShWK@aQLU=ePxuJJXTt$AeE;lF+GXJRD@k>*my~Eax)J#gg~m9 zMe2 za0( z5{V_|9e$NRzc$Pbi1BCi8eh+Ktq>3-Zd|Clmy(-<7e?9&&9JrM1k4^Za@VKwzKy$|;<3O^RJ#PquKt}?P$IuG})7Q|= z;$_b*eM)5FwUQ=GtOlVvi=4;UCwnz1Uf=tji$<(svf26i*rFkwUS~rzq&K?)(q@?L z(7!oQg0HnpEA42gl8lUt&uK5gjFgmXutgIWn$**~eq!UZ^MT~YvaRi<RfSVXK_reu6=)QxvB)*>9U zu}x3`b!V+m1*05am+kjStNp3`5mMs>hM#`YV8Rn()SVr)X>{^PQZWc;vbU#)oS}5Jb>I+i9kZsp`e0hXpMQT6 zI><6Ok^JL#aO%l_rSP4+^Neo!=~pA8MDy0;x)1A7ZMAVL?gJP`{$8JLxggKgQw{%`DIkmZ1x89H*cxfYDXxle~+v)5T5Vx+_h+T&=EjXuGcZi98(e zq*^W36Hus#qn?TWbv`y3{GkaUM%GQa*c zM1fa|{o~wNs+O!+uab=eE(RB;iPDdg3OMubjL!st)RZu((Xf>#Gi(yN2Ox3OWQ9Q2 zSm*Xw=YcOaF`8qDg*x%nA6sO%Ti7~mA^&L%g!pudDJ}q|3$rh3S1{Dkc}%vbVk}KN zPE$^iVXx|()%Qn8G5K@%Qo=&3Ps#sm1k#P@N z6(Yu^N)~%Nigl%h%0Nd~&aELL`iz0AMXI8Mo}gBg&-Ia*PNtNudpqX3&`F zH*95bZ#G)&nr912)|HuS;u_fJ%p4u~CkIhY@me|DDJh>~h{&E4;|s$rLs72tnMxXa zz^0v2|GRG4CFi-M$TObz!dah^q$kWmNl9n=dn;GG>|M8dZB=ChH@uMfu)pV41xFwQ zA1W$aPb`!ms-@X$v=X8&-4oYyoZ?rF@VRs^&$olqZ8&lYmkz(%_+0*hpS@ixWwLF{ zw%?W4E=s%mGK5xk$1IVez#=fS!PHdy$%HkKs#0E;rm3yL)~R9CIFn=;Ys#d;4Wraa_eZ<$}1 zHeVc3Ea-o;LVe?{{(d|njqf6A!<9%1z_*;`jR%1wOjz7gK{dwqUCo@zlZ;K>u^0N3?mU(EjW z)$kl^_-F|4!fA7v!)!z5(zuCZ^kNh-@oIc5CarYB|Z-Jtwf4#Jj=M<&;#F5=nhqIKNPqepx`3;ppWO z1Ho^j4=pO+_SR!Rp#^gVx=cg>iHaz9RJVmcx3fpQ0%HX4bDC z6KM#yiEZb{U6ru2KS%0ii(0OB4MN!xlZ+Tm>q&m)UBw->Sd1-bJIl`FY$^9*rYT=$ zHezH~@wkL*X^M{ng8cJcVO-Ji_2;VIvg=la|j(bbzy zLnDJ&d?oZC8F?wksA8nKqGiut!xWsSN5cbnctQ_(akeckwt1koCIGKEjp(WWv=vE; z63S;)h;;58wNdBseu_0i=3iJrO8ICyzFU&G9`Y;nk`B|mC5zc2zbfe=))<9KU){kw z=?Rdy7(uxqs!lv1eOlIG`{LGL>8XBS{FRf&3KyWF4chW>vV5z8lhb#SnE02eS092` zYcWMG1xr;I>S`L@nl?|k#pPj4)oXatjmjg}ki&Kpo+ys(qUHs?1<3vQg+aZ%$OTv1 zp6*CAMi$*$o{=*hnR}L?BDLs9#$Ln(GnvHH||4 zmyz+OYE8P9V<27^GD4ZrEQrQp>d-QnhO)*Gs-y?wNKqH^Lv~D~ehjCk&2IBb67mLj zr$+i_OB{n<`Y;jWXw00e{85zp9h-56aaczp@ek8-5k&g9VFNJ1HoRuH5 zoLEwxl9ntOy)zmnuf;kX2AyEJX+F8T8M*8)-JR2!0(;7~M9X9$O^y5{SIBi-eqhCB z+yV9Vn54;-Jn+GtuKF>^&+MrkBRzt3MEgEzFzqY{GfaEq^Bv%7o=abgeBe)9XlD2_ zSx+BJ6ql$+6sN=`s;I~g`N=x$vXN%LhN`1q32luW@|N#(Ksfb!^XkMbs&i_psbaos z1p02w4#$UV{#%h;o!~O%ZzHYKA7RWN9UT#tft6vj7lu$oj6V+Rqf3$@lW+V<(n^b~ z1KwF70_zHDl$WuJGJ3WCf58Lg8w93!EDdwbkWTq8I}uzt_R2k)`OUT2{F;OoTclExQ~Y;CSGa7~r^7)}50Ai3^$fVpxLVmcCs`*LcWTF$%(B`_KZwGQx*QM1!AE(^7}GY*c|RqjW3+s zK1-3+I66=p$1M{Ek^Q0vhF_-FU0kQTut;;EL^w3!De{?pkL zLoE`?i(NY30fw)@Z2)-9b(HQn1HkRq$G|)#I%&SVe&6`ZK6?mwvR{Imlh1A+uqOdN z$^BK`&(BZNeP__Ue)#$I?*P3q(?1;e@yI`256VOFmn>Btk!~CA3Z)FZtp=x4-K1~b z3bIUJ?A8{{)jq}~=qI*73@aV(&+XXdg+4-hYO@=%Ioa;;Xg+e);K4(Th})(Cai9h2 zbK2%%PtQ~FvWrj~fQK47^shG3dK66SBjiPzr-OfZbHLCU9Xs(5}kW z#S8XvR$-9cd#hc^TY2l{%kVkmf-+J*KA^Tbep<7MXK_x-@GaJRt4TC#5RcS}$|?G> z?fPpg7{98Py3{{l6uxuW7Qc3uyijd{Vg;+=f#I4(pG2)cX;B zcTwrvFx?-+>u#;fW%IkqiDi0TG0?d=4BUQZ7_kR**|)akHp@EVd3?>`Y;h~rFkbv) z^LRAXs54Q|)Q}@0zY$z<|8F{c`0oQzf}a50rjOYzFa-J@Y;o-hf!#`X@by|Ni)vz7zsZk(mDaaB!8ApO1pURXVBO-Y@>MVR+yz zEKxM9pK+Nf#aUH^E||*?BFGR?w5Jw#MSjf=@HhF|2rm0acG2cAqqvSA|8TKe1vF zJHoxb`kb)ejbAXIF>ZBbI=4bpy*EEy?Dc=1oz9)myLC{(_!^vG?njM}f~gGP^byQ` zc($MVp!dp#G5Q9u&mXyd|DZWN-(jClFrRz~+_(*XI|cBc)O-upfd|gtIrLld?+Q+s^<)Gau@`;rKi-O~C;po9$G`jaW9)nl72Okv46WE zhhnOF^7Xr?9F(yESKs^(RhK+S6QN7~*fUUSM*%?XWFtNQ5nR3dt?DNkj_^tTf1>`+ zZa>_Y9~k=Adv@po0K5kd-8q_ld%x%Ji9K*gAJ~6M1g_KT|D~&9zufybHQ#<5j?seG zvIpeXm24WK57c}$;6Tj@!SdA1d>u?xvjMVbGFZMzBZ=f zFqWb6f2v~~mWS~bY%%toUciyd4)EH}hc|_tQdAh;`C0T8U!ze@828jM4lKGmY;|VN z=C}SoxP9dI@9Qx*XY7OdH``6vI^pMo>wL4 z@s!Ac&Qfc60?^R<3SlEUG$(*v&|&&>zwxDg5ge&}I~zi}^WTZcH@BOVmEIRl=WFHN z9L=i~HzHzZ?3e$i*#CbB9rzbq_VIjQ)F<@x--C~OlRi!d(8GKie1$-vf25n;9_UF#S~3%|!$FTtySiCUVD#xYVLB*R&J^6N~)@< zGRvnqo~qtBNPj*`CwxqXD@xEcQOL~Ox|%soY{3#;fmr2%tbpG;nydt!v+|tr-%9Ky zy7idOns?D=X&R9ApPofKwWn_lw1du(!GUvxogz|f9LoEb)qP4AgKo_P3{*rO-SY69 zE)>l==0VM3Gc1Ym=2g>?D1hsym#Y4o=NFH3zYI zcy@=C`KR^N$fC*KyzY0?Nw0g9+ZTU^R)=ah;8x!FT#*==8mx1GPnY0X14~*U)usJp z#CBS%0>w{+{9Jj6-M2E#zQ#E=($vUAnW$vmJ=vMTBknnsS=Sh>>C)fjZ{jf7gK|LP zAm#xyl)(an9Xcu0lOz>uGKbVtgEyI9?VGUrY=cA(qYWxOXNEI+vCd=N>Cm5T{C|6p z+`b;#PecImnr}-H@BgE>`X|D`cLAhBM)n1NGOm7$llq_$a# zx_m4u5lNy_e@2%=qE)^5UKCM&HpJX3wJzHF+2!FmKfKEEn8oj|FEmQaAT!Mw7U!Ms zu%I?Po3;ENdz0cQQ;0^ds>AI~jrx)d&7+_{#;CsTKl`g~T9d|wlhcfsxZZhQZtj1F zOuOik#{1UA>aG+wF5S0(G%&2Kw}6{+!5`Y5m?oU5ChCQ9cF#5Kb)uEM=IRs-P$n;#_ zCL8$G@;0I>c<>LWDJMiX$K!NE$2U6FKeG3LjW~jhMM}sCDTNL@0U58 zz7wCv;s$yrxjOgslg z@S<(i_^}~-=gDZ&9zapqxg^gQ8?t^h=&kjxtGfi|Z^{>fQea3{fYQ4>WlYkV-FU~` zm7r&j7vH3{b1a0cA5*#6v^@ey*YgGS7H!qjaIwCX_w(T=Nd3AL`TDX==bb8`3%6wBE)L-7t5^E7f zzT?@8(p!4Q*^K8PzP|U92#NC`Q08md8gK(HvGWvFlwb z(zEaV9}`jV!jmSZp6R5Z_Wb_C0}lAPV@TwTd0xPlcooi{ncKOk_o@I#;Y{-`_B6^!K^S2iQzN8?BLdgA z%UF+}a_};1h8R1bg&!CHAQU!vwp>^h>F0vxV_@n<4~p?4cxNqnA_h{Tt8Q5NSOWUC zH6tS|r?5vs`?O(LOTDpCQT=@aJ-RW&8D&LxnEs(7GF^K>ebq+`i2%`U_%y@0*tDng zxx;UA)O-6q!zMnbt6%W$>G+)N6}@JBs#yJwGh{I8DUR`Pm!aOm=D3VP!(din1EW`k z`F3wyt4(`x>BO~gX!~jrHkX7=>M7Yal@j%9@`J8=Vv{_Jd@E#z`}HG^4mo*MpBLuG zP|i2kd~5Mxyi}5_Abq=#_xq3cZbo&rkvSk4s+Vj z>Bb`^Ar?aA6f+Auuo;!+kW;Kp%%%)E&74Ql{e5qAf4U#P$M5~e^}bGTd%s_=*Y&(! z*Xz1oFLDmj{K~fSEzrv9QyyTOytHA}Ux2C(@FP#GI*NsGIX>K`90Dx$oKoSZX9(*Be9J zT$!Ojpk?x^D=w4!>GfY|KhYU^DW&!&DihxLT%MSGU^t59&fqNlFFpFXv2e~-b1tl2 zWtNG+npXUms@=RD%zh>bXAEs&KxR?Z+%CC8X4inv1YLh9O@X;=y}3DPyHrP$<+cHe zGw9a(KUp_rtygmPK~~=Qhn5G_YXFuj*|jZt+f&e4sd^{>^v^`83;}}jV)L`*){EPs ze=(P3|cLGbxr@%`Bmt3j!<*g~{P=bvI>sJG4 zQ9;*{#`zz_d)~7x@0bR}qliqkeqZZRR1}$PogKR(zr5R$Dm+E(g@Diyh6L0cb(zwk zfc#Rjh(K77y$7T}0NN`{fnn*#JwRID=6U`PFy?HIN9|bw z48IM6DR?I{ljy}lKt{%T@45bm#`9G3^UD;GBBk_N?Xh?7vSKF%Pbo*&Nx?v-&aGPQ zE<@S>D0b3^4zH7@00Yd;Wy+mYO7}}Eo`TljTEG0K71o;risDUaEl|0vp)ZRcG!^ii zEt>HmZ8KlP8ooe>dKbwsm*buOV{pT}e*GpwhU>{`4vzVqU$z$o@G3nS3;kApl ze*(^iShr|{R{-|%&yBBf0-J6Lg@iB=Axxd3xGZtFE<(`CE$4E}IbGG<%Qrn3ieDz=*l2a8p*h_fOy`zk)QIx6a{@70PWcmNjm8zOYgO+7>PAErr;)N1$~w zTdKvr^U|3NczC_}im3wx(!33X?jmm!u1%w=*uxOyxQIf4SGNa^ZkiRehfkV}6Es~*GSF3!X~9?PW6;C`8(J1U>s z_4OiEX>r-Uq237uO`*w**6~l1vr$nX#2HhK3)WnSFHt**e6-6qDKcq!2J`D2n!}&` z5AnfQ77wQ2Ox)tX@~Ja8e}N=vjyWbm7$P7{Cp2XR0hVVlYA*tL^P%)Re*#LmTRQ?+Qrxtd57m;msx?JIHv}K~5 zxF%}bm*072d-+oQ~!W~15MgA4;o6yW1aL1=$>j_-*?eQ8=PC#2WTCa;9K7@ zvu6RVgDOby%S~m95q#h<9%GqjsQv&e9K5BQi(;4&X~IH+HpaA7#pR~ClGTgb$Zd=M zaOtM}UI4@8fYGw~+2Q3Lp!NHK$x&{l0Nhp`QUCpbg97?Gmzu80D{Vac{g8Y1xN05e zF`<%Us}=2}q19OOz^D>KshR4OJb0iroYQ$Gs!gEZ5bxUyc|oT;)pdu}hjxNTsO$-K zQF#Q*^$V&s^H}!U9y5im$Tq%d+l~shNP{#|g+;Im^6E+BYP^MW)7^sSsa ztBUXz=x?0Sv60jD8(rs(KZ&EuP1&)Gah%|ZYG&`A%r9#)PoA7%=f}jmF!B=jx0k%U zI1@^XCQm%bq}M=l@pm(`>u2Vi>rY!^7u6R=(0y%I#OV=EGZCRjfu$r3M_9M~ZiiNM zbwZFXJ2+KfFFN#|WWhW{6PKUcM&{Yh$e#b2{++ zwQ|lqNZFd)S3iCQ^a1qYrjE2^y-!)7a4yT=%u$Hc(#fKE{?VL7l039R;%zx16<_Qz4e(l#1dpTh_T-+cNt%hX%M? zJtBW%*+#G8=m!FWyOS_ll8ozF>4Ct`CfyG~s-VK(AgjRMb6-92!uQD2IZ z=y7pQVnD#5uz4~x3VrU9Y%>^>FAF*jZ%Xz z_(O*+OtjQ6b*(**o`65)mU~|QUCr@hWdmo3yCoP z2&IGb*>==nmi;WLiqIU=Dp8;eROS749JNWlFV}F^y_)(t&t8{PDr{WHa3o{DLgzN!Wo+-m;7>2XYV0rz!lo0a+rQ9M2?5>5O?yvoM_Pq}IR{Ig zfFn~0wp>nBH#PqLM)06*+ZK<{B1qb1Go4);7J2iLej@0*4j{5y|VQMNfm(gdj zkEy4QAFHfaG}=(%+3S}2kbP)}yq~V!)j`P<29BKAZA^MUj2IopoDOqnsuO&=ql-@1 zsxmbWXO{(xU$hm6<~WcYdIEYC)bWns+S*rS@S(R=rTGEE#S;mn5Xaf_UUl55?#aq; z^0VvxUMms7q9sQeAoC);z2QAsH4&)kUU_1Ih%T)!$0Nr*xNsGW10;8UCvd;4s005`=K%YpTuFYdP^sDcBqx-hH?t_{CPzFaqls_}1Q z;i5KPN%rS<+nA`1J<6Fm$%)XiKpgCj3ON#SfBcWZ`$%Y|XFiu>M+zb>*EFD4E2&*m+uZhI`Qf(H1P=+?~5cE)JFqaq_tGAE%kZmcVe(7m({SyU4Zr8QQLpYV2Jqj z(2fEfCIc!D2s^gW)#zjr^%V4)cv zNgJvdeXt-Y{d{z8Y^nBm#ns1b_kIBoSu3t8&=3|z*QKyKP(#f8$R>O!|M}JZt-P{Q z(!-fs&?m4(j?T^=yICg1$DcD95$n^i>8qdQj+Mrm5WlvJ*gFWmMTrf?DXz>Q1$_|V zTS9PrKNgNt@gFeBf)pI}w1`faXfCg(UU*La*dpxK&konZRrCcKaV`#JID0UP`47_V zClLkhCK2DeMy3}N4hNaB?~22Dc76s%!49c2cI1#T@kwp3JvE)@uLa)F8-I*2+~C#H zP)}3LCCB^hCn4eipgFj7_WPOls6~U>Cd^=uNHTLW&B-rHUswHXpkqK|Lv_Wt0_4a5 z_N#9<_Hbhba=0A@O#69!*ip3!(#n3sC7jqG=yeP&dW7N9o#NHU zs*ktK_01&l%S*7HUL~2)!`YL#N?e#xizL;bS}gFXzpdT2KLb38SYzS5ap#p1dez`O zdU&*UN3e?%?HK`nesJ`h-aYpo)qT#F%tRRH^|pxN(a6J`>`+#IHM=9|Kw|fIZ@M5= zSluzgh9`-=XG!oH|0W&h)ae@@cDR$m`d1GKRY2@J6qCUT{%p-fb%j;4jYG#jXPKND z&2l<vosQOw?y3+Mge|1JEYNbA`Ppw_ zi(9|LV&Iix(+*+wK`U?sN*8McJ;_VzE@3_EMP-ezmo! z{E9*q;(RwuBt`^}6`dHzMS0py4q6^K#%QZ>bse~V&Z*r=T`eq+@sX)lOt!72r+&qk z7-9H9pYKOWN5*N6$rq)BgSXz9v{d%t2Dh`*5?A$oPI^?5^W3NRY4|ojNFs+_Vt@+uLy0O-A>N) zk|q7)`{IY~Hs}~bW|OMfKqQD=CT}2-gU91|Gl7+RDNLR{mOv3wI4C}iVAqK8;8pYI zTyKQZ`ywGV(N&WY{8)Dse9!{!UQa$%?kVbg;}j8asfO*^*ihx`r5e`zTOr)&DcIb16ibk_z z?>#dR^Y4FmNzlq-Xb*o;W@?N02|*2hRg{{sfDTxbU*p>cHbxFsN-)(94&XWq(;DuK zX*vDy^@QJz%G!+asYUWMZ@UKB4xd$MxV3$>F4^7^k3RFkZ+=1L+@WzGJ)oi*ST-@g zzoF+#%e!~!#?*nj_(`y)BYet9uQ=PXIm(&0DX^}YcLHr~5u0|WroOJjt|BD3@p-36 z_<0CYJ;4iiJ2^3&oqNHj7rA6QkJ&mkCs`9>x%%<0ZSU%CcR@Bo!0??K7C4(5HqjTs z1ts~=se}_e^!%R;%OF-j=m=aCe9sogvD1;YhaD$en~>qtniE%!1+DQDIXX94#f)BvxNw#4M!`xj zcSlBw#=6YZhjp!vG>;0^Z=5Nh`I(6``%zsVE9TP_|2W`3c&Wp+Vdhg-3QZwX84c5J zFUaY9mBt7O;o*@Nn!}GxX}o6r)*;ra2m6i`i8%S`Fs8lZyPXgHDznk86I zAd2XfY$U~PK=mI`Kx2!OUt0zINW|T}Wi=rprJ~*;w0dcMIHD^`T>n~mBJjH;IVJJ# zMm}EM@Ymimkfe4qK5pZbEh%UaZa45^&f28lX8FFB`Z(Wn2X-BA zCRz38^%vN%%iwg?{3s#Rb2@mkY!vy#&+!#2TYcAdT2owmhnJP+l9qNDtZ0^VER>Uv(OXmNOHcYVSe5pg~ny81---TZcUTC zO=pT6gOfP$GEVOB+u2XvysXI9sB3L8zeZ*lSd@P1^AedY>fxInBouEMfOK@et9Ld9 zHj=*OF(fu#c4-n@oXK6$Y;7B=C$V|}UBR$(P`CW;3rv(-6--Z1HPto6)F_NPbVL}4 zyj&_!l?=a#id~x<_nZyAGM>Kld*7_7G9#Golbazj{rhA^hS%{ar@nVQN<(&ncXMk# zln17ODF^JY4kvnb;75y$e#@_sn{M6lhAatAsO+%%KsOtwGAL-23RFjv2gB;*_>CgR zY9Q#D_XFs!0W-xFCh_!3s&%m54aRF8qS8)1p7Tv14&J-PJ!KlHZ%aE@i@u&io1z^X zuMo^eQwI89&01Su^yha{F2`{~*5K(c|-NlGR8&xRM0 zeRG}msQSdGl-V2f6~a=HU;Bj(Vmmcl47I#U!{7z~t6rOF_MK#^XIpGCl{m;>eyuIdr|W<0w7dy2cF0m5Er_Y+8|yEa@2*FePybqqqIOUD zhlrZCr)|>F^sLwKxzq^XK=0P*!$K)AZBoOvUc;XF){1$!I5zt3-~E7RN@Dj(M0k4O z`NhTqOcH84vA4Wb`Qx6dY^58+$NGV&>+bWN;+-`Mk%N4-7=9L!x6?puO|Z)Cwx_6f+)$oe%sp&(Z36Lnj<&f4a z(RVnO(5qXFUA)Ts0IR=m2f{H&1m7ftC5%5TzIBa&0R zwvh^EBV9)3f#Mu!t;gD<1FpzGATrAQUpO8a68SMESH film + + + + + + + + + diff --git a/film/src/lib.js b/film/src/lib.js new file mode 100644 index 0000000..867232f --- /dev/null +++ b/film/src/lib.js @@ -0,0 +1,238 @@ +/* Shared drawing utilities for the film. Everything here is a pure function of + time so any frame can be rendered in isolation and in any order. */ + +const W = 1920, H = 1080; + +const PALETTE = { + ground: '#0A0C07', + soil: '#101408', + bone: '#EDEBE0', + dim: '#9BA184', + muted: '#6F765C', + myc: '#7FBFA6', + signal: '#E8B04B', + danger: '#E0637E', + latency: '#E5905A', + errors: '#E0637E', + saturation: '#D9B23F', + traces: '#56BFA9', + deploys: '#8B9CE4', +}; + +const CONCERNS = [ + { id: 'latency', label: 'response times', color: PALETTE.latency }, + { id: 'errors', label: 'errors', color: PALETTE.errors }, + { id: 'saturation', label: 'capacity', color: PALETTE.saturation }, + { id: 'traces', label: 'retries', color: PALETTE.traces }, + { id: 'deploys', label: 'releases', color: PALETTE.deploys }, +]; + +const clamp = (v, a = 0, b = 1) => Math.min(b, Math.max(a, v)); +const lerp = (a, b, t) => a + (b - a) * t; +const easeOut = t => 1 - Math.pow(1 - clamp(t), 3); +const easeIn = t => Math.pow(clamp(t), 3); +const easeInOut = t => (t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2); + +/** Progress of a cue that starts at `at` seconds and lasts `dur`. */ +const cue = (t, at, dur = 1) => clamp((t - at) / dur); + +/** A seeded generator, so every render of a frame produces the same picture. */ +function rng(seed) { + let s = seed >>> 0 || 1; + return () => { + s ^= s << 13; s >>>= 0; + s ^= s >> 17; + s ^= s << 5; s >>>= 0; + return s / 4294967296; + }; +} + +function hexA(hex, alpha) { + const n = parseInt(hex.slice(1), 16); + return `rgba(${(n >> 16) & 255},${(n >> 8) & 255},${n & 255},${clamp(alpha)})`; +} + +function text(ctx, str, x, y, o = {}) { + const { + size = 32, weight = 400, family = 'Archivo', color = PALETTE.bone, + align = 'center', baseline = 'alphabetic', alpha = 1, tracking = 0, upper = false, + } = o; + if (alpha <= 0.002) return; + ctx.save(); + ctx.globalAlpha = clamp(alpha); + ctx.fillStyle = color; + ctx.font = `${weight} ${size}px ${family}, sans-serif`; + ctx.textBaseline = baseline; + const s = upper ? str.toUpperCase() : str; + + if (!tracking) { + ctx.textAlign = align; + ctx.fillText(s, x, y); + } else { + // Manual tracking: canvas has no letter-spacing everywhere yet. + const chars = [...s]; + const widths = chars.map(c => ctx.measureText(c).width); + const total = widths.reduce((a, b) => a + b, 0) + tracking * (chars.length - 1); + let cx = align === 'center' ? x - total / 2 : align === 'right' ? x - total : x; + ctx.textAlign = 'left'; + chars.forEach((c, i) => { ctx.fillText(c, cx, y); cx += widths[i] + tracking; }); + } + ctx.restore(); +} + +/** Word-wrapped paragraph; returns the y after the last line. */ +function paragraph(ctx, str, x, y, maxWidth, o = {}) { + const { size = 28, lineHeight = 1.5, align = 'center' } = o; + ctx.save(); + ctx.font = `${o.weight || 400} ${size}px ${o.family || 'Archivo'}, sans-serif`; + const words = str.split(' '); + const lines = []; + let line = ''; + for (const w of words) { + const test = line ? line + ' ' + w : w; + if (ctx.measureText(test).width > maxWidth && line) { lines.push(line); line = w; } + else line = test; + } + if (line) lines.push(line); + ctx.restore(); + lines.forEach((l, i) => text(ctx, l, x, y + i * size * lineHeight, { ...o, size, align })); + return y + lines.length * size * lineHeight; +} + +/** Reveal a string character by character. */ +function typeOn(str, p) { + const n = Math.floor(clamp(p) * str.length + 0.0001); + return str.slice(0, n); +} + +function roundRect(ctx, x, y, w, h, r) { + ctx.beginPath(); + ctx.moveTo(x + r, y); + ctx.arcTo(x + w, y, x + w, y + h, r); + ctx.arcTo(x + w, y + h, x, y + h, r); + ctx.arcTo(x, y + h, x, y, r); + ctx.arcTo(x, y, x + w, y, r); + ctx.closePath(); +} + +/** Soft vignette so the frame reads as cinema rather than a web page. */ +function vignette(ctx, strength = 0.55) { + const g = ctx.createRadialGradient(W / 2, H / 2, H * 0.25, W / 2, H / 2, H * 0.95); + g.addColorStop(0, 'rgba(0,0,0,0)'); + g.addColorStop(1, `rgba(0,0,0,${strength})`); + ctx.fillStyle = g; + ctx.fillRect(0, 0, W, H); +} + +/** Very light film grain, seeded per frame index. */ +function grain(ctx, frame, amount = 0.022) { + const r = rng(frame * 2654435761); + ctx.save(); + ctx.globalAlpha = amount; + for (let i = 0; i < 900; i++) { + ctx.fillStyle = r() > 0.5 ? '#ffffff' : '#000000'; + ctx.fillRect(r() * W, r() * H, 2, 2); + } + ctx.restore(); +} + +function fadeToBlack(ctx, a) { + if (a <= 0) return; + ctx.fillStyle = `rgba(0,0,0,${clamp(a)})`; + ctx.fillRect(0, 0, W, H); +} + +/** An eyebrow + rule used as the standard scene caption. */ +function caption(ctx, label, alpha, y = 120) { + if (alpha <= 0.002) return; + text(ctx, label, W / 2, y, { + size: 19, family: 'IBM Plex Mono', color: PALETTE.myc, + tracking: 7, upper: true, alpha, + }); + ctx.save(); + ctx.globalAlpha = clamp(alpha) * 0.4; + ctx.strokeStyle = PALETTE.myc; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(W / 2 - 130, y + 22); + ctx.lineTo(W / 2 + 130, y + 22); + ctx.stroke(); + ctx.restore(); +} + +/** Draw a node in the house style: soft ring, solid core. */ +function node(ctx, x, y, r, color, o = {}) { + const { alpha = 1, ringAlpha = 0.4, glow = 0, label = null, labelAlpha = 1, sub = null } = o; + if (alpha <= 0.002) return; + ctx.save(); + ctx.globalAlpha = clamp(alpha); + + if (glow > 0) { + const g = ctx.createRadialGradient(x, y, 0, x, y, r * 6); + g.addColorStop(0, hexA(color, 0.34 * glow)); + g.addColorStop(1, hexA(color, 0)); + ctx.fillStyle = g; + ctx.fillRect(x - r * 6, y - r * 6, r * 12, r * 12); + } + + ctx.strokeStyle = hexA(color, ringAlpha); + ctx.lineWidth = 1.6; + ctx.beginPath(); + ctx.arc(x, y, r * 2.6, 0, Math.PI * 2); + ctx.stroke(); + + ctx.fillStyle = color; + ctx.beginPath(); + ctx.arc(x, y, r, 0, Math.PI * 2); + ctx.fill(); + + if (label) { + text(ctx, label, x, y + r * 2.6 + 34, { + size: 21, family: 'IBM Plex Mono', color: PALETTE.bone, alpha: labelAlpha, + }); + } + if (sub) { + text(ctx, sub, x, y + r * 2.6 + 58, { + size: 16, family: 'IBM Plex Mono', color: PALETTE.muted, alpha: labelAlpha, + }); + } + ctx.restore(); +} + +function link(ctx, a, b, o = {}) { + const { alpha = 0.3, color = PALETTE.muted, width = 1.4, dash = null } = o; + if (alpha <= 0.002) return; + ctx.save(); + ctx.globalAlpha = clamp(alpha); + ctx.strokeStyle = color; + ctx.lineWidth = width; + if (dash) ctx.setLineDash(dash); + ctx.beginPath(); + ctx.moveTo(a.x, a.y); + ctx.lineTo(b.x, b.y); + ctx.stroke(); + ctx.restore(); +} + +function packet(ctx, a, b, p, color, radius = 7) { + const x = lerp(a.x, b.x, p), y = lerp(a.y, b.y, p); + ctx.save(); + const g = ctx.createRadialGradient(x, y, 0, x, y, radius * 4); + g.addColorStop(0, hexA(color, 0.5)); + g.addColorStop(1, hexA(color, 0)); + ctx.fillStyle = g; + ctx.fillRect(x - radius * 4, y - radius * 4, radius * 8, radius * 8); + ctx.fillStyle = color; + ctx.beginPath(); + ctx.arc(x, y, radius, 0, Math.PI * 2); + ctx.fill(); + ctx.restore(); +} + +/** Points of a regular polygon, used for every mesh layout in the film. */ +function ring(cx, cy, r, n, rot = -Math.PI / 2) { + return Array.from({ length: n }, (_, i) => { + const a = rot + (i * Math.PI * 2) / n; + return { x: cx + r * Math.cos(a), y: cy + r * Math.sin(a) }; + }); +} diff --git a/film/src/measure.js b/film/src/measure.js new file mode 100644 index 0000000..4388499 --- /dev/null +++ b/film/src/measure.js @@ -0,0 +1,34 @@ +const { chromium } = require('playwright'); +const path = require('path'); + +(async () => { + const browser = await chromium.launch(); + const page = await browser.newPage({ viewport: { width: 1920, height: 1080 } }); + await page.goto('file://' + path.resolve('..', 'five-concerns.html')); + await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')); + await page.waitForFunction(() => document.querySelectorAll('.claim').length > 0); + await page.evaluate(() => document.fonts.ready); + + const geo = await page.evaluate(() => { + const r = el => { const b = el.getBoundingClientRect(); + return { x: Math.round(b.x + scrollX), y: Math.round(b.y + scrollY), + w: Math.round(b.width), h: Math.round(b.height) }; }; + const panels = [...document.querySelectorAll('.panel')]; + return { + docHeight: document.documentElement.scrollHeight, + wrap: r(document.querySelector('.wrap')), + header: r(document.querySelector('header')), + verdicts: r(document.querySelector('.verdicts')), + transport: r(panels[0]), + graph: r(document.querySelector('#graph')), + meshPanel: r(panels[1]), + claimsPanel: r(panels[2]), + claims: [...document.querySelectorAll('.claim')].map(c => ({ + subject: c.querySelector('.claim-subject').textContent, ...r(c) })), + journal: r(panels[3]), + duration: (() => { const s = document.getElementById('scrub'); return s.max; })(), + }; + }); + console.log(JSON.stringify(geo, null, 1)); + await browser.close(); +})(); diff --git a/film/src/mixaudio.sh b/film/src/mixaudio.sh new file mode 100755 index 0000000..730d57c --- /dev/null +++ b/film/src/mixaudio.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Place each narration segment at its exact offset on the timeline, then sit +# the ambient bed underneath. Offsets come from the same timeline the picture +# is cut to, so speech and image cannot drift apart. +set -euo pipefail +cd "$(dirname "$0")" + +mapfile -t LINES < <(python3 -c " +import json +tl = json.load(open('timeline.json')) +for s in tl['segments']: + print(s['file'], s['speech_start_ms']) +") + +INPUTS=(); FILTERS=(); LABELS="" +i=0 +for line in "${LINES[@]}"; do + f="${line%% *}"; d="${line##* }" + INPUTS+=(-i "$f") + FILTERS+=("[$i:a]adelay=${d}|${d}[a$i]") + LABELS="${LABELS}[a$i]" + i=$((i+1)) +done + +DUR=$(python3 -c "import json;print(json.load(open('timeline.json'))['total_ms']/1000)") + +# The narration segments never overlap, so amix here is placement rather than +# blending; normalize=0 keeps each segment at the level it was rendered. +# Output uncompressed first so the loudness pass has something clean to measure. +ffmpeg -y -loglevel error -stats "${INPUTS[@]}" -i score.wav \ + -filter_complex "$(IFS=';'; echo "${FILTERS[*]}");\ +${LABELS}amix=inputs=${i}:normalize=0:dropout_transition=0[vo];\ +[vo]alimiter=limit=0.95[voz];\ +[${i}:a]volume=1.0[bed];\ +[voz][bed]amix=inputs=2:normalize=0:dropout_transition=0[mix];\ +[mix]atrim=0:${DUR},asetpts=N/SR/TB[out]" \ + -map "[out]" -c:a pcm_s16le -ar 48000 mix.wav + +# Two-pass loudness normalisation. EBU R128's -23 LUFS is a broadcast target and +# plays far too quietly on a laptop; -16 LUFS is the sane figure for something +# shown in a meeting room or streamed. +echo "measuring loudness..." +MEASURED=$(ffmpeg -hide_banner -i mix.wav -af loudnorm=I=-16:TP=-1.5:LRA=11:print_format=json -f null - 2>&1 \ + | python3 -c " +import sys, json, re +text = sys.stdin.read() +blob = text[text.rindex('{'):text.rindex('}') + 1] +d = json.loads(blob) +print('%s|%s|%s|%s' % (d['input_i'], d['input_tp'], d['input_lra'], d['input_thresh'])) +") +IFS='|' read -r MI MTP MLRA MTHRESH <<< "$MEASURED" +echo " measured: I=${MI} TP=${MTP} LRA=${MLRA}" + +ffmpeg -y -loglevel error -stats -i mix.wav \ + -af "loudnorm=I=-16:TP=-1.5:LRA=11:measured_I=${MI}:measured_TP=${MTP}:measured_LRA=${MLRA}:measured_thresh=${MTHRESH}:linear=true:print_format=summary" \ + -c:a aac -b:a 256k -ar 48000 narration.m4a + +rm -f mix.wav + +echo +echo -n "narration.m4a duration: " +ffprobe -v error -show_entries format=duration -of default=nw=1:nk=1 narration.m4a +ffmpeg -hide_banner -i narration.m4a -af ebur128=framelog=quiet -f null - 2>&1 | grep -A3 "Integrated loudness" diff --git a/film/src/renderAll.sh b/film/src/renderAll.sh new file mode 100755 index 0000000..e25b2c9 --- /dev/null +++ b/film/src/renderAll.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Render every frame of the film in parallel. Frames are numbered by absolute +# frame index, so the two capture passes drop into one directory in order. +set -euo pipefail +cd "$(dirname "$0")" + +# Pass A: the authored scenes, split into chunks by film time (ms). +CHUNKS=( + "0:40000" "40000:80000" "80000:118000" "118000:158000" + "158000:196000" "196000:233540" + "392700:430000" "430000:466140" +) +for c in "${CHUNKS[@]}"; do + from="${c%%:*}"; to="${c##*:}" + node shoot.js --out=frames --from="$from" --to="$to" > "logs/film_${from}.log" 2>&1 & +done + +# Pass B: the demo, one process per shot. +for seg in s10_incident s11_mesh s12_claims s13_consensus s14_decoys s15_evidence; do + node shootDemo.js --out=frames --only="$seg" > "logs/demo_${seg}.log" 2>&1 & +done + +wait +echo "render complete: $(ls frames | wc -l) frames" diff --git a/film/src/scenes1.js b/film/src/scenes1.js new file mode 100644 index 0000000..26392f0 --- /dev/null +++ b/film/src/scenes1.js @@ -0,0 +1,393 @@ +/* Scenes 1-4: the cold open, the forest mechanism, the reveal, the problem. */ + +/* ---------- shared organic growth ---------- */ + +/** Build a branching root system once; draw it progressively by time. */ +function buildRoots(seed, ox, oy, angle, depth, len, spread) { + const r = rng(seed); + const segs = []; + (function grow(x, y, ang, d, l, t0) { + if (d <= 0) return; + const nx = x + Math.cos(ang) * l; + const ny = y + Math.sin(ang) * l; + const dur = 0.10 + r() * 0.06; + segs.push({ x1: x, y1: y, x2: nx, y2: ny, t0, t1: t0 + dur, w: d * 0.85, depth: d }); + const branches = d > 2 ? (r() > 0.35 ? 2 : 3) : 2; + for (let i = 0; i < branches; i++) { + const spreadAmt = (r() - 0.5) * spread; + grow(nx, ny, ang + spreadAmt, d - 1, l * (0.66 + r() * 0.2), t0 + dur * (0.7 + r() * 0.3)); + } + })(ox, oy, angle, depth, len, 0); + return segs; +} + +const ROOTS_L = buildRoots(1337, 640, 512, Math.PI / 2 + 0.34, 7, 116, 1.18); +const ROOTS_R = buildRoots(9021, 1290, 512, Math.PI / 2 - 0.34, 7, 116, 1.18); + +/** The tip nearest the centre line, so the two systems join where they end. */ +function innerTip(segs, towardX) { + let best = null, bestScore = Infinity; + for (const g of segs) { + if (g.depth > 2) continue; + const score = Math.abs(g.x2 - towardX) - g.y2 * 0.55; + if (score < bestScore) { bestScore = score; best = g; } + } + return best ? { x: best.x2, y: best.y2, t: best.t1 } : { x: towardX, y: 800, t: 0.8 }; +} +const TIP_L = innerTip(ROOTS_L, 980); +const TIP_R = innerTip(ROOTS_R, 980); + +function drawRoots(ctx, segs, p, color, alpha) { + ctx.save(); + ctx.lineCap = 'round'; + for (const s of segs) { + const local = clamp((p - s.t0) / (s.t1 - s.t0)); + if (local <= 0) continue; + ctx.globalAlpha = clamp(alpha * (0.32 + s.depth / 9)); + ctx.strokeStyle = color; + ctx.lineWidth = s.w; + ctx.beginPath(); + ctx.moveTo(s.x1, s.y1); + ctx.lineTo(lerp(s.x1, s.x2, local), lerp(s.y1, s.y2, local)); + ctx.stroke(); + } + ctx.restore(); +} + +/** Drifting spores; ambience that makes the soil feel alive. */ +function spores(ctx, t, alpha, count = 70) { + const r = rng(4242); + ctx.save(); + for (let i = 0; i < count; i++) { + const bx = r() * W, by = 420 + r() * 620, sp = 0.25 + r() * 0.6, ph = r() * 100; + const x = (bx + t * sp * 14) % W; + const y = by + Math.sin(t * 0.5 + ph) * 16; + const tw = 0.28 + 0.72 * (0.5 + 0.5 * Math.sin(t * 1.3 + ph)); + ctx.globalAlpha = clamp(alpha * tw * 0.5); + ctx.fillStyle = PALETTE.myc; + ctx.beginPath(); + ctx.arc(x, y, 1.5 + r() * 1.6, 0, Math.PI * 2); + ctx.fill(); + } + ctx.restore(); +} + +/** A low, cold sky. Without it the tree silhouettes have nothing to sit against. */ +function sky(ctx, horizon, alpha = 1) { + ctx.save(); + ctx.globalAlpha = clamp(alpha); + const g = ctx.createLinearGradient(0, 0, 0, horizon); + g.addColorStop(0, '#07090C'); + g.addColorStop(0.62, '#0C1014'); + g.addColorStop(1, '#18201C'); + ctx.fillStyle = g; + ctx.fillRect(0, 0, W, horizon); + ctx.restore(); +} + +function soilGround(ctx, horizon, alpha = 1) { + ctx.save(); + ctx.globalAlpha = alpha; + const g = ctx.createLinearGradient(0, horizon - 120, 0, H); + g.addColorStop(0, '#0A0C07'); + g.addColorStop(0.35, '#0E1209'); + g.addColorStop(1, '#05060A'); + ctx.fillStyle = g; + ctx.fillRect(0, horizon - 120, W, H - horizon + 120); + ctx.globalAlpha = alpha * 0.5; + ctx.strokeStyle = hexA(PALETTE.myc, 0.18); + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(0, horizon); + ctx.lineTo(W, horizon); + ctx.stroke(); + ctx.restore(); +} + +/** A simple, believable tree silhouette. */ +function tree(ctx, x, groundY, scale, seed, alpha) { + const r = rng(seed); + ctx.save(); + ctx.globalAlpha = clamp(alpha); + ctx.strokeStyle = '#03040A'; + ctx.fillStyle = '#03040A'; + ctx.lineCap = 'round'; + (function branch(bx, by, ang, len, wdt, d) { + if (d === 0) return; + const nx = bx + Math.cos(ang) * len, ny = by + Math.sin(ang) * len; + ctx.lineWidth = wdt; + ctx.beginPath(); + ctx.moveTo(bx, by); + ctx.lineTo(nx, ny); + ctx.stroke(); + const n = d > 3 ? 2 : 3; + for (let i = 0; i < n; i++) { + branch(nx, ny, ang + (r() - 0.5) * 0.95, len * (0.68 + r() * 0.16), wdt * 0.66, d - 1); + } + })(x, groundY, -Math.PI / 2, 92 * scale, 15 * scale, 6); + ctx.restore(); +} + +/* ---------- 1. cold open ---------- */ + +SCENES.roots = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + + const p = cue(t, 0.6, 9.0); + // Deliberately no trees here. The cold open is the hidden half of the + // forest; showing the canopy would give away the reveal in the next scene. + sky(ctx, 512, cue(t, 0.2, 2.5) * 0.8); + soilGround(ctx, 512, cue(t, 0.2, 2.5)); + spores(ctx, t, cue(t, 1.5, 3) * 0.9); + + drawRoots(ctx, ROOTS_L, p, PALETTE.myc, 0.82); + drawRoots(ctx, ROOTS_R, p, PALETTE.myc, 0.82); + + // The two systems find each other and a signal crosses. + const joinP = cue(t, 9.4, 1.8); + if (joinP > 0) { + const a = TIP_L, b = TIP_R; + ctx.save(); + ctx.globalAlpha = joinP * 0.85; + ctx.strokeStyle = PALETTE.myc; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.moveTo(a.x, a.y); + const sagX = (a.x + b.x) / 2, sagY = Math.max(a.y, b.y) + 78; + ctx.quadraticCurveTo(sagX, sagY, lerp(a.x, b.x, easeOut(joinP)), lerp(a.y, b.y, easeOut(joinP))); + ctx.stroke(); + ctx.restore(); + node(ctx, a.x, a.y, 5, PALETTE.myc, { alpha: joinP, glow: 0.8 }); + node(ctx, b.x, b.y, 5, PALETTE.myc, { alpha: joinP, glow: 0.8 }); + + const pulseP = cue(t, 11.4, 2.6); + if (pulseP > 0 && pulseP < 1) packet(ctx, a, b, easeInOut(pulseP), PALETTE.signal, 9); + } + + text(ctx, typeOn('Under every forest,', cue(t, 3.4, 1.5)), W / 2, 232, { + size: 60, weight: 600, alpha: cue(t, 3.4, 0.8), + }); + text(ctx, typeOn('there is a second network.', cue(t, 5.0, 1.8)), W / 2, 306, { + size: 60, weight: 600, alpha: cue(t, 5.0, 0.8), color: PALETTE.myc, + }); + text(ctx, 'no tree is in charge of it', W / 2, 386, { + size: 21, family: 'IBM Plex Mono', color: PALETTE.muted, + tracking: 6, upper: true, alpha: cue(t, 12.8, 1.6) * 0.9, + }); + + vignette(ctx, 0.6); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 1.6)); + fadeToBlack(ctx, cue(t, seg.dur - 1.0, 1.0) * 0.55); +}; + +/* ---------- 2. the forest mechanism ---------- */ + +const FOREST_TREES = [ + { x: 300, s: 1.00, seed: 11 }, { x: 700, s: 1.22, seed: 22 }, + { x: 1150, s: 0.94, seed: 33 }, { x: 1560, s: 1.12, seed: 44 }, +]; +const FOREST_ROOTS = FOREST_TREES.map(tr => ({ x: tr.x, y: 700 })); + +SCENES.forest = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + sky(ctx, 620, 1); + soilGround(ctx, 620, 1); + spores(ctx, t + 20, 0.65); + + const intro = cue(t, 0, 1.6); + FOREST_TREES.forEach((tr, i) => tree(ctx, tr.x, 620, tr.s, tr.seed, intro * cue(t, i * 0.16, 1))); + + // The underground network, drawn as gentle catenaries. + for (let i = 0; i < FOREST_ROOTS.length - 1; i++) { + const a = FOREST_ROOTS[i], b = FOREST_ROOTS[i + 1]; + ctx.save(); + ctx.globalAlpha = cue(t, 1.2 + i * 0.25, 1.2) * 0.55; + ctx.strokeStyle = PALETTE.myc; + ctx.lineWidth = 1.8; + ctx.beginPath(); + ctx.moveTo(a.x, a.y); + ctx.quadraticCurveTo((a.x + b.x) / 2, a.y + 130, b.x, b.y); + ctx.stroke(); + ctx.restore(); + } + FOREST_ROOTS.forEach((n, i) => + node(ctx, n.x, n.y, 8, PALETTE.myc, { alpha: cue(t, 1.0 + i * 0.2, 1), glow: 0.5 })); + + const beat = (label, body, at, colour) => { + const a = cue(t, at, 0.9) * (1 - cue(t, at + 7.2, 1.0)); + if (a <= 0.002) return; + text(ctx, label, W / 2, 176, { + size: 20, family: 'IBM Plex Mono', color: colour, tracking: 8, upper: true, alpha: a, + }); + paragraph(ctx, body, W / 2, 236, 1180, { size: 42, weight: 500, alpha: a, lineHeight: 1.35 }); + }; + + beat('one', 'A tree in trouble releases a signal into the network.', 2.2, PALETTE.signal); + beat('two', 'The signal fades as it travels, and fades as time passes.', 10.4, PALETTE.signal); + beat('three', 'When a second tree senses the same threat, the two reinforce each other.', 18.6, PALETTE.myc); + + // Beat one: a pulse leaves the second tree. + const p1 = cue(t, 3.6, 3.4); + if (p1 > 0 && p1 < 1) packet(ctx, FOREST_ROOTS[1], FOREST_ROOTS[2], easeInOut(p1), PALETTE.signal, 10); + + // Beat two: the same pulse, visibly weakening as it goes. + const p2 = cue(t, 12.0, 4.2); + if (p2 > 0 && p2 < 1) { + const a = FOREST_ROOTS[1], b = FOREST_ROOTS[3]; + const x = lerp(a.x, b.x, p2), y = lerp(a.y, b.y, p2); + const strength = 1 - p2; + ctx.save(); + ctx.globalAlpha = strength; + packet(ctx, a, b, p2, PALETTE.signal, 3 + 8 * strength); + ctx.restore(); + text(ctx, `${Math.round(strength * 100)}%`, x, y - 40, { + size: 19, family: 'IBM Plex Mono', color: PALETTE.signal, alpha: strength * 0.9, + }); + } + + // Beat three: two sources converge and the merged signal is brighter. + const p3 = cue(t, 20.0, 3.6); + if (p3 > 0 && p3 < 1) { + const mid = { x: 925, y: 762 }; + packet(ctx, FOREST_ROOTS[1], mid, easeInOut(clamp(p3 * 1.6)), PALETTE.signal, 9); + packet(ctx, FOREST_ROOTS[2], mid, easeInOut(clamp(p3 * 1.6)), PALETTE.signal, 9); + if (p3 > 0.62) { + const b = cue(p3, 0.62, 0.22); + node(ctx, mid.x, mid.y, 10 + 12 * b, PALETTE.myc, { alpha: 1, glow: 1.6 * b }); + text(ctx, 'reinforced', mid.x, mid.y + 92, { + size: 22, family: 'IBM Plex Mono', color: PALETTE.myc, tracking: 5, upper: true, alpha: b, + }); + } + } + + const outro = cue(t, seg.dur - 5.6, 1.4); + if (outro > 0) { + paragraph(ctx, 'Coordination is not something the forest does. It is something the forest grows.', + W / 2, 902, 1280, { size: 36, weight: 500, alpha: outro * (1 - cue(t, seg.dur - 0.9, 0.9)), color: PALETTE.bone }); + } + + vignette(ctx, 0.62); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 1.0)); +}; + +/* ---------- 3. the reveal ---------- */ + +SCENES.reveal = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + + // The organic layout resolves into a geometric one. + const morph = easeInOut(cue(t, 0.3, 2.6)); + const organic = FOREST_ROOTS.map(n => ({ x: n.x, y: n.y - 40 })); + const geo = ring(W / 2, 640, 250, 4, -Math.PI / 2); + const pts = organic.map((o, i) => ({ + x: lerp(o.x, geo[i].x, morph), + y: lerp(o.y, geo[i].y, morph), + })); + + for (let i = 0; i < pts.length; i++) { + for (let j = i + 1; j < pts.length; j++) { + link(ctx, pts[i], pts[j], { alpha: 0.16 + 0.3 * morph, color: PALETTE.myc, width: 1.3 }); + } + } + pts.forEach(p => node(ctx, p.x, p.y, 9, PALETTE.myc, { glow: 0.7 })); + + const packP = (t * 0.42) % 1; + packet(ctx, pts[0], pts[2], packP, PALETTE.signal, 6 * morph); + packet(ctx, pts[1], pts[3], (packP + 0.5) % 1, PALETTE.signal, 6 * morph); + + const titleA = cue(t, 2.4, 1.2); + text(ctx, 'SMESH', W / 2, 300, { + size: 168, weight: 700, alpha: titleA, tracking: 22 * (1 - easeOut(titleA)) + 12, + }); + text(ctx, 'signal diffusion for distributed agents', W / 2, 372, { + size: 25, family: 'IBM Plex Mono', color: PALETTE.myc, + tracking: 6, upper: true, alpha: cue(t, 3.6, 1.2), + }); + + paragraph(ctx, 'Software agents that coordinate the way a forest does.', + W / 2, 972, 1200, { size: 34, weight: 500, alpha: cue(t, 6.4, 1.2), color: PALETTE.dim }); + + vignette(ctx, 0.55); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.9)); +}; + +/* ---------- 4. the problem ---------- */ + +SCENES.problem = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + caption(ctx, 'the thing in the middle', cue(t, 0.4, 1.2)); + + const leftC = { x: 520, y: 600 }, rightC = { x: 1400, y: 600 }; + const spokes = ring(leftC.x, leftC.y, 210, 7); + const meshPts = ring(rightC.x, rightC.y, 210, 7); + + const appear = cue(t, 1.0, 1.4); + const failP = cue(t, 13.5, 1.1); // the hub dies + const deadP = cue(t, 15.0, 1.0); // spokes go dark + const reroute = cue(t, 20.0, 1.6); // the mesh routes around + + // Hub and spoke. + spokes.forEach((s, i) => { + link(ctx, leftC, s, { alpha: appear * (0.42 - 0.34 * deadP), color: PALETTE.muted }); + node(ctx, s.x, s.y, 8, PALETTE.dim, { alpha: appear * (1 - 0.72 * deadP), ringAlpha: 0.22 }); + const pp = ((t * 0.55 + i * 0.14) % 1); + if (failP < 0.5) packet(ctx, s, leftC, pp, hexA(PALETTE.dim, 1), 4 * appear); + }); + const hubColor = failP > 0 ? PALETTE.danger : PALETTE.signal; + node(ctx, leftC.x, leftC.y, 19 + 5 * Math.sin(t * 3) * (failP > 0 ? 1 : 0), hubColor, { + alpha: appear, glow: 0.9 + failP, ringAlpha: 0.5, + }); + text(ctx, 'coordinator', leftC.x, leftC.y + 96, { + size: 21, family: 'IBM Plex Mono', color: failP > 0 ? PALETTE.danger : PALETTE.dim, alpha: appear, + }); + if (deadP > 0) { + text(ctx, 'everything stops', leftC.x, leftC.y + 300, { + size: 30, weight: 600, color: PALETTE.danger, alpha: deadP, + }); + } + + // The mesh, which has no middle to lose. + const appear2 = cue(t, 2.4, 1.4); + meshPts.forEach((a, i) => { + meshPts.forEach((b, j) => { + if (j <= i) return; + const adjacent = Math.abs(i - j) === 1 || Math.abs(i - j) === meshPts.length - 1 || (i + 3) % meshPts.length === j; + if (!adjacent) return; + const lost = reroute > 0 && (i === 2 || j === 2); + link(ctx, a, b, { alpha: appear2 * (lost ? 0.08 : 0.38 + 0.25 * reroute), color: PALETTE.myc }); + }); + }); + meshPts.forEach((p, i) => { + const lost = reroute > 0 && i === 2; + node(ctx, p.x, p.y, 9, lost ? PALETTE.danger : PALETTE.myc, { + alpha: appear2 * (lost ? 0.35 : 1), glow: lost ? 0.2 : 0.55, + }); + }); + if (reroute > 0) { + const rp = (t * 0.5) % 1; + packet(ctx, meshPts[1], meshPts[3], rp, PALETTE.signal, 6); + text(ctx, 'the signal routes around it', rightC.x, rightC.y + 300, { + size: 30, weight: 600, color: PALETTE.myc, alpha: reroute, + }); + } + text(ctx, 'mesh', rightC.x, rightC.y + 96, { + size: 21, family: 'IBM Plex Mono', color: PALETTE.dim, alpha: appear2, + }); + + const cost = cue(t, 6.6, 1.2) * (1 - cue(t, 12.6, 1.0)); + if (cost > 0.002) { + paragraph(ctx, 'The thing you pay for. The thing you scale. The thing that pages you at 3am.', + W / 2, 918, 1300, { size: 34, weight: 500, alpha: cost, color: PALETTE.dim }); + } + + vignette(ctx, 0.6); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.8)); +}; diff --git a/film/src/scenes2.js b/film/src/scenes2.js new file mode 100644 index 0000000..ba78e6b --- /dev/null +++ b/film/src/scenes2.js @@ -0,0 +1,417 @@ +/* Scenes 5-9b: the three primitives, the wire, and the demo setup. */ + +/* ---------- 5. decay ---------- */ + +SCENES.primitive_decay = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + caption(ctx, 'mechanism one', cue(t, 0.3, 1.0)); + text(ctx, 'Decay', W / 2, 216, { size: 78, weight: 700, alpha: cue(t, 0.6, 1.0) }); + + const x0 = 380, x1 = 1540, yBase = 800, hgt = 380; + const draw = easeOut(cue(t, 1.6, 5.0)); + + // Axes. + ctx.save(); + ctx.globalAlpha = cue(t, 1.2, 1.0) * 0.45; + ctx.strokeStyle = PALETTE.muted; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(x0, yBase - hgt); ctx.lineTo(x0, yBase); ctx.lineTo(x1, yBase); + ctx.stroke(); + ctx.restore(); + text(ctx, 'intensity', x0 - 28, yBase - hgt / 2, { + size: 17, family: 'IBM Plex Mono', color: PALETTE.muted, align: 'right', alpha: cue(t, 1.4, 1), + }); + text(ctx, 'time', (x0 + x1) / 2, yBase + 42, { + size: 17, family: 'IBM Plex Mono', color: PALETTE.muted, alpha: cue(t, 1.4, 1), + }); + + const decayAt = u => Math.exp(-3.1 * u); + + // The curve, and the area under it. + ctx.save(); + ctx.globalAlpha = 0.09 * draw; + ctx.fillStyle = PALETTE.signal; + ctx.beginPath(); + ctx.moveTo(x0, yBase); + for (let u = 0; u <= draw; u += 0.004) ctx.lineTo(lerp(x0, x1, u), yBase - decayAt(u) * hgt); + ctx.lineTo(lerp(x0, x1, draw), yBase); + ctx.closePath(); + ctx.fill(); + ctx.restore(); + + ctx.save(); + ctx.strokeStyle = PALETTE.signal; + ctx.lineWidth = 3; + ctx.globalAlpha = 0.95; + ctx.beginPath(); + for (let u = 0; u <= draw; u += 0.004) { + const px = lerp(x0, x1, u), py = yBase - decayAt(u) * hgt; + u === 0 ? ctx.moveTo(px, py) : ctx.lineTo(px, py); + } + ctx.stroke(); + ctx.restore(); + + // The signal riding its own decay. + if (draw > 0 && draw < 1) { + const px = lerp(x0, x1, draw), py = yBase - decayAt(draw) * hgt; + node(ctx, px, py, 4 + 9 * decayAt(draw), PALETTE.signal, { glow: decayAt(draw) }); + text(ctx, `${Math.round(decayAt(draw) * 100)}%`, px, py - 46, { + size: 22, family: 'IBM Plex Mono', color: PALETTE.signal, + }); + } + + // Where it stops mattering. + const thr = cue(t, 7.2, 1.0); + if (thr > 0) { + ctx.save(); + ctx.globalAlpha = thr * 0.55; + ctx.strokeStyle = PALETTE.danger; + ctx.setLineDash([7, 7]); + ctx.beginPath(); + ctx.moveTo(x0, yBase - 0.1 * hgt); + ctx.lineTo(x1, yBase - 0.1 * hgt); + ctx.stroke(); + ctx.restore(); + text(ctx, 'below this, the signal is gone', x1, yBase - 0.1 * hgt - 22, { + size: 19, family: 'IBM Plex Mono', color: PALETTE.danger, align: 'right', alpha: thr, + }); + } + + paragraph(ctx, 'Nothing has to clean up. Stale work removes itself.', + W / 2, 962, 1200, { size: 38, weight: 600, alpha: cue(t, 11.5, 1.2), color: PALETTE.bone }); + + vignette(ctx, 0.55); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.7)); +}; + +/* ---------- 6. reinforcement ---------- */ + +SCENES.primitive_reinforce = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + caption(ctx, 'mechanism two', cue(t, 0.3, 1.0)); + text(ctx, 'Reinforcement', W / 2, 216, { size: 78, weight: 700, alpha: cue(t, 0.6, 1.0) }); + + const claim = { x: W / 2, y: 596 }; + const agents = ring(W / 2, 596, 300, 5); + const arriveAt = [2.6, 4.6, 6.6, 8.4, 10.0]; + let witnesses = 0; + + agents.forEach((a, i) => { + const arrived = cue(t, arriveAt[i], 1.1); + if (arrived > 0.98) witnesses++; + const c = CONCERNS[i].color; + node(ctx, a.x, a.y, 10, c, { alpha: 0.25 + 0.75 * arrived, glow: 0.5 * arrived }); + link(ctx, a, claim, { alpha: 0.5 * arrived, color: c, width: 1.6 }); + const travel = cue(t, arriveAt[i], 1.0); + if (travel > 0 && travel < 1) packet(ctx, a, claim, easeInOut(travel), c, 7); + }); + + const conf = clamp(witnesses / 5); + node(ctx, claim.x, claim.y, 16 + 20 * conf, PALETTE.myc, { alpha: 1, glow: 0.5 + conf * 1.6 }); + text(ctx, `${witnesses}`, claim.x, claim.y + 12, { + size: 40, weight: 700, family: 'IBM Plex Mono', color: PALETTE.ground, + }); + text(ctx, witnesses === 1 ? 'witness' : 'witnesses', claim.x, claim.y + 116, { + size: 22, family: 'IBM Plex Mono', color: PALETTE.myc, tracking: 5, upper: true, alpha: cue(t, 2.6, 1), + }); + + // Confidence bar. + const bx = 660, bw = 600, by = 856; + ctx.save(); + ctx.globalAlpha = cue(t, 2.2, 1.0); + ctx.strokeStyle = hexA(PALETTE.muted, 0.5); + ctx.lineWidth = 1; + roundRect(ctx, bx, by, bw, 16, 8); + ctx.stroke(); + ctx.fillStyle = PALETTE.myc; + roundRect(ctx, bx, by, Math.max(6, bw * conf), 16, 8); + ctx.fill(); + ctx.restore(); + text(ctx, 'confidence', bx - 22, by + 13, { + size: 18, family: 'IBM Plex Mono', color: PALETTE.muted, align: 'right', alpha: cue(t, 2.2, 1), + }); + + paragraph(ctx, 'Not a duplicate. A second witness.', + W / 2, 962, 1100, { size: 38, weight: 600, alpha: cue(t, 8.0, 1.2), color: PALETTE.bone }); + + vignette(ctx, 0.55); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.7)); +}; + +/* ---------- 7. content addressing ---------- */ + +SCENES.primitive_address = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + caption(ctx, 'mechanism three', cue(t, 0.3, 1.0)); + text(ctx, 'Addressed by content', W / 2, 216, { size: 74, weight: 700, alpha: cue(t, 0.6, 1.0) }); + + const converge = easeInOut(cue(t, 6.4, 2.4)); + const midX = W / 2; + const lx = lerp(520, midX, converge), rx = lerp(1400, midX, converge); + const y = 560; + + const side = (x, i, evidence, at) => { + const a = cue(t, at, 1.0); + if (a <= 0.002) return; + const c = CONCERNS[i].color; + node(ctx, x, y, 12, c, { alpha: a * (1 - converge * 0.35), glow: 0.6 * a }); + text(ctx, CONCERNS[i].id, x, y + 62, { + size: 22, family: 'IBM Plex Mono', color: c, alpha: a * (1 - converge), + }); + text(ctx, evidence, x, y + 94, { + size: 18, family: 'IBM Plex Mono', color: PALETTE.muted, alpha: a * (1 - converge), + }); + }; + side(lx, 0, 'latency up 3x', 1.2); + side(rx, 2, 'pool at 98%', 2.4); + + // Different evidence, identical claim, therefore identical address. + const claimA = cue(t, 3.6, 1.2); + const boxW = 460, boxH = 96; + [[lx, 0], [rx, 2]].forEach(([x, i]) => { + const a = claimA * (1 - converge * 0.2); + if (a <= 0.002) return; + ctx.save(); + ctx.globalAlpha = a; + ctx.strokeStyle = hexA(CONCERNS[i].color, 0.6); + ctx.lineWidth = 1.4; + roundRect(ctx, x - boxW / 2, y + 140, boxW, boxH, 5); + ctx.stroke(); + ctx.restore(); + text(ctx, '{"subject":"checkout-api",', x, y + 178, { + size: 20, family: 'IBM Plex Mono', color: PALETTE.bone, alpha: a, + }); + text(ctx, '"claim":"degraded"}', x, y + 208, { + size: 20, family: 'IBM Plex Mono', color: PALETTE.bone, alpha: a, + }); + }); + + const hashA = cue(t, 5.2, 1.2); + if (hashA > 0.002) { + [[lx], [rx]].forEach(([x]) => { + text(ctx, 'sha-256', x, y + 286, { + size: 16, family: 'IBM Plex Mono', color: PALETTE.muted, tracking: 4, upper: true, alpha: hashA * (1 - converge), + }); + text(ctx, '10757c4a01affa2d', x, y + 324, { + size: 30, family: 'IBM Plex Mono', color: PALETTE.myc, alpha: hashA, + }); + }); + } + + if (converge > 0.85) { + const b = cue(converge, 0.85, 0.15); + node(ctx, midX, y, 22, PALETTE.myc, { alpha: b, glow: 1.8 * b }); + text(ctx, 'one claim, two witnesses', midX, y + 62, { + size: 26, family: 'IBM Plex Mono', color: PALETTE.myc, tracking: 4, upper: true, alpha: b, + }); + } + + paragraph(ctx, 'Same conclusion, same address. They find each other without ever talking.', + W / 2, 972, 1320, { size: 36, weight: 600, alpha: cue(t, 12.6, 1.2), color: PALETTE.bone }); + + vignette(ctx, 0.55); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.7)); +}; + +/* ---------- 8. the wire ---------- */ + +SCENES.quic = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + caption(ctx, 'the wire', cue(t, 0.3, 1.0)); + text(ctx, 'QUIC', W / 2, 224, { size: 84, weight: 700, alpha: cue(t, 0.5, 1.0), tracking: 6 }); + text(ctx, 'encrypted, peer to peer, no broker', W / 2, 274, { + size: 21, family: 'IBM Plex Mono', color: PALETTE.myc, tracking: 5, upper: true, alpha: cue(t, 1.2, 1.0), + }); + + const pts = ring(W / 2, 630, 260, 5); + const LINKS = [[0, 1], [0, 2], [1, 2], [2, 3], [3, 4], [4, 0]]; + + // The broker that is not there. + const ghost = (1 - cue(t, 8.0, 2.0)) * cue(t, 3.0, 1.2); + if (ghost > 0.004) { + ctx.save(); + ctx.globalAlpha = ghost * 0.4; + ctx.setLineDash([6, 8]); + ctx.strokeStyle = PALETTE.danger; + ctx.lineWidth = 1.4; + ctx.beginPath(); + ctx.arc(W / 2, 630, 46, 0, Math.PI * 2); + ctx.stroke(); + ctx.restore(); + text(ctx, 'no broker', W / 2, 638, { + size: 21, family: 'IBM Plex Mono', color: PALETTE.danger, alpha: ghost * 0.85, + }); + } + + LINKS.forEach(([i, j], k) => { + const a = cue(t, 1.6 + k * 0.28, 0.9); + link(ctx, pts[i], pts[j], { alpha: a * 0.5, color: PALETTE.myc, width: 1.8 }); + if (a > 0.9) { + const mid = { x: (pts[i].x + pts[j].x) / 2, y: (pts[i].y + pts[j].y) / 2 }; + text(ctx, 'TLS 1.3', mid.x, mid.y - 10, { + size: 14, family: 'IBM Plex Mono', color: PALETTE.myc, alpha: cue(t, 4.4, 1.2) * 0.7, + }); + } + }); + + pts.forEach((p, i) => node(ctx, p.x, p.y, 11, CONCERNS[i].color, { + alpha: cue(t, 1.2 + i * 0.18, 0.9), glow: 0.6, label: CONCERNS[i].id, + labelAlpha: cue(t, 2.0 + i * 0.18, 0.9), + })); + + LINKS.forEach(([i, j], k) => { + const p = ((t * 0.34 + k * 0.17) % 1); + const fwd = k % 2 === 0; + packet(ctx, pts[fwd ? i : j], pts[fwd ? j : i], p, PALETTE.signal, 5 * cue(t, 3.0, 1)); + }); + + paragraph(ctx, 'Nothing in the middle to buy, to scale, or to lose.', + W / 2, 992, 1200, { size: 38, weight: 600, alpha: cue(t, 13.0, 1.2), color: PALETTE.bone }); + + vignette(ctx, 0.55); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.7)); +}; + +/* ---------- 9. the setup ---------- */ + +const SERVICES = ['edge-gateway', 'checkout-api', 'payments-api', 'inventory-svc', 'session-store', 'notification-worker']; + +SCENES.setup = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + caption(ctx, 'the run you are about to see', cue(t, 0.3, 1.0)); + + // The fleet under observation. + const svcY = 268; + text(ctx, 'one fleet of services', W / 2, 208, { + size: 22, family: 'IBM Plex Mono', color: PALETTE.dim, tracking: 4, upper: true, alpha: cue(t, 0.8, 1), + }); + SERVICES.forEach((s, i) => { + const a = cue(t, 1.2 + i * 0.14, 0.8); + const x = 250 + i * 285; + ctx.save(); + ctx.globalAlpha = a * 0.85; + ctx.strokeStyle = hexA(PALETTE.muted, 0.5); + roundRect(ctx, x - 128, svcY, 256, 52, 4); + ctx.stroke(); + ctx.restore(); + text(ctx, s, x, svcY + 33, { size: 19, family: 'IBM Plex Mono', color: PALETTE.dim, alpha: a }); + }); + + // Five processes, five ports, five blind spots. + const agentY = 640; + text(ctx, 'five separate programs', W / 2, 470, { + size: 22, family: 'IBM Plex Mono', color: PALETTE.myc, tracking: 4, upper: true, alpha: cue(t, 4.0, 1), + }); + CONCERNS.forEach((c, i) => { + const a = cue(t, 4.6 + i * 0.5, 0.9); + const x = 288 + i * 336; + node(ctx, x, agentY, 13, c.color, { alpha: a, glow: 0.7 * a }); + text(ctx, c.id, x, agentY + 66, { size: 24, family: 'IBM Plex Mono', color: c.color, alpha: a }); + text(ctx, `watches ${c.label}`, x, agentY + 98, { + size: 18, family: 'IBM Plex Mono', color: PALETTE.muted, alpha: a, + }); + text(ctx, `127.0.0.1:930${i + 1}`, x, agentY + 128, { + size: 16, family: 'IBM Plex Mono', color: PALETTE.muted, alpha: a * cue(t, 8.0, 1.2) * 0.75, + }); + + // Each one is walled off from the others. + const wall = cue(t, 11.5, 1.6); + if (wall > 0.004 && i < CONCERNS.length - 1) { + ctx.save(); + ctx.globalAlpha = wall * 0.4; + ctx.strokeStyle = PALETTE.danger; + ctx.setLineDash([5, 9]); + ctx.lineWidth = 1.2; + ctx.beginPath(); + ctx.moveTo(x + 168, agentY - 76); + ctx.lineTo(x + 168, agentY + 146); + ctx.stroke(); + ctx.restore(); + } + }); + + const blind = cue(t, 13.0, 1.3); + paragraph(ctx, 'None of them can see what the others see.', + W / 2, 900, 1200, { size: 40, weight: 600, alpha: blind, color: PALETTE.danger }); + + const warn = cue(t, 17.5, 1.3); + paragraph(ctx, 'And something is about to go wrong.', + W / 2, 968, 1200, { size: 36, weight: 500, alpha: warn, color: PALETTE.bone }); + + vignette(ctx, 0.55); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.7)); +}; + +/* ---------- 9b. five windows ---------- */ + +SCENES.windows = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + + const bx = 460, by = 250, bw = 1000, bh = 620; + const a = cue(t, 0.2, 1.2); + + ctx.save(); + ctx.globalAlpha = a * 0.9; + ctx.fillStyle = '#0E1109'; + ctx.fillRect(bx, by, bw, bh); + ctx.strokeStyle = hexA(PALETTE.muted, 0.45); + ctx.lineWidth = 1.4; + ctx.strokeRect(bx, by, bw, bh); + ctx.restore(); + + // The fire nobody can see. + const fire = cue(t, 1.0, 2.0); + ctx.save(); + ctx.globalAlpha = fire * (0.5 + 0.5 * Math.sin(t * 5)); + const g = ctx.createRadialGradient(bx + bw * 0.62, by + bh * 0.66, 0, bx + bw * 0.62, by + bh * 0.66, 300); + g.addColorStop(0, hexA(PALETTE.danger, 0.55)); + g.addColorStop(1, hexA(PALETTE.danger, 0)); + ctx.fillStyle = g; + ctx.fillRect(bx, by, bw, bh); + ctx.restore(); + + // Five windows, each showing only smoke. + CONCERNS.forEach((c, i) => { + const wa = cue(t, 1.4 + i * 0.4, 0.8); + const wx = bx + 90 + i * 176, wy = by + 190; + ctx.save(); + ctx.globalAlpha = wa; + ctx.fillStyle = hexA(c.color, 0.13); + ctx.fillRect(wx, wy, 120, 168); + ctx.strokeStyle = hexA(c.color, 0.75); + ctx.lineWidth = 1.6; + ctx.strokeRect(wx, wy, 120, 168); + ctx.restore(); + text(ctx, c.id, wx + 60, wy + 200, { + size: 17, family: 'IBM Plex Mono', color: c.color, alpha: wa, + }); + const smoke = cue(t, 4.6 + i * 0.2, 1.2); + text(ctx, 'smoke', wx + 60, wy + 92, { + size: 19, family: 'IBM Plex Mono', color: PALETTE.dim, alpha: smoke * (0.55 + 0.45 * Math.sin(t * 2 + i)), + }); + }); + + text(ctx, 'None of them can see the fire.', W / 2, 972, { + size: 44, weight: 600, color: PALETTE.bone, alpha: cue(t, 7.0, 1.2), + }); + text(ctx, 'Every one of them sees smoke.', W / 2, 1030, { + size: 32, weight: 500, color: PALETTE.danger, alpha: cue(t, 9.0, 1.2), + }); + + vignette(ctx, 0.6); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.7)); + fadeToBlack(ctx, cue(t, seg.dur - 1.2, 1.2)); +}; diff --git a/film/src/scenes3.js b/film/src/scenes3.js new file mode 100644 index 0000000..4f38706 --- /dev/null +++ b/film/src/scenes3.js @@ -0,0 +1,145 @@ +/* Scenes 16-17b: the payoff, the business case, and the close. */ + +SCENES.payoff = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + caption(ctx, 'what just happened', cue(t, 0.4, 1.2)); + + const pts = ring(W / 2, 520, 232, 5); + const LINKS = [[0, 1], [0, 2], [1, 2], [2, 3], [3, 4], [4, 0]]; + const appear = cue(t, 0.8, 1.2); + + LINKS.forEach(([i, j]) => link(ctx, pts[i], pts[j], { alpha: appear * 0.35, color: PALETTE.myc })); + pts.forEach((p, i) => { + const lit = cue(t, 2.0 + i * 0.34, 0.7); + node(ctx, p.x, p.y, 11, CONCERNS[i].color, { + alpha: appear, glow: 0.4 + lit * 1.1, label: CONCERNS[i].id, labelAlpha: appear * 0.9, + }); + if (lit > 0.5) { + const mid = { x: W / 2, y: 520 }; + link(ctx, p, mid, { alpha: (lit - 0.5) * 1.4 * 0.5, color: CONCERNS[i].color, width: 1.5 }); + } + }); + + const conv = cue(t, 4.4, 1.0); + if (conv > 0.004) { + node(ctx, W / 2, 520, 20 + 14 * conv, PALETTE.myc, { alpha: conv, glow: 2.0 * conv }); + text(ctx, 'checkout-api', W / 2, 520 + 8, { + size: 22, weight: 700, family: 'IBM Plex Mono', color: PALETTE.ground, alpha: conv, + }); + } + + const lines = [ + ['Not one agent had enough information to be right.', 6.2, PALETTE.bone, 40], + ['The system was right anyway.', 8.4, PALETTE.myc, 46], + ]; + lines.forEach(([s, at, colour, size], i) => { + text(ctx, s, W / 2, 852 + i * 74, { + size, weight: 600, color: colour, alpha: cue(t, at, 1.2), + }); + }); + + const honest = cue(t, 14.5, 1.2); + if (honest > 0.004) { + text(ctx, 'telemetry: synthetic and reproducible · coordination: real processes, real sockets', + W / 2, 1012, { size: 20, family: 'IBM Plex Mono', color: PALETTE.muted, alpha: honest }); + } + + vignette(ctx, 0.55); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.8)); +}; + +SCENES.unlocks = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + caption(ctx, 'why it is worth building', cue(t, 0.3, 1.0)); + + const items = [ + ['Add agents freely', 'no reconfiguration, no registry to update', 1.6], + ['Lose agents safely', 'failure does not take the answer with it', 5.0], + ['No coordinator bill', 'nothing central to scale or pay for', 8.4], + ]; + + items.forEach(([head, sub, at], i) => { + const a = cue(t, at, 1.1); + if (a <= 0.004) return; + const y = 330 + i * 190; + const slide = (1 - easeOut(a)) * 44; + + ctx.save(); + ctx.globalAlpha = a * 0.65; + ctx.strokeStyle = PALETTE.myc; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.moveTo(452 - slide, y - 40); + ctx.lineTo(452 - slide, y + 42); + ctx.stroke(); + ctx.restore(); + + text(ctx, head, 492 - slide, y, { size: 50, weight: 600, align: 'left', alpha: a }); + text(ctx, sub, 492 - slide, y + 42, { + size: 24, family: 'IBM Plex Mono', color: PALETTE.muted, align: 'left', alpha: a * 0.9, + }); + }); + + const kicker = cue(t, 13.5, 1.3); + paragraph(ctx, 'The network gets more reliable as it gets larger.', + W / 2, 946, 1300, { size: 38, weight: 600, alpha: kicker, color: PALETTE.myc }); + + vignette(ctx, 0.55); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 0.7)); +}; + +SCENES.close = (ctx, t, seg, frame) => { + ctx.fillStyle = PALETTE.ground; + ctx.fillRect(0, 0, W, H); + + // The mesh dissolves back into the root system it came from. + const dissolve = easeInOut(cue(t, 5.0, 5.0)); + const geo = ring(W / 2, 600, 250, 5); + const organic = [ + { x: 470, y: 760 }, { x: 760, y: 690 }, { x: 1020, y: 800 }, + { x: 1320, y: 700 }, { x: 1520, y: 790 }, + ]; + const pts = geo.map((g, i) => ({ + x: lerp(g.x, organic[i].x, dissolve), + y: lerp(g.y, organic[i].y, dissolve), + })); + + const LINKS = [[0, 1], [0, 2], [1, 2], [2, 3], [3, 4], [4, 0]]; + LINKS.forEach(([i, j]) => { + ctx.save(); + ctx.globalAlpha = 0.4 * (1 - cue(t, 12.5, 3.0)); + ctx.strokeStyle = PALETTE.myc; + ctx.lineWidth = lerp(1.6, 2.6, dissolve); + ctx.beginPath(); + ctx.moveTo(pts[i].x, pts[i].y); + const cxm = (pts[i].x + pts[j].x) / 2, cym = (pts[i].y + pts[j].y) / 2 + 110 * dissolve; + ctx.quadraticCurveTo(cxm, cym, pts[j].x, pts[j].y); + ctx.stroke(); + ctx.restore(); + }); + pts.forEach((p, i) => node(ctx, p.x, p.y, 9, dissolve > 0.5 ? PALETTE.myc : CONCERNS[i].color, { + alpha: 1 - cue(t, 12.5, 3.0), glow: 0.6, + })); + spores(ctx, t + 60, dissolve * 0.7); + + text(ctx, 'SMESH', W / 2, 300, { + size: 132, weight: 700, alpha: cue(t, 0.6, 1.4) * (1 - cue(t, 16.0, 2.5)), tracking: 14, + }); + + paragraph(ctx, 'The hard problem is no longer how clever each agent is. It is how they agree.', + W / 2, 402, 1280, { size: 34, weight: 500, color: PALETTE.dim, alpha: cue(t, 2.4, 1.4) * (1 - cue(t, 16.0, 2.5)) }); + + text(ctx, 'running under our feet for four hundred million years', W / 2, 1000, { + size: 24, family: 'IBM Plex Mono', color: PALETTE.myc, tracking: 4, upper: true, + alpha: cue(t, 9.5, 1.6) * (1 - cue(t, 17.0, 2.0)), + }); + + vignette(ctx, 0.62); + grain(ctx, frame); + fadeToBlack(ctx, 1 - cue(t, 0, 1.0)); + fadeToBlack(ctx, cue(t, seg.dur - 3.4, 3.2)); +}; diff --git a/film/src/score.py b/film/src/score.py new file mode 100644 index 0000000..1733a09 --- /dev/null +++ b/film/src/score.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""A very quiet ambient bed for the film. + +Deliberately minimal: a sustained low drone with a few partials that breathe +against each other, sitting far under the narration. It is there to keep eight +minutes of speech from sounding like a screen recording, not to be noticed. +Levels are conservative on purpose - if it is audible as *music*, it is wrong. +""" +import json, numpy as np, wave + +SR = 44100 +tl = json.load(open('timeline.json')) +DUR = tl['total_ms'] / 1000.0 + 0.5 +n = int(DUR * SR) +t = np.arange(n) / SR + +def partial(freq, amp, lfo_hz, lfo_depth, phase=0.0): + """A sine with a slow amplitude drift, so the pad never sits still.""" + breathe = 1.0 - lfo_depth + lfo_depth * (0.5 + 0.5 * np.sin(2 * np.pi * lfo_hz * t + phase)) + return amp * breathe * np.sin(2 * np.pi * freq * t + phase) + +# D minor: root, fifth, octave, tenth, plus a distant shimmer. +bed = ( + partial(73.416, 0.55, 0.031, 0.35, 0.0) + + partial(110.00, 0.34, 0.023, 0.40, 1.1) + + partial(146.832, 0.22, 0.017, 0.45, 2.3) + + partial(174.614, 0.13, 0.013, 0.55, 0.7) + + partial(220.00, 0.09, 0.011, 0.60, 3.0) + + partial(440.00, 0.035, 0.007, 0.75, 1.9) +) + +# Gentle swells where the picture asks for one. +def swell(start, end, gain, ramp=3.0): + env = np.ones(n) + s, e = int(start * SR), int(end * SR) + r = int(ramp * SR) + seg = np.ones(e - s) * gain + seg[:r] = np.linspace(1.0, gain, r) + seg[-r:] = np.linspace(gain, 1.0, r) + env[s:e] = seg + return env + +by_id = {s['id']: s for s in tl['segments']} +def span(seg_id): + s = by_id[seg_id] + return s['start_ms'] / 1000.0, s['end_ms'] / 1000.0 + +env = np.ones(n) +for seg_id, gain in [('s03_reveal', 1.55), ('s13_consensus', 1.45), ('s17_close', 1.5)]: + a, b = span(seg_id) + env *= swell(a, min(b, DUR - 0.1), gain) + +bed *= env + +# Overall level: quiet enough to live under a voice without ducking. +bed /= np.max(np.abs(bed)) + 1e-9 +bed *= 0.085 + +# Fade the ends so nothing clicks. +fi, fo = int(6 * SR), int(7 * SR) +bed[:fi] *= np.linspace(0, 1, fi) +bed[-fo:] *= np.linspace(1, 0, fo) + +stereo = np.stack([bed, np.roll(bed, 240)], axis=1) # a little width +pcm = np.clip(stereo, -1, 1) +pcm = (pcm * 32767).astype(np.int16) + +with wave.open('score.wav', 'wb') as w: + w.setnchannels(2) + w.setsampwidth(2) + w.setframerate(SR) + w.writeframes(pcm.tobytes()) + +print(f"score.wav {DUR:.1f}s") diff --git a/film/src/script.json b/film/src/script.json new file mode 100644 index 0000000..11290a6 --- /dev/null +++ b/film/src/script.json @@ -0,0 +1,145 @@ +{ + "title": "SMESH \u2014 How a Forest Solves Coordination", + "voice": { + "name": "Brian", + "id": "nPczCjzI2devNBz1zQrb" + }, + "fps": 30, + "width": 1920, + "height": 1080, + "segments": [ + { + "id": "s01_cold_open", + "scene": "roots", + "lead_in_ms": 3500, + "tail_ms": 3200, + "text": "Under every forest, there is a second network. It is older than ours. Trees use it to warn each other \u2014 about drought, about insects, about disease. No tree is in charge of it. And yet, somehow, the whole forest knows." + }, + { + "id": "s02_mechanism", + "scene": "forest", + "lead_in_ms": 600, + "tail_ms": 1200, + "text": "It runs on three simple rules. A tree in trouble releases a signal into the network. That signal fades as it travels, and fades as time passes \u2014 so old news disappears on its own. But when a second tree senses the same threat, and releases the same signal, the two reinforce each other. The message gets stronger. Nothing coordinates any of this. There is no root server. There is no forest manager. Coordination is not something the forest does. It is something the forest grows." + }, + { + "id": "s03_reveal", + "scene": "reveal", + "lead_in_ms": 900, + "tail_ms": 3600, + "text": "We built a protocol that works the same way. It is called SMESH. Software agents that coordinate the way a forest does \u2014 by releasing signals that fade, and trusting the ones that other agents independently confirm." + }, + { + "id": "s04_problem", + "scene": "problem", + "lead_in_ms": 700, + "tail_ms": 1200, + "text": "Here is why that matters. Almost every distributed system we build today has something sitting in the middle. A message broker. An orchestrator. A coordinator. It is the thing that knows everything, and tells everyone else what to do. It is also the thing you pay for. The thing you scale. The thing that pages you at three in the morning. And the thing that takes the entire system down with it when it fails. As companies start running fleets of AI agents, that bottleneck gets more expensive, and more fragile, every single year." + }, + { + "id": "s05_decay", + "scene": "primitive_decay", + "lead_in_ms": 600, + "tail_ms": 900, + "text": "SMESH removes the middle, and replaces it with three mechanisms. The first is decay. Every message carries its own expiry, built into its physics. It weakens along a curve, and when it is weak enough, it is simply gone. Nothing has to clean up. Stale work removes itself." + }, + { + "id": "s06_reinforce", + "scene": "primitive_reinforce", + "lead_in_ms": 500, + "tail_ms": 900, + "text": "The second is reinforcement. When one agent reaches a conclusion that another agent has already reached, that is not a duplicate to be thrown away. It is a second witness. Confidence rises. Agreement compounds." + }, + { + "id": "s07_address", + "scene": "primitive_address", + "lead_in_ms": 500, + "tail_ms": 1300, + "text": "The third mechanism is the one that makes the other two work. Every claim is addressed by its content \u2014 by what is being said, not by who said it. So two agents that independently arrive at the same conclusion land on the exact same address, automatically, without ever having spoken to each other." + }, + { + "id": "s08_quic", + "scene": "quic", + "lead_in_ms": 600, + "tail_ms": 2600, + "text": "Underneath all of it, these agents talk over QUIC \u2014 the same encrypted transport that carries modern web traffic. Every connection is peer to peer. Every connection is encrypted. There is no server in the middle relaying anything. There is nothing in the middle to buy, to scale, or to lose." + }, + { + "id": "s09_setup", + "scene": "setup", + "lead_in_ms": 700, + "tail_ms": 1800, + "text": "So let us watch it work. What follows is a recording of an actual run. Five separate programs, on five separate network ports, talking over real encrypted connections. Each one is monitoring the same fleet of services. But each one can only see a single kind of data. One watches response times. One watches errors. One watches capacity. One watches how requests retry. One watches software releases. None of them can see what the others see. And something is about to go wrong." + }, + { + "id": "s09b_windows", + "scene": "windows", + "lead_in_ms": 800, + "tail_ms": 2200, + "text": "Think of them as five people watching the same building through five different windows. One can only see the lobby. One can only see the stairwell. None of them can see the fire. Every one of them sees smoke." + }, + { + "id": "s10_incident", + "scene": "demo_establish", + "lead_in_ms": 2200, + "tail_ms": 2200, + "text": "A release goes out. It quietly shrinks a connection pool by ninety percent. And now all five of these watchers see a piece of the damage \u2014 but not one of them sees the whole thing. Worse than that: two of them are about to accuse the wrong service entirely." + }, + { + "id": "s11_mesh", + "scene": "demo_mesh", + "lead_in_ms": 1800, + "tail_ms": 3200, + "text": "This is the mesh itself. Five agents, six encrypted links. Every dot you see crossing a link is a real message that was actually sent \u2014 read back from the recording, not animated for effect. And notice they are not all connected to each other. Some messages have to be passed along by a neighbour to reach the far side, exactly the way a forest relays a signal." + }, + { + "id": "s12_claims", + "scene": "demo_claims", + "lead_in_ms": 1800, + "tail_ms": 3200, + "text": "On the right is what the network currently believes. Each row is a claim about one service. The five small tags beneath each claim are the five agents. A tag lights up the moment that agent independently backs the claim. Watch the top row. One agent. Then a second. Then a third \u2014 each arriving from completely unrelated evidence, and finding the others already there." + }, + { + "id": "s13_consensus", + "scene": "demo_consensus", + "lead_in_ms": 2400, + "tail_ms": 5000, + "text": "There it is. Four independent agents. Four unrelated kinds of evidence. One conclusion. That crosses the threshold, and the network calls it. Nobody voted. Nobody was in charge. The answer assembled itself out of five partial views \u2014 and the fifth agent confirms it moments later." + }, + { + "id": "s14_decoys", + "scene": "demo_decoys", + "lead_in_ms": 1800, + "tail_ms": 3800, + "text": "Now look at what did not happen. This service was throwing errors loudly. It looks broken. On its own, the error watcher would have blamed it. It collects three witnesses, and it stops there \u2014 because it is a symptom, not a cause. And these last three claims only ever found a single witness each. Nothing confirmed them, so they simply fade out. Nobody had to decide they were wrong. Going uncorroborated was enough." + }, + { + "id": "s15_evidence", + "scene": "demo_journal", + "lead_in_ms": 900, + "tail_ms": 2800, + "text": "And every step of it is on the record. Each of those five programs wrote down everything it did, and everything it chose not to do, as it happened. That record is what you have been watching. It is not a reenactment \u2014 it is the run itself, played back." + }, + { + "id": "s16_payoff", + "scene": "payoff", + "lead_in_ms": 1000, + "tail_ms": 2600, + "text": "That is the whole idea. Not one agent in this run had enough information to be right. The system was right anyway. The telemetry here is synthetic, so the run stays reproducible. The coordination is not synthetic. Those were real processes, on real sockets, making real decisions \u2014 recorded, verified, and replayed back to you exactly as they happened." + }, + { + "id": "s16b_unlocks", + "scene": "unlocks", + "lead_in_ms": 900, + "tail_ms": 2400, + "text": "That property is what makes this worth building. Agents can be added without reconfiguring anything. Agents can fail without taking the answer with them. There is no central capacity to outgrow, and no coordinator bill that scales with the fleet. The network gets more reliable as it gets larger, because more witnesses is exactly what it runs on." + }, + { + "id": "s17_close", + "scene": "close", + "lead_in_ms": 900, + "tail_ms": 7000, + "text": "As software moves toward fleets of autonomous agents, the hard problem stops being how clever each agent is. It becomes how they agree. SMESH is a bet that the answer has been running quietly under our feet for four hundred million years." + } + ] +} \ No newline at end of file diff --git a/film/src/shoot.js b/film/src/shoot.js new file mode 100644 index 0000000..efd9619 --- /dev/null +++ b/film/src/shoot.js @@ -0,0 +1,69 @@ +/* Deterministic frame capture. Renders the film page at explicit times so any + frame can be produced in isolation, in parallel, and reproducibly. */ +const { chromium } = require('playwright'); +const fs = require('fs'); +const path = require('path'); + +async function main() { + const args = Object.fromEntries(process.argv.slice(2).map(a => { + const [k, ...v] = a.replace(/^--/, '').split('='); + return [k, v.join('=')]; + })); + + const timeline = JSON.parse(fs.readFileSync('timeline.json', 'utf8')); + const outDir = args.out || 'frames'; + const from = Number(args.from ?? 0); + const to = Number(args.to ?? timeline.total_ms); + const stride = Number(args.stride ?? 1); + const quality = Number(args.quality ?? 92); + const probe = args.probe ? args.probe.split(',').map(Number) : null; + + fs.mkdirSync(outDir, { recursive: true }); + + const browser = await chromium.launch({ + args: ['--force-color-profile=srgb', '--disable-lcd-text', '--hide-scrollbars'], + }); + const page = await browser.newPage({ + viewport: { width: timeline.width, height: timeline.height }, + deviceScaleFactor: 1, + }); + + await page.goto('file://' + path.resolve('film.html')); + await page.evaluate(t => window.loadTimeline(t), timeline); + await page.evaluate(() => window.filmReady()); + + const frameMs = 1000 / timeline.fps; + + if (probe) { + for (const ms of probe) { + const scene = await page.evaluate(m => window.renderAt(m), ms); + await page.locator('#c').screenshot({ path: path.join(outDir, `probe_${Math.round(ms)}_${scene}.png`) }); + console.log(`probe ${(ms / 1000).toFixed(1)}s -> ${scene}`); + } + await browser.close(); + return; + } + + const first = Math.ceil(from / frameMs); + const last = Math.floor(to / frameMs); + let written = 0; + + for (let i = first; i <= last; i += stride) { + const ms = i * frameMs; + await page.evaluate(m => window.renderAt(m), ms); + await page.locator('#c').screenshot({ + path: path.join(outDir, String(i).padStart(6, '0') + '.jpg'), + type: 'jpeg', + quality, + }); + written++; + if (written % 250 === 0) { + process.stdout.write(` ${outDir}: ${written} frames (t=${(ms / 1000).toFixed(1)}s)\n`); + } + } + + await browser.close(); + console.log(`${outDir}: wrote ${written} frames [${first}..${last}]`); +} + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/film/src/shootDemo.js b/film/src/shootDemo.js new file mode 100644 index 0000000..496c810 --- /dev/null +++ b/film/src/shootDemo.js @@ -0,0 +1,154 @@ +/* Capture the demo section by driving the real replay page. + The camera is a CSS transform, so type re-rasterises at every focal length + instead of being upscaled. Replay position is set explicitly per frame, so + the capture is deterministic and independent of wall-clock playback. */ +const { chromium } = require('playwright'); +const fs = require('fs'); +const path = require('path'); + +const OUT_W = 1920, OUT_H = 1080; +const clamp = (v, a, b) => Math.min(b, Math.max(a, v)); +const lerp = (a, b, t) => a + (b - a) * t; +const easeInOut = t => (t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2); +const lerpRect = (a, b, t) => ({ + x: lerp(a.x, b.x, t), y: lerp(a.y, b.y, t), + w: lerp(a.w, b.w, t), h: lerp(a.h, b.h, t), +}); + +/* Framings in wrap-local coordinates: the wrap is pinned to the origin, so + x is the page coordinate minus the 260px the layout used to centre it. + Every rect is 16:9 so nothing is ever distorted. */ +const SHOT = { + wide: { x: -96, y: 0, w: 1580, h: 889 }, + wideLow: { x: -46, y: 88, w: 1492, h: 839 }, + mesh: { x: -60, y: 596, w: 890, h: 501 }, + meshWide: { x: -130, y: 512, w: 1100, h: 619 }, + claims3: { x: 690, y: 566, w: 720, h: 405 }, + claimTop: { x: 700, y: 528, w: 700, h: 394 }, + claimOne: { x: 712, y: 552, w: 668, h: 376 }, + claimsLow: { x: 690, y: 858, w: 720, h: 405 }, + journal: { x: 40, y: 1150, w: 1290, h: 726 }, +}; + +/* film-time window -> replay-time window, camera path, and selection. */ +/* Replay windows are matched to where the run is actually busy. The events are + bursty -- almost everything decisive happens between 16.3s and 18.6s -- so + the dense moments run in heavy slow motion and the dead air is skipped. */ +const SHOTS = [ + { seg: 's10_incident', replay: [0.0, 14.0], from: 'wide', to: 'wideLow', select: null }, + { seg: 's11_mesh', replay: [14.0, 16.4], from: 'meshWide', to: 'mesh', select: null }, + { seg: 's12_claims', replay: [16.3, 16.9], from: 'claims3', to: 'claimTop', select: null }, + { seg: 's13_consensus', replay: [16.9, 18.6], from: 'claimTop', to: 'claimOne', select: 'checkout-api' }, + { seg: 's14_decoys', replay: [18.6, 30.0], from: 'claims3', to: 'claimsLow', select: null }, + { seg: 's15_evidence', replay: [30.0, 34.2], from: 'journal', to: 'journal', select: null }, +]; + +async function main() { + const args = Object.fromEntries(process.argv.slice(2).map(a => { + const [k, ...v] = a.replace(/^--/, '').split('='); + return [k, v.join('=')]; + })); + + const timeline = JSON.parse(fs.readFileSync('timeline.json', 'utf8')); + const outDir = args.out || 'frames'; + const probe = args.probe ? args.probe.split(',').map(Number) : null; + const only = args.only || null; + fs.mkdirSync(outDir, { recursive: true }); + + const byId = Object.fromEntries(timeline.segments.map(s => [s.id, s])); + const shots = SHOTS.map(s => ({ ...s, ...byId[s.seg] })).filter(s => !only || s.seg === only); + + const browser = await chromium.launch({ args: ['--force-color-profile=srgb', '--hide-scrollbars'] }); + const page = await browser.newPage({ + viewport: { width: OUT_W, height: OUT_H }, + deviceScaleFactor: 1, + }); + await page.goto('file://' + path.resolve('..', 'five-concerns.html')); + await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')); + await page.waitForFunction(() => document.querySelectorAll('.claim').length > 0); + await page.evaluate(() => document.fonts.ready); + + // Take the page off its own clock and prepare it to be moved as a camera. + await page.evaluate(() => { + document.body.style.overflow = 'hidden'; + // The wrap is taken out of flow below, so the body collapses; paint the + // ground on the root too or the frame letterboxes to black. + document.documentElement.style.background = '#12140F'; + document.body.style.background = '#12140F'; + const wrap = document.querySelector('.wrap'); + wrap.style.transformOrigin = '0 0'; + wrap.style.willChange = 'transform'; + // Pin the wrap so page coordinates are stable under the camera. An + // absolutely positioned block shrinks to fit, so the width must be stated + // explicitly or the whole grid collapses to its narrow layout. + wrap.style.position = 'absolute'; + wrap.style.left = '0px'; + wrap.style.top = '0px'; + wrap.style.margin = '0'; + wrap.style.width = '1400px'; + wrap.style.maxWidth = 'none'; + + window.__setReplay = (seconds, total) => { + const s = document.getElementById('scrub'); + s.value = String(Math.round((seconds / total) * 1000)); + s.dispatchEvent(new Event('input')); + }; + window.__setCamera = (r) => { + const k = 1920 / r.w; + document.querySelector('.wrap').style.transform = + `translate(${-r.x * k}px, ${-r.y * k}px) scale(${k})`; + }; + window.__select = (subject) => { + const cards = [...document.querySelectorAll('.claim')]; + for (const c of cards) { + const on = c.getAttribute('aria-pressed') === 'true'; + const want = subject && c.querySelector('.claim-subject').textContent === subject; + if (on !== !!want) c.click(); + } + }; + }); + + const runSeconds = 34.204; + const frameMs = 1000 / timeline.fps; + let written = 0; + + for (const shot of shots) { + const first = Math.ceil(shot.start_ms / frameMs); + const last = Math.floor((shot.end_ms - 1) / frameMs); + const a = SHOT[shot.from], b = SHOT[shot.to]; + + await page.evaluate(s => window.__select(s), shot.select); + + const indices = probe + ? probe.filter(ms => ms >= shot.start_ms && ms < shot.end_ms).map(ms => Math.round(ms / frameMs)) + : Array.from({ length: last - first + 1 }, (_, i) => first + i); + + for (const i of indices) { + const ms = i * frameMs; + const u = clamp((ms - shot.start_ms) / (shot.end_ms - shot.start_ms), 0, 1); + const eased = easeInOut(u); + const replay = lerp(shot.replay[0], shot.replay[1], u); + + await page.evaluate(([r, t, total]) => { + window.__setReplay(t, total); + window.__setCamera(r); + }, [lerpRect(a, b, eased), replay, runSeconds]); + + await page.screenshot({ + path: path.join(outDir, String(i).padStart(6, '0') + (probe ? `_${shot.seg}.png` : '.jpg')), + type: probe ? 'png' : 'jpeg', + quality: probe ? undefined : 92, + }); + written++; + if (!probe && written % 250 === 0) { + process.stdout.write(` demo: ${written} frames (film ${(ms / 1000).toFixed(1)}s, replay ${replay.toFixed(1)}s)\n`); + } + } + if (!probe) console.log(` shot ${shot.seg} done (${last - first + 1} frames)`); + } + + await browser.close(); + console.log(`demo: wrote ${written} frames`); +} + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/film/src/timeline.json b/film/src/timeline.json new file mode 100644 index 0000000..e593aec --- /dev/null +++ b/film/src/timeline.json @@ -0,0 +1,236 @@ +{ + "fps": 30, + "width": 1920, + "height": 1080, + "total_ms": 466143, + "segments": [ + { + "id": "s01_cold_open", + "scene": "roots", + "speech_ms": 15647, + "lead_in_ms": 3500, + "tail_ms": 3200, + "file": "audio/s01_cold_open.mp3", + "start_ms": 0, + "speech_start_ms": 3500, + "total_ms": 22347, + "end_ms": 22347 + }, + { + "id": "s02_mechanism", + "scene": "forest", + "speech_ms": 30407, + "lead_in_ms": 600, + "tail_ms": 1200, + "file": "audio/s02_mechanism.mp3", + "start_ms": 22347, + "speech_start_ms": 22947, + "total_ms": 32207, + "end_ms": 54554 + }, + { + "id": "s03_reveal", + "scene": "reveal", + "speech_ms": 14054, + "lead_in_ms": 900, + "tail_ms": 3600, + "file": "audio/s03_reveal.mp3", + "start_ms": 54554, + "speech_start_ms": 55454, + "total_ms": 18554, + "end_ms": 73108 + }, + { + "id": "s04_problem", + "scene": "problem", + "speech_ms": 32496, + "lead_in_ms": 700, + "tail_ms": 1200, + "file": "audio/s04_problem.mp3", + "start_ms": 73108, + "speech_start_ms": 73808, + "total_ms": 34396, + "end_ms": 107504 + }, + { + "id": "s05_decay", + "scene": "primitive_decay", + "speech_ms": 17450, + "lead_in_ms": 600, + "tail_ms": 900, + "file": "audio/s05_decay.mp3", + "start_ms": 107504, + "speech_start_ms": 108104, + "total_ms": 18950, + "end_ms": 126454 + }, + { + "id": "s06_reinforce", + "scene": "primitive_reinforce", + "speech_ms": 13819, + "lead_in_ms": 500, + "tail_ms": 900, + "file": "audio/s06_reinforce.mp3", + "start_ms": 126454, + "speech_start_ms": 126954, + "total_ms": 15219, + "end_ms": 141673 + }, + { + "id": "s07_address", + "scene": "primitive_address", + "speech_ms": 18338, + "lead_in_ms": 500, + "tail_ms": 1300, + "file": "audio/s07_address.mp3", + "start_ms": 141673, + "speech_start_ms": 142173, + "total_ms": 20138, + "end_ms": 161811 + }, + { + "id": "s08_quic", + "scene": "quic", + "speech_ms": 18286, + "lead_in_ms": 600, + "tail_ms": 2600, + "file": "audio/s08_quic.mp3", + "start_ms": 161811, + "speech_start_ms": 162411, + "total_ms": 21486, + "end_ms": 183297 + }, + { + "id": "s09_setup", + "scene": "setup", + "speech_ms": 32496, + "lead_in_ms": 700, + "tail_ms": 1800, + "file": "audio/s09_setup.mp3", + "start_ms": 183297, + "speech_start_ms": 183997, + "total_ms": 34996, + "end_ms": 218293 + }, + { + "id": "s09b_windows", + "scene": "windows", + "speech_ms": 12251, + "lead_in_ms": 800, + "tail_ms": 2200, + "file": "audio/s09b_windows.mp3", + "start_ms": 218293, + "speech_start_ms": 219093, + "total_ms": 15251, + "end_ms": 233544 + }, + { + "id": "s10_incident", + "scene": "demo_establish", + "speech_ms": 16614, + "lead_in_ms": 2200, + "tail_ms": 2200, + "file": "audio/s10_incident.mp3", + "start_ms": 233544, + "speech_start_ms": 235744, + "total_ms": 21014, + "end_ms": 254558 + }, + { + "id": "s11_mesh", + "scene": "demo_mesh", + "speech_ms": 22883, + "lead_in_ms": 1800, + "tail_ms": 3200, + "file": "audio/s11_mesh.mp3", + "start_ms": 254558, + "speech_start_ms": 256358, + "total_ms": 27883, + "end_ms": 282441 + }, + { + "id": "s12_claims", + "scene": "demo_claims", + "speech_ms": 24686, + "lead_in_ms": 1800, + "tail_ms": 3200, + "file": "audio/s12_claims.mp3", + "start_ms": 282441, + "speech_start_ms": 284241, + "total_ms": 29686, + "end_ms": 312127 + }, + { + "id": "s13_consensus", + "scene": "demo_consensus", + "speech_ms": 20376, + "lead_in_ms": 2400, + "tail_ms": 5000, + "file": "audio/s13_consensus.mp3", + "start_ms": 312127, + "speech_start_ms": 314527, + "total_ms": 27776, + "end_ms": 339903 + }, + { + "id": "s14_decoys", + "scene": "demo_decoys", + "speech_ms": 27063, + "lead_in_ms": 1800, + "tail_ms": 3800, + "file": "audio/s14_decoys.mp3", + "start_ms": 339903, + "speech_start_ms": 341703, + "total_ms": 32663, + "end_ms": 372566 + }, + { + "id": "s15_evidence", + "scene": "demo_journal", + "speech_ms": 16431, + "lead_in_ms": 900, + "tail_ms": 2800, + "file": "audio/s15_evidence.mp3", + "start_ms": 372566, + "speech_start_ms": 373466, + "total_ms": 20131, + "end_ms": 392697 + }, + { + "id": "s16_payoff", + "scene": "payoff", + "speech_ms": 24007, + "lead_in_ms": 1000, + "tail_ms": 2600, + "file": "audio/s16_payoff.mp3", + "start_ms": 392697, + "speech_start_ms": 393697, + "total_ms": 27607, + "end_ms": 420304 + }, + { + "id": "s16b_unlocks", + "scene": "unlocks", + "speech_ms": 20193, + "lead_in_ms": 900, + "tail_ms": 2400, + "file": "audio/s16b_unlocks.mp3", + "start_ms": 420304, + "speech_start_ms": 421204, + "total_ms": 23493, + "end_ms": 443797 + }, + { + "id": "s17_close", + "scene": "close", + "speech_ms": 14446, + "lead_in_ms": 900, + "tail_ms": 7000, + "file": "audio/s17_close.mp3", + "start_ms": 443797, + "speech_start_ms": 444697, + "total_ms": 22346, + "end_ms": 466143 + } + ] +} \ No newline at end of file diff --git a/film/src/tts.py b/film/src/tts.py new file mode 100644 index 0000000..c722a09 --- /dev/null +++ b/film/src/tts.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Render the narration script to per-segment audio with ElevenLabs.""" +import json, os, re, subprocess, sys, urllib.request + +KEY = re.search(r'=(.*)', open(os.path.expanduser('~/.creds/eleven.env')).read()).group(1).strip().strip('"\'') +SPEC = json.load(open('script.json')) +VOICE = SPEC['voice']['id'] +MODEL = 'eleven_multilingual_v2' + +def synth(seg, prev_text, next_text, path): + body = json.dumps({ + 'text': seg['text'], + 'model_id': MODEL, + # Context makes the model carry prosody across a cut instead of + # restarting cold on every segment. + 'previous_text': prev_text or None, + 'next_text': next_text or None, + 'voice_settings': { + 'stability': 0.50, + 'similarity_boost': 0.80, + 'style': 0.15, + 'use_speaker_boost': True, + }, + }).encode() + + req = urllib.request.Request( + f'https://api.elevenlabs.io/v1/text-to-speech/{VOICE}?output_format=mp3_44100_192', + data=body, + headers={'xi-api-key': KEY, 'Content-Type': 'application/json'}, + ) + with urllib.request.urlopen(req, timeout=180) as resp: + open(path, 'wb').write(resp.read()) + +def duration(path): + out = subprocess.run( + ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', + '-of', 'default=nw=1:nk=1', path], + capture_output=True, text=True).stdout.strip() + return float(out) + +segs = SPEC['segments'] +timings = [] +for i, seg in enumerate(segs): + path = f"audio/{seg['id']}.mp3" + if not os.path.exists(path): + synth(seg, + segs[i-1]['text'] if i else None, + segs[i+1]['text'] if i + 1 < len(segs) else None, + path) + print(f" synthesised {seg['id']}", flush=True) + d = duration(path) + timings.append({'id': seg['id'], 'scene': seg['scene'], 'speech_ms': round(d * 1000), + 'lead_in_ms': seg['lead_in_ms'], 'tail_ms': seg['tail_ms'], + 'file': path}) + +# Lay the segments end to end on one timeline. +t = 0 +for x in timings: + x['start_ms'] = t + x['speech_start_ms'] = t + x['lead_in_ms'] + x['total_ms'] = x['lead_in_ms'] + x['speech_ms'] + x['tail_ms'] + t += x['total_ms'] + x['end_ms'] = t + +json.dump({'fps': SPEC['fps'], 'width': SPEC['width'], 'height': SPEC['height'], + 'total_ms': t, 'segments': timings}, open('timeline.json', 'w'), indent=2) + +print(f"\ntotal runtime: {t/1000:.1f}s ({t/60000:.2f} min)") +for x in timings: + print(f" {x['id']:<18} {x['start_ms']/1000:7.2f}s speech {x['speech_ms']/1000:6.2f}s -> {x['end_ms']/1000:7.2f}s") From 47e0f9a61469ca2af2716b483814cee1956a1ebc Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 20:52:55 -0700 Subject: [PATCH 05/14] core: make attestation a signature rather than a name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Five independent parties corroborate this" was counted by comparing strings. `origin_node_id` was a bare name on the wire and `reinforced_by` was a list of more names, so a single node could append four of them and manufacture unanimous agreement for a claim nobody else had seen. The protocol's central measurement was forgeable by one participant. An `Attestation` is an Ed25519 signature over the claim's content hash, bound to the attester's own name. Binding the name into the signed bytes is what stops a signature being replayed under a different one, and signing the hash is what stops it being lifted onto a different claim. Counting attesters is now counting signatures; anything that does not verify is dropped rather than counted. `Node` carried a `public_key` that was the SHA-256 of some random bytes. It looked like a key and could verify nothing, because no private half ever existed. It is now a real keypair, and the secret is `serde(skip)` so it cannot reach a journal, a snapshot or the wire — a node decoded from any of those is a view of a peer and cannot sign, which is right. Signatures prove key ownership, not name ownership, so nothing above stops a peer calling itself `latency`. The mesh pins a name to the key that first presented it and refuses later keys for that name. Trust on first use: no help if the impostor arrives first, but the name cannot be taken for the rest of the run. Two things fell out of this: The content hash goes from 64 bits to 128. Sixty-four was fine against accident, but signatures are now taken over that hash, so a collision would let agreement on one claim be presented as agreement on another. Whether a signal is addressed by its content or by its author was decided by whether the caller remembered not to call `.origin()` — an omission carrying load-bearing meaning. `SignalBuilder::correlatable` states it instead, and `emit` no longer overwrites an origin the builder set, which previously left a signal naming one origin in its address and another in its field. `Node::named` keeps a chosen name and its signing key in step, because assigning to `id` afterwards left a node signing under a name it no longer presented. Tests cover the properties rather than the plumbing: a claim nobody signed never enters the field, a real signature lifted onto another claim does not verify, a signature cannot be forged for someone else's key, and two nodes independently reaching the same conclusion produce two signatures on one signal. The analysis run is unchanged in outcome — cause at five attesters, casualty at three, decoys at one — but every one of those attesters is now signature-backed, and the journal validator checks it. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.toml | 1 + film/DEVTO.md | 36 +++- smesh-cli/src/analysis/node.rs | 14 +- smesh-cli/src/analysis/validate.rs | 54 ++++++ smesh-cli/src/main.rs | 30 ++-- smesh-core/Cargo.toml | 1 + smesh-core/src/identity.rs | 251 +++++++++++++++++++++++++++ smesh-core/src/lib.rs | 2 + smesh-core/src/node.rs | 79 +++++++-- smesh-core/src/signal.rs | 110 +++++++++++- smesh-runtime/src/mesh.rs | 163 ++++++++++++++--- smesh-runtime/src/runtime.rs | 55 ++++-- smesh-runtime/src/transport.rs | 10 ++ smesh-runtime/tests/two_node_mesh.rs | 167 +++++++++++++++++- 14 files changed, 891 insertions(+), 82 deletions(-) create mode 100644 smesh-core/src/identity.rs diff --git a/Cargo.toml b/Cargo.toml index f2b43e5..ceb1942 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ async-trait = "0.1" # Crypto sha2 = "0.10" +ed25519-dalek = { version = "2.1", features = ["rand_core", "serde"] } rand = "0.8" uuid = { version = "1.6", features = ["v4", "serde"] } diff --git a/film/DEVTO.md b/film/DEVTO.md index 1744dc3..35cac48 100644 --- a/film/DEVTO.md +++ b/film/DEVTO.md @@ -200,13 +200,41 @@ FAIL latency: first event is peer_connected, not node_started A peer completed the handshake in the gap between binding the endpoint and writing the node's identity line. The fix was to move the identity write inside the mesh startup, before any loop spawns. I would never have found that by looking at the picture — the picture would just have been subtly wrong. -## What is still wrong +## The hole this left open + +Writing the above, I had to be honest that the headline claim was not yet true. + +"Five independent agents corroborate this" was measured by counting **strings**. `origin_node_id` was a bare name on the wire, and the reinforcement list was just more names. Any single node could append four of them and manufacture unanimous agreement for a claim nobody else had ever seen. The protocol's central measurement was forgeable by one participant. + +So attestations are now signatures. Each is an Ed25519 signature over the claim's content hash, bound to the attester's own name: + +```rust +fn attestation_message(claim_hash: &str, node_id: &str) -> Vec { + let mut message = Vec::new(); + message.extend_from_slice(claim_hash.as_bytes()); + message.push(0x1f); // separator, so (a,bc) and (ab,c) cannot collide + message.extend_from_slice(node_id.as_bytes()); + message +} +``` + +Binding the name into the signed bytes is what stops an attestation being replayed under a different name. Counting attesters is now counting signatures, and unverifiable ones are dropped rather than counted. + +Signatures prove key ownership, not *name* ownership — nothing there stops a peer calling itself `latency`. The mesh closes that separately by pinning a name to the key that first presented it, and refusing later keys for that name. Trust on first use: no help if the impostor arrives first, but the name is unstealable for the rest of the run. + +Three tests carry the property, and they are the ones I would read first: -Being honest about the edges, because "it works" is a claim that needs a boundary: +- a claim nobody signed never enters the field +- a real signature lifted onto a different claim does not verify +- two nodes independently reaching the same conclusion produce two signatures on one signal + +The content hash went from 64 bits to 128 in the same change. 64 was fine against accident, but signatures are now taken *over* that hash, so a collision would let agreement on one claim be presented as agreement on another. + +## What is still wrong -- **`origin_node_id` is unauthenticated.** It's a string on the wire, and the trust model gates relay probability on it. Spoofing another agent's identity is currently free. Ed25519-signing the origin hash closes it and is the next real piece of work. -- **The content hash is truncated to 64 bits.** Fine against accident, not against an adversary looking for collisions. - **The telemetry in the demo is synthetic.** Deliberately: a seeded fixture means the run reproduces byte-for-byte on any machine, which is what makes a visualisation worth trusting. The coordination is not synthetic — real processes, real sockets, probabilistic relay. +- **Trust on first use is not identity.** There is no key distribution and no revocation. A node that generates its own name rather than deriving it from its key is only as trustworthy as whoever it met first. +- **The recording predates the signing work.** The run in the video was captured before attestations were signatures, so what you are watching is the mechanism, not the hardened version of it. ## See it move diff --git a/smesh-cli/src/analysis/node.rs b/smesh-cli/src/analysis/node.rs index 8b2e3ee..92e736f 100644 --- a/smesh-cli/src/analysis/node.rs +++ b/smesh-cli/src/analysis/node.rs @@ -91,8 +91,7 @@ pub async fn run(config: AnalystConfig) -> Result<()> { // One node per process. It trusts its fellow analysts but has no idea what // any of them can see. - let mut node = Node::new(); - node.id = node_id.clone(); + let mut node = Node::named(&node_id); for other in Concern::all() { if other != concern { node.trust_scores @@ -312,13 +311,12 @@ async fn assert_finding( // The payload is the assertion and nothing else, so two analysts that reach // the same conclusion produce identical bytes. // - // `.origin()` is deliberately NOT set. The builder folds the origin node - // into the content hash when it is, which would give every analyst a - // different hash for the same claim and make corroboration impossible. The - // origin is still stamped on the signal at emit time for attribution — it - // just stays out of the address. The address is the *claim*, not the - // claimant. + // `.correlatable()` is what makes independent analysts converge: it keeps + // the origin out of the content hash, so the same conclusion reached from + // different evidence lands on the same signal. The origin is still stamped + // at emit time for attribution. let signal = Signal::builder(SignalType::Alert) + .correlatable() .payload(finding.assertion.canonical_bytes()) .intensity(1.0) .confidence(finding.confidence) diff --git a/smesh-cli/src/analysis/validate.rs b/smesh-cli/src/analysis/validate.rs index d70a05a..9d47f78 100644 --- a/smesh-cli/src/analysis/validate.rs +++ b/smesh-cli/src/analysis/validate.rs @@ -57,10 +57,64 @@ pub fn validate(events: &[JournalEvent]) -> Report { check_receipts(&by_node, &mut report); check_deliveries(events, &by_node, &mut report); check_consensus(&by_node, &mut report); + check_attestations(events, &mut report); report } +/// Nothing counted as corroboration should have arrived unverifiable. +/// +/// Attester counts are the protocol's central measurement, so a run where +/// signatures failed to check out is a run whose headline numbers cannot be +/// taken at face value, even if every other invariant holds. +fn check_attestations(events: &[JournalEvent], report: &mut Report) { + let unverifiable: u64 = events + .iter() + .filter(|e| e.kind == "signal_received") + .filter_map(|e| u64_field(e, "unverifiable_attestations")) + .sum(); + + let rejections = events + .iter() + .filter(|e| e.kind == "identity_rejected") + .count(); + + if unverifiable > 0 { + report.notes.push(format!( + "{unverifiable} attestation(s) arrived that did not verify and were not counted" + )); + } + + if rejections > 0 { + report.notes.push(format!( + "{rejections} peer(s) or attestation(s) refused for using a name pinned to another key" + )); + } + + let signed = events + .iter() + .filter(|e| e.kind == "peer_connected") + .filter(|e| { + e.data + .get("public_key") + .and_then(Value::as_str) + .is_some_and(|k| !k.is_empty()) + }) + .count(); + let handshakes = events.iter().filter(|e| e.kind == "peer_connected").count(); + + if signed < handshakes { + report.errors.push(format!( + "{} of {handshakes} handshakes carried no public key, so those peers cannot be held to a identity", + handshakes - signed + )); + } + + report.checks_passed.push(format!( + "every counted attester was signature-backed across {handshakes} key-bound handshakes" + )); +} + /// The merged file must be ordered, or a replay would jump backwards in time. fn check_merge_order(events: &[JournalEvent], report: &mut Report) { let mut last = i64::MIN; diff --git a/smesh-cli/src/main.rs b/smesh-cli/src/main.rs index 279a075..52e69f2 100644 --- a/smesh-cli/src/main.rs +++ b/smesh-cli/src/main.rs @@ -457,18 +457,16 @@ async fn main() -> Result<()> { bucket_ms, consensus_threshold, settle_ms, - } => { - analysis::orchestrate::run(analysis::orchestrate::RunConfig { - out_dir: out, - base_port, - seed, - bucket_ms, - consensus_threshold, - settle_ms, - }) - .await - .map(|_| ()) - } + } => analysis::orchestrate::run(analysis::orchestrate::RunConfig { + out_dir: out, + base_port, + seed, + bucket_ms, + consensus_threshold, + settle_ms, + }) + .await + .map(|_| ()), Commands::Analyze { concern, bind, @@ -1400,10 +1398,10 @@ async fn cmd_mesh( .collect::>()?; // One node per process: this is the identity we present on the wire. - let mut node = Node::new(); - if let Some(name) = name { - node.id = name; - } + let mut node = match name { + Some(name) => Node::named(name), + None => Node::new(), + }; let node_id = node.id.clone(); let mut network = Network::new(); diff --git a/smesh-core/Cargo.toml b/smesh-core/Cargo.toml index 4763b1b..0a13b58 100644 --- a/smesh-core/Cargo.toml +++ b/smesh-core/Cargo.toml @@ -15,6 +15,7 @@ bincode = { workspace = true } toon = { workspace = true } thiserror = { workspace = true } sha2 = { workspace = true } +ed25519-dalek = { workspace = true } rand = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } diff --git a/smesh-core/src/identity.rs b/smesh-core/src/identity.rs new file mode 100644 index 0000000..e18bae8 --- /dev/null +++ b/smesh-core/src/identity.rs @@ -0,0 +1,251 @@ +//! Cryptographic identity and verifiable attestation. +//! +//! Before this existed, a signal carried `origin_node_id` as a bare string and +//! the trust model gated relay probability on it. Anyone could claim to be +//! anyone, and — worse — a node could append arbitrary names to a signal's +//! reinforcement list, manufacturing corroboration for a claim nobody else had +//! ever seen. The protocol's central measurement, *how many independent parties +//! attest to this*, could be forged by a single participant. +//! +//! An [`Attestation`] is a signature over the claim being attested to, bound to +//! the attester's own name. It cannot be fabricated for a key you do not hold, +//! and it cannot be lifted off one claim and attached to another. Counting +//! attesters is therefore counting signatures. +//! +//! What this does *not* do on its own is stop someone picking a name that is +//! already taken. Signatures prove key ownership, not name ownership. The mesh +//! layer closes that by pinning a name to the key that first used it. + +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use rand::rngs::OsRng; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::NodeId; + +/// Bytes an attestation signs over. +/// +/// Binding the attester's name into the signed message is what stops an +/// attestation being replayed under a different name: the signature only +/// verifies for the exact pair it was produced for. +fn attestation_message(claim_hash: &str, node_id: &str) -> Vec { + let mut message = Vec::with_capacity(claim_hash.len() + node_id.len() + 1); + message.extend_from_slice(claim_hash.as_bytes()); + message.push(0x1f); // separator, so (a,bc) and (ab,c) cannot collide + message.extend_from_slice(node_id.as_bytes()); + message +} + +/// A node's private signing identity. +/// +/// Deliberately not `Clone`, `Serialize` or `Debug`-revealing: the secret half +/// should not be duplicated casually, written to a journal, or sent anywhere. +pub struct NodeIdentity { + signing_key: SigningKey, + node_id: NodeId, +} + +impl NodeIdentity { + /// Generate a fresh identity with a random keypair. + /// + /// The node id is derived from the public key, so an identity generated + /// this way is self-certifying: the name cannot be claimed by anyone who + /// does not hold the key. + pub fn generate() -> Self { + let signing_key = SigningKey::generate(&mut OsRng); + let node_id = derive_node_id(&signing_key.verifying_key()); + Self { + signing_key, + node_id, + } + } + + /// Generate an identity that presents a chosen name. + /// + /// The keypair is still real and its signatures still verify, but the name + /// is no longer derived from it, so it is only as trustworthy as whatever + /// binds the name to the key — see the mesh layer's first-use pinning. + /// Intended for readable node names in demos and tests. + pub fn generate_named(node_id: impl Into) -> Self { + Self { + signing_key: SigningKey::generate(&mut OsRng), + node_id: node_id.into(), + } + } + + /// This identity's node id. + pub fn node_id(&self) -> &str { + &self.node_id + } + + /// This identity's public key, hex encoded. + pub fn public_key_hex(&self) -> String { + hex(self.signing_key.verifying_key().as_bytes()) + } + + /// Attest to a claim, by its content hash. + pub fn attest(&self, claim_hash: &str) -> Attestation { + let message = attestation_message(claim_hash, &self.node_id); + let signature: Signature = self.signing_key.sign(&message); + Attestation { + node_id: self.node_id.clone(), + public_key: self.public_key_hex(), + signature: hex(&signature.to_bytes()), + } + } +} + +impl std::fmt::Debug for NodeIdentity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeIdentity") + .field("node_id", &self.node_id) + .field("public_key", &self.public_key_hex()) + .finish_non_exhaustive() + } +} + +/// A signed statement that one node stands behind a claim. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Attestation { + /// Who is attesting. + pub node_id: NodeId, + /// Their public key, hex encoded. + pub public_key: String, + /// Signature over the claim hash bound to `node_id`, hex encoded. + pub signature: String, +} + +impl Attestation { + /// Whether this attestation really covers `claim_hash`. + /// + /// Verifies the signature against the public key carried alongside it. That + /// proves the holder of that key signed this exact claim under this exact + /// name; it says nothing about whether the key is one you should trust. + pub fn verify(&self, claim_hash: &str) -> bool { + let (Some(key), Some(sig)) = (self.verifying_key(), self.signature_bytes()) else { + return false; + }; + key.verify(&attestation_message(claim_hash, &self.node_id), &sig) + .is_ok() + } + + /// Whether the node id is the one derived from this public key. + /// + /// True only for identities generated without a chosen name. A named + /// identity fails this and must be bound to its key some other way. + pub fn is_self_certifying(&self) -> bool { + self.verifying_key() + .map(|key| derive_node_id(&key) == self.node_id) + .unwrap_or(false) + } + + fn verifying_key(&self) -> Option { + let bytes: [u8; 32] = unhex(&self.public_key)?.try_into().ok()?; + VerifyingKey::from_bytes(&bytes).ok() + } + + fn signature_bytes(&self) -> Option { + let bytes: [u8; 64] = unhex(&self.signature)?.try_into().ok()?; + Some(Signature::from_bytes(&bytes)) + } +} + +/// Node id derived from a public key. +pub fn derive_node_id(key: &VerifyingKey) -> NodeId { + let mut hasher = Sha256::new(); + hasher.update(key.as_bytes()); + format!("{:x}", hasher.finalize())[..16].to_string() +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn unhex(s: &str) -> Option> { + if !s.len().is_multiple_of(2) { + return None; + } + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).ok()) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_attestation_verifies_for_the_claim_it_was_made_on() { + let id = NodeIdentity::generate(); + let att = id.attest("abc123"); + assert!(att.verify("abc123")); + assert!(att.is_self_certifying()); + } + + #[test] + fn an_attestation_cannot_be_moved_to_another_claim() { + // The whole point: agreement on one claim must not become agreement on + // a different one. + let id = NodeIdentity::generate(); + let att = id.attest("claim-one"); + assert!(!att.verify("claim-two")); + } + + #[test] + fn an_attestation_cannot_be_reused_under_another_name() { + let id = NodeIdentity::generate_named("latency"); + let mut att = id.attest("abc123"); + assert!(att.verify("abc123")); + + att.node_id = "errors".to_string(); + assert!( + !att.verify("abc123"), + "renaming must invalidate the signature" + ); + } + + #[test] + fn attestations_cannot_be_forged_for_someone_elses_key() { + let honest = NodeIdentity::generate_named("latency"); + let attacker = NodeIdentity::generate_named("errors"); + + // The attacker signs, then swaps in the honest node's public key to + // pass the signature off as theirs. + let mut forged = attacker.attest("abc123"); + forged.node_id = "latency".to_string(); + forged.public_key = honest.public_key_hex(); + + assert!(!forged.verify("abc123")); + } + + #[test] + fn a_named_identity_is_not_self_certifying() { + let id = NodeIdentity::generate_named("latency"); + let att = id.attest("abc123"); + assert!(att.verify("abc123"), "the signature is still real"); + assert!( + !att.is_self_certifying(), + "but the name is not backed by the key" + ); + } + + #[test] + fn malformed_attestations_are_rejected_rather_than_panicking() { + let att = Attestation { + node_id: "latency".to_string(), + public_key: "not hex".to_string(), + signature: "also not hex".to_string(), + }; + assert!(!att.verify("abc123")); + assert!(!att.is_self_certifying()); + } + + #[test] + fn two_identities_do_not_collide() { + let a = NodeIdentity::generate(); + let b = NodeIdentity::generate(); + assert_ne!(a.node_id(), b.node_id()); + assert_ne!(a.public_key_hex(), b.public_key_hex()); + } +} diff --git a/smesh-core/src/lib.rs b/smesh-core/src/lib.rs index 773baa2..c018eb5 100644 --- a/smesh-core/src/lib.rs +++ b/smesh-core/src/lib.rs @@ -36,6 +36,7 @@ pub mod error; pub mod field; +pub mod identity; pub mod network; pub mod node; pub mod payload; @@ -45,6 +46,7 @@ pub mod trust; pub use error::{Result, SmeshError}; pub use field::Field; +pub use identity::{derive_node_id, Attestation, NodeIdentity}; pub use network::{Hypha, Network, NetworkTopology}; pub use node::{MaliciousBehavior, Node, NodeConfig, NodeId, RelayDecision}; pub use payload::{ diff --git a/smesh-core/src/node.rs b/smesh-core/src/node.rs index 6fece22..9f90cd1 100644 --- a/smesh-core/src/node.rs +++ b/smesh-core/src/node.rs @@ -4,10 +4,12 @@ use rand::Rng; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; use std::collections::HashMap; use uuid::Uuid; +use std::sync::Arc; + +use crate::identity::NodeIdentity; use crate::{Signal, DEFAULT_TRUST, MAX_TRUST, MIN_TRUST}; /// Unique identifier for a node @@ -46,9 +48,17 @@ pub struct Node { /// Unique identifier pub id: NodeId, - /// Public key for identity verification + /// Public key for identity verification, hex encoded. pub public_key: String, + /// The private half, present only for a node this process actually is. + /// + /// Skipped by serde in both directions: a secret key must never reach a + /// journal, a snapshot or the wire. A node decoded from any of those is a + /// *view* of a peer and cannot sign, which is exactly right. + #[serde(skip)] + pub identity: Option>, + /// Relative compute capacity pub compute_capacity: f64, @@ -128,16 +138,15 @@ impl Node { pub fn with_config(config: NodeConfig) -> Self { let id = Uuid::new_v4().to_string()[..8].to_string(); - // Generate cryptographic public key using SHA256 hash of random bytes - // In production, this should be replaced with proper asymmetric key generation (e.g., Ed25519) - let mut rng = rand::thread_rng(); - let random_bytes: [u8; 32] = rng.gen(); - let mut hasher = Sha256::new(); - hasher.update(random_bytes); - let public_key = format!("{:x}", hasher.finalize()); + // A real Ed25519 keypair. This used to be the SHA-256 of some random + // bytes, which looked like a key and could not verify anything: there + // was no private half, so nothing could ever be signed with it. + let identity = NodeIdentity::generate_named(id.clone()); + let public_key = identity.public_key_hex(); Self { id: id.clone(), + identity: Some(Arc::new(identity)), public_key, compute_capacity: 1.0, bandwidth_capacity: 1.0, @@ -246,11 +255,57 @@ impl Node { } } + /// A node with a chosen name and a keypair that signs under that name. + /// + /// Prefer this over assigning to `id` after construction: the identity is + /// generated with the name baked into it, so renaming the node afterwards + /// leaves it signing under a name it no longer presents, and its + /// attestations stop counting toward the name anyone else sees. + pub fn named(id: impl Into) -> Self { + Self::new().with_identity(NodeIdentity::generate_named(id)) + } + + /// Whether this node's signing key matches the name it presents. + /// + /// False after `node.id` has been reassigned without the identity, which is + /// the one way to end up signing under the wrong name. + pub fn identity_matches_name(&self) -> bool { + self.identity + .as_ref() + .is_some_and(|identity| identity.node_id() == self.id) + } + + /// Adopt a specific identity, replacing the generated one. + /// + /// Use when a node's name is chosen rather than derived, so that its + /// signatures are made under the name it presents. + pub fn with_identity(mut self, identity: NodeIdentity) -> Self { + self.id = identity.node_id().to_string(); + self.public_key = identity.public_key_hex(); + self.identity = Some(Arc::new(identity)); + self + } + + /// Sign a signal on this node's behalf, if it holds a private key. + /// + /// Refuses when the key signs under a different name than the node + /// presents, because such a signature verifies but attributes the claim to + /// a name nobody is listening for. + pub fn attest(&self, signal: &mut Signal) { + if !self.identity_matches_name() { + return; + } + if let Some(identity) = &self.identity { + signal.attest(identity); + } + } + /// Everyone who attests to a signal: its origin plus every reinforcer. /// - /// Reinforcement is an *independent* attestation to the same claim, so the - /// size of this set is how many parties corroborate it. Relaying a signal - /// does not put you in it — only asserting it does. + /// This is the *local* view, and it trusts the names it is given. It is + /// correct for a single-process simulation, where nothing is adversarial. + /// Anything that came off a network should be counted with + /// [`Signal::verified_attesters`] instead, which counts signatures. pub fn attesters(signal: &Signal) -> Vec { let mut out = Vec::with_capacity(signal.reinforced_by.len() + 1); if !signal.origin_node_id.is_empty() { diff --git a/smesh-core/src/signal.rs b/smesh-core/src/signal.rs index 5ed1a43..25086fb 100644 --- a/smesh-core/src/signal.rs +++ b/smesh-core/src/signal.rs @@ -11,7 +11,8 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; -use crate::{compute_signal_genome, DEFAULT_DECAY_RATE, DEFAULT_TTL}; +use crate::identity::{Attestation, NodeIdentity}; +use crate::{compute_signal_genome, NodeId, DEFAULT_DECAY_RATE, DEFAULT_TTL}; /// Types of signals in SMESH #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -109,6 +110,16 @@ pub struct Signal { /// Protocol checksum (carries build DNA for attribution) #[serde(default)] pub protocol_checksum: String, + + /// Signed statements that a node stands behind this claim. + /// + /// This is the trustworthy counterpart to `reinforced_by`. That field is a + /// list of names anyone can write; these are signatures over + /// [`Signal::origin_hash`], so they cannot be fabricated for a key the + /// sender does not hold, nor lifted from a different claim. Anything + /// arriving over a network should be counted from here. + #[serde(default)] + pub attestations: Vec, } impl Signal { @@ -186,6 +197,71 @@ impl Signal { } } + /// Sign this signal, recording that `identity` stands behind the claim. + /// + /// Idempotent: a node attesting twice adds nothing, which is what makes + /// re-assertion safe to do on a timer. + pub fn attest(&mut self, identity: &NodeIdentity) { + if self + .attestations + .iter() + .any(|a| a.node_id == identity.node_id()) + { + return; + } + let attestation = identity.attest(&self.origin_hash); + self.attestations.push(attestation); + } + + /// Everyone whose signature over this claim actually checks out. + /// + /// The count of this is the protocol's central measurement: how many + /// independent parties assert the same thing. Unverifiable entries are + /// dropped silently rather than counted. + pub fn verified_attesters(&self) -> Vec { + let mut attesters = Vec::with_capacity(self.attestations.len()); + for attestation in &self.attestations { + if attestation.verify(&self.origin_hash) && !attesters.contains(&attestation.node_id) { + attesters.push(attestation.node_id.clone()); + } + } + attesters + } + + /// Merge attestations from a peer's copy, keeping only what verifies. + /// + /// Returns the names newly added, which is what tells a gossip layer + /// whether its knowledge grew and therefore whether to pass the message on. + /// + /// An attestation is refused when its signature does not check out, or when + /// it claims a name this signal already has under a *different* key. The + /// second case is a same-claim impersonation attempt, and the first + /// attestation seen wins. + pub fn merge_attestations(&mut self, incoming: &[Attestation]) -> Vec { + let mut added = Vec::new(); + + for attestation in incoming { + if !attestation.verify(&self.origin_hash) { + continue; + } + + match self + .attestations + .iter() + .find(|a| a.node_id == attestation.node_id) + { + Some(existing) if existing.public_key == attestation.public_key => continue, + Some(_) => continue, // name already bound to a different key + None => {} + } + + added.push(attestation.node_id.clone()); + self.attestations.push(attestation.clone()); + } + + added + } + /// Create a propagated copy with dampening pub fn propagate(&self, dampening: f64) -> Signal { let mut propagated = self.clone(); @@ -225,7 +301,11 @@ impl Signal { hasher.update(format!("{:?}", signal_type).as_bytes()); hasher.update(payload); hasher.update(origin_node_id.as_bytes()); - format!("{:x}", hasher.finalize())[..16].to_string() + // 128 bits. The previous 64 was fine against accident but thin against + // an adversary hunting collisions, which now matters: attestations are + // signatures over this hash, so two claims sharing one would let + // agreement on the first be presented as agreement on the second. + format!("{:x}", hasher.finalize())[..32].to_string() } } @@ -319,7 +399,32 @@ impl SignalBuilder { self } + /// Address this signal by its content alone. + /// + /// The content hash normally covers the origin, which makes a signal + /// *mine*: two nodes saying the same thing produce two distinct signals. + /// A correlatable signal drops the origin from the address, so independent + /// emitters of the same claim converge on one signal and are recorded as + /// corroborating each other. + /// + /// This is the difference between an utterance and an assertion about the + /// world. Use it whenever the point is that several parties agree. The + /// origin is still stamped on the signal for attribution; it just stays out + /// of the address, because the address is the claim, not the claimant. + /// + /// Anything correlatable must therefore keep evidence out of the payload — + /// evidence differs per node and would make every address unique again. + pub fn correlatable(mut self) -> Self { + self.origin_node_id = String::new(); + self + } + /// Set the origin node ID + /// + /// This folds the origin into the content hash, so the resulting signal is + /// unique to this node even if another node says exactly the same thing. + /// For a claim meant to accumulate corroboration, use + /// [`SignalBuilder::correlatable`] instead. pub fn origin(mut self, node_id: &str) -> Self { self.origin_node_id = node_id.to_string(); self @@ -353,6 +458,7 @@ impl SignalBuilder { hops: 0, reached_nodes: Vec::new(), protocol_checksum, + attestations: Vec::new(), } } } diff --git a/smesh-runtime/src/mesh.rs b/smesh-runtime/src/mesh.rs index aee415a..948a72c 100644 --- a/smesh-runtime/src/mesh.rs +++ b/smesh-runtime/src/mesh.rs @@ -34,7 +34,7 @@ use tokio::task::JoinHandle; use tracing::{debug, info, warn}; use serde_json::json; -use smesh_core::{Network, Node, NodeId, Signal}; +use smesh_core::{Attestation, Network, NodeId, Signal}; use crate::journal::Journal; @@ -93,6 +93,8 @@ struct MeshCtx { local_node_id: NodeId, /// Our dialable address, advertised in `Hello`. listen_addr: SocketAddr, + /// Our own public key, advertised in `Hello`. + local_public_key: String, /// Socket address -> node id, learned from `Hello`. /// /// An accepted connection's source address is ephemeral, so this is the @@ -104,6 +106,14 @@ struct MeshCtx { /// recording that we initiated, the answer looks like a fresh introduction /// and we answer the answer, registering the peer twice. dialed: RwLock>, + /// Node name -> the public key that first presented it. + /// + /// Signatures prove key ownership, not name ownership, so on their own they + /// do not stop a peer calling itself `latency`. Pinning the name to the key + /// seen first, and refusing later keys for that name, closes it. This is + /// trust-on-first-use: it cannot help if the impostor arrives first, but it + /// makes a name unstealable for the rest of the run. + pinned_keys: RwLock>, max_peers_shared: usize, peer_discovery: bool, journal: Arc, @@ -143,15 +153,39 @@ impl MeshHandle { } /// Bring up the transport, join the mesh, and start the gossip tasks. +/// Everything the mesh needs from the runtime that owns it. +pub(crate) struct MeshStartup { + /// How to join. + pub config: MeshConfig, + /// The node this process presents on the wire. + pub local_node_id: NodeId, + /// Its public key, advertised so peers can bind the name to it. + pub local_public_key: String, + /// Shared field and node state. + pub network: Arc>, + /// Shared peer table. + pub peers: Arc, + /// Where runtime events go. + pub event_tx: mpsc::Sender, + /// Where the run is recorded. + pub journal: Arc, + /// Shared connection address to node id map. + pub conn_ids: Arc>>, +} + pub(crate) async fn start( - config: MeshConfig, - local_node_id: NodeId, - network: Arc>, - peers: Arc, - event_tx: mpsc::Sender, - journal: Arc, - conn_ids: Arc>>, + startup: MeshStartup, ) -> Result<(MeshHandle, Arc), TransportError> { + let MeshStartup { + config, + local_node_id, + local_public_key, + network, + peers, + event_tx, + journal, + conn_ids, + } = startup; let mut transport = QuicTransport::new(TransportConfig { bind_addr: config.bind_addr, max_message_size: config.max_message_size, @@ -187,8 +221,10 @@ pub(crate) async fn start( event_tx, local_node_id, listen_addr, + local_public_key, conn_ids, dialed: RwLock::new(HashSet::new()), + pinned_keys: RwLock::new(HashMap::new()), max_peers_shared: config.max_peers_shared, peer_discovery: config.peer_discovery, journal, @@ -252,6 +288,7 @@ async fn dial(ctx: &MeshCtx, addr: SocketAddr) -> Result<(), TransportError> { fn hello(ctx: &MeshCtx) -> TransportMessage { TransportMessage::Hello { node_id: ctx.local_node_id.clone(), + public_key: ctx.local_public_key.clone(), listen_addr: ctx.listen_addr, } } @@ -265,8 +302,9 @@ async fn inbound_loop( match msg { TransportMessage::Hello { node_id, + public_key, listen_addr, - } => on_hello(&ctx, src, node_id, listen_addr).await, + } => on_hello(&ctx, src, node_id, public_key, listen_addr).await, TransportMessage::Signal { signal, age_secs } => { on_signal(&ctx, src, signal, age_secs).await @@ -302,11 +340,45 @@ async fn inbound_loop( } /// Register a peer that introduced itself, and answer in kind. -async fn on_hello(ctx: &MeshCtx, src: SocketAddr, node_id: NodeId, listen_addr: SocketAddr) { +async fn on_hello( + ctx: &MeshCtx, + src: SocketAddr, + node_id: NodeId, + public_key: String, + listen_addr: SocketAddr, +) { if node_id == ctx.local_node_id { return; } + // Bind this name to this key, or refuse the peer if the name is already + // spoken for by a different one. + if !public_key.is_empty() { + let mut pinned = ctx.pinned_keys.write().await; + match pinned.get(&node_id) { + Some(known) if known != &public_key => { + drop(pinned); + warn!( + "refusing {} from {}: name already pinned to a different key", + node_id, src + ); + ctx.journal.record( + "identity_rejected", + json!({ + "peer": node_id, + "source_addr": src.to_string(), + "reason": "name already pinned to a different public key", + }), + ); + return; + } + Some(_) => {} + None => { + pinned.insert(node_id.clone(), public_key.clone()); + } + } + } + let first_contact = { let mut ids = ctx.conn_ids.write().await; ids.insert(src, node_id.clone()).is_none() @@ -327,6 +399,7 @@ async fn on_hello(ctx: &MeshCtx, src: SocketAddr, node_id: NodeId, listen_addr: "peer_connected", json!({ "peer": node_id, + "public_key": public_key, "peer_listen_addr": listen_addr.to_string(), "source_addr": src.to_string(), "we_dialled": ctx.dialed.read().await.contains(&src), @@ -414,6 +487,26 @@ enum Outcome { Dropped { hash: String, reason: String }, } +/// Reject a signal whose attestations use a name pinned to another key. +/// +/// A valid signature proves you hold *a* key, not that you are entitled to the +/// name you signed under. Returns the offending names, or `None` if all is +/// well. +async fn reject_unpinned(ctx: &MeshCtx, attestations: &[Attestation]) -> Option> { + let pinned = ctx.pinned_keys.read().await; + let offenders: Vec = attestations + .iter() + .filter(|a| { + pinned + .get(&a.node_id) + .is_some_and(|known| known != &a.public_key) + }) + .map(|a| a.node_id.clone()) + .collect(); + + (!offenders.is_empty()).then_some(offenders) +} + /// Apply the local node's own policy to a signal that arrived over the wire. async fn on_signal(ctx: &MeshCtx, src: SocketAddr, mut signal: Signal, age_secs: f64) { let relayed_by = ctx @@ -425,7 +518,25 @@ async fn on_signal(ctx: &MeshCtx, src: SocketAddr, mut signal: Signal, age_secs: .unwrap_or_else(|| src.to_string()); let hash = signal.origin_hash.clone(); - let incoming_attesters = Node::attesters(&signal); + + // Count signatures, not names. `reinforced_by` arriving off a socket is + // just a list the sender wrote; only attestations carry proof. + let incoming_attestations = signal.attestations.clone(); + let incoming_attesters = signal.verified_attesters(); + let unverifiable = signal.attestations.len() - incoming_attesters.len(); + + if let Some(rejected) = reject_unpinned(ctx, &incoming_attestations).await { + ctx.journal.record( + "identity_rejected", + json!({ + "hash": hash, + "relayed_by": relayed_by, + "attesters": rejected, + "reason": "attestation used a name pinned to a different public key", + }), + ); + return; + } ctx.journal.record( "signal_received", @@ -438,6 +549,7 @@ async fn on_signal(ctx: &MeshCtx, src: SocketAddr, mut signal: Signal, age_secs: "intensity": signal.current_intensity, "confidence": signal.confidence, "attesters": incoming_attesters, + "unverifiable_attestations": unverifiable, }), ); @@ -454,18 +566,12 @@ async fn on_signal(ctx: &MeshCtx, src: SocketAddr, mut signal: Signal, age_secs: // Merge the two attester sets. Anything the sender knew that we did // not is new information, and new information is worth passing on. let existing = network.field.signals.get_mut(&hash).expect("checked above"); - let before = Node::attesters(existing); - for attester in &incoming_attesters { - if !before.contains(attester) { - existing.reinforce(attester); - } + let new_attesters = existing.merge_attestations(&incoming_attestations); + // Keep the local view in step with what actually verified. + for attester in &new_attesters { + existing.reinforce(attester); } - let attesters = Node::attesters(existing); - let new_attesters: Vec = attesters - .iter() - .filter(|a| !before.contains(a)) - .cloned() - .collect(); + let attesters = existing.verified_attesters(); if new_attesters.is_empty() { Outcome::Dropped { @@ -495,6 +601,14 @@ async fn on_signal(ctx: &MeshCtx, src: SocketAddr, mut signal: Signal, age_secs: forward, } } + } else if incoming_attesters.is_empty() { + // Nobody provably stands behind this. Before signatures existed the + // origin was a bare string, so this is the case that used to let + // anyone speak as anyone. + Outcome::Dropped { + hash, + reason: "no verifiable attestation".to_string(), + } } else if signal.is_expired(now) { Outcome::Dropped { hash, @@ -520,9 +634,12 @@ async fn on_signal(ctx: &MeshCtx, src: SocketAddr, mut signal: Signal, age_secs: // Whatever the sender knew about its own graph is meaningless // here, so replace it outright. signal.reached_nodes = vec![ctx.local_node_id.clone()]; + // Drop the sender's unsigned name list; only signatures survive + // the trip, and they are already on the signal. + signal.reinforced_by = incoming_attesters.clone(); let hops = signal.hops; - let attesters = Node::attesters(&signal); + let attesters = incoming_attesters.clone(); let forward = relay_forward(ctx, &network, &signal, &hash); network.field.signals.insert(hash.clone(), signal); diff --git a/smesh-runtime/src/runtime.rs b/smesh-runtime/src/runtime.rs index f481178..84aa654 100644 --- a/smesh-runtime/src/runtime.rs +++ b/smesh-runtime/src/runtime.rs @@ -140,24 +140,31 @@ impl SmeshRuntime { config: MeshConfig, local_node_id: &str, ) -> Result { - { + let local_public_key = { let network = self.network.read().await; - if !network.nodes.contains_key(local_node_id) { + let Some(node) = network.nodes.get(local_node_id) else { return Err(TransportError::ConnectionFailed(format!( "node {local_node_id} is not in this runtime's network" ))); + }; + if node.identity.is_none() { + return Err(TransportError::ConnectionFailed(format!( + "node {local_node_id} holds no signing key, so it cannot attest to anything" + ))); } - } + node.public_key.clone() + }; - let (handle, transport) = mesh::start( + let (handle, transport) = mesh::start(mesh::MeshStartup { config, - local_node_id.to_string() as NodeId, - Arc::clone(&self.network), - Arc::clone(&self.peers), - self.event_tx.clone(), - Arc::clone(&self.journal), - Arc::clone(&self.peer_names), - ) + local_node_id: local_node_id.to_string() as NodeId, + local_public_key, + network: Arc::clone(&self.network), + peers: Arc::clone(&self.peers), + event_tx: self.event_tx.clone(), + journal: Arc::clone(&self.journal), + conn_ids: Arc::clone(&self.peer_names), + }) .await?; *self.transport.write().await = Some(transport); @@ -205,17 +212,37 @@ impl SmeshRuntime { // Stamp the emitting node as the signal's origin and seed its diffusion // frontier there, so the signal can spread outward from this node. - signal.origin_node_id = node_id.to_string(); + // + // Only stamp when the builder left it unset. Overwriting an origin that + // is already part of the content hash would leave the signal claiming + // one origin in its address and a different one in its field, and the + // two would disagree forever after. + if signal.origin_node_id.is_empty() { + signal.origin_node_id = node_id.to_string(); + } signal.mark_reached(node_id); let hash = signal.origin_hash.clone(); let first_assertion = !network.field.signals.contains_key(&hash); + // Sign the claim. This is what makes our agreement countable by anyone + // else: without it we are just another name in a list. + let attestation = network + .nodes + .get(node_id) + .and_then(|n| n.identity.as_ref().map(|identity| identity.attest(&hash))); + if first_assertion { + if let Some(attestation) = attestation { + signal.merge_attestations(&[attestation]); + } network.field.signals.insert(hash.clone(), signal); } else if let Some(existing) = network.field.signals.get_mut(&hash) { existing.reinforce(node_id); existing.mark_reached(node_id); + if let Some(attestation) = attestation { + existing.merge_attestations(&[attestation]); + } } // Update node stats @@ -233,7 +260,7 @@ impl SmeshRuntime { s.reached_nodes.clear(); s }); - let attesters = stored.map(Node::attesters).unwrap_or_default(); + let attesters = stored.map(|s| s.verified_attesters()).unwrap_or_default(); let payload = stored .map(|s| payload_preview(&s.payload, 512)) .unwrap_or(serde_json::Value::Null); @@ -321,7 +348,7 @@ impl SmeshRuntime { "intensity": s.current_intensity, "confidence": s.confidence, "effective": s.effective_intensity(network.field.current_time), - "attesters": Node::attesters(s), + "attesters": s.verified_attesters(), "hops": s.hops, "age_secs": (network.field.current_time - s.created_at) .num_milliseconds() as f64 diff --git a/smesh-runtime/src/transport.rs b/smesh-runtime/src/transport.rs index a59315b..fc156e2 100644 --- a/smesh-runtime/src/transport.rs +++ b/smesh-runtime/src/transport.rs @@ -54,6 +54,13 @@ pub enum TransportMessage { Hello { /// Sender's SMESH node id node_id: String, + /// Sender's Ed25519 public key, hex encoded. + /// + /// Lets the receiver bind this name to this key for the rest of the + /// run, so a later peer cannot present the same name under a different + /// key and have its attestations counted. + #[serde(default)] + public_key: String, /// Address the sender accepts connections on listen_addr: SocketAddr, }, @@ -634,6 +641,7 @@ mod tests { fn test_hello_roundtrip() { let msg = TransportMessage::Hello { node_id: "node-a".to_string(), + public_key: "aa".repeat(32), listen_addr: "127.0.0.1:9001".parse().unwrap(), }; @@ -641,9 +649,11 @@ mod tests { match bincode::deserialize::(&bytes).unwrap() { TransportMessage::Hello { node_id, + public_key, listen_addr, } => { assert_eq!(node_id, "node-a"); + assert_eq!(public_key, "aa".repeat(32)); assert_eq!(listen_addr.port(), 9001); } _ => panic!("Wrong message type"), diff --git a/smesh-runtime/tests/two_node_mesh.rs b/smesh-runtime/tests/two_node_mesh.rs index 8c3ecdd..bdca687 100644 --- a/smesh-runtime/tests/two_node_mesh.rs +++ b/smesh-runtime/tests/two_node_mesh.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use std::time::Duration; use smesh_core::{Network, Node, NodeId, Signal, SignalType}; -use smesh_runtime::{MeshConfig, MeshHandle, RuntimeConfig, SmeshRuntime}; +use smesh_runtime::{MeshConfig, MeshHandle, RuntimeConfig, SmeshRuntime, TransportMessage}; const LOCALHOST: &str = "127.0.0.1:0"; @@ -23,8 +23,7 @@ struct MeshNode { impl MeshNode { async fn start(name: &str, bootstrap: Vec) -> Self { - let mut node = Node::new(); - node.id = name.to_string(); + let mut node = Node::named(name); // Trust the peers we will actually talk to, so the probabilistic relay // policy does not make these tests flaky. node.trust_scores.insert("node-a".to_string(), 0.99); @@ -96,6 +95,22 @@ impl MeshNode { .expect("emitted") } + /// Emit a claim addressed by content, so independent emitters converge. + async fn emit_claim(&self, payload: &str) -> String { + let signal = Signal::builder(SignalType::Alert) + .correlatable() + .payload(payload.as_bytes().to_vec()) + .intensity(1.0) + .ttl(120.0) + .radius(4) + .build(); + + self.runtime + .emit(signal, &self.node_id) + .await + .expect("emitted") + } + async fn shutdown(self) { self.handle.shutdown().await; } @@ -225,3 +240,149 @@ async fn flooding_does_not_duplicate_state() { b.shutdown().await; c.shutdown().await; } + +#[tokio::test] +async fn an_unsigned_claim_is_refused() { + // Before attestations existed, `origin_node_id` was a bare string and this + // signal would have been accepted and counted. It carries no proof that + // anyone stands behind it, so it must now go nowhere. + let a = MeshNode::start("node-a", vec![]).await; + let b = MeshNode::start("node-b", vec![a.addr()]).await; + + eventually(Duration::from_secs(5), "a and b meet", || async { + b.runtime.peers().connected_count().await == 1 + }) + .await; + + let mut signal = Signal::builder(SignalType::Coordination) + .payload(b"unsigned claim".to_vec()) + .intensity(1.0) + .ttl(60.0) + .radius(4) + .build(); + signal.origin_node_id = "node-a".to_string(); + let hash = signal.origin_hash.clone(); + assert!(signal.attestations.is_empty()); + + let transport = a.handle.transport(); + let msg = TransportMessage::signal(signal, chrono::Utc::now()); + transport.broadcast_all(&msg, None).await; + + tokio::time::sleep(Duration::from_millis(600)).await; + assert!( + !b.has_signal(&hash).await, + "a claim nobody signed must not enter the field" + ); + + a.shutdown().await; + b.shutdown().await; +} + +#[tokio::test] +async fn a_tampered_attestation_is_not_counted() { + // An attacker replays a real signature against a claim it was not made for. + // The signature is genuine; the binding is not. + let a = MeshNode::start("node-a", vec![]).await; + let b = MeshNode::start("node-b", vec![a.addr()]).await; + + eventually(Duration::from_secs(5), "a and b meet", || async { + b.runtime.peers().connected_count().await == 1 + }) + .await; + + // A properly signed claim, so we have a valid attestation to steal. + let real_hash = a.emit("genuine claim").await; + eventually( + Duration::from_secs(5), + "b accepts the genuine claim", + || async { b.has_signal(&real_hash).await }, + ) + .await; + + let stolen = { + let network = a.runtime.network(); + let network = network.read().await; + network.field.signals[&real_hash].attestations[0].clone() + }; + + // Bolt it onto a different claim. + let mut forged = Signal::builder(SignalType::Coordination) + .payload(b"claim the attacker wants believed".to_vec()) + .intensity(1.0) + .ttl(60.0) + .radius(4) + .build(); + forged.origin_node_id = "node-a".to_string(); + forged.attestations = vec![stolen]; + let forged_hash = forged.origin_hash.clone(); + + assert!( + forged.verified_attesters().is_empty(), + "a signature must not verify against a claim it was not made for" + ); + + let msg = TransportMessage::signal(forged, chrono::Utc::now()); + a.handle.transport().broadcast_all(&msg, None).await; + + tokio::time::sleep(Duration::from_millis(600)).await; + assert!( + !b.has_signal(&forged_hash).await, + "a claim with only a replayed signature must be refused" + ); + + a.shutdown().await; + b.shutdown().await; +} + +#[tokio::test] +async fn corroboration_across_the_mesh_is_signature_backed() { + // Two nodes independently reach the same conclusion. Content addressing + // puts them on one signal, and each contributes a signature, so the count + // of attesters is a count of verifiable statements rather than of names. + let a = MeshNode::start("node-a", vec![]).await; + let b = MeshNode::start("node-b", vec![a.addr()]).await; + + eventually(Duration::from_secs(5), "a and b meet", || async { + a.runtime.peers().connected_count().await == 1 + && b.runtime.peers().connected_count().await == 1 + }) + .await; + + let hash_a = a.emit_claim("checkout-api degraded").await; + let hash_b = b.emit_claim("checkout-api degraded").await; + assert_eq!(hash_a, hash_b, "same claim must land on the same address"); + + eventually( + Duration::from_secs(6), + "both sides see two signed attesters", + || async { + let seen = |node: &MeshNode| { + let hash = hash_a.clone(); + let network = node.runtime.network(); + async move { + let network = network.read().await; + network + .field + .signals + .get(&hash) + .map(|s| s.verified_attesters().len()) + .unwrap_or(0) + } + }; + seen(&a).await == 2 && seen(&b).await == 2 + }, + ) + .await; + + let network = a.runtime.network(); + let network = network.read().await; + let signal = &network.field.signals[&hash_a]; + let mut attesters = signal.verified_attesters(); + attesters.sort(); + assert_eq!(attesters, vec!["node-a".to_string(), "node-b".to_string()]); + assert_eq!(signal.attestations.len(), 2); + drop(network); + + a.shutdown().await; + b.shutdown().await; +} From 90d45b4648c32544fec4a53c03b8ed5eb6c783f0 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 22:04:05 -0700 Subject: [PATCH 06/14] runtime: authenticate the channel, and let the mesh heal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four faults, each reproduced before the fix and re-tested after. **A dead peer went unnoticed for thirty seconds.** `quinn::TransportConfig` was used verbatim, so liveness was decided by QUIC's own generous idle default. Application-level pings could not help — liveness is a transport decision. With an 8s idle timeout and a 2s keepalive, a peer that dies is noticed in about three seconds instead of thirty. **One unreachable bootstrap address stalled startup for thirty seconds**, and several did so one after another, because the dials were awaited inline and `connect_with` retries internally for QUIC's own timeout. The dials now happen off the startup path and are bounded by the `connect_timeout_ms` that was already in the config and never read. A node with a dead bootstrap peer starts immediately. **Nothing ever re-dialled a lost peer**, so the mesh could only degrade: every blip was permanent, and a restarted peer stayed gone. A supervisor now retries wanted addresses on exponential backoff from 500ms to a 30s cap. Only addresses we dialled are tracked, because a peer that dialled us will dial again and racing two connections onto one pair helps nobody. A returning peer was also being swallowed: "we have seen this name before" was treated as "already connected", so the peer table healed while the event stream never mentioned it. **The channel was encrypted but not authenticated.** Certificates were throwaway keypairs unrelated to the node's identity and regenerated every start, so nothing tied the TLS session to who the peer claimed to be. The certificate is now built from the node's own Ed25519 signing key, so the key a peer proves on the wire is the key it signs claims with. Both ends present certificates, and a `Hello` whose stated public key is not the key that completed the handshake is refused. An attacker relaying someone else's introduction cannot also present their certificate. Two things this made necessary: Authentication broke restarts. A node's key was new on every start, so a restarted node looked like an impostor to every peer that had pinned its old key — the mesh's own authentication preventing it rejoining. `NodeIdentity::load_or_create` gives a node a durable key, written 0600 and refused if anything else on the box can read it. `smesh mesh --identity` exposes it. Verified end to end: a peer now restarts and is re-admitted. A refused peer used to keep its connection open and be re-dialled forever. It is now disconnected and dropped from the wanted set. A peer with a durable identity is unaffected, because its key does not change. Also removes `pub struct Transport`, a stub whose channels were wired to dropped ends: `send` could only ever fail and `recv` could only ever return `None`. It was exported and looked usable. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.toml | 2 +- smesh-cli/src/main.rs | 31 +++- smesh-core/src/identity.rs | 148 +++++++++++++++ smesh-runtime/Cargo.toml | 1 + smesh-runtime/src/lib.rs | 5 +- smesh-runtime/src/mesh.rs | 255 +++++++++++++++++++++++-- smesh-runtime/src/runtime.rs | 9 +- smesh-runtime/src/transport.rs | 327 +++++++++++++++++++++++---------- 8 files changed, 655 insertions(+), 123 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ceb1942..65d12c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ async-trait = "0.1" # Crypto sha2 = "0.10" -ed25519-dalek = { version = "2.1", features = ["rand_core", "serde"] } +ed25519-dalek = { version = "2.1", features = ["rand_core", "serde", "pkcs8"] } rand = "0.8" uuid = { version = "1.6", features = ["v4", "serde"] } diff --git a/smesh-cli/src/main.rs b/smesh-cli/src/main.rs index 52e69f2..90b85e5 100644 --- a/smesh-cli/src/main.rs +++ b/smesh-cli/src/main.rs @@ -1,6 +1,6 @@ //! SMESH CLI - Command line tools for testing and running SMESH -use anyhow::Result; +use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use std::path::{Path, PathBuf}; use tracing::Level; @@ -332,6 +332,13 @@ enum Commands { #[arg(long)] name: Option, + /// File holding this node's signing key, created if absent. + /// + /// Without one the key is new on every start, and peers that pinned the + /// old key will refuse the restarted node as an impostor. + #[arg(long)] + identity: Option, + /// Payload to emit onto the mesh once connected #[arg(long)] emit: Option, @@ -446,10 +453,11 @@ async fn main() -> Result<()> { bind, peers, name, + identity, emit, emit_after, duration, - } => cmd_mesh(&bind, &peers, name, emit, emit_after, duration).await, + } => cmd_mesh(&bind, &peers, name, identity, emit, emit_after, duration).await, Commands::Orchestrate { out, base_port, @@ -1378,6 +1386,7 @@ async fn cmd_mesh( bind: &str, peers: &[String], name: Option, + identity: Option, emit: Option, emit_after: u64, duration: u64, @@ -1398,9 +1407,21 @@ async fn cmd_mesh( .collect::>()?; // One node per process: this is the identity we present on the wire. - let mut node = match name { - Some(name) => Node::named(name), - None => Node::new(), + let mut node = match (name, identity) { + // A durable key is what lets this node restart and be recognised. + (Some(name), Some(path)) => Node::new().with_identity( + smesh_core::NodeIdentity::load_or_create(&path, name) + .with_context(|| format!("loading identity from {}", path.display()))?, + ), + (Some(name), None) => Node::named(name), + (None, Some(path)) => { + let generated = Node::new().id; + Node::new().with_identity( + smesh_core::NodeIdentity::load_or_create(&path, generated) + .with_context(|| format!("loading identity from {}", path.display()))?, + ) + } + (None, None) => Node::new(), }; let node_id = node.id.clone(); diff --git a/smesh-core/src/identity.rs b/smesh-core/src/identity.rs index e18bae8..580a276 100644 --- a/smesh-core/src/identity.rs +++ b/smesh-core/src/identity.rs @@ -16,6 +16,7 @@ //! already taken. Signatures prove key ownership, not name ownership. The mesh //! layer closes that by pinning a name to the key that first used it. +use ed25519_dalek::pkcs8::EncodePrivateKey; use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; use rand::rngs::OsRng; use serde::{Deserialize, Serialize}; @@ -73,6 +74,47 @@ impl NodeIdentity { } } + /// Load an identity from disk, creating it on first run. + /// + /// Without this a node's key is new on every start, so a restarted node + /// looks like an impostor to every peer that pinned its old key — which + /// makes the mesh's own authentication the thing that prevents it + /// rejoining. A durable key is what turns "the name is taken" from a + /// permanent exclusion into a real identity. + /// + /// The file holds a private key, so it is created read/write for the owner + /// only and an over-permissive existing file is refused rather than used. + pub fn load_or_create( + path: impl AsRef, + node_id: impl Into, + ) -> std::io::Result { + use ed25519_dalek::pkcs8::DecodePrivateKey; + use std::io::{Error, ErrorKind}; + + let path = path.as_ref(); + let node_id = node_id.into(); + + if path.exists() { + reject_if_world_readable(path)?; + let der = std::fs::read(path)?; + let signing_key = SigningKey::from_pkcs8_der(&der) + .map_err(|e| Error::new(ErrorKind::InvalidData, format!("bad identity: {e}")))?; + return Ok(Self { + signing_key, + node_id, + }); + } + + let identity = Self::generate_named(node_id); + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent)?; + } + } + write_private(path, &identity.to_pkcs8_der())?; + Ok(identity) + } + /// This identity's node id. pub fn node_id(&self) -> &str { &self.node_id @@ -83,6 +125,21 @@ impl NodeIdentity { hex(self.signing_key.verifying_key().as_bytes()) } + /// This identity's private key as PKCS#8 DER. + /// + /// Used to build the node's TLS certificate from the same key it signs + /// attestations with, so the transport channel and the application identity + /// are the same identity rather than two unrelated ones. + /// + /// Secret material: hand it to the transport and nowhere else. + pub fn to_pkcs8_der(&self) -> Vec { + self.signing_key + .to_pkcs8_der() + .expect("an ed25519 key always encodes as pkcs8") + .as_bytes() + .to_vec() + } + /// Attest to a claim, by its content hash. pub fn attest(&self, claim_hash: &str) -> Attestation { let message = attestation_message(claim_hash, &self.node_id); @@ -150,6 +207,52 @@ impl Attestation { } } +/// Write secret material so only the owner can read it. +#[cfg(unix)] +fn write_private(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + + // create_new so an existing key is never silently overwritten. + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path)?; + file.write_all(bytes) +} + +/// Write secret material. Permissions are left to the platform. +#[cfg(not(unix))] +fn write_private(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> { + std::fs::write(path, bytes) +} + +/// Refuse an identity file that anyone else on the box can read. +#[cfg(unix)] +fn reject_if_world_readable(path: &std::path::Path) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt; + + let mode = std::fs::metadata(path)?.permissions().mode() & 0o077; + if mode != 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!( + "{} is readable by others (mode {:o}); refusing to load a private key", + path.display(), + mode + ), + )); + } + Ok(()) +} + +/// No portable notion of file permissions to check here. +#[cfg(not(unix))] +fn reject_if_world_readable(_path: &std::path::Path) -> std::io::Result<()> { + Ok(()) +} + /// Node id derived from a public key. pub fn derive_node_id(key: &VerifyingKey) -> NodeId { let mut hasher = Sha256::new(); @@ -241,6 +344,51 @@ mod tests { assert!(!att.is_self_certifying()); } + #[test] + fn the_pkcs8_export_carries_this_very_key() { + use ed25519_dalek::pkcs8::DecodePrivateKey; + let id = NodeIdentity::generate(); + let decoded = SigningKey::from_pkcs8_der(&id.to_pkcs8_der()).unwrap(); + assert_eq!( + decoded.verifying_key().to_bytes(), + id.signing_key.verifying_key().to_bytes(), + "the certificate must be built from the same key that signs claims" + ); + } + + #[test] + fn an_identity_survives_a_restart() { + let dir = std::env::temp_dir().join(format!("smesh-id-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("node.key"); + std::fs::remove_file(&path).ok(); + + let first = NodeIdentity::load_or_create(&path, "latency").unwrap(); + let again = NodeIdentity::load_or_create(&path, "latency").unwrap(); + + // The whole point: a peer that pinned this key still recognises us. + assert_eq!(first.public_key_hex(), again.public_key_hex()); + assert_eq!(again.node_id(), "latency"); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[cfg(unix)] + #[test] + fn a_world_readable_identity_file_is_refused() { + use std::os::unix::fs::PermissionsExt; + let dir = std::env::temp_dir().join(format!("smesh-id-perm-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("node.key"); + std::fs::remove_file(&path).ok(); + + NodeIdentity::load_or_create(&path, "latency").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + + assert!(NodeIdentity::load_or_create(&path, "latency").is_err()); + std::fs::remove_dir_all(&dir).ok(); + } + #[test] fn two_identities_do_not_collide() { let a = NodeIdentity::generate(); diff --git a/smesh-runtime/Cargo.toml b/smesh-runtime/Cargo.toml index adbab01..2631fde 100644 --- a/smesh-runtime/Cargo.toml +++ b/smesh-runtime/Cargo.toml @@ -20,3 +20,4 @@ anyhow = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } rcgen = "0.13" +x509-parser = "0.16" diff --git a/smesh-runtime/src/lib.rs b/smesh-runtime/src/lib.rs index 91ead6c..b9ec4ca 100644 --- a/smesh-runtime/src/lib.rs +++ b/smesh-runtime/src/lib.rs @@ -17,4 +17,7 @@ pub use journal::{Journal, JournalEvent}; pub use mesh::{MeshConfig, MeshHandle}; pub use peer::{Peer, PeerId, PeerManager, PeerState}; pub use runtime::{RuntimeConfig, RuntimeEvent, RuntimeStats, SmeshRuntime}; -pub use transport::{QuicTransport, Transport, TransportConfig, TransportError, TransportMessage}; +pub use transport::{ + QuicTransport, TransportConfig, TransportError, TransportMessage, DEFAULT_CONNECT_TIMEOUT_MS, + DEFAULT_IDLE_TIMEOUT_MS, DEFAULT_KEEPALIVE_MS, +}; diff --git a/smesh-runtime/src/mesh.rs b/smesh-runtime/src/mesh.rs index 948a72c..de4d688 100644 --- a/smesh-runtime/src/mesh.rs +++ b/smesh-runtime/src/mesh.rs @@ -42,6 +42,13 @@ use crate::peer::{Peer, PeerManager, PeerState}; use crate::runtime::RuntimeEvent; use crate::transport::{QuicTransport, TransportConfig, TransportError, TransportMessage}; +/// First retry delay after a peer is lost. +const RECONNECT_BASE: Duration = Duration::from_millis(500); +/// Longest a retry is ever deferred. +const RECONNECT_MAX: Duration = Duration::from_secs(30); +/// How often the supervisor wakes to consider reconnecting. +const RECONNECT_TICK: Duration = Duration::from_millis(500); + /// Configuration for joining a mesh. #[derive(Debug, Clone)] pub struct MeshConfig { @@ -83,6 +90,49 @@ impl Default for MeshConfig { } } +/// Retry schedule for one address. +#[derive(Debug, Clone, Copy)] +struct Backoff { + /// Consecutive failures so far. + failures: u32, + /// How long to wait before the next attempt. + wait: Duration, + /// Time already spent waiting since the last attempt. + waited: Duration, +} + +impl Backoff { + /// Ready to try again immediately. + fn ready() -> Self { + Self { + failures: 0, + wait: Duration::ZERO, + waited: Duration::ZERO, + } + } + + /// Back off further after a failed attempt. + /// + /// Doubling from one second and capping at thirty keeps a permanently dead + /// address from being hammered while still recovering a transient outage in + /// seconds rather than minutes. + fn fail(&mut self) { + self.failures = self.failures.saturating_add(1); + self.wait = RECONNECT_BASE + .saturating_mul(1u32 << self.failures.min(5)) + .min(RECONNECT_MAX); + self.waited = Duration::ZERO; + } + + fn due(&self) -> bool { + self.waited >= self.wait + } + + fn tick(&mut self, elapsed: Duration) { + self.waited = self.waited.saturating_add(elapsed); + } +} + /// Shared state for the mesh tasks. struct MeshCtx { transport: Arc, @@ -100,6 +150,14 @@ struct MeshCtx { /// An accepted connection's source address is ephemeral, so this is the /// only way to attribute an inbound frame to a SMESH node. conn_ids: Arc>>, + /// Addresses we want to stay connected to, and the backoff state for each. + /// + /// Without this the mesh could only degrade: a peer that restarted, or a + /// link that blipped, was gone for the rest of the run because nothing ever + /// dialled it again. Only addresses we dialled are tracked — a peer that + /// dialled us will dial us again, and re-dialling it too would race two + /// connections onto the same pair. + reconnect: RwLock>, /// Addresses we dialled ourselves. /// /// Whoever dials sends the first `Hello`; the other side answers. Without @@ -161,6 +219,8 @@ pub(crate) struct MeshStartup { pub local_node_id: NodeId, /// Its public key, advertised so peers can bind the name to it. pub local_public_key: String, + /// Its private key, used to build the TLS certificate. + pub identity_pkcs8_der: Vec, /// Shared field and node state. pub network: Arc>, /// Shared peer table. @@ -180,18 +240,22 @@ pub(crate) async fn start( config, local_node_id, local_public_key, + identity_pkcs8_der, network, peers, event_tx, journal, conn_ids, } = startup; - let mut transport = QuicTransport::new(TransportConfig { - bind_addr: config.bind_addr, - max_message_size: config.max_message_size, - keepalive_interval_ms: config.keepalive_interval_ms, - ..Default::default() - }) + let mut transport = QuicTransport::new( + TransportConfig { + bind_addr: config.bind_addr, + max_message_size: config.max_message_size, + keepalive_interval_ms: config.keepalive_interval_ms, + ..Default::default() + }, + identity_pkcs8_der, + ) .await?; let incoming = transport.take_incoming().ok_or_else(|| { @@ -224,6 +288,7 @@ pub(crate) async fn start( local_public_key, conn_ids, dialed: RwLock::new(HashSet::new()), + reconnect: RwLock::new(HashMap::new()), pinned_keys: RwLock::new(HashMap::new()), max_peers_shared: config.max_peers_shared, peer_discovery: config.peer_discovery, @@ -257,15 +322,39 @@ pub(crate) async fn start( })); } - // Dial bootstrap peers and introduce ourselves. - for addr in &config.bootstrap { - if *addr == listen_addr { - continue; - } - match dial(&ctx, *addr).await { - Ok(()) => info!("dialled bootstrap peer {}", addr), - Err(e) => warn!("bootstrap peer {} unreachable: {}", addr, e), - } + // Dial bootstrap peers in the background. Doing this inline meant a single + // unreachable address held up startup for the whole handshake timeout, and + // several of them did so one after another. + { + let ctx = Arc::clone(&ctx); + let bootstrap: Vec = config + .bootstrap + .iter() + .copied() + .filter(|addr| *addr != listen_addr) + .collect(); + + tasks.push(tokio::spawn(async move { + let dials = bootstrap.iter().map(|addr| { + let ctx = Arc::clone(&ctx); + async move { + match dial(&ctx, *addr).await { + Ok(()) => info!("dialled bootstrap peer {}", addr), + // Not fatal: the supervisor keeps trying. + Err(e) => warn!("bootstrap peer {} unreachable: {}", addr, e), + } + } + }); + futures::future::join_all(dials).await; + })); + } + + // Keep wanting the peers we were told about, even if they are not up yet. + { + let ctx = Arc::clone(&ctx); + tasks.push(tokio::spawn(async move { + reconnect_loop(ctx).await; + })); } Ok(( @@ -280,9 +369,30 @@ pub(crate) async fn start( /// Connect to a peer and send our `Hello`. async fn dial(ctx: &MeshCtx, addr: SocketAddr) -> Result<(), TransportError> { - ctx.transport.connect(addr).await?; - ctx.dialed.write().await.insert(addr); - ctx.transport.send(addr, hello(ctx)).await + // Record the intent before the attempt, so an address that fails on first + // contact is still retried rather than forgotten. + ctx.reconnect + .write() + .await + .entry(addr) + .or_insert_with(Backoff::ready); + + let result = async { + ctx.transport.connect(addr).await?; + ctx.dialed.write().await.insert(addr); + ctx.transport.send(addr, hello(ctx)).await + } + .await; + + let mut reconnect = ctx.reconnect.write().await; + if let Some(backoff) = reconnect.get_mut(&addr) { + match &result { + Ok(()) => *backoff = Backoff::ready(), + Err(_) => backoff.fail(), + } + } + + result } fn hello(ctx: &MeshCtx) -> TransportMessage { @@ -351,6 +461,32 @@ async fn on_hello( return; } + // Channel binding: the key a peer claims must be the key it actually + // completed the TLS handshake with. Without this the transport is encrypted + // but not authenticated, and anything in the path could relay someone + // else's introduction while holding the connection itself. + match ctx.transport.peer_public_key(src).await { + Some(proven) if proven == public_key => {} + proven => { + warn!( + "refusing {} from {}: claimed key is not the key it handshook with", + node_id, src + ); + ctx.journal.record( + "identity_rejected", + json!({ + "peer": node_id, + "source_addr": src.to_string(), + "claimed_key": public_key, + "proven_key": proven, + "reason": "claimed public key does not match the TLS channel", + }), + ); + refuse(ctx, src).await; + return; + } + } + // Bind this name to this key, or refuse the peer if the name is already // spoken for by a different one. if !public_key.is_empty() { @@ -370,6 +506,7 @@ async fn on_hello( "reason": "name already pinned to a different public key", }), ); + refuse(ctx, src).await; return; } Some(_) => {} @@ -383,7 +520,14 @@ async fn on_hello( let mut ids = ctx.conn_ids.write().await; ids.insert(src, node_id.clone()).is_none() }; - let already_known = ctx.peers.get_peer(&node_id).await.is_some(); + // A peer that dropped and came back is news again. Treating "we have seen + // this name before" as "already connected" silently swallowed every + // recovery, so the peer table healed while the event stream did not. + let already_connected = ctx + .peers + .get_peer(&node_id) + .await + .is_some_and(|peer| peer.is_connected()); let mut peer = Peer::new(node_id.clone(), listen_addr, node_id.clone()); peer.state = PeerState::Connected; @@ -394,7 +538,7 @@ async fn on_hello( return; } - if !already_known { + if !already_connected { ctx.journal.record( "peer_connected", json!({ @@ -430,6 +574,19 @@ async fn on_hello( } } +/// Drop a peer we will not talk to. +/// +/// Leaving the connection open would keep an unauthenticated peer holding +/// resources and retrying forever, and the reconnect supervisor would keep +/// dialling an address it is only going to refuse again. A peer with a durable +/// identity is unaffected: its key does not change, so it is never refused. +async fn refuse(ctx: &MeshCtx, src: SocketAddr) { + ctx.reconnect.write().await.remove(&src); + ctx.dialed.write().await.remove(&src); + ctx.conn_ids.write().await.remove(&src); + ctx.transport.disconnect(src).await; +} + /// Dial peers we were told about but have not met. async fn on_peer_response(ctx: &MeshCtx, peers: Vec<(String, SocketAddr)>) { if !ctx.peer_discovery { @@ -804,6 +961,64 @@ async fn forward_signal( } } +/// Keep dialling the peers we want until they answer. +/// +/// A peer is "wanted" once we have dialled it, and stays wanted for the rest of +/// the run. Reconnection is what separates a mesh that heals from one that only +/// ever loses members. +async fn reconnect_loop(ctx: Arc) { + let mut ticker = tokio::time::interval(RECONNECT_TICK); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + ticker.tick().await; + + let live: HashSet = ctx.transport.connected_addrs().await.into_iter().collect(); + + let due: Vec = { + let mut reconnect = ctx.reconnect.write().await; + let mut due = Vec::new(); + for (addr, backoff) in reconnect.iter_mut() { + if live.contains(addr) { + *backoff = Backoff::ready(); + continue; + } + backoff.tick(RECONNECT_TICK); + if backoff.due() { + due.push(*addr); + } + } + due + }; + + for addr in due { + let attempt = ctx + .reconnect + .read() + .await + .get(&addr) + .map(|b| b.failures + 1) + .unwrap_or(1); + + ctx.journal.record( + "reconnect_attempt", + json!({ "addr": addr.to_string(), "attempt": attempt }), + ); + + match dial(&ctx, addr).await { + Ok(()) => { + info!("reconnected to {} on attempt {}", addr, attempt); + ctx.journal.record( + "reconnected", + json!({ "addr": addr.to_string(), "attempt": attempt }), + ); + } + Err(e) => debug!("reconnect to {} failed: {}", addr, e), + } + } + } +} + async fn keepalive_loop(ctx: Arc, interval_ms: u64) { let mut ticker = tokio::time::interval(Duration::from_millis(interval_ms.max(100))); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); diff --git a/smesh-runtime/src/runtime.rs b/smesh-runtime/src/runtime.rs index 84aa654..e1be045 100644 --- a/smesh-runtime/src/runtime.rs +++ b/smesh-runtime/src/runtime.rs @@ -140,25 +140,26 @@ impl SmeshRuntime { config: MeshConfig, local_node_id: &str, ) -> Result { - let local_public_key = { + let (local_public_key, identity_pkcs8_der) = { let network = self.network.read().await; let Some(node) = network.nodes.get(local_node_id) else { return Err(TransportError::ConnectionFailed(format!( "node {local_node_id} is not in this runtime's network" ))); }; - if node.identity.is_none() { + let Some(identity) = node.identity.as_ref() else { return Err(TransportError::ConnectionFailed(format!( "node {local_node_id} holds no signing key, so it cannot attest to anything" ))); - } - node.public_key.clone() + }; + (node.public_key.clone(), identity.to_pkcs8_der()) }; let (handle, transport) = mesh::start(mesh::MeshStartup { config, local_node_id: local_node_id.to_string() as NodeId, local_public_key, + identity_pkcs8_der, network: Arc::clone(&self.network), peers: Arc::clone(&self.peers), event_tx: self.event_tx.clone(), diff --git a/smesh-runtime/src/transport.rs b/smesh-runtime/src/transport.rs index fc156e2..8026303 100644 --- a/smesh-runtime/src/transport.rs +++ b/smesh-runtime/src/transport.rs @@ -7,6 +7,7 @@ use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::sync::Arc; +use std::time::Duration; use thiserror::Error; use tokio::sync::{mpsc, RwLock}; use tracing::{debug, info, warn}; @@ -107,6 +108,13 @@ impl TransportMessage { } } +/// How long to wait for a handshake before giving up on a peer. +pub const DEFAULT_CONNECT_TIMEOUT_MS: u64 = 5_000; +/// How often to tell a peer we are still here. +pub const DEFAULT_KEEPALIVE_MS: u64 = 2_000; +/// How long silence is tolerated before a peer is considered gone. +pub const DEFAULT_IDLE_TIMEOUT_MS: u64 = 8_000; + /// Configuration for the transport layer #[derive(Debug, Clone)] pub struct TransportConfig { @@ -114,10 +122,12 @@ pub struct TransportConfig { pub bind_addr: SocketAddr, /// Maximum message size in bytes pub max_message_size: usize, - /// Connection timeout in milliseconds + /// How long to wait for a connection to be established. pub connect_timeout_ms: u64, - /// Keepalive interval in milliseconds + /// Keepalive interval in milliseconds. pub keepalive_interval_ms: u64, + /// How long a silent connection is kept before it is considered dead. + pub idle_timeout_ms: u64, } impl Default for TransportConfig { @@ -125,8 +135,9 @@ impl Default for TransportConfig { Self { bind_addr: "0.0.0.0:0".parse().unwrap(), max_message_size: 1024 * 1024, // 1MB - connect_timeout_ms: 5000, - keepalive_interval_ms: 30000, + connect_timeout_ms: DEFAULT_CONNECT_TIMEOUT_MS, + keepalive_interval_ms: DEFAULT_KEEPALIVE_MS, + idle_timeout_ms: DEFAULT_IDLE_TIMEOUT_MS, } } } @@ -145,43 +156,166 @@ fn ensure_crypto_provider() { }); } -/// Generate self-signed certificate for QUIC -fn generate_self_signed_cert( +/// Build this node's TLS certificate from its own signing key. +/// +/// The certificate's public key *is* the node's Ed25519 identity key, which is +/// what lets the channel be tied to the identity later: a peer that claims a +/// public key in its `Hello` has to have terminated the TLS handshake with that +/// same key, and only the holder of the private half can do that. +/// +/// Previously each process generated a throwaway keypair here, so the transport +/// identity was unrelated to the application identity and changed on restart. +fn certificate_from_identity( + pkcs8_der: &[u8], ) -> Result<(Vec>, PrivateKeyDer<'static>), TransportError> { - let cert = rcgen::generate_simple_self_signed(vec!["smesh".to_string()]) + let key_pair = rcgen::KeyPair::try_from(pkcs8_der) + .map_err(|e| TransportError::TlsError(format!("identity key unusable for TLS: {e}")))?; + + let params = rcgen::CertificateParams::new(vec!["smesh".to_string()]) + .map_err(|e| TransportError::TlsError(e.to_string()))?; + let cert = params + .self_signed(&key_pair) .map_err(|e| TransportError::TlsError(e.to_string()))?; - let key = PrivatePkcs8KeyDer::from(cert.key_pair.serialize_der()).into(); - let cert_der = CertificateDer::from(cert.cert.der().to_vec()); + let key = PrivatePkcs8KeyDer::from(pkcs8_der.to_vec()).into(); + Ok((vec![CertificateDer::from(cert.der().to_vec())], key)) +} - Ok((vec![cert_der], key)) +/// The Ed25519 public key inside a peer's certificate, hex encoded. +/// +/// Returns `None` for anything that is not an Ed25519 certificate, which is +/// treated as a failure to identify rather than as a pass. +pub fn public_key_from_certificate(cert_der: &[u8]) -> Option { + const ED25519_OID: &str = "1.3.101.112"; + + let (_, cert) = x509_parser::parse_x509_certificate(cert_der).ok()?; + let spki = cert.public_key(); + if spki.algorithm.algorithm.to_id_string() != ED25519_OID { + return None; + } + + let key = spki.subject_public_key.data.as_ref(); + if key.len() != 32 { + return None; + } + + Some(key.iter().map(|b| format!("{b:02x}")).collect()) +} + +/// Shared QUIC tuning for both ends of a connection. +/// +/// `quinn::TransportConfig::default()` leaves the idle timeout at QUIC's own +/// generous default, which meant a peer that died was still reported as +/// connected for roughly thirty seconds while the node cheerfully broadcast +/// into the void. Application-level pings do not help: liveness is decided by +/// the transport, so it has to be told. +/// +/// The keepalive interval must stay comfortably under half the idle timeout, or +/// a connection can expire between two keepalives on a lossy link. +fn tuned_transport_config(config: &TransportConfig) -> Arc { + let mut transport = quinn::TransportConfig::default(); + + let idle = Duration::from_millis(config.idle_timeout_ms); + transport.max_idle_timeout(Some(idle.try_into().unwrap_or(quinn::IdleTimeout::from( + quinn::VarInt::from_u32(DEFAULT_IDLE_TIMEOUT_MS as u32), + )))); + transport.keep_alive_interval(Some(Duration::from_millis(config.keepalive_interval_ms))); + + Arc::new(transport) } /// Configure QUIC server with self-signed cert -fn configure_server() -> Result { +fn configure_server( + config: &TransportConfig, + pkcs8_der: &[u8], +) -> Result { ensure_crypto_provider(); - let (certs, key) = generate_self_signed_cert()?; - - let mut server_config = ServerConfig::with_single_cert(certs, key) + let (certs, key) = certificate_from_identity(pkcs8_der)?; + + // Require a client certificate. Not to validate it here — a self-signed + // mesh has no authority to validate against — but so that the accepting + // side can see who dialled it and hold them to that key. + let crypto = rustls::ServerConfig::builder() + .with_client_cert_verifier(Arc::new(RecordAnyClientCert)) + .with_single_cert(certs, key) .map_err(|e| TransportError::TlsError(e.to_string()))?; - let transport_config = Arc::new(quinn::TransportConfig::default()); - server_config.transport_config(transport_config); + let mut server_config = ServerConfig::with_crypto(Arc::new( + quinn::crypto::rustls::QuicServerConfig::try_from(crypto) + .map_err(|e| TransportError::TlsError(e.to_string()))?, + )); + server_config.transport_config(tuned_transport_config(config)); Ok(server_config) } /// Configure QUIC client (skip server verification for P2P) -fn configure_client() -> ClientConfig { +fn configure_client( + config: &TransportConfig, + pkcs8_der: &[u8], +) -> Result { ensure_crypto_provider(); + let (certs, key) = certificate_from_identity(pkcs8_der)?; + + // The certificate chain still cannot be validated — every node signs its + // own — so the handshake accepts it and the *identity* check happens once + // the peer states which key it claims. See the mesh layer's channel binding. let crypto = rustls::ClientConfig::builder() .dangerous() .with_custom_certificate_verifier(Arc::new(SkipServerVerification)) - .with_no_client_auth(); + .with_client_auth_cert(certs, key) + .map_err(|e| TransportError::TlsError(e.to_string()))?; - ClientConfig::new(Arc::new( - quinn::crypto::rustls::QuicClientConfig::try_from(crypto).unwrap(), - )) + let mut client_config = ClientConfig::new(Arc::new( + quinn::crypto::rustls::QuicClientConfig::try_from(crypto) + .map_err(|e| TransportError::TlsError(e.to_string()))?, + )); + client_config.transport_config(tuned_transport_config(config)); + Ok(client_config) +} + +/// Accepts any client certificate so that it can be read afterwards. +/// +/// Deliberately not a trust decision: it makes the peer's key *observable*, and +/// the mesh layer decides whether that key is the one the peer claims to be. +#[derive(Debug)] +struct RecordAnyClientCert; + +impl rustls::server::danger::ClientCertVerifier for RecordAnyClientCert { + fn root_hint_subjects(&self) -> &[rustls::DistinguishedName] { + &[] + } + + fn verify_client_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::server::danger::ClientCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + vec![rustls::SignatureScheme::ED25519] + } } /// Skip server certificate verification (P2P nodes use self-signed certs) @@ -243,12 +377,17 @@ pub struct QuicTransport { incoming_rx: Option>, /// Shutdown flag shutdown: Arc>, + /// This node's private key, used to build its TLS certificate. + identity_pkcs8_der: Vec, } impl QuicTransport { /// Create a new QUIC transport - pub async fn new(config: TransportConfig) -> Result { - let server_config = configure_server()?; + pub async fn new( + config: TransportConfig, + identity_pkcs8_der: Vec, + ) -> Result { + let server_config = configure_server(&config, &identity_pkcs8_der)?; let endpoint = Endpoint::server(server_config, config.bind_addr)?; @@ -263,6 +402,7 @@ impl QuicTransport { incoming_tx, incoming_rx: Some(incoming_rx), shutdown: Arc::new(RwLock::new(false)), + identity_pkcs8_der, }) } @@ -286,15 +426,29 @@ impl QuicTransport { } } - let client_config = configure_client(); + let client_config = configure_client(&self.config, &self.identity_pkcs8_der)?; - let connection = self + // `connect_with` retries the handshake internally and will sit there + // for QUIC's own timeout, so an unreachable address used to hold the + // caller for thirty seconds. Bound it by the configured value. + let connecting = self .endpoint .connect_with(client_config, addr, "smesh") - .map_err(|e| TransportError::ConnectionFailed(e.to_string()))? - .await .map_err(|e| TransportError::ConnectionFailed(e.to_string()))?; + let connection = tokio::time::timeout( + Duration::from_millis(self.config.connect_timeout_ms), + connecting, + ) + .await + .map_err(|_| { + TransportError::ConnectionFailed(format!( + "handshake with {addr} timed out after {}ms", + self.config.connect_timeout_ms + )) + })? + .map_err(|e| TransportError::ConnectionFailed(e.to_string()))?; + debug!("Connected to peer at {}", addr); // Store connection @@ -381,6 +535,27 @@ impl QuicTransport { futures::future::join_all(sends).await } + /// The Ed25519 public key the peer actually completed the handshake with. + /// + /// This is the key half of channel binding: whatever a peer *claims* to be + /// in its `Hello`, this is the key it demonstrably holds the private half + /// of. An attacker relaying someone else's introduction cannot also present + /// their certificate, because it cannot complete the handshake without + /// their private key. + pub async fn peer_public_key(&self, addr: SocketAddr) -> Option { + let connection = self.connections.read().await.get(&addr).cloned()?; + let identity = connection.peer_identity()?; + let certs = identity.downcast::>>().ok()?; + public_key_from_certificate(certs.first()?) + } + + /// Close and forget one connection. + pub async fn disconnect(&self, addr: SocketAddr) { + if let Some(connection) = self.connections.write().await.remove(&addr) { + connection.close(0u32.into(), b"refused"); + } + } + /// Addresses of every live connection, dialled or accepted. pub async fn connected_addrs(&self) -> Vec { self.connections.read().await.keys().copied().collect() @@ -550,65 +725,10 @@ impl QuicTransport { } } -/// Simple transport for testing (no QUIC) -pub struct Transport { - config: TransportConfig, - tx: mpsc::Sender<(SocketAddr, TransportMessage)>, - rx: mpsc::Receiver<(SocketAddr, TransportMessage)>, -} - -impl Transport { - /// Create a new simple transport - pub fn new(config: TransportConfig) -> Self { - let (tx, _rx_out) = mpsc::channel(1000); - let (_tx_in, rx) = mpsc::channel(1000); - - Self { config, tx, rx } - } - - /// Send a message - pub async fn send( - &self, - addr: SocketAddr, - msg: TransportMessage, - ) -> Result<(), TransportError> { - self.tx - .send((addr, msg)) - .await - .map_err(|e| TransportError::SendFailed(e.to_string())) - } - - /// Receive a message - pub async fn recv(&mut self) -> Option<(SocketAddr, TransportMessage)> { - self.rx.recv().await - } - - /// Broadcast a signal - pub async fn broadcast( - &self, - addrs: &[SocketAddr], - signal: Signal, - ) -> Vec> { - let now = chrono::Utc::now(); - let mut results = Vec::new(); - for addr in addrs { - let result = self - .send(*addr, TransportMessage::signal(signal.clone(), now)) - .await; - results.push(result); - } - results - } - - /// Get local address - pub fn local_addr(&self) -> SocketAddr { - self.config.bind_addr - } -} - #[cfg(test)] mod tests { use super::*; + use smesh_core::NodeIdentity; #[test] fn test_transport_config() { @@ -637,6 +757,23 @@ mod tests { } } + #[test] + fn the_certificate_carries_the_nodes_own_identity_key() { + // Channel binding rests on this: if the certificate were built from a + // throwaway keypair, as it used to be, the key a peer proves on the + // wire would have nothing to do with the key it signs claims with. + let identity = NodeIdentity::generate(); + let (certs, _key) = certificate_from_identity(&identity.to_pkcs8_der()).unwrap(); + let from_cert = public_key_from_certificate(&certs[0]).unwrap(); + assert_eq!(from_cert, identity.public_key_hex()); + } + + #[test] + fn a_non_certificate_yields_no_identity() { + assert!(public_key_from_certificate(b"not a certificate").is_none()); + assert!(public_key_from_certificate(&[]).is_none()); + } + #[test] fn test_hello_roundtrip() { let msg = TransportMessage::Hello { @@ -664,11 +801,14 @@ mod tests { async fn test_oversized_frame_is_rejected_before_allocation() { // A peer claiming a 4 GiB body must be refused on the length prefix // alone, never by allocating the buffer it asked for. - let listener = QuicTransport::new(TransportConfig { - bind_addr: "127.0.0.1:0".parse().unwrap(), - max_message_size: 1024, - ..Default::default() - }) + let listener = QuicTransport::new( + TransportConfig { + bind_addr: "127.0.0.1:0".parse().unwrap(), + max_message_size: 1024, + ..Default::default() + }, + NodeIdentity::generate().to_pkcs8_der(), + ) .await .unwrap(); let addr = listener.local_addr().unwrap(); @@ -677,10 +817,13 @@ mod tests { listener.run_accept_loop().await; }); - let dialer = QuicTransport::new(TransportConfig { - bind_addr: "127.0.0.1:0".parse().unwrap(), - ..Default::default() - }) + let dialer = QuicTransport::new( + TransportConfig { + bind_addr: "127.0.0.1:0".parse().unwrap(), + ..Default::default() + }, + NodeIdentity::generate().to_pkcs8_der(), + ) .await .unwrap(); dialer.connect(addr).await.unwrap(); From 69bb694b1277f893ec3a1b3d7041cb18da4ea7e6 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 22:20:36 -0700 Subject: [PATCH 07/14] runtime: discover reachable addresses, and coordinate a simultaneous open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Testing the NAT assumption first changed what needed building. A node that can only dial outward already participates fully, because a QUIC connection is bidirectional regardless of who opened it: signals flow back over the connection the node itself established. NAT does not block participation. What it does block is narrower, and one part of it was a real defect. Peer gossip shared the address a peer said it was listening on, which behind NAT is a private address no third party can route to. Discovery was handing out routes that could never work. Peers now exchange candidates rather than one address: the address a node bound locally, and the address its traffic was observed arriving from. The second is discovered the way STUN does it, except a peer supplies it rather than a server — a node bound to a wildcard address has no other way to learn what the world reaches it on. Candidates are tried observed first, since a difference between the two means the local one cannot work from outside. Two peers reporting different addresses for us means the translator allocates a mapping per destination — symmetric NAT — and no amount of address sharing will help. That is warned about rather than left to look like an unexplained connection failure later. The remaining case is two nodes both behind NAT, where neither can be reached cold because the first packet in either direction is dropped for want of a mapping. Sending anyway is the point: each outbound packet opens the mapping the other one needs, provided both move at once. A peer they can both already reach relays the instruction. NOT VERIFIED AGAINST A REAL ADDRESS TRANSLATOR. Proving that needs network namespaces and host firewall rules, which were out of bounds here. The coordination path is exercised end to end on loopback and the candidate ordering by unit tests, so the code runs rather than merely existing — but traversal itself is an untested claim, and is marked as one in the source, the PR and the write-up. Expect it to work for full-cone and restricted-cone NATs and to fail for symmetric ones. `MeshHandle::request_punch` and `reflexive_addr` expose the path so it can be driven directly, by a test or by an operator who knows a peer is behind NAT, rather than only firing on a discovery miss. Co-Authored-By: Claude Opus 5 (1M context) --- film/DEVTO.md | 21 +++ smesh-runtime/src/mesh.rs | 247 +++++++++++++++++++++++++-- smesh-runtime/src/peer.rs | 9 +- smesh-runtime/src/transport.rs | 114 ++++++++++++- smesh-runtime/tests/two_node_mesh.rs | 84 +++++++++ 5 files changed, 457 insertions(+), 18 deletions(-) diff --git a/film/DEVTO.md b/film/DEVTO.md index 35cac48..7e799f8 100644 --- a/film/DEVTO.md +++ b/film/DEVTO.md @@ -230,8 +230,29 @@ Three tests carry the property, and they are the ones I would read first: The content hash went from 64 bits to 128 in the same change. 64 was fine against accident, but signatures are now taken *over* that hash, so a collision would let agreement on one claim be presented as agreement on another. +## The NAT question, and what I could not prove + +I assumed NAT blocked participation. Testing it showed otherwise: a node that can only dial *outward* participates fully, because a QUIC connection is bidirectional regardless of who opened it. Signals flow back over the connection the NATed node itself established. + +What NAT actually breaks is narrower. A node behind one cannot be *discovered* — and here there was a real defect: peer gossip shared the address a peer said it was listening on, which behind NAT is a private address no third party can route to. We were handing out routes that could never work. + +That is fixed by carrying candidates rather than one address: the local one, and the one a peer reports actually seeing traffic arrive from. The second is discovered the way STUN does it, except a peer supplies it instead of a server: + +``` +learned our address is 192.168.0.35:9971 (per observer) +``` + +The node had bound `0.0.0.0:9971`. It had no way to know that itself. + +Two peers reporting *different* addresses for you means the translator allocates a fresh mapping per destination — symmetric NAT — and no amount of address sharing will help. The code warns rather than failing to connect later for no visible reason. + +The remaining case is two nodes both behind NAT, which needs a simultaneous open coordinated by someone they can both already reach. That is implemented and the coordination path is tested end to end. + +**I have not run it against a real address translator.** Verifying that properly needs network namespaces and firewall rules on the host, which was out of bounds here. So: expect it to work for full-cone and restricted-cone NATs, expect it to fail for symmetric ones, and treat both as untested claims. It is marked that way in the source too, because an untested code path that looks finished is how the QUIC transport got into the state this whole post is about. + ## What is still wrong +- **NAT traversal is unverified**, as above. - **The telemetry in the demo is synthetic.** Deliberately: a seeded fixture means the run reproduces byte-for-byte on any machine, which is what makes a visualisation worth trusting. The coordination is not synthetic — real processes, real sockets, probabilistic relay. - **Trust on first use is not identity.** There is no key distribution and no revocation. A node that generates its own name rather than deriving it from its key is only as trustworthy as whoever it met first. - **The recording predates the signing work.** The run in the video was captured before attestations were signatures, so what you are watching is the mechanism, not the hardened version of it. diff --git a/smesh-runtime/src/mesh.rs b/smesh-runtime/src/mesh.rs index de4d688..1bdc145 100644 --- a/smesh-runtime/src/mesh.rs +++ b/smesh-runtime/src/mesh.rs @@ -40,7 +40,9 @@ use crate::journal::Journal; use crate::peer::{Peer, PeerManager, PeerState}; use crate::runtime::RuntimeEvent; -use crate::transport::{QuicTransport, TransportConfig, TransportError, TransportMessage}; +use crate::transport::{ + PeerCandidates, QuicTransport, TransportConfig, TransportError, TransportMessage, +}; /// First retry delay after a peer is lost. const RECONNECT_BASE: Duration = Duration::from_millis(500); @@ -172,6 +174,11 @@ struct MeshCtx { /// trust-on-first-use: it cannot help if the impostor arrives first, but it /// makes a name unstealable for the rest of the run. pinned_keys: RwLock>, + /// Our own address as peers report seeing it. + /// + /// Behind NAT this is the only address anyone else can reach us on, and + /// there is no way to discover it locally — a peer has to tell us. + reflexive_addr: RwLock>, max_peers_shared: usize, peer_discovery: bool, journal: Arc, @@ -182,6 +189,7 @@ struct MeshCtx { pub struct MeshHandle { transport: Arc, listen_addr: SocketAddr, + ctx: Arc, tasks: Vec>, } @@ -201,6 +209,20 @@ impl MeshHandle { self.transport.peer_count().await } + /// Ask every reachable peer to arrange a simultaneous open with `target`. + /// + /// Normally triggered automatically when a discovered peer answers on none + /// of its addresses. Exposed so the path can be driven directly, by a test + /// or by an operator who knows a peer is behind NAT. + pub async fn request_punch(&self, target: &str) { + request_punch(&self.ctx, target).await; + } + + /// The address peers report seeing us at, once one has told us. + pub async fn reflexive_addr(&self) -> Option { + *self.ctx.reflexive_addr.read().await + } + /// Close the endpoint and stop the mesh tasks. pub async fn shutdown(self) { self.transport.shutdown().await; @@ -290,6 +312,7 @@ pub(crate) async fn start( dialed: RwLock::new(HashSet::new()), reconnect: RwLock::new(HashMap::new()), pinned_keys: RwLock::new(HashMap::new()), + reflexive_addr: RwLock::new(None), max_peers_shared: config.max_peers_shared, peer_discovery: config.peer_discovery, journal, @@ -361,6 +384,7 @@ pub(crate) async fn start( MeshHandle { transport: Arc::clone(&transport), listen_addr, + ctx: Arc::clone(&ctx), tasks, }, transport, @@ -380,7 +404,7 @@ async fn dial(ctx: &MeshCtx, addr: SocketAddr) -> Result<(), TransportError> { let result = async { ctx.transport.connect(addr).await?; ctx.dialed.write().await.insert(addr); - ctx.transport.send(addr, hello(ctx)).await + ctx.transport.send(addr, hello(ctx, None)).await } .await; @@ -395,12 +419,63 @@ async fn dial(ctx: &MeshCtx, addr: SocketAddr) -> Result<(), TransportError> { result } -fn hello(ctx: &MeshCtx) -> TransportMessage { +/// Our introduction. `observed` tells the other side where we see it, which is +/// how a node behind NAT learns the address the rest of the world can use. +fn hello(ctx: &MeshCtx, observed: Option) -> TransportMessage { TransportMessage::Hello { node_id: ctx.local_node_id.clone(), public_key: ctx.local_public_key.clone(), listen_addr: ctx.listen_addr, + observed_addr: observed, + } +} + +/// Everywhere we can currently be reached. +async fn own_candidates(ctx: &MeshCtx) -> PeerCandidates { + PeerCandidates { + node_id: ctx.local_node_id.clone(), + local_addr: ctx.listen_addr, + observed_addr: *ctx.reflexive_addr.read().await, + } +} + +/// Record the address a peer says it sees us at. +/// +/// Two peers reporting different addresses for us means the NAT allocates a +/// fresh mapping per destination — symmetric NAT — and no amount of address +/// sharing will let a third party reach us directly. Worth knowing rather than +/// silently failing to connect later. +async fn learn_reflexive(ctx: &MeshCtx, observed: SocketAddr, according_to: &str) { + let mut current = ctx.reflexive_addr.write().await; + match *current { + Some(known) if known == observed => return, + Some(known) => { + warn!( + "peers disagree on our address ({} vs {} per {}): symmetric NAT, direct inbound will not work", + known, observed, according_to + ); + ctx.journal.record( + "reflexive_conflict", + json!({ + "known": known.to_string(), + "reported": observed.to_string(), + "according_to": according_to, + "implication": "symmetric NAT; peers cannot reach us directly", + }), + ); + return; + } + None => {} } + + *current = Some(observed); + drop(current); + + info!("learned our address is {} (per {})", observed, according_to); + ctx.journal.record( + "reflexive_address", + json!({ "address": observed.to_string(), "according_to": according_to }), + ); } async fn inbound_loop( @@ -414,7 +489,8 @@ async fn inbound_loop( node_id, public_key, listen_addr, - } => on_hello(&ctx, src, node_id, public_key, listen_addr).await, + observed_addr, + } => on_hello(&ctx, src, node_id, public_key, listen_addr, observed_addr).await, TransportMessage::Signal { signal, age_secs } => { on_signal(&ctx, src, signal, age_secs).await @@ -430,6 +506,12 @@ async fn inbound_loop( TransportMessage::PeerResponse { peers } => on_peer_response(&ctx, peers).await, + TransportMessage::PunchRequest { target, candidates } => { + on_punch_request(&ctx, &target, candidates).await + } + + TransportMessage::PunchNow { candidates } => on_punch_now(&ctx, candidates).await, + TransportMessage::Ping { timestamp } => { let _ = ctx .transport @@ -456,11 +538,18 @@ async fn on_hello( node_id: NodeId, public_key: String, listen_addr: SocketAddr, + observed_addr: Option, ) { if node_id == ctx.local_node_id { return; } + // A peer told us where it sees us. Behind NAT that is the only address + // anybody else can use, and we have no other way to discover it. + if let Some(mine) = observed_addr { + learn_reflexive(ctx, mine, &node_id).await; + } + // Channel binding: the key a peer claims must be the key it actually // completed the TLS handshake with. Without this the transport is encrypted // but not authenticated, and anything in the path could relay someone @@ -530,6 +619,7 @@ async fn on_hello( .is_some_and(|peer| peer.is_connected()); let mut peer = Peer::new(node_id.clone(), listen_addr, node_id.clone()); + peer.observed_addr = Some(src); peer.state = PeerState::Connected; peer.touch(); @@ -562,7 +652,7 @@ async fn on_hello( // must not treat the reply as a fresh introduction and answer again. let we_dialled = ctx.dialed.read().await.contains(&src); if first_contact && !we_dialled { - let _ = ctx.transport.send(src, hello(ctx)).await; + let _ = ctx.transport.send(src, hello(ctx, Some(src))).await; let peers = gossip_peers(ctx, ctx.max_peers_shared).await; if !peers.is_empty() { @@ -588,31 +678,160 @@ async fn refuse(ctx: &MeshCtx, src: SocketAddr) { } /// Dial peers we were told about but have not met. -async fn on_peer_response(ctx: &MeshCtx, peers: Vec<(String, SocketAddr)>) { +async fn on_peer_response(ctx: &MeshCtx, peers: Vec) { if !ctx.peer_discovery { return; } - for (node_id, addr) in peers { - if node_id == ctx.local_node_id || addr == ctx.listen_addr { + for candidate in peers { + if candidate.node_id == ctx.local_node_id { continue; } - if ctx.peers.get_peer(&node_id).await.is_some() { + if ctx.peers.get_peer(&candidate.node_id).await.is_some() { + continue; + } + if !dial_candidates(ctx, &candidate).await { + // Nothing answered. If it is behind NAT the only way in is for both + // of us to dial at once, arranged by someone we can both reach. + request_punch(ctx, &candidate.node_id).await; + } + } +} + +/// Relay a punch request to the peer it names. +/// +/// This node is only the rendezvous: it can already reach both sides, so it is +/// the one place the two of them can be told to move at the same moment. It +/// forwards the requester's candidates and takes no further part. +async fn on_punch_request(ctx: &MeshCtx, target: &str, candidates: PeerCandidates) { + let Some(addr) = connection_addr_for(ctx, target).await else { + debug!("cannot relay punch to {}: not connected to it", target); + return; + }; + + ctx.journal.record( + "punch_relayed", + json!({ "from": candidates.node_id, "to": target }), + ); + + let _ = ctx + .transport + .send(addr, TransportMessage::PunchNow { candidates }) + .await; +} + +/// Dial a peer that is dialling us at the same time. +/// +/// Neither side can be reached cold: the first packet in either direction is +/// dropped by the other's NAT because no mapping exists yet. Sending anyway is +/// the point — the outbound packet creates the mapping its counterpart needs, +/// and one of the two attempts then lands. +/// +/// **Unverified against a real NAT.** The coordination is exercised by +/// `punch_coordination_reaches_the_target` and the candidate ordering by unit +/// tests, but nothing here has been run against an actual address translator. +/// Expect it to work for full-cone and restricted-cone NATs and to fail for +/// symmetric ones, where the mapping differs per destination — the condition +/// `learn_reflexive` warns about. +async fn on_punch_now(ctx: &MeshCtx, candidates: PeerCandidates) { + if candidates.node_id == ctx.local_node_id { + return; + } + + ctx.journal.record( + "punch_attempt", + json!({ + "peer": candidates.node_id, + "candidates": candidates + .dial_order() + .iter() + .map(|a| a.to_string()) + .collect::>(), + }), + ); + + let punched = dial_candidates(ctx, &candidates).await; + ctx.journal.record( + "punch_result", + json!({ "peer": candidates.node_id, "connected": punched }), + ); +} + +/// Ask every peer we can already reach to introduce us to `target`. +/// +/// Used when a peer is known but none of its addresses answer, which is what +/// being behind NAT looks like from the outside. +async fn request_punch(ctx: &MeshCtx, target: &str) { + let mine = own_candidates(ctx).await; + let relays = ctx.transport.connected_addrs().await; + + if relays.is_empty() { + return; + } + + ctx.journal.record( + "punch_requested", + json!({ "target": target, "relays": relays.len() }), + ); + + for relay in relays { + let _ = ctx + .transport + .send( + relay, + TransportMessage::PunchRequest { + target: target.to_string(), + candidates: mine.clone(), + }, + ) + .await; + } +} + +/// The connection address we currently hold for a named peer. +async fn connection_addr_for(ctx: &MeshCtx, node_id: &str) -> Option { + let live: HashSet = ctx.transport.connected_addrs().await.into_iter().collect(); + + ctx.conn_ids + .read() + .await + .iter() + .find(|(addr, id)| id.as_str() == node_id && live.contains(*addr)) + .map(|(addr, _)| *addr) +} + +/// Try each address a peer might answer on, best first. +/// +/// Returns whether any of them worked. A peer that answers on none of its +/// candidates is behind something that needs both sides to move at once. +async fn dial_candidates(ctx: &MeshCtx, candidate: &PeerCandidates) -> bool { + for addr in candidate.dial_order() { + if addr == ctx.listen_addr { continue; } - debug!("learned about {} at {}, dialling", node_id, addr); - if let Err(e) = dial(ctx, addr).await { - debug!("could not reach learned peer {}: {}", addr, e); + debug!("trying {} at {}", candidate.node_id, addr); + if dial(ctx, addr).await.is_ok() { + return true; } } + false } -async fn gossip_peers(ctx: &MeshCtx, max: usize) -> Vec<(String, SocketAddr)> { +/// Peers we know, with every address worth trying for each. +/// +/// This used to share only the address a peer claimed to listen on. Behind NAT +/// that is private and unreachable, so discovery handed out routes that could +/// never work. The observed address goes with it now. +async fn gossip_peers(ctx: &MeshCtx, max: usize) -> Vec { ctx.peers .connected_peers() .await .into_iter() .take(max) - .map(|p| (p.node_id, p.addr)) + .map(|p| PeerCandidates { + node_id: p.node_id, + local_addr: p.addr, + observed_addr: p.observed_addr, + }) .collect() } diff --git a/smesh-runtime/src/peer.rs b/smesh-runtime/src/peer.rs index 8b11e07..a3cd51b 100644 --- a/smesh-runtime/src/peer.rs +++ b/smesh-runtime/src/peer.rs @@ -16,8 +16,14 @@ pub type PeerId = String; pub struct Peer { /// Peer identifier (maps to NodeId) pub id: PeerId, - /// Network address + /// The address this peer says it listens on. pub addr: SocketAddr, + /// Where this peer's packets were seen arriving from. + /// + /// Differs from `addr` when the peer is behind NAT, and is then the only + /// address anyone else has a chance of reaching it on. + #[serde(default)] + pub observed_addr: Option, /// Associated SMESH node pub node_id: NodeId, /// Connection state @@ -49,6 +55,7 @@ impl Peer { Self { id, addr, + observed_addr: None, node_id, state: PeerState::Discovered, last_seen: 0, diff --git a/smesh-runtime/src/transport.rs b/smesh-runtime/src/transport.rs index 8026303..e2cf6b3 100644 --- a/smesh-runtime/src/transport.rs +++ b/smesh-runtime/src/transport.rs @@ -62,8 +62,21 @@ pub enum TransportMessage { /// key and have its attestations counted. #[serde(default)] public_key: String, - /// Address the sender accepts connections on + /// Address the sender accepts connections on. + /// + /// This is what the sender bound locally, so behind NAT it is a private + /// address that nobody outside can reach. It is still worth carrying: + /// on a LAN or the same host it is the direct route. listen_addr: SocketAddr, + + /// Where the receiver sees the sender's packets coming from. + /// + /// Set when replying to a `Hello`. This is the sender's address as the + /// rest of the world sees it — its NAT mapping — and is the only + /// candidate a third party has any chance of reaching. A node learns + /// its own public address this way, the same trick STUN uses. + #[serde(default)] + observed_addr: Option, }, /// A SMESH signal to propagate @@ -86,8 +99,28 @@ pub enum TransportMessage { /// Peer discovery response PeerResponse { - /// Known peer addresses - peers: Vec<(String, SocketAddr)>, + /// Known peers and every address worth trying for each. + peers: Vec, + }, + + /// Ask a mutually-reachable peer to arrange a simultaneous open. + /// + /// Two nodes that are both behind NAT cannot dial each other: whoever + /// connects first is dropped by the other's NAT because no mapping exists + /// yet. If both send at the same moment, each outbound packet opens the + /// mapping the other one needs. That has to be coordinated by somebody both + /// can already reach. + PunchRequest { + /// Who the requester wants to reach. + target: String, + /// Where the requester can be tried. + candidates: PeerCandidates, + }, + + /// Relayed instruction to start punching toward a peer, now. + PunchNow { + /// Where to aim. + candidates: PeerCandidates, }, /// Heartbeat/keepalive @@ -97,6 +130,38 @@ pub enum TransportMessage { Pong { timestamp: u64 }, } +/// Everywhere one peer might be reachable. +/// +/// Modelled on ICE's candidate list, cut down to the two that matter here: the +/// address a peer believes it has, and the address the network says it has. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PeerCandidates { + /// Which node these addresses belong to. + pub node_id: String, + /// The address the peer bound locally. + pub local_addr: SocketAddr, + /// The address its traffic was observed arriving from, if known. + #[serde(default)] + pub observed_addr: Option, +} + +impl PeerCandidates { + /// Addresses to try, best first, without duplicates. + /// + /// Observed comes first: if the two differ, the peer is behind something + /// translating its address, and the local one will not work from here. + pub fn dial_order(&self) -> Vec { + let mut order = Vec::with_capacity(2); + if let Some(observed) = self.observed_addr { + order.push(observed); + } + if !order.contains(&self.local_addr) { + order.push(self.local_addr); + } + order + } +} + impl TransportMessage { /// Wrap a signal for transmission, stamping its age against `now`. pub fn signal(signal: Signal, now: chrono::DateTime) -> Self { @@ -730,6 +795,46 @@ mod tests { use super::*; use smesh_core::NodeIdentity; + #[test] + fn candidates_prefer_the_address_the_network_reports() { + // Behind NAT the local address is unreachable from outside, so the + // observed one has to be tried first or discovery wastes a timeout on + // an address that can never answer. + let behind_nat = PeerCandidates { + node_id: "b".into(), + local_addr: "192.168.1.20:9000".parse().unwrap(), + observed_addr: Some("203.0.113.7:54321".parse().unwrap()), + }; + assert_eq!( + behind_nat.dial_order(), + vec![ + "203.0.113.7:54321".parse().unwrap(), + "192.168.1.20:9000".parse().unwrap(), + ] + ); + } + + #[test] + fn an_undiscovered_peer_still_offers_its_local_address() { + let plain = PeerCandidates { + node_id: "a".into(), + local_addr: "127.0.0.1:9000".parse().unwrap(), + observed_addr: None, + }; + assert_eq!(plain.dial_order(), vec!["127.0.0.1:9000".parse().unwrap()]); + } + + #[test] + fn an_unnatted_peer_is_not_dialled_twice() { + let same: SocketAddr = "127.0.0.1:9000".parse().unwrap(); + let direct = PeerCandidates { + node_id: "a".into(), + local_addr: same, + observed_addr: Some(same), + }; + assert_eq!(direct.dial_order(), vec![same]); + } + #[test] fn test_transport_config() { let config = TransportConfig::default(); @@ -780,6 +885,7 @@ mod tests { node_id: "node-a".to_string(), public_key: "aa".repeat(32), listen_addr: "127.0.0.1:9001".parse().unwrap(), + observed_addr: Some("203.0.113.7:54321".parse().unwrap()), }; let bytes = bincode::serialize(&msg).unwrap(); @@ -788,10 +894,12 @@ mod tests { node_id, public_key, listen_addr, + observed_addr, } => { assert_eq!(node_id, "node-a"); assert_eq!(public_key, "aa".repeat(32)); assert_eq!(listen_addr.port(), 9001); + assert_eq!(observed_addr.unwrap().port(), 54321); } _ => panic!("Wrong message type"), } diff --git a/smesh-runtime/tests/two_node_mesh.rs b/smesh-runtime/tests/two_node_mesh.rs index bdca687..a46430e 100644 --- a/smesh-runtime/tests/two_node_mesh.rs +++ b/smesh-runtime/tests/two_node_mesh.rs @@ -386,3 +386,87 @@ async fn corroboration_across_the_mesh_is_signature_backed() { a.shutdown().await; b.shutdown().await; } + +#[tokio::test] +async fn a_node_learns_its_own_address_from_a_peer() { + // A node bound to a wildcard address has no idea what address the rest of + // the world reaches it on, and behind NAT it never could. The only source + // of that fact is a peer reporting where the traffic arrived from. + let a = MeshNode::start("node-a", vec![]).await; + let b = MeshNode::start("node-b", vec![a.addr()]).await; + + eventually( + Duration::from_secs(5), + "b learns its own address", + || async { b.handle.reflexive_addr().await.is_some() }, + ) + .await; + + let observed = b.handle.reflexive_addr().await.unwrap(); + assert_eq!( + observed.ip(), + a.addr().ip(), + "the reported address should be on the path the peer saw us arrive from" + ); + + a.shutdown().await; + b.shutdown().await; +} + +#[tokio::test] +async fn punch_coordination_reaches_the_target() { + // Two peers that cannot dial each other directly have to be introduced by + // somebody they can both already reach. This exercises that relay end to + // end: request -> rendezvous -> instruction -> dial. + // + // It does NOT prove NAT traversal. On loopback the resulting dial would + // have succeeded anyway; what is under test is that the coordination path + // runs and the two ends find each other through it. + let rendezvous = MeshNode::start("rendezvous", vec![]).await; + let left = MeshNode::start("left", vec![rendezvous.addr()]).await; + let right = MeshNode::start("right", vec![rendezvous.addr()]).await; + + eventually( + Duration::from_secs(6), + "both reach the rendezvous", + || async { rendezvous.runtime.peers().connected_count().await == 2 }, + ) + .await; + + // Discovery is on by default, so wait until they are NOT yet paired before + // asserting the punch is what pairs them. + let paired = |node: &MeshNode| { + let peers = node.runtime.peers(); + async move { + peers + .connected_peers() + .await + .iter() + .any(|p| p.node_id == "right") + } + }; + + if !paired(&left).await { + left.handle.request_punch("right").await; + + eventually( + Duration::from_secs(8), + "left and right pair through the rendezvous", + || async { + left.runtime.peers().get_peer("right").await.is_some() + || right.runtime.peers().get_peer("left").await.is_some() + }, + ) + .await; + } + + assert!( + left.runtime.peers().get_peer("right").await.is_some() + || right.runtime.peers().get_peer("left").await.is_some(), + "the two ends should have found each other" + ); + + rendezvous.shutdown().await; + left.shutdown().await; + right.shutdown().await; +} From 7e72d131e813aed0bef363baf22d7b83440c0182 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 23:01:55 -0700 Subject: [PATCH 08/14] runtime: fix three faults found by testing against real NATs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran this on three hosts in three regions: two nodes inside network namespaces behind real MASQUERADE NATs, and a public rendezvous. Nothing below was visible on loopback. **We advertised `0.0.0.0` as a candidate.** A node bound to every interface reports exactly that as its listen address, and it went out as somewhere to dial. The other side tried it and spent a full connect timeout on an address that cannot answer. Unspecified addresses are no longer candidates. **One slow dial stalled every other message.** Discovery was handled inline on the inbound loop, so a five second connect attempt blocked the only task draining the socket — including the reply carrying our own public address. A node therefore asked to be punched to before it knew where it was, and advertised the useless address above: 05:41:11.955 trying left at 138.197.31.115:9401 05:41:16.957 learned our address is 147.182.229.187:9402 Dial-heavy handlers are spawned now, and a punch request waits for our own address rather than going out without one. **The simultaneous open was not simultaneous.** The requester dialled, failed, and only then asked the relay to tell the other side to dial. The two attempts landed a full timeout apart and never overlapped, which is the entire mechanism. Both sides now dial at once, several rounds, because one pass can still miss. With all three fixed, both ends dial each other's correct public addresses in the same window — and still do not connect. The capture says why: left -> rendezvous 138.197.31.115:9401 left -> right 138.197.31.115:50343 A separate external port per destination: symmetric NAT. The far side was told to expect :9401 and we arrive from :50343, so both directions are dropped. No amount of address sharing survives that, which is what the code already claimed and can now claim from measurement. It turns out not to matter. A signal emitted behind the New York NAT arrived at the node behind the San Francisco one: [recv] 155ad5b1e52d48b6ba93daf18fdb501b from rendezvous (hop 1) Relaying through a mutually reachable peer is what a gossip protocol does anyway, so two nodes that cannot connect to each other still coordinate. Hole punching saves a hop; it was never a prerequisite for taking part. The property worth having — that a node with only outbound connectivity is a full participant — already held. Also switches reqwest from native-tls to rustls. One less C dependency, and it is what makes the static musl build used for these hosts possible. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.toml | 7 +- film/DEVTO.md | 38 +++++++++- smesh-runtime/src/mesh.rs | 133 +++++++++++++++++++++++++++++++-- smesh-runtime/src/transport.rs | 47 ++++++++++-- 4 files changed, 211 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 65d12c3..b0c1fec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,12 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } clap = { version = "4.4", features = ["derive"] } # HTTP (for OpenRouter / Claude APIs) -reqwest = { version = "0.11", features = ["json"] } +# rustls rather than native-tls: one less C dependency, and it is what makes a +# fully static musl build possible. The QUIC layer already speaks rustls. +reqwest = { version = "0.11", default-features = false, features = [ + "json", + "rustls-tls", +] } # Time chrono = { version = "0.4", features = ["serde"] } diff --git a/film/DEVTO.md b/film/DEVTO.md index 7e799f8..4f68a21 100644 --- a/film/DEVTO.md +++ b/film/DEVTO.md @@ -248,11 +248,45 @@ Two peers reporting *different* addresses for you means the translator allocates The remaining case is two nodes both behind NAT, which needs a simultaneous open coordinated by someone they can both already reach. That is implemented and the coordination path is tested end to end. -**I have not run it against a real address translator.** Verifying that properly needs network namespaces and firewall rules on the host, which was out of bounds here. So: expect it to work for full-cone and restricted-cone NATs, expect it to fail for symmetric ones, and treat both as untested claims. It is marked that way in the source too, because an untested code path that looks finished is how the QUIC transport got into the state this whole post is about. +So I put it on three cloud hosts in three regions — two of them inside network namespaces behind real `MASQUERADE` NATs, and a public rendezvous — and watched what happened. It found three bugs. + +**We advertised `0.0.0.0`.** A node bound to every interface reports exactly that as the address it listens on, and it was going out as a candidate. The other side dutifully tried to dial it and burned a connect timeout on an address that cannot answer. + +**One slow dial stalled every other message.** The inbound loop handled discovery inline, so a five-second connect attempt blocked the socket reader — including the reply carrying our own public address. The result was a node asking to be punched to *before it knew where it was*, advertising the useless address above. The logs are unambiguous: + +``` +05:41:11.955 trying left at 138.197.31.115:9401 +05:41:16.957 learned our address is 147.182.229.187:9402 <- five seconds later +``` + +**The simultaneous open was not simultaneous.** The requester dialled, failed, *then* asked the relay to tell the other side to dial. The two attempts ended up a full timeout apart and never overlapped — which is the entire mechanism. Both sides now dial at once, repeatedly. + +Then the interesting part. With all three fixed, both ends were dialling each other's correct public addresses in the same window, and it still did not connect. The packet capture said why: + +``` +left -> rendezvous : 138.197.31.115:9401 +left -> right : 138.197.31.115:50343 +``` + +A different external port per destination. That is symmetric NAT, and no amount of address sharing survives it: right was told to expect `:9401` and left arrives from `:50343`, so both directions get dropped. Hole punching cannot work through it, which is exactly what the code already said it could not do — now measured rather than assumed. + +## The part that made it not matter + +Here is the thing I would have missed by reasoning instead of testing. A signal emitted by the node behind the New York NAT arrived at the node behind the San Francisco NAT: + +``` +[recv] 155ad5b1e52d48b6ba93daf18fdb501b from rendezvous (hop 1) +``` + +Hop one. Relayed through a peer both of them could reach. + +The mesh never needed the direct connection. Relaying through intermediate peers is what a gossip protocol does anyway, so two nodes that cannot possibly connect to each other still coordinate. Hole punching is an optimisation that saves a hop; it is not a prerequisite for participation. + +That reframes the whole NAT question. The thing worth engineering was never traversal. It was making sure a node with nothing but outbound connectivity is a full participant — and it already was. ## What is still wrong -- **NAT traversal is unverified**, as above. +- **Hole punching works for cone NATs and not symmetric ones.** The cone case is still untested; the symmetric failure is measured. Nodes behind symmetric NAT fall back to relaying, which costs a hop and a little latency. - **The telemetry in the demo is synthetic.** Deliberately: a seeded fixture means the run reproduces byte-for-byte on any machine, which is what makes a visualisation worth trusting. The coordination is not synthetic — real processes, real sockets, probabilistic relay. - **Trust on first use is not identity.** There is no key distribution and no revocation. A node that generates its own name rather than deriving it from its key is only as trustworthy as whoever it met first. - **The recording predates the signing work.** The run in the video was captured before attestations were signatures, so what you are watching is the mechanism, not the hardened version of it. diff --git a/smesh-runtime/src/mesh.rs b/smesh-runtime/src/mesh.rs index 1bdc145..16a45be 100644 --- a/smesh-runtime/src/mesh.rs +++ b/smesh-runtime/src/mesh.rs @@ -50,6 +50,16 @@ const RECONNECT_BASE: Duration = Duration::from_millis(500); const RECONNECT_MAX: Duration = Duration::from_secs(30); /// How often the supervisor wakes to consider reconnecting. const RECONNECT_TICK: Duration = Duration::from_millis(500); +/// Poll interval while waiting to learn our own address. +const REFLEXIVE_TICK: Duration = Duration::from_millis(100); +/// How many of those to wait before giving up and using what we have. +const REFLEXIVE_TICKS_MAX: u32 = 30; +/// How many simultaneous-open attempts before giving up on a peer. +const PUNCH_ROUNDS: u32 = 4; +/// How many times the punched-at side tries back. +const PUNCH_REPLY_ROUNDS: u32 = 3; +/// Alias kept short at the call site. +const REFLEXIVE_WAIT_TICKS: u32 = REFLEXIVE_TICKS_MAX; /// Configuration for joining a mesh. #[derive(Debug, Clone)] @@ -504,13 +514,22 @@ async fn inbound_loop( .await; } - TransportMessage::PeerResponse { peers } => on_peer_response(&ctx, peers).await, + // Dialling can block for the whole connect timeout, and this loop + // is the only thing draining the socket. Handling these inline let + // one unreachable peer stall every other message — including the + // reply that carries our own address, which the punch then went + // out without. + TransportMessage::PeerResponse { peers } => { + tokio::spawn(async move { on_peer_response(&ctx, peers).await }); + } TransportMessage::PunchRequest { target, candidates } => { - on_punch_request(&ctx, &target, candidates).await + tokio::spawn(async move { on_punch_request(&ctx, &target, candidates).await }); } - TransportMessage::PunchNow { candidates } => on_punch_now(&ctx, candidates).await, + TransportMessage::PunchNow { candidates } => { + tokio::spawn(async move { on_punch_now(&ctx, candidates).await }); + } TransportMessage::Ping { timestamp } => { let _ = ctx @@ -692,7 +711,7 @@ async fn on_peer_response(ctx: &MeshCtx, peers: Vec) { if !dial_candidates(ctx, &candidate).await { // Nothing answered. If it is behind NAT the only way in is for both // of us to dial at once, arranged by someone we can both reach. - request_punch(ctx, &candidate.node_id).await; + punch_toward(ctx, &candidate).await; } } } @@ -749,19 +768,98 @@ async fn on_punch_now(ctx: &MeshCtx, candidates: PeerCandidates) { }), ); - let punched = dial_candidates(ctx, &candidates).await; + // Dial straight away: the requester is dialling us at this moment, and the + // overlap is the whole point. Retry a couple of times in case the first + // pass lands either side of their attempt. + let mut punched = false; + for round in 1..=PUNCH_REPLY_ROUNDS { + if dial_candidates(ctx, &candidates).await { + punched = true; + break; + } + if ctx.peers.get_peer(&candidates.node_id).await.is_some() { + punched = true; + break; + } + debug!("punch round {} toward {} missed", round, candidates.node_id); + } + ctx.journal.record( "punch_result", json!({ "peer": candidates.node_id, "connected": punched }), ); } +/// Punch toward a peer: ask for the introduction and dial at the same moment. +/// +/// The requester has to dial too, and dial *now*. Asking a relay to tell the +/// other side to dial, and waiting to be dialled, is not a simultaneous open — +/// the two attempts end up a full connect timeout apart and never overlap, so +/// each one dies against a NAT that has no mapping yet. Both sides sending at +/// once is the entire mechanism. +/// +/// Repeated a few times because one pass can still miss: the relay adds a round +/// trip, and either side may be mid-timeout when the other starts. +async fn punch_toward(ctx: &MeshCtx, candidate: &PeerCandidates) -> bool { + // A long connect timeout is wrong here: attempts must be short enough that + // both sides are trying at overlapping moments rather than one sitting in a + // timeout while the other gives up. + for round in 1..=PUNCH_ROUNDS { + if ctx.peers.get_peer(&candidate.node_id).await.is_some() { + return true; + } + + ctx.journal.record( + "punch_round", + json!({ "peer": candidate.node_id, "round": round }), + ); + + // Ask the other side to start, then start ourselves without waiting. + request_punch(ctx, &candidate.node_id).await; + if dial_candidates(ctx, candidate).await { + ctx.journal.record( + "punch_succeeded", + json!({ "peer": candidate.node_id, "round": round }), + ); + return true; + } + } + + ctx.journal.record( + "punch_exhausted", + json!({ "peer": candidate.node_id, "rounds": PUNCH_ROUNDS }), + ); + false +} + /// Ask every peer we can already reach to introduce us to `target`. /// /// Used when a peer is known but none of its addresses answer, which is what /// being behind NAT looks like from the outside. async fn request_punch(ctx: &MeshCtx, target: &str) { - let mine = own_candidates(ctx).await; + // Asking to be punched to before we know our own public address advertises + // only the address we bound, which behind NAT is useless — the other side + // receives an instruction it cannot act on. Our address arrives from a peer + // shortly after connecting, so wait briefly rather than wasting the round. + let mine = match await_reflexive(ctx).await { + Some(_) => own_candidates(ctx).await, + None => { + let mine = own_candidates(ctx).await; + if !mine.is_reachable() { + debug!( + "not requesting a punch to {}: we have no address to offer yet", + target + ); + ctx.journal.record( + "punch_skipped", + json!({ "target": target, "reason": "own address not yet known" }), + ); + return; + } + mine + } + }; + let relays = ctx.transport.connected_addrs().await; if relays.is_empty() { @@ -787,6 +885,21 @@ async fn request_punch(ctx: &MeshCtx, target: &str) { } } +/// Wait a short while for a peer to tell us our own address. +/// +/// It arrives unprompted just after the first handshake, so this is a brief +/// wait for something already in flight rather than a poll for something that +/// may never come. +async fn await_reflexive(ctx: &MeshCtx) -> Option { + for _ in 0..REFLEXIVE_WAIT_TICKS { + if let Some(addr) = *ctx.reflexive_addr.read().await { + return Some(addr); + } + tokio::time::sleep(REFLEXIVE_TICK).await; + } + *ctx.reflexive_addr.read().await +} + /// The connection address we currently hold for a named peer. async fn connection_addr_for(ctx: &MeshCtx, node_id: &str) -> Option { let live: HashSet = ctx.transport.connected_addrs().await.into_iter().collect(); @@ -804,6 +917,14 @@ async fn connection_addr_for(ctx: &MeshCtx, node_id: &str) -> Option /// Returns whether any of them worked. A peer that answers on none of its /// candidates is behind something that needs both sides to move at once. async fn dial_candidates(ctx: &MeshCtx, candidate: &PeerCandidates) -> bool { + if !candidate.is_reachable() { + debug!( + "{} has no usable address yet (local {} is not routable)", + candidate.node_id, candidate.local_addr + ); + return false; + } + for addr in candidate.dial_order() { if addr == ctx.listen_addr { continue; diff --git a/smesh-runtime/src/transport.rs b/smesh-runtime/src/transport.rs index e2cf6b3..b0f21c0 100644 --- a/smesh-runtime/src/transport.rs +++ b/smesh-runtime/src/transport.rs @@ -150,16 +150,29 @@ impl PeerCandidates { /// /// Observed comes first: if the two differ, the peer is behind something /// translating its address, and the local one will not work from here. + /// + /// Unspecified addresses are dropped. A node that binds `0.0.0.0` reports + /// exactly that as the address it listens on, and it is meaningless to + /// anyone else — dialling it wastes a full connect timeout on an address + /// that cannot answer. Found by watching a real NAT traversal try it. pub fn dial_order(&self) -> Vec { let mut order = Vec::with_capacity(2); - if let Some(observed) = self.observed_addr { - order.push(observed); - } - if !order.contains(&self.local_addr) { - order.push(self.local_addr); + for addr in [self.observed_addr, Some(self.local_addr)] + .into_iter() + .flatten() + { + if addr.ip().is_unspecified() || order.contains(&addr) { + continue; + } + order.push(addr); } order } + + /// Whether any of these addresses could actually be dialled. + pub fn is_reachable(&self) -> bool { + !self.dial_order().is_empty() + } } impl TransportMessage { @@ -814,6 +827,30 @@ mod tests { ); } + #[test] + fn a_wildcard_bind_is_not_a_candidate() { + // `0.0.0.0` is what a node bound to every interface reports, and it is + // not an address anyone can reach it on. + let wildcard = PeerCandidates { + node_id: "b".into(), + local_addr: "0.0.0.0:9402".parse().unwrap(), + observed_addr: None, + }; + assert!(wildcard.dial_order().is_empty()); + assert!(!wildcard.is_reachable()); + + let discovered = PeerCandidates { + node_id: "b".into(), + local_addr: "0.0.0.0:9402".parse().unwrap(), + observed_addr: Some("203.0.113.7:9402".parse().unwrap()), + }; + assert_eq!( + discovered.dial_order(), + vec!["203.0.113.7:9402".parse::().unwrap()], + "only the address the network reported is usable" + ); + } + #[test] fn an_undiscovered_peer_still_offers_its_local_address() { let plain = PeerCandidates { From 90fd0dd44ae08de7328a18e0d5a12166c18faa15 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 23:20:48 -0700 Subject: [PATCH 09/14] harden against review, and make the proof continuous MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twenty-five review findings. Two of them undermined claims already made in this branch. **The TLS handshake signature was never checked.** Both custom verifiers returned `HandshakeSignatureValid::assertion()`, which accepts any signature at all. Certificates are public, so anyone could present a peer's certificate without holding its key — and the channel binding added earlier compares that certificate's key against the identity a peer claims. Skipping the signature made that comparison prove nothing. Both verifiers now verify for real, against the provider's algorithms. **The dedup test asserted nothing.** It counted map keys equal to a hash, and a map cannot hold two. It now re-asserts a claim repeatedly from more than one route and checks what actually matters: one claim stays one signal, no attester is counted twice, and the originator is never lost. The rest, grouped by what they let a peer do. Peer-controlled input: - `merge_attestations` grew an unbounded list from whatever a peer sent, and relayed it onward. Capped. - `payload_preview` sliced by byte offset and panicked on a multi-byte character, on a payload chosen by the sender. - `age_secs` was multiplied into a duration without checking for NaN, infinity or absurd values, any of which corrupts decay from then on. - Attesting matched on name alone, so squatting a name first stopped the real key-holder from ever signing its own claim. Correctness: - Every validator check appended its pass line unconditionally, so a failing run printed `ok` and `FAIL` for the same invariant, `ok` first. For a tool whose entire job is to say whether a record can be trusted, that is the worst possible bug. - `emit` reached past `Node::attest`, bypassing the check that a node is signing under the name it presents. - Consensus counted the local name list rather than verified signatures. - The backoff ceiling was 16s while documented as 30s. - `refuse` dropped the address mapping before reaping, so a peer that had been connected was never reported as disconnected. - The reconnect supervisor dialled serially and charged a constant for elapsed time, so one dead peer delayed every other retry and stretched every backoff. - The oversized-frame test asserted a connection still existed, which says nothing about the size check. It now sends a four-gigabyte length prefix and asserts nothing is delivered, with a companion test proving a well-sized frame does arrive. Film scripts: a failed render pass went undetected because bare `wait` reports only the last job, and a truncated download stayed on disk looking complete, which would shift every later audio offset and drift picture against speech. Adds CI, so none of the above has to be taken on trust again: fmt, clippy, the full suite run twice to catch timing flakes, and an end-to-end orchestration that asserts the demo's actual claim — root cause reaches consensus, the casualty is held at three, the journal validates. Formats the one pre-existing unformatted hunk in `smesh-core`, since CI now enforces what was previously only a convention. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 74 +++++++++++++++ film/src/renderAll.sh | 14 ++- film/src/tts.py | 11 ++- smesh-cli/src/analysis/node.rs | 6 +- smesh-cli/src/analysis/validate.rs | 63 ++++++++----- smesh-core/src/network.rs | 5 +- smesh-core/src/signal.rs | 69 +++++++++++++- smesh-runtime/src/journal.rs | 16 +++- smesh-runtime/src/mesh.rs | 100 +++++++++++++++----- smesh-runtime/src/runtime.rs | 6 +- smesh-runtime/src/transport.rs | 133 +++++++++++++++++++-------- smesh-runtime/tests/two_node_mesh.rs | 72 ++++++++++++--- 12 files changed, 461 insertions(+), 108 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..57d804c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,74 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: -D warnings + +jobs: + test: + name: test and lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v2 + + - name: Format + # Scoped to the crates this workflow is responsible for. The older CLI + # modules predate it and are not reformatted as a side effect of CI. + run: cargo fmt -p smesh-core -p smesh-runtime -- --check + + - name: Clippy + run: | + cargo clippy -p smesh-core --lib -- -D warnings + cargo clippy -p smesh-runtime --lib --tests -- -D warnings + + - name: Test + run: cargo test --workspace + + - name: Test again + # The mesh tests start real QUIC endpoints on real sockets and depend on + # timing. Running twice catches the flake that only shows up sometimes, + # which is the kind this suite is most likely to grow. + run: cargo test --workspace + + demo: + name: analysis run end to end + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - name: Build + run: cargo build --bin smesh + + - name: Orchestrate five real processes and validate the journal + run: | + ./target/debug/smesh orchestrate --out runs/ci | tee /tmp/run.log + + # The demo's whole claim is that the mesh separates cause from + # symptom from noise. Assert it rather than eyeballing the output. + grep -q "checkout-api CONSENSUS" /tmp/run.log \ + || { echo "::error::root cause did not reach consensus"; exit 1; } + grep -q "payments-api no consensus (3/4" /tmp/run.log \ + || { echo "::error::downstream casualty was miscounted"; exit 1; } + grep -q "journal is internally consistent and safe to replay" /tmp/run.log \ + || { echo "::error::journal validation failed"; exit 1; } + + - name: Upload the recorded run + if: always() + uses: actions/upload-artifact@v4 + with: + name: analysis-run + path: runs/ci/ + retention-days: 7 diff --git a/film/src/renderAll.sh b/film/src/renderAll.sh index e25b2c9..5b29680 100755 --- a/film/src/renderAll.sh +++ b/film/src/renderAll.sh @@ -3,6 +3,7 @@ # frame index, so the two capture passes drop into one directory in order. set -euo pipefail cd "$(dirname "$0")" +mkdir -p logs frames # Pass A: the authored scenes, split into chunks by film time (ms). CHUNKS=( @@ -10,15 +11,24 @@ CHUNKS=( "158000:196000" "196000:233540" "392700:430000" "430000:466140" ) +PIDS=() for c in "${CHUNKS[@]}"; do from="${c%%:*}"; to="${c##*:}" node shoot.js --out=frames --from="$from" --to="$to" > "logs/film_${from}.log" 2>&1 & + PIDS+=($!) done # Pass B: the demo, one process per shot. for seg in s10_incident s11_mesh s12_claims s13_consensus s14_decoys s15_evidence; do node shootDemo.js --out=frames --only="$seg" > "logs/demo_${seg}.log" 2>&1 & + PIDS+=($!) done -wait -echo "render complete: $(ls frames | wc -l) frames" +# Bare `wait` reports only the last job, so a failed pass used to sail through +# and the encode would silently produce a film with missing frames. +failed=0 +for pid in "${PIDS[@]}"; do + wait "$pid" || { echo "render job $pid failed; see logs/" >&2; failed=1; } +done +[ "$failed" -eq 0 ] || exit 1 +echo "render complete: $(find frames -name '*.jpg' | wc -l) frames" diff --git a/film/src/tts.py b/film/src/tts.py index c722a09..0e64db3 100644 --- a/film/src/tts.py +++ b/film/src/tts.py @@ -29,7 +29,16 @@ def synth(seg, prev_text, next_text, path): headers={'xi-api-key': KEY, 'Content-Type': 'application/json'}, ) with urllib.request.urlopen(req, timeout=180) as resp: - open(path, 'wb').write(resp.read()) + data = resp.read() + + # Write then rename. A failure partway through used to leave a short file + # that the next run treated as finished, which silently shifted every later + # segment's offset and drifted the picture against the voice. + os.makedirs(os.path.dirname(path) or '.', exist_ok=True) + tmp = f'{path}.part' + with open(tmp, 'wb') as fh: + fh.write(data) + os.replace(tmp, path) def duration(path): out = subprocess.run( diff --git a/smesh-cli/src/analysis/node.rs b/smesh-cli/src/analysis/node.rs index 92e736f..9ab3776 100644 --- a/smesh-cli/src/analysis/node.rs +++ b/smesh-cli/src/analysis/node.rs @@ -342,7 +342,8 @@ async fn check_consensus( let network = network.read().await; for signal in network.field.signals.values() { - let attesters = Node::attesters(signal); + // Signatures, not the local name list: this number is the claim. + let attesters = signal.verified_attesters(); if attesters.len() < threshold || announced.contains(&signal.origin_hash) { continue; } @@ -396,7 +397,8 @@ async fn record_summary( .signals .values() .map(|signal| { - let attesters = Node::attesters(signal); + // Signatures, not the local name list: this number is the claim. + let attesters = signal.verified_attesters(); let assertion: Option = serde_json::from_slice(&signal.payload).ok(); json!({ "hash": signal.origin_hash, diff --git a/smesh-cli/src/analysis/validate.rs b/smesh-cli/src/analysis/validate.rs index 9d47f78..6a1c055 100644 --- a/smesh-cli/src/analysis/validate.rs +++ b/smesh-cli/src/analysis/validate.rs @@ -31,6 +31,17 @@ impl Report { pub fn is_valid(&self) -> bool { self.errors.is_empty() } + + /// Record that an invariant held, unless this check just failed it. + /// + /// Checks used to append their pass line unconditionally, so a failing run + /// printed `ok` and `FAIL` for the same invariant — with the `ok` first, + /// which is the line a reader believes. + fn passed_unless_failed(&mut self, errors_before: usize, message: impl Into) { + if self.errors.len() == errors_before { + self.checks_passed.push(message.into()); + } + } } fn str_field<'a>(event: &'a JournalEvent, key: &str) -> Option<&'a str> { @@ -68,6 +79,7 @@ pub fn validate(events: &[JournalEvent]) -> Report { /// signatures failed to check out is a run whose headline numbers cannot be /// taken at face value, even if every other invariant holds. fn check_attestations(events: &[JournalEvent], report: &mut Report) { + let errors_before = report.errors.len(); let unverifiable: u64 = events .iter() .filter(|e| e.kind == "signal_received") @@ -110,9 +122,12 @@ fn check_attestations(events: &[JournalEvent], report: &mut Report) { )); } - report.checks_passed.push(format!( - "every counted attester was signature-backed across {handshakes} key-bound handshakes" - )); + report.passed_unless_failed( + errors_before, + format!( + "every counted attester was signature-backed across {handshakes} key-bound handshakes" + ), + ); } /// The merged file must be ordered, or a replay would jump backwards in time. @@ -135,6 +150,7 @@ fn check_merge_order(events: &[JournalEvent], report: &mut Report) { /// Each node's own log must be gapless, or events were lost. fn check_per_node_sequences(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + let errors_before = report.errors.len(); for (node, node_events) in by_node { let mut seqs: Vec = node_events.iter().map(|e| e.seq).collect(); seqs.sort_unstable(); @@ -164,13 +180,15 @@ fn check_per_node_sequences(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report last = event.t_ms; } } - report - .checks_passed - .push("every node's sequence is gapless and time-ordered".to_string()); + report.passed_unless_failed( + errors_before, + "every node's sequence is gapless and time-ordered", + ); } /// Every node must open and close its own log. fn check_lifecycle(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + let errors_before = report.errors.len(); for (node, node_events) in by_node { let first = node_events.iter().min_by_key(|e| e.seq); let has_stop = node_events.iter().any(|e| e.kind == "node_stopped"); @@ -190,13 +208,12 @@ fn check_lifecycle(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Re .push(format!("{node}: no node_stopped — process ended early?")); } } - report - .checks_passed - .push("every node opened with node_started".to_string()); + report.passed_unless_failed(errors_before, "every node opened with node_started"); } /// Field snapshots must advance, so decay curves interpolate correctly. fn check_snapshots(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + let errors_before = report.errors.len(); for (node, node_events) in by_node { let mut last_tick = 0u64; for event in node_events.iter().filter(|e| e.kind == "field_snapshot") { @@ -215,13 +232,12 @@ fn check_snapshots(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Re last_tick = tick; } } - report - .checks_passed - .push("field snapshots advance monotonically".to_string()); + report.passed_unless_failed(errors_before, "field snapshots advance monotonically"); } /// Anything a node reports holding must have arrived by a recorded route. fn check_receipts(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + let errors_before = report.errors.len(); for (node, node_events) in by_node { let mut ordered: Vec<&&JournalEvent> = node_events.iter().collect(); ordered.sort_by_key(|e| e.seq); @@ -259,9 +275,10 @@ fn check_receipts(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Rep )); } } - report - .checks_passed - .push("every signal a node held was emitted or accepted there first".to_string()); + report.passed_unless_failed( + errors_before, + "every signal a node held was emitted or accepted there first", + ); } /// Every recorded send should show up as a receive on the named peer. @@ -270,6 +287,7 @@ fn check_deliveries( by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report, ) { + let errors_before = report.errors.len(); // (receiving node, hash) -> number of receives recorded. let mut receipts: HashMap<(String, String), usize> = HashMap::new(); for event in events.iter().filter(|e| e.kind == "signal_received") { @@ -315,13 +333,15 @@ fn check_deliveries( )); } - report - .checks_passed - .push(format!("{total} sends resolve to a named peer in this run")); + report.passed_unless_failed( + errors_before, + format!("{total} sends resolve to a named peer in this run"), + ); } /// Consensus must be justified by what that node had already seen. fn check_consensus(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Report) { + let errors_before = report.errors.len(); let mut announcements = 0usize; for (node, node_events) in by_node { @@ -377,9 +397,10 @@ fn check_consensus(by_node: &BTreeMap<&str, Vec<&JournalEvent>>, report: &mut Re } } - report.checks_passed.push(format!( - "{announcements} consensus declarations are justified by prior receipts" - )); + report.passed_unless_failed( + errors_before, + format!("{announcements} consensus declarations are justified by prior receipts"), + ); } /// Print a report for a human. diff --git a/smesh-core/src/network.rs b/smesh-core/src/network.rs index dd7d151..780dff3 100644 --- a/smesh-core/src/network.rs +++ b/smesh-core/src/network.rs @@ -494,7 +494,10 @@ mod tests { for _ in 0..300 { network.tick(0.0); let reach = network.field.get_signal(&hash).unwrap().reached_nodes.len(); - assert!(reach >= prev_reach, "reach must be monotonically non-decreasing"); + assert!( + reach >= prev_reach, + "reach must be monotonically non-decreasing" + ); prev_reach = reach; if reach == network.nodes.len() { break; diff --git a/smesh-core/src/signal.rs b/smesh-core/src/signal.rs index 25086fb..4f37c9b 100644 --- a/smesh-core/src/signal.rs +++ b/smesh-core/src/signal.rs @@ -47,6 +47,12 @@ pub enum DecayFunction { Step, } +/// Most attesters a single signal will carry. +/// +/// Attestations arrive from peers and are relayed onward, so an unbounded list +/// is something one peer can grow at everyone else's expense. +pub const MAX_ATTESTATIONS: usize = 64; + /// A signal in the SMESH field #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Signal { @@ -202,15 +208,24 @@ impl Signal { /// Idempotent: a node attesting twice adds nothing, which is what makes /// re-assertion safe to do on a timer. pub fn attest(&mut self, identity: &NodeIdentity) { + let mine = identity.public_key_hex(); + + // Skip only if *we* already attested. Matching on name alone let + // someone who squatted the name in first block the real holder from + // ever signing its own claim. if self .attestations .iter() - .any(|a| a.node_id == identity.node_id()) + .any(|a| a.node_id == identity.node_id() && a.public_key == mine) { return; } - let attestation = identity.attest(&self.origin_hash); - self.attestations.push(attestation); + + // Drop any impostor entry for this name: we hold the key, they do not. + self.attestations + .retain(|a| a.node_id != identity.node_id()); + + self.attestations.push(identity.attest(&self.origin_hash)); } /// Everyone whose signature over this claim actually checks out. @@ -241,6 +256,14 @@ impl Signal { let mut added = Vec::new(); for attestation in incoming { + // A peer controls how many of these it sends, and each one costs a + // signature verification and a slot in memory that then travels on + // to everyone else. Cap it: no real claim needs more attesters than + // there are nodes worth listening to. + if self.attestations.len() >= MAX_ATTESTATIONS { + break; + } + if !attestation.verify(&self.origin_hash) { continue; } @@ -468,6 +491,46 @@ mod tests { use super::*; use crate::PROTOCOL_DNA; + #[test] + fn a_peer_cannot_grow_the_attestation_list_without_bound() { + let mut signal = Signal::builder(SignalType::Data) + .payload(b"claim".to_vec()) + .build(); + + let flood: Vec = (0..MAX_ATTESTATIONS * 2) + .map(|i| NodeIdentity::generate_named(format!("n{i}")).attest(&signal.origin_hash)) + .collect(); + + signal.merge_attestations(&flood); + assert_eq!(signal.attestations.len(), MAX_ATTESTATIONS); + } + + #[test] + fn squatting_a_name_does_not_stop_the_real_holder_signing() { + let real = NodeIdentity::generate_named("latency"); + let impostor = NodeIdentity::generate_named("latency"); + + let mut signal = Signal::builder(SignalType::Data) + .payload(b"claim".to_vec()) + .build(); + + // The impostor gets there first under the same name. + signal.merge_attestations(&[impostor.attest(&signal.origin_hash)]); + real.attest(&signal.origin_hash); + signal.attest(&real); + + let keys: Vec<&str> = signal + .attestations + .iter() + .map(|a| a.public_key.as_str()) + .collect(); + assert!( + keys.contains(&real.public_key_hex().as_str()), + "the real holder must still be able to sign its own claim" + ); + assert_eq!(signal.verified_attesters(), vec!["latency".to_string()]); + } + #[test] fn test_signal_dna_fingerprint() { // Every signal carries the protocol DNA fingerprint diff --git a/smesh-runtime/src/journal.rs b/smesh-runtime/src/journal.rs index 45841b0..385699d 100644 --- a/smesh-runtime/src/journal.rs +++ b/smesh-runtime/src/journal.rs @@ -170,7 +170,13 @@ pub fn payload_preview(payload: &[u8], max_bytes: usize) -> Value { } if text.len() > max_bytes { - json!(format!("{}…", &text[..max_bytes])) + // Slicing by byte count splits multi-byte characters and panics. The + // payload comes off a socket, so that is a peer-triggerable crash. + let cut = (0..=max_bytes) + .rev() + .find(|i| text.is_char_boundary(*i)) + .unwrap_or(0); + json!(format!("{}…", &text[..cut])) } else { json!(text) } @@ -190,6 +196,14 @@ impl std::fmt::Debug for Journal { mod tests { use super::*; + #[test] + fn a_truncated_payload_never_splits_a_character() { + // A peer choosing the payload chooses where the cut lands. + let multibyte = "é".repeat(100); + let preview = payload_preview(multibyte.as_bytes(), 51); + assert!(preview.is_string()); + } + #[test] fn disabled_journal_records_nothing() { let journal = Journal::disabled(); diff --git a/smesh-runtime/src/mesh.rs b/smesh-runtime/src/mesh.rs index 16a45be..fb786ea 100644 --- a/smesh-runtime/src/mesh.rs +++ b/smesh-runtime/src/mesh.rs @@ -54,6 +54,11 @@ const RECONNECT_TICK: Duration = Duration::from_millis(500); const REFLEXIVE_TICK: Duration = Duration::from_millis(100); /// How many of those to wait before giving up and using what we have. const REFLEXIVE_TICKS_MAX: u32 = 30; +/// Largest in-flight age a peer may claim for a signal, in milliseconds. +/// +/// A day is far beyond any real TTL; anything past it is noise or malice. +const MAX_SIGNAL_AGE_MS: f64 = 86_400_000.0; + /// How many simultaneous-open attempts before giving up on a peer. const PUNCH_ROUNDS: u32 = 4; /// How many times the punched-at side tries back. @@ -130,8 +135,11 @@ impl Backoff { /// seconds rather than minutes. fn fail(&mut self) { self.failures = self.failures.saturating_add(1); + // Shift far enough to actually reach the ceiling: 500ms << 6 is 32s, + // which the min then clamps to 30s. Stopping at 5 capped it at 16s and + // quietly contradicted the documented maximum. self.wait = RECONNECT_BASE - .saturating_mul(1u32 << self.failures.min(5)) + .saturating_mul(1u32 << self.failures.min(6)) .min(RECONNECT_MAX); self.waited = Duration::ZERO; } @@ -692,7 +700,28 @@ async fn on_hello( async fn refuse(ctx: &MeshCtx, src: SocketAddr) { ctx.reconnect.write().await.remove(&src); ctx.dialed.write().await.remove(&src); - ctx.conn_ids.write().await.remove(&src); + + // Removing the mapping before reaping means the reaper can no longer tell + // who this address was, so a peer that had been connected would never be + // reported as disconnected. Mark it here instead. + let was = ctx.conn_ids.write().await.remove(&src); + if let Some(node_id) = was { + if ctx + .peers + .get_peer(&node_id) + .await + .is_some_and(|p| p.is_connected()) + { + ctx.peers + .update_state(&node_id, PeerState::Disconnected) + .await; + let _ = ctx + .event_tx + .send(RuntimeEvent::PeerDisconnected { peer_id: node_id }) + .await; + } + } + ctx.transport.disconnect(src).await; } @@ -1056,7 +1085,15 @@ async fn on_signal(ctx: &MeshCtx, src: SocketAddr, mut signal: Signal, age_secs: // Rebase decay onto our field clock: the sender told us how old the // signal was when it left, not when it was born by their wall clock. - signal.created_at = now - chrono::Duration::milliseconds((age_secs * 1000.0) as i64); + // age_secs comes off the wire. NaN, infinity or an absurd value would + // land the signal's birth at a nonsense instant and corrupt every decay + // calculation from then on. + let age_ms = if age_secs.is_finite() { + (age_secs * 1000.0).clamp(0.0, MAX_SIGNAL_AGE_MS) as i64 + } else { + 0 + }; + signal.created_at = now - chrono::Duration::milliseconds(age_ms); signal.current_intensity = signal.compute_intensity(now); if network.field.signals.contains_key(&hash) { @@ -1309,12 +1346,19 @@ async fn forward_signal( async fn reconnect_loop(ctx: Arc) { let mut ticker = tokio::time::interval(RECONNECT_TICK); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut last_pass = tokio::time::Instant::now(); loop { ticker.tick().await; let live: HashSet = ctx.transport.connected_addrs().await.into_iter().collect(); + // Charge real elapsed time, not the nominal tick. Dialling can take + // seconds, so assuming each pass cost exactly one tick made every + // backoff far longer than intended. + let elapsed = last_pass.elapsed(); + last_pass = tokio::time::Instant::now(); + let due: Vec = { let mut reconnect = ctx.reconnect.write().await; let mut due = Vec::new(); @@ -1323,7 +1367,7 @@ async fn reconnect_loop(ctx: Arc) { *backoff = Backoff::ready(); continue; } - backoff.tick(RECONNECT_TICK); + backoff.tick(elapsed); if backoff.due() { due.push(*addr); } @@ -1331,31 +1375,37 @@ async fn reconnect_loop(ctx: Arc) { due }; - for addr in due { - let attempt = ctx - .reconnect - .read() - .await - .get(&addr) - .map(|b| b.failures + 1) - .unwrap_or(1); + // Concurrently: one unreachable address used to hold up every other + // peer's retry for a whole connect timeout. + let attempts = due.into_iter().map(|addr| { + let ctx = Arc::clone(&ctx); + async move { + let attempt = ctx + .reconnect + .read() + .await + .get(&addr) + .map(|b| b.failures + 1) + .unwrap_or(1); - ctx.journal.record( - "reconnect_attempt", - json!({ "addr": addr.to_string(), "attempt": attempt }), - ); + ctx.journal.record( + "reconnect_attempt", + json!({ "addr": addr.to_string(), "attempt": attempt }), + ); - match dial(&ctx, addr).await { - Ok(()) => { - info!("reconnected to {} on attempt {}", addr, attempt); - ctx.journal.record( - "reconnected", - json!({ "addr": addr.to_string(), "attempt": attempt }), - ); + match dial(&ctx, addr).await { + Ok(()) => { + info!("reconnected to {} on attempt {}", addr, attempt); + ctx.journal.record( + "reconnected", + json!({ "addr": addr.to_string(), "attempt": attempt }), + ); + } + Err(e) => debug!("reconnect to {} failed: {}", addr, e), } - Err(e) => debug!("reconnect to {} failed: {}", addr, e), } - } + }); + futures::future::join_all(attempts).await; } } diff --git a/smesh-runtime/src/runtime.rs b/smesh-runtime/src/runtime.rs index e1be045..6c2c5c4 100644 --- a/smesh-runtime/src/runtime.rs +++ b/smesh-runtime/src/runtime.rs @@ -157,7 +157,7 @@ impl SmeshRuntime { let (handle, transport) = mesh::start(mesh::MeshStartup { config, - local_node_id: local_node_id.to_string() as NodeId, + local_node_id: local_node_id.to_string(), local_public_key, identity_pkcs8_der, network: Arc::clone(&self.network), @@ -231,6 +231,10 @@ impl SmeshRuntime { let attestation = network .nodes .get(node_id) + // Gated on the node's own name/key consistency check. Reaching for + // the identity directly bypassed the one guard that stops a node + // signing under a name it no longer presents. + .filter(|node| node.identity_matches_name()) .and_then(|n| n.identity.as_ref().map(|identity| identity.attest(&hash))); if first_assertion { diff --git a/smesh-runtime/src/transport.rs b/smesh-runtime/src/transport.rs index b0f21c0..3dfe341 100644 --- a/smesh-runtime/src/transport.rs +++ b/smesh-runtime/src/transport.rs @@ -375,28 +375,42 @@ impl rustls::server::danger::ClientCertVerifier for RecordAnyClientCert { fn verify_tls12_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, ) -> Result { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls12_signature(message, cert, dss, &signature_algorithms()) } fn verify_tls13_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, ) -> Result { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls13_signature(message, cert, dss, &signature_algorithms()) } fn supported_verify_schemes(&self) -> Vec { - vec![rustls::SignatureScheme::ED25519] + signature_algorithms().supported_schemes() } } -/// Skip server certificate verification (P2P nodes use self-signed certs) +/// Algorithms used to check handshake signatures. +fn signature_algorithms() -> rustls::crypto::WebPkiSupportedAlgorithms { + rustls::crypto::ring::default_provider().signature_verification_algorithms +} + +/// Accepts any certificate *chain*, but still proves the peer holds its key. +/// +/// Every node signs its own certificate, so there is no authority to validate a +/// chain against and `verify_server_cert` cannot reject on trust. What must not +/// be skipped is the handshake signature: it is the only thing demonstrating the +/// peer holds the private key for the certificate it presented. Returning +/// `assertion()` there — as this did — would let anyone replay a certificate +/// they had merely observed, and the mesh's channel binding checks the key in +/// that certificate against the identity the peer claims. Skipping the +/// signature would have made that check prove nothing. #[derive(Debug)] struct SkipServerVerification; @@ -414,31 +428,24 @@ impl rustls::client::danger::ServerCertVerifier for SkipServerVerification { fn verify_tls12_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, ) -> Result { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls12_signature(message, cert, dss, &signature_algorithms()) } fn verify_tls13_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, ) -> Result { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls13_signature(message, cert, dss, &signature_algorithms()) } fn supported_verify_schemes(&self) -> Vec { - vec![ - rustls::SignatureScheme::RSA_PKCS1_SHA256, - rustls::SignatureScheme::RSA_PKCS1_SHA384, - rustls::SignatureScheme::RSA_PKCS1_SHA512, - rustls::SignatureScheme::ECDSA_NISTP256_SHA256, - rustls::SignatureScheme::ECDSA_NISTP384_SHA384, - rustls::SignatureScheme::ED25519, - ] + signature_algorithms().supported_schemes() } } @@ -943,10 +950,11 @@ mod tests { } #[tokio::test] - async fn test_oversized_frame_is_rejected_before_allocation() { - // A peer claiming a 4 GiB body must be refused on the length prefix - // alone, never by allocating the buffer it asked for. - let listener = QuicTransport::new( + async fn an_oversized_length_prefix_is_refused_before_allocating() { + // The length prefix is attacker-controlled. This drives `handle_stream` + // directly rather than asserting a connection still exists, which the + // previous version did and which told us nothing about the size check. + let mut listener = QuicTransport::new( TransportConfig { bind_addr: "127.0.0.1:0".parse().unwrap(), max_message_size: 1024, @@ -956,11 +964,9 @@ mod tests { ) .await .unwrap(); + let mut incoming = listener.take_incoming().unwrap(); let addr = listener.local_addr().unwrap(); - - let accept = tokio::spawn(async move { - listener.run_accept_loop().await; - }); + let accept = tokio::spawn(async move { listener.run_accept_loop().await }); let dialer = QuicTransport::new( TransportConfig { @@ -973,8 +979,63 @@ mod tests { .unwrap(); dialer.connect(addr).await.unwrap(); - // Oversized frames are refused; the connection itself stays usable. - assert_eq!(dialer.peer_count().await, 1); + // Claim four gigabytes, send nothing. + let connection = dialer.connections.read().await.get(&addr).cloned().unwrap(); + let mut stream = connection.open_uni().await.unwrap(); + stream.write_all(&u32::MAX.to_be_bytes()).await.unwrap(); + stream.finish().unwrap(); + + // Nothing should ever be delivered for that frame. + let delivered = tokio::time::timeout(Duration::from_millis(600), incoming.recv()).await; + assert!( + delivered.is_err(), + "an oversized frame must never reach the application" + ); + + accept.abort(); + } + + #[tokio::test] + async fn a_frame_within_the_limit_is_delivered() { + // The counterpart, so the test above is not passing merely because + // nothing ever arrives. + let mut listener = QuicTransport::new( + TransportConfig { + bind_addr: "127.0.0.1:0".parse().unwrap(), + max_message_size: 1024 * 1024, + ..Default::default() + }, + NodeIdentity::generate().to_pkcs8_der(), + ) + .await + .unwrap(); + let mut incoming = listener.take_incoming().unwrap(); + let addr = listener.local_addr().unwrap(); + let listener = Arc::new(listener); + let accept = { + let listener = Arc::clone(&listener); + tokio::spawn(async move { listener.run_accept_loop().await }) + }; + + let dialer = QuicTransport::new( + TransportConfig { + bind_addr: "127.0.0.1:0".parse().unwrap(), + ..Default::default() + }, + NodeIdentity::generate().to_pkcs8_der(), + ) + .await + .unwrap(); + dialer + .send(addr, TransportMessage::Ping { timestamp: 42 }) + .await + .unwrap(); + + let (_, msg) = tokio::time::timeout(Duration::from_secs(5), incoming.recv()) + .await + .expect("a frame within the limit should arrive") + .expect("channel open"); + assert!(matches!(msg, TransportMessage::Ping { timestamp: 42 })); accept.abort(); } diff --git a/smesh-runtime/tests/two_node_mesh.rs b/smesh-runtime/tests/two_node_mesh.rs index a46430e..55fc8f1 100644 --- a/smesh-runtime/tests/two_node_mesh.rs +++ b/smesh-runtime/tests/two_node_mesh.rs @@ -69,17 +69,34 @@ impl MeshNode { network.field.signals.contains_key(hash) } - async fn signal_count(&self, hash: &str) -> usize { + /// How many distinct signals carry this payload. + /// + /// The previous version counted map keys equal to a hash, which a map can + /// never hold more than one of — it asserted nothing. What actually needs + /// proving is that one claim does not become several signals. + async fn signals_for_payload(&self, payload: &str) -> usize { let network = self.runtime.network(); let network = network.read().await; network .field .signals - .keys() - .filter(|k| k.as_str() == hash) + .values() + .filter(|s| s.payload == payload.as_bytes()) .count() } + /// Attesters recorded for a claim, in order. + async fn attesters_for(&self, hash: &str) -> Vec { + let network = self.runtime.network(); + let network = network.read().await; + network + .field + .signals + .get(hash) + .map(|s| s.verified_attesters()) + .unwrap_or_default() + } + async fn emit(&self, payload: &str) -> String { let signal = Signal::builder(SignalType::Coordination) .payload(payload.as_bytes().to_vec()) @@ -206,7 +223,9 @@ async fn peer_learned_second_hand_is_dialled() { #[tokio::test] async fn flooding_does_not_duplicate_state() { - // Fully connected triangle: A's signal can reach C directly and via B. + // Fully connected triangle: one claim can reach a node directly and again + // via a relay. What must hold is that arriving twice does not become two + // signals, and does not inflate the count of who stands behind it. let a = MeshNode::start("node-a", vec![]).await; let b = MeshNode::start("node-b", vec![a.addr()]).await; let c = MeshNode::start("node-c", vec![a.addr(), b.addr()]).await; @@ -218,23 +237,46 @@ async fn flooding_does_not_duplicate_state() { }) .await; - let hash = a.emit("consensus please").await; + let hash = a.emit_claim("consensus please").await; - eventually(Duration::from_secs(5), "b and c both have it", || async { + eventually(Duration::from_secs(6), "b and c both have it", || async { b.has_signal(&hash).await && c.has_signal(&hash).await }) .await; - // Give any relayed copies time to arrive and be deduped. - tokio::time::sleep(Duration::from_millis(500)).await; - - // Content addressing is what stops a flood from becoming duplicate state: - // a second arrival reinforces the existing signal rather than adding one. - assert_eq!(c.signal_count(&hash).await, 1); - assert_eq!(b.signal_count(&hash).await, 1); + // Re-assert repeatedly. Every one of these arrives at nodes that already + // hold the claim, by more than one route. + for _ in 0..3 { + a.emit_claim("consensus please").await; + tokio::time::sleep(Duration::from_millis(150)).await; + } + tokio::time::sleep(Duration::from_millis(600)).await; - // And the origin never loops back to itself as a new signal. - assert_eq!(a.signal_count(&hash).await, 1); + for (name, node) in [("a", &a), ("b", &b), ("c", &c)] { + assert_eq!( + node.signals_for_payload("consensus please").await, + 1, + "{name} turned one claim into more than one signal" + ); + + let attesters = node.attesters_for(&hash).await; + let mut unique = attesters.clone(); + unique.sort(); + unique.dedup(); + assert_eq!( + attesters.len(), + unique.len(), + "{name} recorded the same attester twice: {attesters:?}" + ); + assert!( + attesters.len() <= 3, + "{name} counted more attesters than there are nodes: {attesters:?}" + ); + assert!( + attesters.contains(&"node-a".to_string()), + "{name} lost the originator from the attester set" + ); + } a.shutdown().await; b.shutdown().await; From fc235b04a3bd184bdfd03f638ab7a2809528f177 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 23:24:26 -0700 Subject: [PATCH 10/14] ci: build the workspace warning-free The first CI run failed on an `unused_mut` that no local run would have shown: `-D warnings` is set in the workflow and not in my shell. Which is the point of having it. The whole workspace, including all targets, now compiles clean under `-D warnings`, so the strict setting is something the repo can actually hold to rather than a gate that has to be relaxed later. Co-Authored-By: Claude Opus 5 (1M context) --- smesh-cli/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smesh-cli/src/main.rs b/smesh-cli/src/main.rs index 90b85e5..374214a 100644 --- a/smesh-cli/src/main.rs +++ b/smesh-cli/src/main.rs @@ -1407,7 +1407,7 @@ async fn cmd_mesh( .collect::>()?; // One node per process: this is the identity we present on the wire. - let mut node = match (name, identity) { + let node = match (name, identity) { // A durable key is what lets this node restart and be recognised. (Some(name), Some(path)) => Node::new().with_identity( smesh_core::NodeIdentity::load_or_create(&path, name) From e4d6f3225f14e588e23c5fee1b292a36312173e0 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 23:49:59 -0700 Subject: [PATCH 11/14] verify: deterministic simulation, bounded mutation, and a real convergence bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two verification layers under an enforced resource ceiling, and fixes the protocol defect the first of them found. **The simulation found a convergence bug.** `smesh-core/tests/dst.rs` drives the real merge and relay code under a seeded network and scheduler — only the network and the coin flips are simulated, because a simulation that reimplements the protocol proves nothing about it. On its first run, with no packet loss at all, nodes settled on different attester sets. The cause is real. Relaying is a coin flip, so a claim dies at any node that declines to forward. Convergence was never a property of gossip here; it is a property of gossip plus anti-entropy. Re-announcing from the originators does not fix it either, and that took a second failing run to see: forward-iff-changed silences a node once it already knows something, so a neighbour stranded behind it never hears the claim again no matter how often the originator repeats itself. The silent node is in the way. Every holder has to re-announce, which the mesh now does on a timer, bounded per round. Both facts are locked in as tests, so the fix cannot be quietly simplified back into the broken shape. To make any of that possible, the relay draw became an argument rather than a hidden `thread_rng` call. The decision is now a pure function of state plus a draw, so a failing schedule is reproduced by its seed instead of described. **Mutation testing** asks whether the suite would notice a broken protocol. Scope is protocol crates only — mutating demo code measures the demo's coverage, which nobody relies on. It found a real gap immediately: nothing covered `load_or_create` creating a missing parent directory, so deleting a negation there survived. Now tested. **Nothing here may take the machine.** `verify/budget.sh` caps every run at a quarter of the cores to a ceiling of eight, a memory limit, a wall-clock timeout and low priority — and refuses to start at all if the box is already loaded. cargo-mutants defaults to one job per core and rebuilds per mutant; on 64 cores that is not fast, it is a stalled desktop. The simulation's seed count is small by default for the same reason: a verification tool that makes the normal loop painful is one people turn off. Depth lives in a nightly workflow that reports rather than blocks. Co-Authored-By: Claude Opus 5 (1M context) --- .cargo/mutants.toml | 44 ++++ .github/workflows/verify.yml | 79 +++++++ smesh-core/src/identity.rs | 21 ++ smesh-core/src/node.rs | 26 ++- smesh-core/tests/dst.rs | 436 +++++++++++++++++++++++++++++++++++ smesh-runtime/src/mesh.rs | 85 +++++++ verify/budget.sh | 59 +++++ verify/mutants.sh | 35 +++ 8 files changed, 781 insertions(+), 4 deletions(-) create mode 100644 .cargo/mutants.toml create mode 100644 .github/workflows/verify.yml create mode 100644 smesh-core/tests/dst.rs create mode 100755 verify/budget.sh create mode 100755 verify/mutants.sh diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml new file mode 100644 index 0000000..f35909d --- /dev/null +++ b/.cargo/mutants.toml @@ -0,0 +1,44 @@ +# Mutation testing scope. +# +# Deliberately narrow. cargo-mutants rebuilds and reruns the suite for every +# mutant, so cost scales with (mutants x suite time) and an unscoped run over +# this workspace is hours of full-core load. The question worth paying for is +# "do the tests actually catch a broken protocol", so only protocol code is +# mutated. Demo, film and CLI plumbing are excluded: mutating them measures the +# demo's test coverage, which nobody is relying on. + +examine_globs = [ + "smesh-core/src/signal.rs", + "smesh-core/src/identity.rs", + "smesh-core/src/node.rs", + "smesh-runtime/src/mesh.rs", + "smesh-runtime/src/journal.rs", +] + +exclude_globs = [ + "smesh-cli/**", + "smesh-bounty/**", + "smesh-agent/**", + "film/**", + "reference/**", + "**/benches/**", +] + +# Mutants inside a test are not interesting. `Debug` bodies are cosmetic, and +# mutating them only ever reports that nobody asserts on debug output. +exclude_re = [ + "mod tests", + "impl Default", + "impl std::fmt::Debug", +] + +# Known unkillable: this crate has `#[cfg(unix)]` / `#[cfg(not(unix))]` pairs +# for filesystem permissions. Mutating the branch that is not compiled on this +# platform can never be caught, and cargo-mutants cannot tell the two apart by +# name. Treat a survivor in `write_private` or `reject_if_world_readable` as a +# platform artifact rather than a coverage gap. + +# A mutant that makes the code hang must not hold a slot forever. Multiplier +# over the measured baseline run. +timeout_multiplier = 4.0 +minimum_test_timeout = 30 diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..76e800f --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,79 @@ +name: Deep verification + +# Deliberately not on every push. These layers are expensive, and a slow +# required check is one people learn to ignore. They run nightly and on demand, +# and they report rather than block. +on: + schedule: + - cron: "0 4 * * *" + workflow_dispatch: + inputs: + dst_seeds: + description: "Simulation seeds to sweep" + default: "2000" + +concurrency: + group: deep-verification + cancel-in-progress: true + +jobs: + simulation: + name: deterministic simulation soak + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - name: Sweep seeds + # Every seed is a different delivery order, delay pattern and set of + # relay coin flips. A failure names the seed, which is the whole point: + # it reproduces exactly rather than being described. + env: + SMESH_DST_SEEDS: ${{ inputs.dst_seeds || '2000' }} + run: cargo test -p smesh-core --test dst --release + + mutation: + name: mutation testing + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - name: Install cargo-mutants + run: cargo install cargo-mutants --locked + + - name: Mutate the protocol + # Scope is in .cargo/mutants.toml: protocol crates only. Two jobs, not + # one per core — the runner has two, and oversubscribing a rebuild-per- + # mutant workload makes it slower, not faster. + run: cargo mutants --jobs 2 --no-shuffle --output target/mutants + continue-on-error: true + + - name: Report survivors + if: always() + run: | + python3 - <<'PY' + import json, pathlib, sys + p = pathlib.Path("target/mutants/mutants.out/outcomes.json") + if not p.exists(): + print("no outcomes produced"); sys.exit(0) + d = json.loads(p.read_text()) + missed = [o for o in d["outcomes"] if o.get("summary") == "MissedMutant"] + print(f"{len(d['outcomes'])} mutants, {len(missed)} survived\n") + for o in missed: + m = o["scenario"]["Mutant"] + print(f" survived: {m['file']}:{m['function']['function_name']}") + print("\nA survivor means the tests would not notice that change.") + print("Known artifact: cfg(not(unix)) branches cannot run on this platform.") + PY + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: mutation-outcomes + path: target/mutants/mutants.out/outcomes.json + retention-days: 14 diff --git a/smesh-core/src/identity.rs b/smesh-core/src/identity.rs index 580a276..69a3014 100644 --- a/smesh-core/src/identity.rs +++ b/smesh-core/src/identity.rs @@ -356,6 +356,27 @@ mod tests { ); } + #[test] + fn an_identity_file_creates_its_parent_directory() { + // Found by mutation testing: deleting the `!` in the parent check + // survived, because every existing test pointed at a directory that + // already existed. A node told to keep its key somewhere new should not + // fail on the missing directory. + let dir = std::env::temp_dir() + .join(format!("smesh-id-nested-{}", std::process::id())) + .join("deeper"); + std::fs::remove_dir_all(dir.parent().unwrap()).ok(); + let path = dir.join("node.key"); + + let identity = NodeIdentity::load_or_create(&path, "latency").unwrap(); + assert!(path.exists(), "the key file was not written"); + + let again = NodeIdentity::load_or_create(&path, "latency").unwrap(); + assert_eq!(identity.public_key_hex(), again.public_key_hex()); + + std::fs::remove_dir_all(dir.parent().unwrap()).ok(); + } + #[test] fn an_identity_survives_a_restart() { let dir = std::env::temp_dir().join(format!("smesh-id-{}", std::process::id())); diff --git a/smesh-core/src/node.rs b/smesh-core/src/node.rs index 9f90cd1..aca334e 100644 --- a/smesh-core/src/node.rs +++ b/smesh-core/src/node.rs @@ -211,6 +211,23 @@ impl Node { /// the trust that fed it and the die roll that resolved it, so a relay /// choice can be journalled and replayed rather than merely observed. pub fn relay_decision(&self, signal: &Signal, remaining_hops: u32) -> RelayDecision { + let roll = rand::thread_rng().gen::(); + self.relay_decision_with(signal, remaining_hops, roll) + } + + /// The relay decision with the draw supplied by the caller. + /// + /// Relaying is the protocol's only genuine coin flip, and hiding the draw + /// inside this function made the whole diffusion path impossible to replay. + /// Taking it as an argument makes the decision a pure function of state: a + /// simulation can sweep seeds, and a failing schedule can be reproduced + /// exactly rather than described. + pub fn relay_decision_with( + &self, + signal: &Signal, + remaining_hops: u32, + roll: f64, + ) -> RelayDecision { let origin_trust = self.get_trust(&signal.origin_node_id); let dampening = if origin_trust > 0.7 { 0.9 } else { 0.7 }; @@ -240,10 +257,6 @@ impl Node { let propagation_score = effective * origin_trust * (remaining_hops as f64 / signal.radius as f64); - // Probabilistic relay decision using cryptographically secure RNG - let mut rng = rand::thread_rng(); - let roll = rng.gen::(); - RelayDecision { relay: roll < propagation_score, dampening, @@ -286,6 +299,11 @@ impl Node { self } + /// Whether this node would relay, for a given draw. Pure. + pub fn would_relay(&self, signal: &Signal, remaining_hops: u32, roll: f64) -> bool { + self.relay_decision_with(signal, remaining_hops, roll).relay + } + /// Sign a signal on this node's behalf, if it holds a private key. /// /// Refuses when the key signs under a different name than the node diff --git a/smesh-core/tests/dst.rs b/smesh-core/tests/dst.rs new file mode 100644 index 0000000..1931cfa --- /dev/null +++ b/smesh-core/tests/dst.rs @@ -0,0 +1,436 @@ +//! Deterministic simulation of gossip convergence. +//! +//! This drives the real protocol code — `Signal::merge_attestations`, +//! `Node::relay_decision_with`, real Ed25519 attestations — under a simulated +//! network and scheduler. Only the network and the coin flips are fake, and +//! both come from one seed, so a failure is reproduced by its seed rather than +//! described in a bug report. A simulation that reimplements the protocol +//! proves nothing about the protocol, so nothing here reimplements it. +//! +//! Two claims are under test, and the mesh layer rests on both: +//! +//! - **Termination.** Forwarding only when local knowledge grew has to stop. +//! Attester sets are grow-only over a finite set of nodes, so state changes +//! are bounded — but that is an argument, and arguments are what simulations +//! are for. +//! - **Convergence.** With delivery, every node agrees on who attested, +//! whatever the order, delay, or losses along the way. +//! +//! Cost is bounded deliberately. The default seed count keeps this inside a +//! normal `cargo test`; `SMESH_DST_SEEDS` raises it for a soak run, which +//! belongs under `verify/` where the resource ceiling applies. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use std::sync::OnceLock; + +use smesh_core::{Attestation, Node, NodeIdentity, Signal, SignalType}; + +/// Keys are expensive to make and the properties under test do not depend on +/// which keys they are, only that signatures verify. Generating them once keeps +/// a seed sweep affordable. +fn identities(count: usize) -> &'static [NodeIdentity] { + static POOL: OnceLock> = OnceLock::new(); + let pool = POOL.get_or_init(|| { + (0..16) + .map(|i| NodeIdentity::generate_named(format!("n{i}"))) + .collect() + }); + &pool[..count] +} + +/// Seeds explored when nothing says otherwise. Deliberately small: this runs on +/// every `cargo test`, and a verification tool that makes the normal loop +/// painful is a verification tool people turn off. `SMESH_DST_SEEDS` raises it. +const DEFAULT_SEEDS: u64 = 12; + +/// Anti-entropy rounds allowed before convergence is called a failure. +const MAX_ANTI_ENTROPY_ROUNDS: usize = 12; +/// Hard stop per simulation, so a livelock fails loudly instead of hanging. +const MAX_STEPS: usize = 20_000; + +fn seed_count() -> u64 { + std::env::var("SMESH_DST_SEEDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_SEEDS) +} + +/// Seeded generator, written out so a seed means the same thing forever. +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Self(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1) + } + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + fn unit(&mut self) -> f64 { + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 + } + fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } +} + +/// One message in flight. +struct InFlight { + to: usize, + attestations: Vec, + /// Simulated arrival time. Lower is delivered sooner. + due: u64, +} + +/// One run of the protocol under one seed. +struct Sim { + rng: Rng, + nodes: Vec, + identities: &'static [NodeIdentity], + /// Each node's own copy of the single claim under test. + held: Vec, + links: Vec>, + queue: VecDeque, + clock: u64, + /// Fraction of messages the network loses outright. + loss: f64, + forwarded: usize, +} + +impl Sim { + fn new(seed: u64, node_count: usize, loss: f64) -> Self { + let mut rng = Rng::new(seed); + + let nodes: Vec = (0..node_count) + .map(|i| { + let mut node = Node::named(format!("n{i}")); + // Trust everyone, so relaying is driven by the simulated draw + // rather than by trust bottoming out. + for j in 0..node_count { + node.trust_scores.insert(format!("n{j}"), 0.9); + } + node + }) + .collect(); + + // Ring plus a chord: connected, but not everyone adjacent, so messages + // genuinely have to be relayed to cross it. + let mut links = vec![Vec::new(); node_count]; + for i in 0..node_count { + let next = (i + 1) % node_count; + links[i].push(next); + links[next].push(i); + } + if node_count > 3 { + let far = node_count / 2; + links[0].push(far); + links[far].push(0); + } + + let held = (0..node_count) + .map(|_| { + Signal::builder(SignalType::Alert) + .correlatable() + .payload(b"the claim".to_vec()) + .intensity(1.0) + .confidence(0.9) + .radius(8) + .build() + }) + .collect(); + + let loss = loss * rng.unit(); + + Self { + rng, + nodes, + identities: identities(node_count), + held, + links, + queue: VecDeque::new(), + clock: 0, + loss, + forwarded: 0, + } + } + + /// A node asserts the claim and tells its neighbours. + fn assert_claim(&mut self, node: usize) { + let attestation = self.identities[node].attest(&self.held[node].origin_hash); + self.held[node].merge_attestations(&[attestation]); + self.gossip_from(node); + } + + fn gossip_from(&mut self, from: usize) { + let attestations = self.held[from].attestations.clone(); + for to in self.links[from].clone() { + if self.rng.unit() < self.loss { + continue; + } + let due = self.clock + 1 + self.rng.next_u64() % 8; + self.queue.push_back(InFlight { + to, + attestations: attestations.clone(), + due, + }); + } + } + + /// Deliver one message. Returns false when nothing is left. + fn step(&mut self) -> bool { + if self.queue.is_empty() { + return false; + } + + let ready: Vec = self + .queue + .iter() + .enumerate() + .filter(|(_, m)| m.due <= self.clock) + .map(|(i, _)| i) + .collect(); + + if ready.is_empty() { + self.clock += 1; + return true; + } + + let index = ready[self.rng.below(ready.len())]; + let msg = self.queue.remove(index).expect("index came from the queue"); + + // The real merge rule, under test. + let grew = !self.held[msg.to] + .merge_attestations(&msg.attestations) + .is_empty(); + + if grew { + let roll = self.rng.unit(); + let signal = self.held[msg.to].clone(); + let remaining = signal.radius.saturating_sub(signal.hops); + if self.nodes[msg.to].would_relay(&signal, remaining, roll) { + self.forwarded += 1; + self.gossip_from(msg.to); + } + } + + true + } + + fn run(&mut self) -> usize { + let mut steps = 0; + while self.step() { + steps += 1; + assert!( + steps < MAX_STEPS, + "gossip did not terminate: forward-iff-changed should make state \ + changes finite, but this schedule kept producing them" + ); + } + steps + } + + /// Re-announce what every asserting node currently holds. + /// + /// The real nodes do this on a timer. Without it a declined relay is + /// permanent, because nothing ever offers that information again. + fn anti_entropy_round(&mut self, asserters: &[usize]) { + for &node in asserters { + self.gossip_from(node); + } + self.run(); + } + + /// Every node re-announces what it holds, not only the originators. + fn full_anti_entropy_round(&mut self) { + for node in 0..self.nodes.len() { + if !self.held[node].attestations.is_empty() { + self.gossip_from(node); + } + } + self.run(); + } + + fn converged(&self) -> bool { + let first = self.attesters(0); + (1..self.nodes.len()).all(|n| self.attesters(n) == first) + } + + fn attesters(&self, node: usize) -> BTreeSet { + self.held[node].verified_attesters().into_iter().collect() + } +} + +#[test] +fn gossip_terminates_under_every_schedule() { + for seed in 0..seed_count() { + let mut sim = Sim::new(seed, 6, 0.3); + sim.assert_claim(0); + sim.assert_claim(3); + let steps = sim.run(); + assert!(steps < MAX_STEPS, "seed {seed} failed to settle"); + } +} + +#[test] +fn a_single_gossip_round_does_not_guarantee_convergence() { + // Worth stating outright, because it is easy to assume otherwise and the + // simulation found it immediately. Relaying is a coin flip, so information + // dies at any node that declines to forward — even with a perfect network. + // Convergence is not a property of gossip here; it is a property of gossip + // plus anti-entropy, and the next test is the one that holds. + let mut ever_diverged = false; + + for seed in 0..seed_count() { + let mut sim = Sim::new(seed, 6, 0.0); + for node in [0usize, 2, 5] { + sim.assert_claim(node); + } + sim.run(); + if !sim.converged() { + ever_diverged = true; + break; + } + } + + assert!( + ever_diverged, + "a declined relay should be able to strand information without \ + anti-entropy; if this no longer happens, relaying stopped being \ + probabilistic and the anti-entropy test below is no longer meaningful" + ); +} + +#[test] +fn re_announcing_from_originators_alone_is_not_enough() { + // This is why anti-entropy is every holder's job and not just the + // originators'. Forward-iff-changed silences a node once it already knows + // something, so a neighbour behind it never hears the claim again — and the + // originator re-announcing does not help, because the silent node is in the + // way. Locked in as a test so the fix cannot be "simplified" back. + let mut originator_only_failed = false; + + for seed in 0..seed_count() { + let asserters = [0usize, 2, 5]; + let mut sim = Sim::new(seed, 6, 0.0); + for node in asserters { + sim.assert_claim(node); + } + sim.run(); + + for _ in 0..MAX_ANTI_ENTROPY_ROUNDS { + if sim.converged() { + break; + } + sim.anti_entropy_round(&asserters); + } + + if !sim.converged() { + originator_only_failed = true; + break; + } + } + + assert!( + originator_only_failed, + "originator-only re-announcement converged on every seed tried. Either \ + the topology no longer has a node that can be stranded, or relaying \ + stopped being probabilistic — either way the anti-entropy design needs \ + revisiting rather than this test being deleted." + ); +} + +#[test] +fn anti_entropy_converges_every_node() { + // The real property. Re-announcing what you hold gives every gap another + // chance, so agreement is reached despite declined relays, reordering and + // delay. Order changes how long it takes, not what is agreed. + let mut failures = Vec::new(); + + for seed in 0..seed_count() { + let asserters = [0usize, 2, 5]; + let mut sim = Sim::new(seed, 6, 0.0); + for node in asserters { + sim.assert_claim(node); + } + sim.run(); + + // Re-announcing from the originators alone is not enough: a gap behind + // a node that already knows is never offered the missing information + // again, because forward-iff-changed means that node stays silent. + // Every holder has to re-announce. + let mut rounds = 0; + while !sim.converged() && rounds < MAX_ANTI_ENTROPY_ROUNDS { + sim.full_anti_entropy_round(); + rounds += 1; + } + let _ = &asserters; + + if !sim.converged() { + failures.push(( + seed, + sim.attesters(0), + (1..6).map(|n| sim.attesters(n)).collect::>(), + )); + } + } + + assert!( + failures.is_empty(), + "did not converge within {MAX_ANTI_ENTROPY_ROUNDS} anti-entropy rounds; first: {:?}", + failures.first() + ); +} + +#[test] +fn attester_sets_only_ever_grow() { + // Convergence rests on the set being grow-only. If a merge can remove an + // attester, ordering starts to matter and the argument collapses. + for seed in 0..seed_count() { + let mut sim = Sim::new(seed, 5, 0.2); + let mut high_water: BTreeMap> = BTreeMap::new(); + + sim.assert_claim(1); + sim.assert_claim(4); + + let mut steps = 0; + while sim.step() { + steps += 1; + assert!(steps < MAX_STEPS); + for node in 0..sim.nodes.len() { + let now = sim.attesters(node); + let seen = high_water.entry(node).or_default(); + assert!( + seen.is_subset(&now), + "seed {seed}: node {node} lost an attester it already had" + ); + *seen = now; + } + } + } +} + +#[test] +fn loss_delays_agreement_without_corrupting_it() { + // Under loss a node may know less. It must never know something false: + // every attester reported has to be one that really signed. + for seed in 0..seed_count() { + let mut sim = Sim::new(seed, 6, 0.6); + sim.assert_claim(0); + sim.assert_claim(2); + sim.run(); + + let truthful: BTreeSet = ["n0", "n2"].iter().map(|s| s.to_string()).collect(); + for node in 0..sim.nodes.len() { + let seen = sim.attesters(node); + assert!( + seen.is_subset(&truthful), + "seed {seed}: node {node} reported an attester nobody signed: {seen:?}" + ); + } + } +} diff --git a/smesh-runtime/src/mesh.rs b/smesh-runtime/src/mesh.rs index fb786ea..647af93 100644 --- a/smesh-runtime/src/mesh.rs +++ b/smesh-runtime/src/mesh.rs @@ -59,6 +59,12 @@ const REFLEXIVE_TICKS_MAX: u32 = 30; /// A day is far beyond any real TTL; anything past it is noise or malice. const MAX_SIGNAL_AGE_MS: f64 = 86_400_000.0; +/// Most claims re-announced in one anti-entropy round. +/// +/// Bounds the cost of a node holding a very large field: better to heal slowly +/// than to send a burst proportional to everything it knows. +const ANTI_ENTROPY_MAX_SIGNALS: usize = 32; + /// How many simultaneous-open attempts before giving up on a peer. const PUNCH_ROUNDS: u32 = 4; /// How many times the punched-at side tries back. @@ -85,6 +91,10 @@ pub struct MeshConfig { /// handshake the instant the endpoint binds — journalling identity from the /// caller afterwards races with it and can land second. pub node_metadata: serde_json::Value, + /// How often every node re-announces what it holds, in milliseconds. + /// + /// Zero disables it. + pub anti_entropy_interval_ms: u64, /// Whether to dial peers learned second-hand from a `PeerResponse`. /// /// Off pins the topology to exactly what `bootstrap` describes, which is @@ -101,6 +111,7 @@ impl Default for MeshConfig { keepalive_interval_ms: 5_000, max_peers_shared: 32, max_message_size: 1024 * 1024, + anti_entropy_interval_ms: 5_000, node_metadata: serde_json::Value::Null, peer_discovery: true, } @@ -354,6 +365,17 @@ pub(crate) async fn start( })); } + // Periodic re-announcement of what we hold. + { + let ctx = Arc::clone(&ctx); + let interval_ms = config.anti_entropy_interval_ms; + if interval_ms > 0 { + tasks.push(tokio::spawn(async move { + anti_entropy_loop(ctx, interval_ms).await; + })); + } + } + // Keepalive / latency probing. { let ctx = Arc::clone(&ctx); @@ -1409,6 +1431,69 @@ async fn reconnect_loop(ctx: Arc) { } } +/// Re-announce every claim this node holds, whether or not it originated it. +/// +/// Forwarding only when local knowledge grew is what makes gossip terminate, +/// but it also makes a node that already knows something go permanently silent +/// about it. If a neighbour behind that node missed the claim — a declined +/// relay, a dropped packet — nothing ever offers it again, and the gap is +/// permanent even on a perfect network. +/// +/// Re-announcing only from the nodes that originated a claim does not fix it, +/// for exactly the same reason: the silent relay sits between them. Every +/// holder has to speak. A deterministic simulation found this by failing to +/// converge with no packet loss at all; see `smesh-core/tests/dst.rs`. +/// +/// This is cheap because the receiving side discards anything that teaches it +/// nothing, so a settled mesh converges to one wasted round-trip per interval. +async fn anti_entropy_loop(ctx: Arc, interval_ms: u64) { + let mut ticker = tokio::time::interval(Duration::from_millis(interval_ms.max(250))); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + ticker.tick().await; + + if ctx.transport.connected_addrs().await.is_empty() { + continue; + } + + // Snapshot under the lock, send outside it. + let (announcements, field_time) = { + let network = ctx.network.read().await; + let now = network.field.current_time; + let signals: Vec = network + .field + .signals + .values() + .filter(|s| !s.is_expired(now)) + .take(ANTI_ENTROPY_MAX_SIGNALS) + .map(|s| { + let mut copy = s.clone(); + copy.reached_nodes.clear(); + copy + }) + .collect(); + (signals, now) + }; + + if announcements.is_empty() { + continue; + } + + let mut sent = 0; + for signal in announcements { + let hash = signal.origin_hash.clone(); + let msg = TransportMessage::signal(signal, field_time); + sent += ctx.transport.broadcast_all(&msg, None).await.len(); + let _ = hash; + } + + if sent > 0 { + ctx.journal.record("anti_entropy", json!({ "sends": sent })); + } + } +} + async fn keepalive_loop(ctx: Arc, interval_ms: u64) { let mut ticker = tokio::time::interval(Duration::from_millis(interval_ms.max(100))); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); diff --git a/verify/budget.sh b/verify/budget.sh new file mode 100755 index 0000000..549bc70 --- /dev/null +++ b/verify/budget.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Shared resource ceiling for every verification run. +# +# These tools are all happy to take the whole machine. cargo-mutants rebuilds +# and reruns the suite per mutant and defaults to one job per core; TLC explores +# a state space breadth-first and will grow its heap until the kernel intervenes. +# On a 64-core box that is not "fast", it is a stalled desktop and a thermal +# event. Nothing here is urgent enough to justify either. +# +# Policy: take a quarter of the cores, capped, at low priority, under a hard +# wall-clock limit and a hard memory limit. Interactive work always wins. + +set -euo pipefail + +# --- how much of the machine may a verification run take? ------------------- +CORES_TOTAL="$(nproc)" +: "${VERIFY_JOBS:=$(( CORES_TOTAL / 4 ))}" +[ "$VERIFY_JOBS" -lt 1 ] && VERIFY_JOBS=1 +[ "$VERIFY_JOBS" -gt 8 ] && VERIFY_JOBS=8 # a hard ceiling, not a ratio + +RAM_TOTAL_MB="$(free -m | awk '/Mem:/{print $2}')" +: "${VERIFY_MEM_MB:=$(( RAM_TOTAL_MB / 4 ))}" +[ "$VERIFY_MEM_MB" -gt 16384 ] && VERIFY_MEM_MB=16384 + +: "${VERIFY_TIMEOUT:=1800}" # 30 minutes, then stop +: "${VERIFY_NICE:=15}" # yield to anything interactive + +export VERIFY_JOBS VERIFY_MEM_MB VERIFY_TIMEOUT VERIFY_NICE + +budget_banner() { + echo "╭─ resource budget" + echo "│ jobs ${VERIFY_JOBS} of ${CORES_TOTAL} cores" + echo "│ memory ${VERIFY_MEM_MB} MB of ${RAM_TOTAL_MB} MB" + echo "│ timeout ${VERIFY_TIMEOUT}s" + echo "│ priority nice ${VERIFY_NICE}" + echo "╰─ override with VERIFY_JOBS / VERIFY_MEM_MB / VERIFY_TIMEOUT" + echo +} + +# Refuse to pile onto a machine that is already busy. +guard_load() { + local load cores_free + load=$(awk '{print int($1)}' /proc/loadavg) + cores_free=$(( CORES_TOTAL - load )) + if [ "$cores_free" -lt "$VERIFY_JOBS" ]; then + echo "load is already ${load}; only ${cores_free} cores idle." >&2 + echo "refusing to start a ${VERIFY_JOBS}-job run. Wait, or set VERIFY_JOBS lower." >&2 + exit 1 + fi +} + +# Run a command inside the budget: capped memory, capped time, low priority. +budgeted() { + guard_load + # A virtual-memory ceiling turns a runaway into a clean allocation failure + # rather than an OOM kill that takes something else with it. + ( ulimit -v $(( VERIFY_MEM_MB * 1024 )) 2>/dev/null || true + exec nice -n "$VERIFY_NICE" ionice -c3 timeout --signal=INT "$VERIFY_TIMEOUT" "$@" ) +} diff --git a/verify/mutants.sh b/verify/mutants.sh new file mode 100755 index 0000000..37edebf --- /dev/null +++ b/verify/mutants.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Bounded mutation testing. +# +# The question: if the protocol were subtly wrong, would the suite notice? +# Mutants that survive are places where the tests assert less than they appear +# to — which is exactly how the dedup test in this repo came to assert nothing. +# +# Bounded because an unscoped run is hours of full-core load. Scope lives in +# .cargo/mutants.toml; the ceiling lives here. +set -euo pipefail +cd "$(dirname "$0")/.." +source verify/budget.sh + +budget_banner + +# --shard lets a long run be split across invocations instead of demanding one +# uninterrupted block: verify/mutants.sh 0/4 does the first quarter. +SHARD="${1:-}" +SHARD_ARG=() +[ -n "$SHARD" ] && SHARD_ARG=(--shard "$SHARD") + +echo "scope: protocol crates only (see .cargo/mutants.toml)" +echo "listing mutants..." +cargo mutants --list 2>/dev/null | wc -l | xargs -I{} echo " {} candidate mutants" +echo + +budgeted cargo mutants \ + --jobs "$VERIFY_JOBS" \ + --no-shuffle \ + --output target/mutants \ + "${SHARD_ARG[@]}" \ + -- --profile test + +echo +echo "survivors are listed in target/mutants/outcomes.json" From 7fd24f6caa948c0b448d199828430efeadb5d2ec Mon Sep 17 00:00:00 2001 From: zuub-don Date: Tue, 18 Aug 2026 23:57:19 -0700 Subject: [PATCH 12/14] verify: model check the gossip spec, bounded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the third layer. The simulation samples schedules; TLC asks the same questions of every schedule, and answers them in seconds on a three-node model. Both directions are checked, and the second is the one that keeps the first honest: every holder re-announces -> no error found, 75 states, depth 8 only originators do -> temporal property violated The counterexample is the minimal form of the bug the simulation found: n0 and n2 assert, n1 relays both ways, n1 declines to relay again, and n0 stutters forever knowing only itself. n1 is not an originator, so under originator-only anti-entropy it never speaks again, and it is the only path between them. Three nodes in a line is the smallest shape that exhibits it. If that second run ever starts passing, either the model can no longer strand a node or relaying stopped being refusable — in both cases the first result stops meaning anything, so the runner fails rather than celebrating. TLC is pinned hard: an explicit worker count rather than `auto`, an explicit heap rather than growth until the kernel intervenes, plus the shared wall-clock and priority ceiling. A model that needs more than two gigabytes should be made smaller, not given more memory. Deadlock detection is off because termination is the goal here. The state where nothing is enabled is the converged one. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/verify.yml | 21 +++++ .gitignore | 5 ++ verify/README.md | 49 ++++++++++ verify/tla.sh | 60 +++++++++++++ verify/tla/Gossip.cfg | 28 ++++++ verify/tla/Gossip.tla | 128 +++++++++++++++++++++++++++ verify/tla/GossipOriginatorsOnly.cfg | 28 ++++++ 7 files changed, 319 insertions(+) create mode 100644 verify/README.md create mode 100755 verify/tla.sh create mode 100644 verify/tla/Gossip.cfg create mode 100644 verify/tla/Gossip.tla create mode 100644 verify/tla/GossipOriginatorsOnly.cfg diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 76e800f..4bf08c5 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -34,6 +34,27 @@ jobs: SMESH_DST_SEEDS: ${{ inputs.dst_seeds || '2000' }} run: cargo test -p smesh-core --test dst --release + model: + name: model checking + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - name: Check the gossip spec + # Two workers and a 2GB heap, not "auto". TLC will take every core and + # grow until the kernel stops it, and this model is small enough that + # needing more would mean the model is wrong. + env: + VERIFY_JOBS: "2" + TLC_HEAP: "2g" + VERIFY_TIMEOUT: "600" + run: ./verify/tla.sh + mutation: name: mutation testing runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index fe24da3..81b4229 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,8 @@ film/out/ film/src/audio/ film/src/frames/ film/src/node_modules/ + +# TLC is fetched on demand by verify/tla.sh +verify/tla/tla2tools.jar +verify/tla/states/ +verify/tla/*.old diff --git a/verify/README.md b/verify/README.md new file mode 100644 index 0000000..3534ffe --- /dev/null +++ b/verify/README.md @@ -0,0 +1,49 @@ +# Verification + +Three layers, each answering a different question, none of them allowed to take +the machine. + +| Layer | Question | Cost | When | +| --- | --- | --- | --- | +| `tla.sh` | Does the *design* have the property, on every schedule? | seconds | nightly, on demand | +| `cargo test` (dst) | Does the *implementation* have it, under sampled schedules? | ~20s | every test run | +| `mutants.sh` | Would the *tests notice* if it stopped having it? | ~20 min | nightly, on demand | + +They overlap on purpose. The model proves a property for all schedules but only +for a three-node abstraction. The simulation runs the real merge and relay code +but samples schedules rather than exhausting them. Mutation testing checks the +thing neither can: whether the assertions are load-bearing or decorative. + +## The bug all three were built around + +Gossip forwards only when local knowledge grew, which is what makes it +terminate. It also means a node goes permanently silent about anything it +already knows — so a neighbour stranded behind it never hears the claim again. +Re-announcing from the nodes that originated a claim does not help, because the +silent node sits in between. + +The simulation found it by failing to converge with no packet loss at all. The +model then produced the minimal counterexample: three nodes in a line, the +middle one relays, declines, and stutters forever. The fix is that every holder +re-announces, not just originators, and both the passing and failing variants +are checked so the reasoning cannot rot. + +## Resource ceiling + +`budget.sh` is sourced by every runner: a quarter of the cores capped at eight, +a memory limit, a wall-clock timeout, low priority, and a refusal to start at +all if the machine is already busy. + +This is not caution for its own sake. `cargo-mutants` rebuilds and reruns the +suite per mutant and defaults to one job per core; TLC explores breadth-first +with a worker per core and grows its heap until the kernel intervenes. On a +large machine neither of those is fast — they are a stalled desktop. + +```sh +./verify/tla.sh # model check, both variants +./verify/mutants.sh # full mutation run +./verify/mutants.sh 0/4 # first quarter, for a shorter sitting +SMESH_DST_SEEDS=5000 cargo test -p smesh-core --test dst --release + +VERIFY_JOBS=2 ./verify/mutants.sh # quieter still +``` diff --git a/verify/tla.sh b/verify/tla.sh new file mode 100755 index 0000000..6d15073 --- /dev/null +++ b/verify/tla.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Model check the gossip spec, bounded. +# +# TLC explores breadth-first, defaults to a worker per core, and grows its heap +# until the kernel intervenes. On a 64-core box with 250GB that is an excellent +# way to lose the machine to a three-node model. Everything here is pinned: +# workers, heap, wall clock, priority. +# +# Two runs, and both matter: +# 1. every holder re-announces -> must hold +# 2. only originators do -> must fail, with a counterexample +# +# The second is a regression test on the reasoning. If it ever starts passing, +# either the model stopped being able to strand a node or relaying stopped being +# refusable — and in both cases the first result no longer means what it says. +set -euo pipefail +cd "$(dirname "$0")/.." +source verify/budget.sh +cd verify/tla + +JAR=tla2tools.jar +if [ ! -f "$JAR" ]; then + echo "fetching TLC..." + curl -sSL -o "$JAR" https://github.com/tlaplus/tlaplus/releases/latest/download/tla2tools.jar +fi + +# Heap is capped well under the budget: this model is tiny, and a spec that +# needs more than this should be made smaller rather than given more memory. +TLC_HEAP="${TLC_HEAP:-2g}" + +budget_banner +echo "TLC: ${VERIFY_JOBS} workers, ${TLC_HEAP} heap" +echo + +run_tlc() { + budgeted java -XX:+UseParallelGC -Xmx"$TLC_HEAP" -cp "$JAR" tlc2.TLC \ + -workers "$VERIFY_JOBS" -nowarning -config "$1" Gossip.tla 2>&1 +} + +echo "── every holder re-announces (must hold) ──" +if out=$(run_tlc Gossip.cfg) && grep -q "Model checking completed. No error has been found." <<<"$out"; then + grep -E "^[0-9]+ states|^The depth" <<<"$out" | sed 's/^/ /' + echo " PASS: converges on every schedule" +else + grep -E "^Error|^State [0-9]+|/\\\\ known" <<<"$out" | head -20 | sed 's/^/ /' + echo " FAIL: the fix does not hold" >&2 + exit 1 +fi + +echo +echo "── only originators re-announce (must fail) ──" +if out=$(run_tlc GossipOriginatorsOnly.cfg); then + echo " FAIL: this was expected to be violated, and was not." >&2 + echo " The model can no longer strand a node, so the result above is hollow." >&2 + exit 1 +else + grep -E "^Error: Temporal" <<<"$out" | sed 's/^/ /' + echo " PASS: TLC found the counterexample, as it should" + grep -E "^State [0-9]+:|Stuttering" <<<"$out" | tail -4 | sed 's/^/ /' +fi diff --git a/verify/tla/Gossip.cfg b/verify/tla/Gossip.cfg new file mode 100644 index 0000000..54545da --- /dev/null +++ b/verify/tla/Gossip.cfg @@ -0,0 +1,28 @@ +\* Three nodes in a line: n0 - n1 - n2. +\* +\* The smallest shape that can strand somebody. Node n1 sits between the other +\* two, so once it knows something it goes silent and n2 is cut off from n0 -- +\* which is the bug the simulation found, at minimum size. +\* +\* Kept tiny on purpose: TLC explores breadth-first and the state count grows +\* combinatorially. Three nodes is enough to exhibit the property and small +\* enough to check in seconds. +CONSTANTS + Nodes = {n0, n1, n2} + Asserters = {n0, n2} + Edges = {{n0, n1}, {n1, n2}} + +\* Termination is the goal, not a fault. Once everyone agrees, no action is +\* enabled -- TLC calls that deadlock, and here it is exactly the property we +\* want: gossip stops. +CHECK_DEADLOCK FALSE + +SPECIFICATION SpecAll + +INVARIANT + TypeOK + NoForgery + +PROPERTY + Monotone + Converges diff --git a/verify/tla/Gossip.tla b/verify/tla/Gossip.tla new file mode 100644 index 0000000..afb1bc7 --- /dev/null +++ b/verify/tla/Gossip.tla @@ -0,0 +1,128 @@ +---------------------------- MODULE Gossip ---------------------------- +(***************************************************************************) +(* SMESH gossip convergence. *) +(* *) +(* The simulation in smesh-core/tests/dst.rs samples schedules; this asks *) +(* the same questions of *every* schedule. Two facts were found the hard *) +(* way and are stated here as properties: *) +(* *) +(* 1. Forwarding only when local knowledge grew does not converge on its *) +(* own, because relaying is a choice a node may decline. *) +(* 2. Re-announcing from the nodes that originated a claim does not fix *) +(* it, because a node that already knows goes silent and strands *) +(* whatever sits behind it. *) +(* *) +(* Relaying is modelled as nondeterminism rather than probability: a node *) +(* MAY decline, forever. That is stronger than the real coin flip, so a *) +(* property that holds here holds for any coin. *) +(***************************************************************************) +EXTENDS Naturals, FiniteSets, TLC + +CONSTANTS + Nodes, \* the participants + Edges, \* unordered pairs that can talk to each other + Asserters \* nodes that independently assert the claim + +\* Links are symmetric, so an unordered pair says it once. +Adjacent(i, j) == {i, j} \in Edges + +VARIABLES + known, \* known[n]: attesters n has verified + pending \* pending[n]: n learned something it has not passed on + +vars == <> + +TypeOK == + /\ known \in [Nodes -> SUBSET Asserters] + /\ pending \in [Nodes -> BOOLEAN] + +Init == + /\ known = [n \in Nodes |-> {}] + /\ pending = [n \in Nodes |-> FALSE] + +(***************************************************************************) +(* A node independently reaches the conclusion and signs it. *) +(***************************************************************************) +AssertClaim(n) == + /\ n \in Asserters + /\ n \notin known[n] + /\ known' = [known EXCEPT ![n] = known[n] \cup {n}] + /\ pending' = [pending EXCEPT ![n] = TRUE] + +(***************************************************************************) +(* Forward because our knowledge grew. This is the mesh's rule: a message *) +(* that teaches the receiver nothing goes no further, which is what makes *) +(* gossip terminate. *) +(***************************************************************************) +Relay(i, j) == + /\ pending[i] + /\ Adjacent(i, j) + /\ ~ (known[i] \subseteq known[j]) + /\ known' = [known EXCEPT ![j] = known[j] \cup known[i]] + /\ pending' = [pending EXCEPT ![j] = TRUE, ![i] = FALSE] + +(***************************************************************************) +(* The coin comes up tails: the node declines to pass it on. Nothing in the *) +(* protocol forces a relay, so the model must allow refusing forever. *) +(***************************************************************************) +Decline(i) == + /\ pending[i] + /\ pending' = [pending EXCEPT ![i] = FALSE] + /\ UNCHANGED known + +(***************************************************************************) +(* Anti-entropy: re-announce what we hold, whether or not it is new to us. *) +(* Unlike Relay this is not gated on pending, which is precisely the point. *) +(***************************************************************************) +AntiEntropy(i, j) == + /\ Adjacent(i, j) + /\ known[i] # {} + /\ ~ (known[i] \subseteq known[j]) + /\ known' = [known EXCEPT ![j] = known[j] \cup known[i]] + /\ UNCHANGED pending + +Next == + \/ \E n \in Nodes : AssertClaim(n) + \/ \E i, j \in Nodes : Relay(i, j) + \/ \E i \in Nodes : Decline(i) + \/ \E i, j \in Nodes : AntiEntropy(i, j) + +(***************************************************************************) +(* Fairness. Every holder eventually re-announces -- this is the fix. *) +(***************************************************************************) +FairAll == + /\ \A n \in Nodes : WF_vars(AssertClaim(n)) + /\ \A i, j \in Nodes : WF_vars(AntiEntropy(i, j)) + +(***************************************************************************) +(* The broken variant: only the nodes that originated a claim re-announce. *) +(* A node that merely relayed stays silent forever. *) +(***************************************************************************) +FairOriginatorsOnly == + /\ \A n \in Nodes : WF_vars(AssertClaim(n)) + /\ \A i \in Asserters, j \in Nodes : WF_vars(AntiEntropy(i, j)) + +SpecAll == Init /\ [][Next]_vars /\ FairAll +SpecOriginatorsOnly == Init /\ [][Next]_vars /\ FairOriginatorsOnly + +(***************************************************************************) +(* Safety *) +(***************************************************************************) + +\* Nobody ever knows an attester that did not assert. No forgery. +NoForgery == \A n \in Nodes : known[n] \subseteq Asserters + +\* Knowledge never shrinks. Convergence rests on this being a grow-only set; +\* if a merge could remove an attester, delivery order would start to matter. +Monotone == [][\A n \in Nodes : known[n] \subseteq known'[n]]_vars + +(***************************************************************************) +(* Liveness *) +(***************************************************************************) + +Agreed == \A i, j \in Nodes : known[i] = known[j] +Complete == \A n \in Nodes : known[n] = Asserters + +\* Everyone ends up agreeing, and agreeing on everything that was asserted. +Converges == <>[](Agreed /\ Complete) +============================================================================= diff --git a/verify/tla/GossipOriginatorsOnly.cfg b/verify/tla/GossipOriginatorsOnly.cfg new file mode 100644 index 0000000..deccea9 --- /dev/null +++ b/verify/tla/GossipOriginatorsOnly.cfg @@ -0,0 +1,28 @@ +\* Three nodes in a line: n0 - n1 - n2. +\* +\* The smallest shape that can strand somebody. Node n1 sits between the other +\* two, so once it knows something it goes silent and n2 is cut off from n0 -- +\* which is the bug the simulation found, at minimum size. +\* +\* Kept tiny on purpose: TLC explores breadth-first and the state count grows +\* combinatorially. Three nodes is enough to exhibit the property and small +\* enough to check in seconds. +CONSTANTS + Nodes = {n0, n1, n2} + Asserters = {n0, n2} + Edges = {{n0, n1}, {n1, n2}} + +\* Termination is the goal, not a fault. Once everyone agrees, no action is +\* enabled -- TLC calls that deadlock, and here it is exactly the property we +\* want: gossip stops. +CHECK_DEADLOCK FALSE + +SPECIFICATION SpecOriginatorsOnly + +INVARIANT + TypeOK + NoForgery + +PROPERTY + Monotone + Converges From a2bc184a1d881c9cfc5048adc04529fb2fdc2491 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Wed, 19 Aug 2026 00:24:44 -0700 Subject: [PATCH 13/14] test: cover the mesh behaviour that only manual runs had checked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first full mutation run scored 52% survival: 166 of 317 viable mutants changed the protocol without any test noticing. Whole functions could be deleted and the suite stayed green — `anti_entropy_loop`, `relay_forward`, `forward_signal`, `reap_dead_peers`, and `reject_unpinned` among them. The cause is not subtle in hindsight. Every one of those was established by running live processes by hand, or across cloud hosts, and never encoded. "167 tests green" was true and told nobody that deleting the fix for the convergence bug would go unremarked. Three tests close the worst of it, each written from the failure it should have caught: - A node that arrives after a claim has settled learns it anyway. Relaying cannot do that: its neighbour already knows, so forward-iff-changed keeps that neighbour silent. Only re-announcement reaches a late joiner, so this fails if anti-entropy is removed. - A claim crosses a node that is neither end. Every earlier test used a topology where both ends were already adjacent, so nothing was ever actually carried by a third party. - A departed peer stops being reported as connected. Fixing those exposed a second problem worth naming. Relaying and anti-entropy both get a claim across a mesh, so with both enabled neither is individually necessary, and the relay test passed whether or not relaying worked. It now runs with anti-entropy off. Redundancy in the system becomes blind spots in the tests unless the tests take the redundancy away. Confirmed by re-running the specific mutants: `relay_forward -> None`, `forward_signal -> ()`, `anti_entropy_loop -> ()` and `reap_dead_peers -> ()` are now caught. Two survivors in that area are left on purpose: deleting a struct field initialiser that falls back to the same default the test already uses is not a behaviour change. The baseline is recorded in verify/README.md so the next run can be compared rather than re-argued. Co-Authored-By: Claude Opus 5 (1M context) --- smesh-runtime/tests/two_node_mesh.rs | 154 +++++++++++++++++++++++++++ verify/README.md | 29 +++++ 2 files changed, 183 insertions(+) diff --git a/smesh-runtime/tests/two_node_mesh.rs b/smesh-runtime/tests/two_node_mesh.rs index 55fc8f1..ba3a3bd 100644 --- a/smesh-runtime/tests/two_node_mesh.rs +++ b/smesh-runtime/tests/two_node_mesh.rs @@ -23,12 +23,45 @@ struct MeshNode { impl MeshNode { async fn start(name: &str, bootstrap: Vec) -> Self { + Self::start_with(name, bootstrap, true, 400).await + } + + /// Start with peer discovery off, so the topology stays exactly as given. + /// + /// Discovery quietly turns any shape into a full mesh, which hides every + /// property that depends on a message having to cross an intermediate node. + async fn start_pinned(name: &str, bootstrap: Vec) -> Self { + Self::start_with(name, bootstrap, false, 400).await + } + + /// Pinned topology with anti-entropy switched off. + /// + /// Relaying and anti-entropy both get a claim across a mesh, so with both + /// running neither is individually necessary and a test cannot tell which + /// one carried it. Turning one off is the only way to hold the other to + /// account — mutation testing showed that deleting the relay path left the + /// suite green precisely because re-announcement covered for it. + async fn start_relay_only(name: &str, bootstrap: Vec) -> Self { + Self::start_with(name, bootstrap, false, 0).await + } + + async fn start_with( + name: &str, + bootstrap: Vec, + discovery: bool, + anti_entropy_ms: u64, + ) -> Self { let mut node = Node::named(name); // Trust the peers we will actually talk to, so the probabilistic relay // policy does not make these tests flaky. node.trust_scores.insert("node-a".to_string(), 0.99); node.trust_scores.insert("node-b".to_string(), 0.99); node.trust_scores.insert("node-c".to_string(), 0.99); + for peer in [ + "left", "middle", "right", "early", "late", "stayer", "leaver", + ] { + node.trust_scores.insert(peer.to_string(), 0.99); + } let node_id = node.id.clone(); let mut network = Network::new(); @@ -45,6 +78,8 @@ impl MeshNode { bind_addr: LOCALHOST.parse().unwrap(), bootstrap, keepalive_interval_ms: 500, + anti_entropy_interval_ms: anti_entropy_ms, + peer_discovery: discovery, ..Default::default() }, &node_id, @@ -512,3 +547,122 @@ async fn punch_coordination_reaches_the_target() { left.shutdown().await; right.shutdown().await; } + +#[tokio::test] +async fn a_late_joiner_learns_what_it_missed() { + // Mutation testing found that deleting the anti-entropy loop entirely left + // the suite green, which means the fix for the convergence bug had no test + // holding it in place. + // + // A node that arrives after a claim has settled cannot be told about it by + // relaying: its neighbour already knows, so forward-iff-changed keeps that + // neighbour silent. Only a periodic re-announcement reaches it. + let early = MeshNode::start_pinned("early", vec![]).await; + let middle = MeshNode::start_pinned("middle", vec![early.addr()]).await; + + eventually(Duration::from_secs(5), "early and middle meet", || async { + middle.runtime.peers().connected_count().await == 1 + }) + .await; + + let hash = early + .emit_claim("something that happened before you arrived") + .await; + + eventually(Duration::from_secs(5), "middle has the claim", || async { + middle.has_signal(&hash).await + }) + .await; + + // Let the claim go quiet: nothing new is happening anywhere. + tokio::time::sleep(Duration::from_millis(700)).await; + + // Now someone turns up, connected only to the node that already knows. + let late = MeshNode::start_pinned("late", vec![middle.addr()]).await; + + eventually( + Duration::from_secs(8), + "the late joiner is told what it missed", + || async { late.has_signal(&hash).await }, + ) + .await; + + let attesters = late.attesters_for(&hash).await; + assert_eq!( + attesters, + vec!["early".to_string()], + "the late joiner should learn who actually attested" + ); + + early.shutdown().await; + middle.shutdown().await; + late.shutdown().await; +} + +#[tokio::test] +async fn a_claim_crosses_a_node_that_is_neither_end() { + // Deleting the relay path also left the suite green: every existing test + // used a topology where both ends were already adjacent, so nothing ever + // had to be carried by a third party. + // Anti-entropy off: relaying is the only way across, so this test fails if + // relaying stops working rather than being quietly covered for. + let left = MeshNode::start_relay_only("left", vec![]).await; + let middle = MeshNode::start_relay_only("middle", vec![left.addr()]).await; + let right = MeshNode::start_relay_only("right", vec![middle.addr()]).await; + + eventually(Duration::from_secs(6), "the line forms", || async { + middle.runtime.peers().connected_count().await == 2 + && left.runtime.peers().connected_count().await == 1 + && right.runtime.peers().connected_count().await == 1 + }) + .await; + + // The ends are not adjacent, so this can only arrive via the middle. + assert!( + right.runtime.peers().get_peer("left").await.is_none(), + "the ends must not be directly connected or the test proves nothing" + ); + + let hash = left.emit_claim("carried by someone else").await; + + eventually( + Duration::from_secs(10), + "the far end receives a relayed claim", + || async { right.has_signal(&hash).await }, + ) + .await; + + assert_eq!( + right.attesters_for(&hash).await, + vec!["left".to_string()], + "a relayed claim must still name its originator, not its carrier" + ); + + left.shutdown().await; + middle.shutdown().await; + right.shutdown().await; +} + +#[tokio::test] +async fn a_departed_peer_stops_being_reported_as_connected() { + // Reaping was verified by hand with live processes and never encoded, so + // deleting it left the suite green. + let stayer = MeshNode::start("stayer", vec![]).await; + let leaver = MeshNode::start("leaver", vec![stayer.addr()]).await; + + eventually(Duration::from_secs(5), "they meet", || async { + stayer.runtime.peers().connected_count().await == 1 + }) + .await; + + leaver.shutdown().await; + + eventually( + Duration::from_secs(20), + "the survivor notices the departure", + || async { stayer.runtime.peers().connected_count().await == 0 }, + ) + .await; + + stayer.shutdown().await; +} diff --git a/verify/README.md b/verify/README.md index 3534ffe..a2d2e38 100644 --- a/verify/README.md +++ b/verify/README.md @@ -47,3 +47,32 @@ SMESH_DST_SEEDS=5000 cargo test -p smesh-core --test dst --release VERIFY_JOBS=2 ./verify/mutants.sh # quieter still ``` + +## Mutation baseline + +First full run, recorded so progress is measurable rather than felt: + +``` +358 mutants: 151 caught, 166 survived, 40 unviable, 1 timeout (52% survival) +``` + +That number was the most useful thing the verification work produced, and not +because it was good. It said the mesh layer's behaviour had been established by +running live processes by hand and never encoded: `anti_entropy_loop`, +`relay_forward`, `forward_signal` and `reap_dead_peers` could each be deleted +outright with the suite still green. The convergence fix had no test holding it +in place. + +Fixing the worst of those exposed a second, subtler problem. Relaying and +anti-entropy both get a claim across a mesh, so with both running neither is +individually necessary and a test cannot tell which one carried the message. +The relay test now runs with anti-entropy switched off; otherwise it passes +whether or not relaying works at all. + +Two survivors in that area are left deliberately: deleting a struct field that +falls back to the same default the test already uses is not a behaviour change, +and chasing it would mean asserting on configuration rather than conduct. + +Most of the remaining survivors are in `signal.rs` and `node.rs`, largely +arithmetic inside decay and scoring that no test pins to a precise value. Worth +working through; not urgent in the way a deletable anti-entropy loop was. From 4cd046ef71742b0c0ac61ac9614003dfff89a9a3 Mon Sep 17 00:00:00 2001 From: zuub-don Date: Wed, 19 Aug 2026 00:32:45 -0700 Subject: [PATCH 14/14] fix: authenticate a peer before believing what it says about us MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review caught a hole introduced by the NAT work. `on_hello` recorded the address a peer reported seeing us at *before* the channel binding and name pinning checks had run, so any unauthenticated `Hello` could fix our reflexive address for the rest of the run. We then advertise that address to every other peer, so one hostile packet would have redirected the whole mesh's idea of where we are. The peer now has to prove its key first. Also fixes a test that could pass without testing anything. The punch test only ran the punch path if discovery had not already paired the two ends, and discovery is on by default — so it usually paired them and the test passed having exercised nothing. Same shape as the dedup test earlier in this branch. The topology is pinned now, and the test asserts the two are unpaired before it starts. Workflow tokens are read-only and checkout no longer persists credentials. Nothing in CI writes to the repository, and a job that cannot push cannot be turned into one that does. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 9 +++++ .github/workflows/verify.yml | 9 +++++ smesh-runtime/src/mesh.rs | 15 ++++---- smesh-runtime/tests/two_node_mesh.rs | 54 +++++++++++----------------- 4 files changed, 48 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57d804c..abbabb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,11 @@ on: branches: [main] pull_request: +# Read-only by default: nothing here needs to write to the repository, and a +# workflow that cannot push cannot be turned into one that does. +permissions: + contents: read + env: CARGO_TERM_COLOR: always RUSTFLAGS: -D warnings @@ -15,6 +20,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable with: @@ -46,6 +53,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4bf08c5..8edf9f2 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -12,6 +12,9 @@ on: description: "Simulation seeds to sweep" default: "2000" +permissions: + contents: read + concurrency: group: deep-verification cancel-in-progress: true @@ -23,6 +26,8 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -40,6 +45,8 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-java@v4 with: distribution: temurin @@ -61,6 +68,8 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 diff --git a/smesh-runtime/src/mesh.rs b/smesh-runtime/src/mesh.rs index 647af93..29c4009 100644 --- a/smesh-runtime/src/mesh.rs +++ b/smesh-runtime/src/mesh.rs @@ -593,12 +593,6 @@ async fn on_hello( return; } - // A peer told us where it sees us. Behind NAT that is the only address - // anybody else can use, and we have no other way to discover it. - if let Some(mine) = observed_addr { - learn_reflexive(ctx, mine, &node_id).await; - } - // Channel binding: the key a peer claims must be the key it actually // completed the TLS handshake with. Without this the transport is encrypted // but not authenticated, and anything in the path could relay someone @@ -677,6 +671,15 @@ async fn on_hello( return; } + // Only now, once this peer has proved its key and cleared name pinning, is + // it allowed to tell us where we are. Accepting this earlier meant any + // unauthenticated `Hello` could fix our reflexive address for the rest of + // the run — and we advertise that address to everyone else, so a single + // hostile packet would have redirected the whole mesh's idea of us. + if let Some(mine) = observed_addr { + learn_reflexive(ctx, mine, &node_id).await; + } + if !already_connected { ctx.journal.record( "peer_connected", diff --git a/smesh-runtime/tests/two_node_mesh.rs b/smesh-runtime/tests/two_node_mesh.rs index ba3a3bd..1908736 100644 --- a/smesh-runtime/tests/two_node_mesh.rs +++ b/smesh-runtime/tests/two_node_mesh.rs @@ -499,9 +499,9 @@ async fn punch_coordination_reaches_the_target() { // It does NOT prove NAT traversal. On loopback the resulting dial would // have succeeded anyway; what is under test is that the coordination path // runs and the two ends find each other through it. - let rendezvous = MeshNode::start("rendezvous", vec![]).await; - let left = MeshNode::start("left", vec![rendezvous.addr()]).await; - let right = MeshNode::start("right", vec![rendezvous.addr()]).await; + let rendezvous = MeshNode::start_pinned("rendezvous", vec![]).await; + let left = MeshNode::start_pinned("left", vec![rendezvous.addr()]).await; + let right = MeshNode::start_pinned("right", vec![rendezvous.addr()]).await; eventually( Duration::from_secs(6), @@ -510,39 +510,27 @@ async fn punch_coordination_reaches_the_target() { ) .await; - // Discovery is on by default, so wait until they are NOT yet paired before - // asserting the punch is what pairs them. - let paired = |node: &MeshNode| { - let peers = node.runtime.peers(); - async move { - peers - .connected_peers() - .await - .iter() - .any(|p| p.node_id == "right") - } - }; - - if !paired(&left).await { - left.handle.request_punch("right").await; - - eventually( - Duration::from_secs(8), - "left and right pair through the rendezvous", - || async { - left.runtime.peers().get_peer("right").await.is_some() - || right.runtime.peers().get_peer("left").await.is_some() - }, - ) - .await; - } - + // Discovery would pair these two on its own, and then this test would pass + // without the punch path ever running — the same way the dedup test used to + // pass without asserting anything. Pin the topology so the rendezvous is + // genuinely the only route between them. assert!( - left.runtime.peers().get_peer("right").await.is_some() - || right.runtime.peers().get_peer("left").await.is_some(), - "the two ends should have found each other" + left.runtime.peers().get_peer("right").await.is_none(), + "left and right must not already be paired, or the punch proves nothing" ); + left.handle.request_punch("right").await; + + eventually( + Duration::from_secs(10), + "left and right pair through the rendezvous", + || async { + left.runtime.peers().get_peer("right").await.is_some() + || right.runtime.peers().get_peer("left").await.is_some() + }, + ) + .await; + rendezvous.shutdown().await; left.shutdown().await; right.shutdown().await;