diff --git a/Cargo.lock b/Cargo.lock index be447bb..cc3a4bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,18 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -44,6 +56,18 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-trait" version = "0.1.89" @@ -52,7 +76,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -63,7 +87,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -90,6 +114,19 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "blake3" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -188,12 +225,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror", +] + [[package]] name = "color_quant" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "countme" version = "3.0.1" @@ -310,7 +362,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -319,6 +371,18 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "equivalent" version = "1.0.2" @@ -335,6 +399,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fdeflate" version = "0.3.7" @@ -419,7 +495,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -473,6 +549,9 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] [[package]] name = "hashbrown" @@ -480,6 +559,15 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "httparse" version = "1.10.1" @@ -650,6 +738,17 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libsqlite3-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "litemap" version = "0.8.2" @@ -691,7 +790,7 @@ dependencies = [ "quote", "regex-automata", "regex-syntax", - "syn", + "syn 2.0.118", ] [[package]] @@ -838,7 +937,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -847,6 +946,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "plotters" version = "0.3.7" @@ -888,6 +993,18 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -995,6 +1112,20 @@ dependencies = [ "text-size", ] +[[package]] +name = "rusqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +dependencies = [ + "bitflags 2.13.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1049,7 +1180,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1073,7 +1204,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1161,6 +1292,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1169,7 +1311,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1178,6 +1320,26 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f18aa187839b2bdb1ad2fa35ead8c4c2976b64e4363c386d45ac0f7ee85c9233" +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "thread_local" version = "1.1.9" @@ -1232,7 +1394,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1338,7 +1500,7 @@ checksum = "84fd902d4e0b9a4b27f2f440108dc034e1758628a9b702f8ec61ad66355422fa" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1366,7 +1528,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1439,6 +1601,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -1487,7 +1661,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -1587,7 +1761,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -1608,7 +1782,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1628,13 +1802,13 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] [[package]] name = "zerosyntax-analysis" -version = "1.3.0" +version = "1.3.1" dependencies = [ "criterion", "rowan", @@ -1646,7 +1820,7 @@ dependencies = [ [[package]] name = "zerosyntax-schema" -version = "1.3.0" +version = "1.3.1" dependencies = [ "serde", "serde_json", @@ -1654,14 +1828,17 @@ dependencies = [ [[package]] name = "zerosyntax-server" -version = "1.3.0" +version = "1.3.1" dependencies = [ "anyhow", "base64", + "blake3", "clap", "dashmap 6.2.1", "percent-encoding", + "postcard", "ropey", + "rusqlite", "serde", "serde_json", "tokio", @@ -1677,7 +1854,7 @@ dependencies = [ [[package]] name = "zerosyntax-syntax" -version = "1.3.0" +version = "1.3.1" dependencies = [ "criterion", "logos", @@ -1686,7 +1863,7 @@ dependencies = [ [[package]] name = "zerosyntax-w3d" -version = "1.3.0" +version = "1.3.1" dependencies = [ "glam", "image", @@ -1722,7 +1899,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index db8078f..aa99fa6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/schema", "crates/syntax", "crates/analysis", "crates/w3d", "crates/server"] [workspace.package] -version = "1.3.0" +version = "1.3.1" edition = "2021" license = "MIT" repository = "https://github.com/ViTeXFTW/ZeroSyntaxV2" @@ -45,6 +45,9 @@ clap = { version = "4", default-features = false, features = [ "usage", "error-context", ] } +rusqlite = { version = "0.31", features = ["bundled"] } +postcard = { version = "1.1", default-features = false, features = ["use-std"] } +blake3 = "=1.5.5" # benches criterion = "0.8" diff --git a/crates/analysis/src/completion.rs b/crates/analysis/src/completion.rs index ccdb8c5..2c3965b 100644 --- a/crates/analysis/src/completion.rs +++ b/crates/analysis/src/completion.rs @@ -71,7 +71,10 @@ pub fn complete( index, (file, offset), ), - PosContext::ModuleName { slot_accepts } => module_name_completions(analyzer, &slot_accepts), + PosContext::ModuleName { + scope_node, + slot_accepts, + } => module_name_completions(analyzer, &scope_node, &slot_accepts), PosContext::SubBlockArg { argument_type } => { completions_for_type(analyzer, &argument_type, 0, None, None, index) } @@ -93,8 +96,10 @@ enum PosContext { first_token: Option, }, /// Completing a module type name after a slot `=`. Carries the slot's - /// accepted interfaces so completions can be filtered to valid modules only. + /// enclosing scope so its snippet can choose an unused-looking module tag, + /// and accepted interfaces so candidates can be filtered to valid modules. ModuleName { + scope_node: SyntaxNode, slot_accepts: Vec, }, /// Completing the argument of a sub-block header. @@ -189,7 +194,8 @@ fn classify_position(analyzer: &Analyzer, root: &SyntaxNode, offset: u32) -> Pos // (before any nested field/scope) and after `=`, and the slot is a real // module slot of the parent block. if on_header_line(&module_node, offset) && after_equals(&module_node, offset) { - let parent = enclosing_scope(&module_node).map(|p| scope_schema(analyzer, &p)); + let scope_node = enclosing_scope(&module_node); + let parent = scope_node.as_ref().map(|p| scope_schema(analyzer, p)); let slot = Module(module_node.clone()).slot(); let slot_accepts = slot.as_ref().and_then(|s| { parent.as_ref().and_then(|p| { @@ -201,6 +207,7 @@ fn classify_position(analyzer: &Analyzer, root: &SyntaxNode, offset: u32) -> Pos }); if let Some(accepts) = slot_accepts { return PosContext::ModuleName { + scope_node: scope_node.unwrap_or_else(|| root.clone()), slot_accepts: accepts, }; } @@ -790,7 +797,12 @@ fn top_level_completions(analyzer: &Analyzer) -> Vec { .collect() } -fn module_name_completions(analyzer: &Analyzer, slot_accepts: &[String]) -> Vec { +fn module_name_completions( + analyzer: &Analyzer, + scope_node: &SyntaxNode, + slot_accepts: &[String], +) -> Vec { + let tag = next_module_tag(analyzer, scope_node); analyzer .schema() .modules @@ -801,7 +813,7 @@ fn module_name_completions(analyzer: &Analyzer, slot_accepts: &[String]) -> Vec< .map(|m| { // Snippet: module name + placeholder tag + indented body + End. // Also satisfies missing-module-tag in one accept. - let insert = Some(format!("{} ${{1:ModuleTag_01}}\n\t$0\nEnd", m.name)); + let insert = Some(format!("{} ${{1:{tag}}}\n\t$0\nEnd", m.name)); Completion { label: m.name.clone(), kind: CompletionKind::Module, @@ -812,6 +824,49 @@ fn module_name_completions(analyzer: &Analyzer, slot_accepts: &[String]) -> Vec< .collect() } +/// Suggest the next numeric tag used by module slots in the enclosing Object. +/// +/// Descriptive tags (such as `ModuleTag_Draw`) deliberately do not affect the +/// numeric sequence. Only genuine module slots are considered: sub-block +/// headers can also have several arguments, but those arguments are not tags. +fn next_module_tag(analyzer: &Analyzer, scope_node: &SyntaxNode) -> String { + const PREFIX: &str = "ModuleTag_"; + let object_node = scope_node + .ancestors() + .find(|node| { + Block(node.clone()) + .keyword() + .is_some_and(|keyword| keyword.text().eq_ignore_ascii_case("Object")) + }) + .unwrap_or_else(|| scope_node.clone()); + let highest = object_node + .descendants() + .filter_map(Module::cast) + .filter(|module| { + let parent = enclosing_scope(&module.0); + let module_slots = parent + .as_ref() + .map(|parent| scope_schema(analyzer, parent).module_slots()) + .unwrap_or_default(); + module.slot().is_some_and(|slot| { + module_slots + .iter() + .any(|module_slot| module_slot.keyword.eq_ignore_ascii_case(slot.text())) + }) + }) + .filter_map(|module| module.tag()) + .filter_map(|tag| { + let text = tag.text(); + text.get(..PREFIX.len()) + .filter(|prefix| prefix.eq_ignore_ascii_case(PREFIX)) + .and_then(|_| text.get(PREFIX.len()..)) + .and_then(|number| number.parse::().ok()) + }) + .max() + .unwrap_or(0); + format!("ModuleTag_{:02}", highest + 1) +} + // --- position helpers --- fn ancestor_of_kind(node: &SyntaxNode, kind: SyntaxKind) -> Option { @@ -1007,6 +1062,52 @@ mod tests { assert!(out.contains(&"ActiveBody".to_string()), "{out:?}"); } + #[test] + fn module_snippet_uses_next_numeric_tag_in_object() { + let src = "Object Tank\n Draw = W3DTankDraw ModuleTag_01\n End\n Behavior = SlowDeathBehavior MODULETAG_03\n End\n Behavior = \nEnd\n"; + let offset = "Object Tank\n Draw = W3DTankDraw ModuleTag_01\n End\n Behavior = SlowDeathBehavior MODULETAG_03\n End\n Behavior = ".len() as u32; + let completion = item(src, offset, "AutoHealBehavior"); + assert_eq!( + completion.insert.as_deref(), + Some("AutoHealBehavior ${1:ModuleTag_04}\n\t$0\nEnd") + ); + } + + #[test] + fn module_snippet_ignores_descriptive_tags_and_sub_block_arguments() { + let src = "Object Tank\n Draw = W3DTankDraw ModuleTag_Draw\n ConditionState = DAMAGED REALLYDAMAGED\n End\n End\n Behavior = \nEnd\n"; + let offset = "Object Tank\n Draw = W3DTankDraw ModuleTag_Draw\n ConditionState = DAMAGED REALLYDAMAGED\n End\n End\n Behavior = ".len() as u32; + let completion = item(src, offset, "AutoHealBehavior"); + assert_eq!( + completion.insert.as_deref(), + Some("AutoHealBehavior ${1:ModuleTag_01}\n\t$0\nEnd") + ); + } + + #[test] + fn module_snippet_in_reentrant_scope_uses_object_wide_sequence() { + let src = "Object Tank\n AddModule\n Behavior = SlowDeathBehavior ModuleTag_04\n End\n Behavior = \n End\nEnd\n"; + let offset = "Object Tank\n AddModule\n Behavior = SlowDeathBehavior ModuleTag_04\n End\n Behavior = ".len() as u32; + let completion = item(src, offset, "AutoHealBehavior"); + assert_eq!( + completion.insert.as_deref(), + Some("AutoHealBehavior ${1:ModuleTag_05}\n\t$0\nEnd") + ); + } + + #[test] + fn module_snippet_continues_past_u32_tag_values() { + let src = "Object Tank\n Behavior = SlowDeathBehavior ModuleTag_4294967295\n End\n Behavior = \nEnd\n"; + let offset = + "Object Tank\n Behavior = SlowDeathBehavior ModuleTag_4294967295\n End\n Behavior = " + .len() as u32; + let completion = item(src, offset, "AutoHealBehavior"); + assert_eq!( + completion.insert.as_deref(), + Some("AutoHealBehavior ${1:ModuleTag_4294967296}\n\t$0\nEnd") + ); + } + #[test] fn model_asset_completions_use_index() { let a = Analyzer::embedded(); diff --git a/crates/analysis/src/semantic.rs b/crates/analysis/src/semantic.rs index b13b79f..fbf4797 100644 --- a/crates/analysis/src/semantic.rs +++ b/crates/analysis/src/semantic.rs @@ -217,6 +217,12 @@ impl<'a> Sem<'a> { ); continue; } + // Coordinates are a single schema value expressed as several raw + // syntax tokens (`X:… Y:… [Z:…]`). Highlight every axis alike. + if matches!(active_ty, Some(ValueType::Coord2D | ValueType::Coord3D)) { + self.set(tok, SemKind::Number); + continue; + } // Token lists classify each position by its own element type. let elem = active_ty.and_then(|ty| ty.token_type_at_input(&input, i)); self.set(tok, value_token_kind(tok, elem)); @@ -333,6 +339,19 @@ mod tests { } } + #[test] + fn classifies_coordinate_axes_as_numbers() { + let src = "Object Test\n Behavior = DefaultProductionExitUpdate ModuleTag_07\n NaturalRallyPoint = X:0.0 Y:-60.0 Z:0.0\n End\nEnd\n"; + let t = toks(src); + for axis in ["X:0.0", "Y:-60.0", "Z:0.0"] { + assert!( + t.iter() + .any(|(kind, text)| *kind == SemKind::Number && text == axis), + "{axis} was not classified as a number" + ); + } + } + #[test] fn remove_module_is_keyword_and_tag_is_reference() { let src = "Object Tank\n Behavior = DestroyDie ModuleTag_01\n End\n RemoveModule ModuleTag_01\nEnd\n"; diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 3c738af..a8b5e7d 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -26,3 +26,6 @@ tracing-subscriber.workspace = true anyhow.workspace = true walkdir.workspace = true clap.workspace = true +rusqlite.workspace = true +postcard.workspace = true +blake3.workspace = true diff --git a/crates/server/src/backend.rs b/crates/server/src/backend.rs index 1e33492..a40e042 100644 --- a/crates/server/src/backend.rs +++ b/crates/server/src/backend.rs @@ -1391,19 +1391,7 @@ impl Backend { } pub async fn index_cache_path(&self) -> Result { - let roots = self - .roots - .lock() - .map(|roots| roots.clone()) - .unwrap_or_default(); - let base_roots = self - .settings - .lock() - .map(|settings| settings.base_ini_roots.clone()) - .unwrap_or_default(); - Ok(index_cache_path(&roots, &base_roots) - .to_string_lossy() - .into_owned()) + Ok(index_cache_path().to_string_lossy().into_owned()) } /// The (kind, name, span) under the cursor — a reference-typed value token @@ -1704,16 +1692,6 @@ impl LanguageServer for Backend { { return Ok(None); } - let roots = self - .roots - .lock() - .map(|roots| roots.clone()) - .unwrap_or_default(); - let base_roots = self - .settings - .lock() - .map(|settings| settings.base_ini_roots.clone()) - .unwrap_or_default(); let mut progress = if params.command == REBUILD_INDEX_CACHE_COMMAND { Some(self.begin_progress(ProgressWork::ManualRebuild).await) } else { @@ -1724,7 +1702,7 @@ impl LanguageServer for Backend { .report("Clearing the persistent index cache", Some(0)) .await; } - let cleared = match clear_index_cache(&roots, &base_roots) { + let cleared = match clear_index_cache() { Ok(cleared) => cleared, Err(error) => { tracing::error!(%error, "asset index cache clear failed"); diff --git a/crates/server/src/cache.rs b/crates/server/src/cache.rs new file mode 100644 index 0000000..7ac239b --- /dev/null +++ b/crates/server/src/cache.rs @@ -0,0 +1,626 @@ +//! Transactional persistent cache for physical workspace inputs. +//! +//! Roots are deliberately absent from the stored identity: they define the +//! current discovery order and base/workspace role, while the expensive scan +//! result belongs to a canonical physical input. One SQLite database provides +//! batched page I/O, atomic updates, and safe sharing between editor windows. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use rusqlite::{params, Connection, ErrorCode, OptionalExtension, TransactionBehavior}; +use zerosyntax_analysis::Analyzer; + +const STORE_FILE: &str = "index-v1.sqlite3"; +const STORE_VERSION: i64 = 1; +/// Bump when `CachedEntry` serialization or any extractor feeding it changes. +/// SQL layout changes instead bump `STORE_VERSION` and the store filename. +const PRODUCER_ABI: &[u8] = b"zerosyntax-physical-input-v2"; +const BUSY_TIMEOUT: Duration = Duration::from_secs(2); +const TOUCH_INTERVAL_SECS: i64 = 24 * 60 * 60; +const MAX_AGE_SECS: i64 = 30 * 24 * 60 * 60; +const MAX_PAYLOAD_BYTES: i64 = 1024 * 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Fingerprint { + pub(crate) len: u64, + pub(crate) modified_secs: u64, + pub(crate) modified_nanos: u32, +} + +#[derive(Clone)] +struct PendingRecord { + path: String, + fingerprint: Fingerprint, + payload: Vec, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct CommitOutcome { + /// At least one analyzed payload was inserted, replaced, or invalidated. + pub(crate) updated: bool, + pub(crate) pruned: usize, +} + +/// A scan-scoped view of the persistent store. +/// +/// The captured epoch prevents a scan that started before `clear()` from +/// repopulating the store after the clear transaction commits. +pub(crate) struct InputCache { + connection: Connection, + cache_dir: PathBuf, + producer: [u8; 32], + epoch: i64, + now: i64, + pending: Vec, + touches: HashSet, + invalid: HashSet, +} + +pub(crate) fn cache_path(cache_dir: &Path) -> PathBuf { + cache_dir.join(STORE_FILE) +} + +pub(crate) fn producer_id(analyzer: &Analyzer) -> Result<[u8; 32]> { + let schema = serde_json::to_vec(analyzer.schema()).context("failed to serialize schema")?; + let mut hasher = blake3::Hasher::new(); + hasher.update(PRODUCER_ABI); + hasher.update(&(schema.len() as u64).to_le_bytes()); + hasher.update(&schema); + Ok(*hasher.finalize().as_bytes()) +} + +impl InputCache { + pub(crate) fn open(cache_dir: &Path, producer: [u8; 32]) -> Result { + std::fs::create_dir_all(cache_dir) + .with_context(|| format!("failed to create cache directory {}", cache_dir.display()))?; + let connection = open_recovering(cache_dir)?; + let epoch = connection.query_row( + "SELECT epoch FROM cache_meta WHERE singleton = 1", + [], + |row| row.get(0), + )?; + Ok(Self { + connection, + cache_dir: cache_dir.to_path_buf(), + producer, + epoch, + now: unix_now(), + pending: Vec::new(), + touches: HashSet::new(), + invalid: HashSet::new(), + }) + } + + pub(crate) fn lookup( + &mut self, + path: &str, + fingerprint: &Fingerprint, + ) -> Result>> { + let row = self + .connection + .prepare_cached( + "SELECT payload, last_used + FROM input_cache + WHERE producer = ?1 AND path = ?2 + AND file_len = ?3 AND modified_secs = ?4 AND modified_nanos = ?5", + )? + .query_row( + params![ + self.producer.as_slice(), + path, + as_sql_int(fingerprint.len)?, + as_sql_int(fingerprint.modified_secs)?, + i64::from(fingerprint.modified_nanos), + ], + |row| Ok((row.get::<_, Vec>(0)?, row.get::<_, i64>(1)?)), + ) + .optional()?; + let Some((payload, last_used)) = row else { + return Ok(None); + }; + if last_used < self.now - TOUCH_INTERVAL_SECS { + self.touches.insert(path.to_owned()); + } + Ok(Some(payload)) + } + + pub(crate) fn invalidate(&mut self, path: &str) { + self.invalid.insert(path.to_owned()); + } + + pub(crate) fn queue_store(&mut self, path: String, fingerprint: Fingerprint, payload: Vec) { + self.pending.push(PendingRecord { + path, + fingerprint, + payload, + }); + } + + pub(crate) fn has_pending_writes(&self) -> bool { + !self.pending.is_empty() || !self.invalid.is_empty() + } + + pub(crate) fn commit(&mut self) -> Result { + let payload_updated = self.has_pending_writes(); + if !payload_updated && self.touches.is_empty() { + return Ok(CommitOutcome::default()); + } + + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let current_epoch: i64 = transaction.query_row( + "SELECT epoch FROM cache_meta WHERE singleton = 1", + [], + |row| row.get(0), + )?; + if current_epoch != self.epoch { + transaction.commit()?; + self.pending.clear(); + self.touches.clear(); + self.invalid.clear(); + return Ok(CommitOutcome::default()); + } + + { + let mut invalidate = transaction + .prepare_cached("DELETE FROM input_cache WHERE producer = ?1 AND path = ?2")?; + for path in &self.invalid { + invalidate.execute(params![self.producer.as_slice(), path])?; + } + } + { + let mut touch = transaction.prepare_cached( + "UPDATE input_cache SET last_used = ?1 + WHERE producer = ?2 AND path = ?3", + )?; + for path in &self.touches { + touch.execute(params![self.now, self.producer.as_slice(), path])?; + } + } + { + let mut upsert = transaction.prepare_cached( + "INSERT INTO input_cache ( + producer, path, file_len, modified_secs, modified_nanos, + payload, payload_bytes, last_used + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(producer, path) DO UPDATE SET + file_len = excluded.file_len, + modified_secs = excluded.modified_secs, + modified_nanos = excluded.modified_nanos, + payload = excluded.payload, + payload_bytes = excluded.payload_bytes, + last_used = excluded.last_used", + )?; + for record in &self.pending { + upsert.execute(params![ + self.producer.as_slice(), + record.path, + as_sql_int(record.fingerprint.len)?, + as_sql_int(record.fingerprint.modified_secs)?, + i64::from(record.fingerprint.modified_nanos), + record.payload, + as_sql_int(record.payload.len() as u64)?, + self.now, + ])?; + } + } + + let pruned = prune(&transaction, self.now, MAX_PAYLOAD_BYTES)?; + transaction.commit()?; + self.pending.clear(); + self.touches.clear(); + self.invalid.clear(); + remove_legacy_caches(&self.cache_dir); + Ok(CommitOutcome { + updated: payload_updated, + pruned, + }) + } +} + +pub(crate) fn clear(cache_dir: &Path) -> Result { + let legacy_existed = legacy_cache_paths(cache_dir).next().is_some(); + std::fs::create_dir_all(cache_dir)?; + let mut connection = open_recovering(cache_dir)?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let cached: i64 = + transaction.query_row("SELECT COUNT(*) FROM input_cache", [], |row| row.get(0))?; + transaction.execute( + "UPDATE cache_meta SET epoch = epoch + 1 WHERE singleton = 1", + [], + )?; + transaction.execute("DELETE FROM input_cache", [])?; + transaction.commit()?; + remove_legacy_caches(cache_dir); + connection.execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")?; + Ok(legacy_existed || cached > 0) +} + +fn open_connection(path: &Path) -> rusqlite::Result { + let mut connection = Connection::open(path)?; + configure(&mut connection)?; + Ok(connection) +} + +fn open_recovering(cache_dir: &Path) -> Result { + let path = cache_path(cache_dir); + match open_connection(&path) { + Ok(connection) => Ok(connection), + Err(error) if is_corrupt(&error) => { + remove_store_files(&path).with_context(|| { + format!("failed to replace corrupt cache at {}", path.display()) + })?; + open_connection(&path) + .with_context(|| format!("failed to recreate corrupt cache at {}", path.display())) + } + Err(error) => { + Err(error).with_context(|| format!("failed to open cache in {}", cache_dir.display())) + } + } +} + +fn configure(connection: &mut Connection) -> rusqlite::Result<()> { + connection.busy_timeout(BUSY_TIMEOUT)?; + connection.execute_batch( + "PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + PRAGMA temp_store = MEMORY; + PRAGMA wal_autocheckpoint = 1000;", + )?; + let version: i64 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; + if version != STORE_VERSION { + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute_batch( + "DROP TABLE IF EXISTS input_cache; + DROP TABLE IF EXISTS cache_meta;", + )?; + create_schema(&transaction)?; + transaction.pragma_update(None, "user_version", STORE_VERSION)?; + transaction.commit()?; + } else { + create_schema(connection)?; + } + Ok(()) +} + +fn create_schema(connection: &Connection) -> rusqlite::Result<()> { + connection.execute_batch( + "CREATE TABLE IF NOT EXISTS cache_meta ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + epoch INTEGER NOT NULL + ); + INSERT OR IGNORE INTO cache_meta(singleton, epoch) VALUES (1, 0); + CREATE TABLE IF NOT EXISTS input_cache ( + producer BLOB NOT NULL, + path TEXT NOT NULL, + file_len INTEGER NOT NULL, + modified_secs INTEGER NOT NULL, + modified_nanos INTEGER NOT NULL, + payload BLOB NOT NULL, + payload_bytes INTEGER NOT NULL, + last_used INTEGER NOT NULL, + PRIMARY KEY (producer, path) + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS input_cache_last_used + ON input_cache(last_used);", + )?; + Ok(()) +} + +fn prune(connection: &Connection, now: i64, max_payload_bytes: i64) -> Result { + let mut pruned = connection.execute( + "DELETE FROM input_cache WHERE last_used < ?1", + params![now - MAX_AGE_SECS], + )?; + let total: i64 = connection.query_row( + "SELECT COALESCE(SUM(payload_bytes), 0) FROM input_cache", + [], + |row| row.get(0), + )?; + if total <= max_payload_bytes { + return Ok(pruned); + } + + let mut reclaimed = 0i64; + let mut victims = Vec::new(); + { + let mut statement = connection.prepare( + "SELECT producer, path, payload_bytes + FROM input_cache + ORDER BY last_used ASC", + )?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, Vec>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + })?; + for row in rows { + let (producer, path, bytes) = row?; + reclaimed += bytes; + victims.push((producer, path)); + if total - reclaimed <= max_payload_bytes { + break; + } + } + } + let mut delete = + connection.prepare_cached("DELETE FROM input_cache WHERE producer = ?1 AND path = ?2")?; + for (producer, path) in victims { + pruned += delete.execute(params![producer, path])?; + } + Ok(pruned) +} + +fn as_sql_int(value: u64) -> Result { + i64::try_from(value).context("input metadata exceeds SQLite integer range") +} + +fn is_corrupt(error: &rusqlite::Error) -> bool { + matches!( + error, + rusqlite::Error::SqliteFailure(code, _) + if matches!(code.code, ErrorCode::DatabaseCorrupt | ErrorCode::NotADatabase) + ) +} + +fn remove_store_files(path: &Path) -> std::io::Result<()> { + let sidecar = |suffix: &str| { + let mut value = path.as_os_str().to_os_string(); + value.push(suffix); + PathBuf::from(value) + }; + for candidate in [path.to_path_buf(), sidecar("-wal"), sidecar("-shm")] { + match std::fs::remove_file(candidate) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(()) +} + +fn unix_now() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 +} + +fn legacy_cache_paths(cache_dir: &Path) -> impl Iterator { + std::fs::read_dir(cache_dir) + .into_iter() + .flatten() + .flatten() + .filter_map(|entry| { + let path = entry.path(); + let name = path.file_name()?.to_str()?; + is_legacy_cache_name(name).then_some(path) + }) +} + +fn remove_legacy_caches(cache_dir: &Path) { + for path in legacy_cache_paths(cache_dir) { + if let Err(error) = std::fs::remove_file(&path) { + tracing::debug!(path = %path.display(), %error, "legacy index cache could not be removed"); + } + } +} + +fn is_legacy_cache_name(name: &str) -> bool { + let Some((version, hash)) = name + .strip_prefix("index-v") + .and_then(|name| name.strip_suffix(".json")) + .and_then(|name| name.split_once('-')) + else { + return false; + }; + version.parse::().is_ok() + && hash.len() == 16 + && hash.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_dir(name: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "zerosyntax-input-cache-{name}-{}-{}", + std::process::id(), + UNIX_EPOCH.elapsed().unwrap().as_nanos() + )) + } + + fn fingerprint(value: u64) -> Fingerprint { + Fingerprint { + len: value, + modified_secs: value + 1, + modified_nanos: value as u32 + 2, + } + } + + #[test] + fn records_are_reused_by_input_not_root_set() { + let dir = temp_dir("reuse"); + let producer = [7; 32]; + let mut first = InputCache::open(&dir, producer).unwrap(); + first.queue_store("base/Weapon.ini".into(), fingerprint(10), vec![1, 2, 3]); + assert!(first.commit().unwrap().updated); + drop(first); + + let mut second = InputCache::open(&dir, producer).unwrap(); + assert_eq!( + second.lookup("base/Weapon.ini", &fingerprint(10)).unwrap(), + Some(vec![1, 2, 3]) + ); + assert!(second + .lookup("base/Weapon.ini", &fingerprint(11)) + .unwrap() + .is_none()); + drop(second); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn producer_identity_isolated_records() { + let dir = temp_dir("producer"); + let mut first = InputCache::open(&dir, [1; 32]).unwrap(); + first.queue_store("Weapon.ini".into(), fingerprint(10), vec![1]); + first.commit().unwrap(); + drop(first); + + let mut other = InputCache::open(&dir, [2; 32]).unwrap(); + assert!(other + .lookup("Weapon.ini", &fingerprint(10)) + .unwrap() + .is_none()); + drop(other); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn clear_epoch_rejects_late_writes() { + let dir = temp_dir("epoch"); + let producer = [3; 32]; + let mut stale_scan = InputCache::open(&dir, producer).unwrap(); + stale_scan.queue_store("Weapon.ini".into(), fingerprint(10), vec![1]); + + clear(&dir).unwrap(); + assert!(!stale_scan.commit().unwrap().updated); + drop(stale_scan); + + let mut after = InputCache::open(&dir, producer).unwrap(); + assert!(after + .lookup("Weapon.ini", &fingerprint(10)) + .unwrap() + .is_none()); + drop(after); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn clear_removes_legacy_caches_but_preserves_unrelated_files() { + let dir = temp_dir("legacy"); + std::fs::create_dir_all(&dir).unwrap(); + let legacy = dir.join("index-v5-0123456789abcdef.json"); + let unrelated = dir.join("notes.txt"); + std::fs::write(&legacy, b"legacy").unwrap(); + std::fs::write(&unrelated, b"keep").unwrap(); + + assert!(clear(&dir).unwrap()); + assert!(!legacy.exists()); + assert!(unrelated.exists()); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn corrupt_database_is_replaced_and_reused() { + let dir = temp_dir("corrupt-database"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(cache_path(&dir), b"not sqlite").unwrap(); + + let mut cache = InputCache::open(&dir, [9; 32]).unwrap(); + cache.queue_store("Weapon.ini".into(), fingerprint(10), vec![4, 5]); + cache.commit().unwrap(); + drop(cache); + + let mut warm = InputCache::open(&dir, [9; 32]).unwrap(); + assert_eq!( + warm.lookup("Weapon.ini", &fingerprint(10)).unwrap(), + Some(vec![4, 5]) + ); + drop(warm); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn concurrent_scan_sessions_commit_a_valid_union() { + let dir = temp_dir("concurrent-union"); + let producer = [5; 32]; + let mut first = InputCache::open(&dir, producer).unwrap(); + let mut second = InputCache::open(&dir, producer).unwrap(); + first.queue_store("Weapon.ini".into(), fingerprint(10), vec![1]); + second.queue_store("Object.ini".into(), fingerprint(20), vec![2]); + first.commit().unwrap(); + second.commit().unwrap(); + drop(first); + drop(second); + + let mut combined = InputCache::open(&dir, producer).unwrap(); + assert_eq!( + combined.lookup("Weapon.ini", &fingerprint(10)).unwrap(), + Some(vec![1]) + ); + assert_eq!( + combined.lookup("Object.ini", &fingerprint(20)).unwrap(), + Some(vec![2]) + ); + drop(combined); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn expired_records_are_pruned_individually() { + let dir = temp_dir("expired"); + let producer = [6; 32]; + let mut cache = InputCache::open(&dir, producer).unwrap(); + cache.queue_store("Old.ini".into(), fingerprint(10), vec![1]); + cache.commit().unwrap(); + cache + .connection + .execute("UPDATE input_cache SET last_used = 0", []) + .unwrap(); + cache.queue_store("Fresh.ini".into(), fingerprint(20), vec![2]); + assert_eq!(cache.commit().unwrap().pruned, 1); + assert!(cache.lookup("Old.ini", &fingerprint(10)).unwrap().is_none()); + assert_eq!( + cache.lookup("Fresh.ini", &fingerprint(20)).unwrap(), + Some(vec![2]) + ); + drop(cache); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn clearing_an_empty_store_reports_no_removed_data() { + let dir = temp_dir("empty-clear"); + assert!(!clear(&dir).unwrap()); + assert!(!clear(&dir).unwrap()); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn payload_budget_prunes_least_recent_records() { + let dir = temp_dir("payload-budget"); + let producer = [8; 32]; + let mut cache = InputCache::open(&dir, producer).unwrap(); + cache.queue_store("Old.ini".into(), fingerprint(10), vec![1; 16]); + cache.commit().unwrap(); + cache + .connection + .execute( + "UPDATE input_cache SET last_used = ?1", + params![unix_now() - 1], + ) + .unwrap(); + cache.queue_store("New.ini".into(), fingerprint(20), vec![2; 16]); + cache.commit().unwrap(); + + let transaction = cache.connection.transaction().unwrap(); + assert_eq!(prune(&transaction, unix_now(), 16).unwrap(), 1); + transaction.commit().unwrap(); + assert!(cache.lookup("Old.ini", &fingerprint(10)).unwrap().is_none()); + assert_eq!( + cache.lookup("New.ini", &fingerprint(20)).unwrap(), + Some(vec![2; 16]) + ); + drop(cache); + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index d9b662b..64beb7e 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -2,6 +2,7 @@ //! INI files. Speaks LSP over stdio. mod backend; +mod cache; mod cli; mod convert; mod progress; diff --git a/crates/server/src/scan.rs b/crates/server/src/scan.rs index 88aa187..c48e52b 100644 --- a/crates/server/src/scan.rs +++ b/crates/server/src/scan.rs @@ -1,14 +1,14 @@ //! Shared filesystem, BIG archive, and W3D workspace scanning. -use std::collections::{hash_map::DefaultHasher, HashMap, HashSet}; -use std::hash::{Hash, Hasher}; +use std::collections::HashSet; use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Instant, UNIX_EPOCH}; use anyhow::{Context, Result}; use percent_encoding::percent_decode_str; +use postcard::ser_flavors::Flavor; use serde::{Deserialize, Serialize}; use tower_lsp::lsp_types::Url; use zerosyntax_analysis::index::{ @@ -18,6 +18,8 @@ use zerosyntax_analysis::index::{ use zerosyntax_analysis::Analyzer; use zerosyntax_w3d::W3dFile; +use crate::cache::{self, Fingerprint, InputCache}; + pub(crate) type ScanEntry = ( String, Vec, @@ -72,18 +74,69 @@ pub(crate) struct ScanOutcome { pub(crate) stats: ScanStats, } -const INDEX_CACHE_VERSION: u32 = 5; -/// How many current-version caches `prune_index_caches` keeps, newest first. -const INDEX_CACHE_RETAINED: usize = 4; -/// How long a cache may sit unused before `prune_index_caches` drops it. -const INDEX_CACHE_MAX_AGE: Duration = Duration::from_secs(30 * 24 * 60 * 60); const MAX_PREVIEW_ASSET_BYTES: u64 = 128 * 1024 * 1024; +const MAX_CACHE_PAYLOAD_BYTES: usize = 256 * 1024 * 1024; + +struct BoundedVec { + bytes: Vec, + limit: usize, +} + +impl BoundedVec { + fn new(limit: usize) -> Self { + Self { + bytes: Vec::new(), + limit, + } + } -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] -struct Fingerprint { - len: u64, - modified_secs: u64, - modified_nanos: u32, + fn grow_to_fit(&mut self, required: usize) -> postcard::Result<()> { + // Grow geometrically for the ordinary hot path, but never request + // capacity beyond the configured payload envelope. The small initial + // allocation avoids retaining 4 KiB for every tiny cached input. + let target = self + .bytes + .capacity() + .saturating_mul(2) + .max(256) + .max(required) + .min(self.limit); + self.bytes + .try_reserve_exact(target - self.bytes.len()) + .map_err(|_| postcard::Error::SerializeBufferFull)?; + Ok(()) + } +} + +impl Flavor for BoundedVec { + type Output = Vec; + + fn try_push(&mut self, byte: u8) -> postcard::Result<()> { + if self.bytes.len() == self.limit { + return Err(postcard::Error::SerializeBufferFull); + } + if self.bytes.len() == self.bytes.capacity() { + self.grow_to_fit(self.bytes.len() + 1)?; + } + self.bytes.push(byte); + Ok(()) + } + + fn try_extend(&mut self, bytes: &[u8]) -> postcard::Result<()> { + if bytes.len() > self.limit - self.bytes.len() { + return Err(postcard::Error::SerializeBufferFull); + } + let required = self.bytes.len() + bytes.len(); + if required > self.bytes.capacity() { + self.grow_to_fit(required)?; + } + self.bytes.extend_from_slice(bytes); + Ok(()) + } + + fn finalize(self) -> postcard::Result { + Ok(self.bytes) + } } #[derive(Serialize, Deserialize)] @@ -131,17 +184,20 @@ impl From for ScanEntry { } } -#[derive(Serialize, Deserialize)] -struct CachedFile { - fingerprint: Fingerprint, - entries: Vec, +fn serialize_cached(entries: &[CachedEntry]) -> Result> { + serialize_bounded(entries, MAX_CACHE_PAYLOAD_BYTES) + .context("failed to encode cache payload within 256 MiB limit") } -#[derive(Serialize, Deserialize)] -struct IndexCache { - version: u32, - schema_hash: u64, - files: HashMap, +fn serialize_bounded(value: &T, limit: usize) -> postcard::Result> { + postcard::serialize_with_flavor(value, BoundedVec::new(limit)) +} + +fn deserialize_cached(payload: &[u8]) -> Result> { + if payload.len() > MAX_CACHE_PAYLOAD_BYTES { + anyhow::bail!("cache payload exceeds 256 MiB"); + } + postcard::from_bytes(payload).context("failed to decode cache payload") } fn cache_dir() -> PathBuf { @@ -166,12 +222,6 @@ fn path_key(path: &Path) -> String { } } -fn schema_hash() -> u64 { - let mut hasher = DefaultHasher::new(); - zerosyntax_schema::EMBEDDED_SCHEMA_JSON.hash(&mut hasher); - hasher.finish() -} - fn fingerprint(path: &Path) -> Option { let metadata = std::fs::metadata(path).ok()?; let modified = metadata.modified().ok()?.duration_since(UNIX_EPOCH).ok()?; @@ -182,118 +232,12 @@ fn fingerprint(path: &Path) -> Option { }) } -/// Refresh the retention timestamp without rewriting a valid cache. -fn refresh_index_cache_last_used(path: &Path) -> bool { - match std::fs::OpenOptions::new() - .write(true) - .open(path) - .and_then(|file| file.set_modified(SystemTime::now())) - { - Ok(()) => true, - Err(error) => { - tracing::debug!(path = %path.display(), %error, "index cache last-used time could not be updated; falling back to cache rewrite"); - false - } - } -} - -pub(crate) fn index_cache_path(workspace_roots: &[PathBuf], base_roots: &[PathBuf]) -> PathBuf { - let mut roots: Vec<_> = workspace_roots - .iter() - .map(|root| format!("workspace:{}", path_key(root))) - .chain( - base_roots - .iter() - .map(|root| format!("base:{}", path_key(root))), - ) - .collect(); - roots.sort_unstable(); - let mut hasher = DefaultHasher::new(); - roots.hash(&mut hasher); - cache_dir().join(format!( - "index-v{INDEX_CACHE_VERSION}-{:016x}.json", - hasher.finish() - )) +pub(crate) fn index_cache_path() -> PathBuf { + cache::cache_path(&cache_dir()) } -/// The cache version encoded in an `index-v-.json` file name, -/// or `None` for anything the server did not write as an index cache. -fn cache_file_version(name: &str) -> Option { - let (version, hash) = name - .strip_prefix("index-v")? - .strip_suffix(".json")? - .split_once('-')?; - (hash.len() == 16 && hash.bytes().all(|byte| byte.is_ascii_hexdigit())) - .then(|| version.parse().ok()) - .flatten() -} - -/// Delete index caches the server can no longer use — earlier cache -/// versions, and current-version caches unused for a while — down to -/// `INDEX_CACHE_RETAINED` files. `keep`, when given, is exempt and reserves a -/// retention slot; pass `None` if the caller didn't just write it -/// successfully, so a failed or partial write can't shield a stale file at -/// the expense of evicting a newer one. Unrecognized files are never -/// touched, and a failed delete only costs disk space. -fn prune_index_caches_in(dir: &Path, keep: Option<&Path>) -> usize { - let Ok(dir) = std::fs::read_dir(dir) else { - return 0; - }; - let now = SystemTime::now(); - let mut pruned = 0; - let mut remove = |path: &Path| match std::fs::remove_file(path) { - Ok(()) => pruned += 1, - Err(error) => { - tracing::debug!(path = %path.display(), %error, "stale index cache could not be removed") - } - }; - let mut current = Vec::new(); - for entry in dir.flatten() { - let path = entry.path(); - let Some(version) = path - .file_name() - .and_then(|name| name.to_str()) - .and_then(cache_file_version) - else { - continue; - }; - if Some(path.as_path()) == keep { - continue; - } - let modified = entry.metadata().and_then(|data| data.modified()).ok(); - let expired = modified - .and_then(|modified| now.duration_since(modified).ok()) - .is_some_and(|age| age > INDEX_CACHE_MAX_AGE); - if version != INDEX_CACHE_VERSION || expired { - remove(&path); - } else { - current.push((modified, path)); - } - } - current.sort_unstable_by_key(|(modified, _)| std::cmp::Reverse(*modified)); - let retained = INDEX_CACHE_RETAINED.saturating_sub(usize::from(keep.is_some())); - for (_, path) in current.into_iter().skip(retained) { - remove(&path); - } - pruned -} - -fn prune_index_caches(keep: Option<&Path>) -> usize { - prune_index_caches_in(&cache_dir(), keep) -} - -pub(crate) fn clear_index_cache( - workspace_roots: &[PathBuf], - base_roots: &[PathBuf], -) -> std::io::Result { - let path = index_cache_path(workspace_roots, base_roots); - let cleared = match std::fs::remove_file(&path) { - Ok(()) => true, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, - Err(error) => return Err(error), - }; - prune_index_caches(None); - Ok(cleared) +pub(crate) fn clear_index_cache() -> Result { + cache::clear(&cache_dir()) } struct BigEntry { @@ -646,42 +590,33 @@ pub(crate) fn scan_with_cache( workspace_roots: &[PathBuf], base_roots: &[PathBuf], progress: &mut impl FnMut(ScanProgress), +) -> ScanOutcome { + scan_with_cache_in( + analyzer, + workspace_roots, + base_roots, + &cache_dir(), + progress, + ) +} + +fn scan_with_cache_in( + analyzer: &Analyzer, + workspace_roots: &[PathBuf], + base_roots: &[PathBuf], + cache_dir: &Path, + progress: &mut impl FnMut(ScanProgress), ) -> ScanOutcome { let started = Instant::now(); progress(ScanProgress::Discovering); - let cache_path = index_cache_path(workspace_roots, base_roots); - let expected_schema_hash = schema_hash(); - let empty_cache = || IndexCache { - version: INDEX_CACHE_VERSION, - schema_hash: expected_schema_hash, - files: HashMap::new(), - }; - let (mut cache, cache_state) = match std::fs::read(&cache_path) { - Err(error) if error.kind() == std::io::ErrorKind::NotFound => (empty_cache(), "absent"), - Err(error) => { - tracing::debug!(path = %cache_path.display(), %error, "index cache could not be read"); - (empty_cache(), "corrupt") - } - Ok(bytes) => match serde_json::from_slice::(&bytes) { - Err(error) => { - tracing::debug!(path = %cache_path.display(), %error, "index cache could not be parsed"); - (empty_cache(), "corrupt") - } - Ok(cache) - if cache.version != INDEX_CACHE_VERSION - || cache.schema_hash != expected_schema_hash => - { - tracing::debug!( - path = %cache_path.display(), - cache_version = cache.version, - expected_version = INDEX_CACHE_VERSION, - "index cache is stale" - ); - (empty_cache(), "stale") - } - Ok(cache) => (cache, "valid"), - }, - }; + let cache_path = cache::cache_path(cache_dir); + let mut persistent_cache = cache::producer_id(analyzer) + .and_then(|producer| InputCache::open(cache_dir, producer)) + .map_err(|error| { + tracing::warn!(path = %cache_path.display(), %error, "persistent input cache unavailable"); + error + }) + .ok(); let mut discovered_inputs = 0; let mut discovery_failures = 0; @@ -716,52 +651,57 @@ pub(crate) fn scan_with_cache( skipped: discovery_failures + fingerprint_failures, }); - let cache_manifest_unchanged = cache_state == "valid" - && discovery_failures == 0 - && fingerprint_failures == 0 - && paths.len() == cache.files.len() - && paths.iter().all(|(_, key, fingerprint, _)| { - cache - .files - .get(key) - .is_some_and(|cached| cached.fingerprint == *fingerprint) - }); - let cache_unchanged = cache_manifest_unchanged && refresh_index_cache_last_used(&cache_path); - - let mut next = if cache_unchanged { - HashMap::new() - } else { - HashMap::with_capacity(paths.len()) - }; let mut scanned = Vec::new(); let total = paths.len(); for (done, (path, key, fingerprint, is_base)) in paths.into_iter().enumerate() { - let entries = match cache.files.remove(&key) { - Some(cached) if cached.fingerprint == fingerprint => { - cache_hits += 1; - cached.entries.into_iter().map(ScanEntry::from).collect() - } - _ => { - cache_misses += 1; - match scan_path(analyzer, &path) { - Ok(entries) => entries, + let cached_entries = if let Some(cache) = persistent_cache.as_mut() { + match cache.lookup(&key, &fingerprint) { + Ok(Some(payload)) => match deserialize_cached(&payload) { + Ok(entries) => Some(entries.into_iter().map(ScanEntry::from).collect()), Err(error) => { - scan_failures += 1; - tracing::debug!(path = %path.display(), %error, "workspace input could not be indexed"); - Vec::new() + cache.invalidate(&key); + tracing::debug!(path = %path.display(), %error, "cached input payload is corrupt"); + None } + }, + Ok(None) => None, + Err(error) => { + tracing::warn!(path = %cache_path.display(), %error, "persistent input cache lookup failed; continuing uncached"); + persistent_cache = None; + None + } + } + } else { + None + }; + + let entries = if let Some(entries) = cached_entries { + cache_hits += 1; + entries + } else { + cache_misses += 1; + match scan_path_stable(analyzer, &path, fingerprint) { + Ok((fingerprint, entries)) => { + if let Some(cache) = persistent_cache.as_mut() { + let cached: Vec<_> = entries.iter().map(CachedEntry::from).collect(); + match serialize_cached(&cached) { + Ok(payload) => cache.queue_store(key, fingerprint, payload), + Err(error) => tracing::debug!( + path = %path.display(), + %error, + "workspace input could not be serialized for caching" + ), + } + } + entries + } + Err(error) => { + scan_failures += 1; + tracing::debug!(path = %path.display(), %error, "workspace input could not be indexed"); + Vec::new() } } }; - if !cache_unchanged { - next.insert( - key, - CachedFile { - fingerprint, - entries: entries.iter().map(CachedEntry::from).collect(), - }, - ); - } scanned.extend(entries.into_iter().map(|entry| (is_base, entry))); progress(ScanProgress::Indexing { done: done + 1, @@ -770,30 +710,24 @@ pub(crate) fn scan_with_cache( cache_misses, }); } - let mut cache_written = cache_unchanged; + let mut cache_written = persistent_cache.is_some(); let mut cache_updated = false; - if !cache_unchanged { - progress(ScanProgress::WritingCache); - let cache = IndexCache { - version: INDEX_CACHE_VERSION, - schema_hash: expected_schema_hash, - files: next, - }; - if let Some(parent) = cache_path.parent() { - if let Err(error) = std::fs::create_dir_all(parent).and_then(|()| { - serde_json::to_vec(&cache) - .map_err(std::io::Error::other) - .and_then(|bytes| std::fs::write(&cache_path, bytes)) - }) { - tracing::debug!(path = %cache_path.display(), %error, "asset index cache write failed"); - tracing::warn!(%error, "could not write asset index cache"); - } else { - cache_written = true; - cache_updated = true; + let mut pruned_cache_records = 0; + if let Some(cache) = persistent_cache.as_mut() { + if cache.has_pending_writes() { + progress(ScanProgress::WritingCache); + } + match cache.commit() { + Ok(outcome) => { + cache_updated = outcome.updated; + pruned_cache_records = outcome.pruned; + } + Err(error) => { + cache_written = false; + tracing::warn!(path = %cache_path.display(), %error, "persistent input cache commit failed"); } } } - let pruned_caches = prune_index_caches(cache_written.then_some(cache_path.as_path())); let skipped_count = fingerprint_failures + scan_failures; if skipped_count > 0 { tracing::warn!( @@ -805,7 +739,6 @@ pub(crate) fn scan_with_cache( } tracing::debug!( path = %cache_path.display(), - cache_state, discovered_inputs, discovery_failures, cache_hits, @@ -815,7 +748,7 @@ pub(crate) fn scan_with_cache( scan_failures, produced_entries = scanned.len(), cache_written, - pruned_caches, + pruned_cache_records, cache_updated, elapsed_ms = started.elapsed().as_millis() as u64, "workspace scan completed" @@ -904,6 +837,26 @@ fn scan_path(analyzer: &Analyzer, path: &Path) -> Result> { } } +/// Analyze one stable physical snapshot. A writer can replace an input after +/// discovery but before it is read; retry once so the payload is never stored +/// under a fingerprint for different bytes. +fn scan_path_stable( + analyzer: &Analyzer, + path: &Path, + mut expected: Fingerprint, +) -> Result<(Fingerprint, Vec)> { + for _ in 0..2 { + let entries = scan_path(analyzer, path)?; + let observed = fingerprint(path) + .with_context(|| format!("could not fingerprint {} after indexing", path.display()))?; + if observed == expected { + return Ok((observed, entries)); + } + expected = observed; + } + anyhow::bail!("{} changed repeatedly while it was indexed", path.display()) +} + pub(crate) fn read_asset_uri(uri: &str) -> Result> { let url = Url::parse(uri).with_context(|| format!("invalid asset URI `{uri}`"))?; if url.scheme() == "file" { @@ -958,206 +911,500 @@ mod tests { )) } + fn comparable(entries: &[(bool, ScanEntry)]) -> Vec<(bool, Vec)> { + entries + .iter() + .map(|(is_base, entry)| { + ( + *is_base, + postcard::to_stdvec(&CachedEntry::from(entry)).unwrap(), + ) + }) + .collect() + } + + fn write_big(path: &Path, entries: &[(&str, &[u8])]) { + let data_offset = 0x10 + + entries + .iter() + .map(|(name, _)| 8 + name.len() + 1) + .sum::(); + let archive_size = data_offset + entries.iter().map(|(_, data)| data.len()).sum::(); + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"BIGF"); + bytes.extend_from_slice(&(archive_size as u32).to_be_bytes()); + bytes.extend_from_slice(&(entries.len() as u32).to_be_bytes()); + bytes.extend_from_slice(&0u32.to_be_bytes()); + let mut offset = data_offset; + for (name, data) in entries { + bytes.extend_from_slice(&(offset as u32).to_be_bytes()); + bytes.extend_from_slice(&(data.len() as u32).to_be_bytes()); + bytes.extend_from_slice(name.as_bytes()); + bytes.push(0); + offset += data.len(); + } + for (_, data) in entries { + bytes.extend_from_slice(data); + } + std::fs::write(path, bytes).unwrap(); + } + #[test] - fn unchanged_warm_cache_is_not_rewritten() { + fn warm_scan_is_identical_and_does_not_rewrite_payloads() { let root = unique_temp_dir("warm-cache"); - std::fs::create_dir_all(&root).unwrap(); - std::fs::write(root.join("Weapon.ini"), "Weapon TestWeapon\nEnd\n").unwrap(); - let workspace_roots = vec![root.clone()]; - - let cold = scan_with_cache(&Analyzer::embedded(), &workspace_roots, &[], &mut |_| {}); + let cache_dir = root.join("cache"); + let workspace = root.join("workspace"); + std::fs::create_dir_all(&workspace).unwrap(); + std::fs::write(workspace.join("Weapon.ini"), "Weapon TestWeapon\nEnd\n").unwrap(); + let workspace_roots = vec![workspace]; + + let cold = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |_| {}, + ); assert_eq!(cold.stats.cache_misses, 1); assert!(cold.stats.cache_updated); - let cache_path = index_cache_path(&workspace_roots, &[]); - let old_last_used = SystemTime::now() - Duration::from_secs(24 * 60 * 60); - std::fs::OpenOptions::new() - .write(true) - .open(&cache_path) - .unwrap() - .set_modified(old_last_used) - .unwrap(); let mut warm_events = Vec::new(); - let warm = scan_with_cache(&Analyzer::embedded(), &workspace_roots, &[], &mut |event| { - warm_events.push(event) - }); + let warm = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |event| warm_events.push(event), + ); assert_eq!(warm.stats.cache_hits, 1); assert_eq!(warm.stats.cache_misses, 0); assert!(warm.stats.cache_written); assert!(!warm.stats.cache_updated); assert!(!warm_events.contains(&ScanProgress::WritingCache)); - assert!( - std::fs::metadata(&cache_path).unwrap().modified().unwrap() > old_last_used, - "using an unchanged cache refreshes its retention timestamp" - ); + assert_eq!(comparable(&cold.entries), comparable(&warm.entries)); - std::fs::write(root.join("Weapon.ini"), "Weapon UpdatedWeapon\nEnd\n").unwrap(); - let changed = scan_with_cache(&Analyzer::embedded(), &workspace_roots, &[], &mut |_| {}); + std::fs::write( + workspace_roots[0].join("Weapon.ini"), + "Weapon UpdatedWeapon\nEnd\n", + ) + .unwrap(); + let changed = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |_| {}, + ); + assert_eq!(changed.stats.cache_hits, 0); assert_eq!(changed.stats.cache_misses, 1); assert!(changed.stats.cache_updated); - let _ = clear_index_cache(&workspace_roots, &[]); std::fs::remove_dir_all(root).unwrap(); } #[test] - fn failed_retention_refresh_rejects_the_no_rewrite_fast_path() { - let missing = unique_temp_dir("missing-cache").join("index.json"); - assert!(!missing.exists()); - assert!(!refresh_index_cache_last_used(&missing)); + fn cache_serialization_stops_at_the_payload_limit() { + let value = vec![7u8; 128]; + assert_eq!( + serialize_bounded(&value, 64).unwrap_err(), + postcard::Error::SerializeBufferFull + ); + + let payload = serialize_bounded(&value, 256).unwrap(); + assert_eq!(postcard::from_bytes::>(&payload).unwrap(), value); } #[test] - fn discovery_failures_reach_progress_and_scan_stats() { - let missing = std::env::temp_dir().join(format!( - "zerosyntax-missing-{}-{}", - std::process::id(), - UNIX_EPOCH.elapsed().unwrap().as_nanos() - )); - assert!(!missing.exists()); - let workspace_roots = vec![missing]; - let mut events = Vec::new(); - let outcome = scan_with_cache(&Analyzer::embedded(), &workspace_roots, &[], &mut |event| { - events.push(event) - }); - - assert_eq!(outcome.stats.discovered_inputs, 0); - assert_eq!(outcome.stats.discovery_failures, 1); - assert_eq!(outcome.stats.skipped_inputs(), 1); - assert!(events.iter().any(|event| matches!( - event, - ScanProgress::InputsDiscovered { - total: 0, - skipped: 1 - } - ))); + fn big_archive_is_one_cached_input_with_many_virtual_entries() { + let root = unique_temp_dir("big-cache"); + let cache_dir = root.join("cache"); + let workspace = root.join("workspace"); + std::fs::create_dir_all(&workspace).unwrap(); + write_big( + &workspace.join("Data.big"), + &[ + ("Data\\INI\\Object.ini", b"Object BigObject\nEnd\n"), + ("Data\\INI\\Weapon.ini", b"Weapon BigWeapon\nEnd\n"), + ], + ); + let workspace_roots = vec![workspace]; + + let cold = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |_| {}, + ); + assert_eq!(cold.stats.cache_misses, 1); + assert_eq!(cold.entries.len(), 2); + + let warm = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |_| {}, + ); + assert_eq!(warm.stats.cache_hits, 1); + assert_eq!(warm.stats.cache_misses, 0); + assert_eq!(comparable(&cold.entries), comparable(&warm.entries)); - let _ = clear_index_cache(&workspace_roots, &[]); + std::fs::remove_dir_all(root).unwrap(); } - fn temp_cache_dir(name: &str) -> PathBuf { - let dir = std::env::temp_dir().join(format!( - "zerosyntax-prune-{name}-{}-{}", - std::process::id(), - UNIX_EPOCH.elapsed().unwrap().as_nanos() - )); - std::fs::create_dir_all(&dir).unwrap(); - dir + #[test] + fn unchanged_base_roots_are_reused_across_workspaces() { + let root = unique_temp_dir("cross-workspace-cache"); + let cache_dir = root.join("cache"); + let base_root = root.join("base"); + let workspace_a = root.join("workspace-a"); + let workspace_b = root.join("workspace-b"); + for directory in [&base_root, &workspace_a, &workspace_b] { + std::fs::create_dir_all(directory).unwrap(); + } + std::fs::write(base_root.join("Weapon.ini"), "Weapon BaseWeapon\nEnd\n").unwrap(); + std::fs::write(workspace_a.join("Object.ini"), "Object WorkspaceA\nEnd\n").unwrap(); + std::fs::write(workspace_b.join("Object.ini"), "Object WorkspaceB\nEnd\n").unwrap(); + let base_roots = vec![base_root.clone()]; + let workspace_a_roots = vec![workspace_a]; + let workspace_b_roots = vec![workspace_b]; + + let cold = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_a_roots, + &base_roots, + &cache_dir, + &mut |_| {}, + ); + assert_eq!(cold.stats.cache_hits, 0); + assert_eq!(cold.stats.cache_misses, 2); + + let switched = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_b_roots, + &base_roots, + &cache_dir, + &mut |_| {}, + ); + assert_eq!( + switched.stats.cache_hits, 1, + "the unchanged base file should be reused from workspace A's cache" + ); + assert_eq!( + switched.stats.cache_misses, 1, + "only workspace B's file should need indexing" + ); + + let added_base_root = root.join("base-added-later"); + std::fs::create_dir_all(&added_base_root).unwrap(); + std::fs::write(added_base_root.join("Armor.ini"), "Armor TestArmor\nEnd\n").unwrap(); + let extended_base_roots = vec![base_root.clone(), added_base_root]; + let extended = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_b_roots, + &extended_base_roots, + &cache_dir, + &mut |_| {}, + ); + assert_eq!( + extended.stats.cache_hits, 2, + "the existing base and workspace files should be reused" + ); + assert_eq!( + extended.stats.cache_misses, 1, + "only the newly configured base file should need indexing" + ); + + std::fs::write( + base_root.join("Weapon.ini"), + "Weapon UpdatedBaseWeapon\nEnd\n", + ) + .unwrap(); + let changed = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_a_roots, + &base_roots, + &cache_dir, + &mut |_| {}, + ); + assert_eq!( + changed.stats.cache_hits, 1, + "the unchanged workspace file should still be reusable" + ); + assert_eq!( + changed.stats.cache_misses, 1, + "a modified base file must not be reused" + ); + + std::fs::remove_dir_all(root).unwrap(); } - /// Write a cache-shaped file `age` old, so retention order is deterministic - /// instead of dependent on filesystem timestamp granularity. - fn write_cache_file(dir: &Path, name: &str, age: Duration) -> PathBuf { - let path = dir.join(name); - let file = std::fs::File::create(&path).unwrap(); - file.set_modified(SystemTime::now() - age).unwrap(); - path + #[test] + fn actual_schema_identity_separates_cached_analysis() { + let root = unique_temp_dir("schema-cache"); + let cache_dir = root.join("cache"); + let base_root = root.join("base"); + let workspace = root.join("workspace"); + for directory in [&base_root, &workspace] { + std::fs::create_dir_all(directory).unwrap(); + } + std::fs::write(base_root.join("Weapon.ini"), "Weapon BaseWeapon\nEnd\n").unwrap(); + std::fs::write(workspace.join("Object.ini"), "Object Workspace\nEnd\n").unwrap(); + let base_roots = vec![base_root]; + let workspace_roots = vec![workspace]; + + scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &base_roots, + &cache_dir, + &mut |_| {}, + ); + let mut schema = zerosyntax_schema::embedded(); + schema.engine_revision.push_str("-custom"); + let custom = Analyzer::new(schema); + let rescanned = scan_with_cache_in( + &custom, + &workspace_roots, + &base_roots, + &cache_dir, + &mut |_| {}, + ); + assert_eq!(rescanned.stats.cache_hits, 0); + assert_eq!(rescanned.stats.cache_misses, 2); + + std::fs::remove_dir_all(root).unwrap(); } #[test] - fn cache_file_version_accepts_only_generated_names() { - assert_eq!( - cache_file_version("index-v5-0123456789abcdef.json"), - Some(5) + fn cached_role_is_taken_from_the_current_scan_plan() { + let root = unique_temp_dir("cache-role"); + let cache_dir = root.join("cache"); + let shared = root.join("shared"); + std::fs::create_dir_all(&shared).unwrap(); + std::fs::write(shared.join("Weapon.ini"), "Weapon SharedWeapon\nEnd\n").unwrap(); + + let base = scan_with_cache_in( + &Analyzer::embedded(), + &[], + std::slice::from_ref(&shared), + &cache_dir, + &mut |_| {}, ); + assert!(base.entries.iter().all(|(is_base, _)| *is_base)); + + let workspace = scan_with_cache_in( + &Analyzer::embedded(), + std::slice::from_ref(&shared), + &[], + &cache_dir, + &mut |_| {}, + ); + assert_eq!(workspace.stats.cache_hits, 1); + assert!(workspace.entries.iter().all(|(is_base, _)| !*is_base)); assert_eq!( - cache_file_version("index-v12-0123456789abcdef.json"), - Some(12) + comparable(&base.entries) + .into_iter() + .map(|(_, payload)| payload) + .collect::>(), + comparable(&workspace.entries) + .into_iter() + .map(|(_, payload)| payload) + .collect::>() ); - for name in [ - "index-v5-0123456789abcde.json", // hash too short - "index-v5-0123456789abcdefg.json", // not hexadecimal - "index-vX-0123456789abcdef.json", - "index-v5-0123456789abcdef.json.bak", - "notes.txt", - ] { - assert_eq!(cache_file_version(name), None, "{name}"); - } + + std::fs::remove_dir_all(root).unwrap(); } #[test] - fn pruning_drops_earlier_versions_and_keeps_recent_caches() { - let dir = temp_cache_dir("versions"); - let unrelated = write_cache_file(&dir, "notes.txt", Duration::ZERO); - let old_version = write_cache_file(&dir, "index-v1-00000000000000ff.json", Duration::ZERO); - let keep = write_cache_file( - &dir, - &format!("index-v{INDEX_CACHE_VERSION}-0000000000000000.json"), - Duration::ZERO, + fn cached_payloads_follow_current_root_order_and_overlap_deduplication() { + let root = unique_temp_dir("cache-order"); + let cache_dir = root.join("cache"); + let first = root.join("first"); + let nested = first.join("nested"); + let second = root.join("second"); + for directory in [&nested, &second] { + std::fs::create_dir_all(directory).unwrap(); + } + std::fs::write(first.join("First.ini"), "Object FirstObject\nEnd\n").unwrap(); + std::fs::write(nested.join("Nested.ini"), "Object NestedObject\nEnd\n").unwrap(); + std::fs::write(second.join("Second.ini"), "Object SecondObject\nEnd\n").unwrap(); + + let cold = scan_with_cache_in( + &Analyzer::embedded(), + &[], + &[first.clone(), second.clone(), nested.clone()], + &cache_dir, + &mut |_| {}, ); - let others: Vec<_> = (1..=INDEX_CACHE_RETAINED as u64 + 2) - .map(|index| { - write_cache_file( - &dir, - &format!("index-v{INDEX_CACHE_VERSION}-{index:016x}.json"), - Duration::from_secs(index * 60), - ) - }) + assert_eq!(cold.stats.discovered_inputs, 3); + assert_eq!(cold.entries.len(), 3, "nested overlap is emitted once"); + + let reordered = scan_with_cache_in( + &Analyzer::embedded(), + &[], + &[second, first, nested], + &cache_dir, + &mut |_| {}, + ); + assert_eq!(reordered.stats.cache_hits, 3); + assert_eq!(reordered.entries.len(), 3); + let files: Vec<_> = reordered + .entries + .iter() + .map(|(_, entry)| entry.0.as_str()) .collect(); + assert!(files[0].ends_with("Second.ini")); + assert!(files[1].ends_with("First.ini")); + assert!(files[2].ends_with("Nested.ini")); - let pruned = prune_index_caches_in(&dir, Some(&keep)); + std::fs::remove_dir_all(root).unwrap(); + } - assert!(keep.exists(), "the cache just written survives"); - assert!(unrelated.exists(), "unrelated files are never touched"); - assert!(!old_version.exists(), "earlier cache versions are dropped"); - let surviving = others.iter().filter(|path| path.exists()).count(); - assert_eq!(surviving, INDEX_CACHE_RETAINED - 1, "newest others survive"); - assert!(others[0].exists() && !others[others.len() - 1].exists()); - assert_eq!(pruned, 1 + others.len() - surviving); - std::fs::remove_dir_all(&dir).unwrap(); + #[test] + fn failed_physical_input_is_retried_instead_of_cached_empty() { + let root = unique_temp_dir("failed-input"); + let cache_dir = root.join("cache"); + let workspace = root.join("workspace"); + std::fs::create_dir_all(&workspace).unwrap(); + std::fs::write(workspace.join("Broken.big"), b"not a BIG archive").unwrap(); + let workspace_roots = vec![workspace]; + + for _ in 0..2 { + let outcome = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |_| {}, + ); + assert_eq!(outcome.stats.cache_hits, 0); + assert_eq!(outcome.stats.cache_misses, 1); + assert_eq!(outcome.stats.scan_failures, 1); + } + + std::fs::remove_dir_all(root).unwrap(); } #[test] - fn pruning_drops_caches_unused_past_the_age_limit() { - let dir = temp_cache_dir("age"); - let fresh = write_cache_file( - &dir, - &format!("index-v{INDEX_CACHE_VERSION}-000000000000000a.json"), - Duration::ZERO, + fn corrupt_payload_invalidates_only_that_input() { + let root = unique_temp_dir("corrupt-payload"); + let cache_dir = root.join("cache"); + let workspace = root.join("workspace"); + std::fs::create_dir_all(&workspace).unwrap(); + std::fs::write(workspace.join("Weapon.ini"), "Weapon TestWeapon\nEnd\n").unwrap(); + std::fs::write(workspace.join("Object.ini"), "Object TestObject\nEnd\n").unwrap(); + let workspace_roots = vec![workspace]; + + scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |_| {}, ); - let expired = write_cache_file( - &dir, - &format!("index-v{INDEX_CACHE_VERSION}-000000000000000b.json"), - INDEX_CACHE_MAX_AGE + Duration::from_secs(60), + let connection = rusqlite::Connection::open(cache::cache_path(&cache_dir)).unwrap(); + connection + .execute( + "UPDATE input_cache SET payload = X'FF' WHERE path LIKE '%weapon.ini'", + [], + ) + .unwrap(); + drop(connection); + + let recovered = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |_| {}, ); + assert_eq!(recovered.stats.cache_hits, 1); + assert_eq!(recovered.stats.cache_misses, 1); + assert!(recovered.stats.cache_updated); + + let warm = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |_| {}, + ); + assert_eq!(warm.stats.cache_hits, 2); + assert_eq!(warm.stats.cache_misses, 0); - // No `keep` (the cache write failed) still prunes. - let pruned = prune_index_caches_in(&dir, None); - - assert!(fresh.exists()); - assert!(!expired.exists()); - assert_eq!(pruned, 1); - std::fs::remove_dir_all(&dir).unwrap(); + std::fs::remove_dir_all(root).unwrap(); } #[test] - fn failed_write_does_not_reserve_a_retention_slot_for_the_stale_file() { - // A write failure leaves the previous file at `cache_path` in place. - // It must compete for a retention slot like any other cache, not - // reserve one and evict a newer file in its place. - let dir = temp_cache_dir("stale-keep"); - let stale = write_cache_file( - &dir, - &format!("index-v{INDEX_CACHE_VERSION}-0000000000000001.json"), - Duration::from_secs(600), + fn many_inputs_share_one_store_and_warm_without_payload_writes() { + let root = unique_temp_dir("many-inputs"); + let cache_dir = root.join("cache"); + let workspace = root.join("workspace"); + std::fs::create_dir_all(&workspace).unwrap(); + const INPUTS: usize = 256; + for index in 0..INPUTS { + std::fs::write( + workspace.join(format!("Object{index:03}.ini")), + format!("Object CachedObject{index:03}\nEnd\n"), + ) + .unwrap(); + } + let workspace_roots = vec![workspace]; + let analyzer = Analyzer::embedded(); + + let cold_started = Instant::now(); + let cold = scan_with_cache_in(&analyzer, &workspace_roots, &[], &cache_dir, &mut |_| {}); + let cold_elapsed = cold_started.elapsed(); + assert_eq!(cold.stats.cache_misses, INPUTS); + assert!(cold.stats.cache_updated); + + let warm_started = Instant::now(); + let warm = scan_with_cache_in(&analyzer, &workspace_roots, &[], &cache_dir, &mut |_| {}); + let warm_elapsed = warm_started.elapsed(); + assert_eq!(warm.stats.cache_hits, INPUTS); + assert_eq!(warm.stats.cache_misses, 0); + assert!(!warm.stats.cache_updated); + assert_eq!(comparable(&cold.entries), comparable(&warm.entries)); + assert!(cache::cache_path(&cache_dir).is_file()); + eprintln!( + "physical input cache: {INPUTS} inputs, cold={cold_elapsed:?}, warm={warm_elapsed:?}" ); - let others: Vec<_> = (2..=INDEX_CACHE_RETAINED as u64 + 1) - .map(|index| { - write_cache_file( - &dir, - &format!("index-v{INDEX_CACHE_VERSION}-{index:016x}.json"), - Duration::from_secs(600 - index * 60), - ) - }) - .collect(); - prune_index_caches_in(&dir, None); + std::fs::remove_dir_all(root).unwrap(); + } - assert!(!stale.exists(), "the oldest file is evicted, not reserved"); - assert!( - others.iter().all(|path| path.exists()), - "newer files are not evicted to make room for the stale one" + #[test] + fn discovery_failures_reach_progress_and_scan_stats() { + let missing = std::env::temp_dir().join(format!( + "zerosyntax-missing-{}-{}", + std::process::id(), + UNIX_EPOCH.elapsed().unwrap().as_nanos() + )); + let cache_dir = unique_temp_dir("missing-root-cache"); + assert!(!missing.exists()); + let workspace_roots = vec![missing]; + let mut events = Vec::new(); + let outcome = scan_with_cache_in( + &Analyzer::embedded(), + &workspace_roots, + &[], + &cache_dir, + &mut |event| events.push(event), ); - std::fs::remove_dir_all(&dir).unwrap(); + + assert_eq!(outcome.stats.discovered_inputs, 0); + assert_eq!(outcome.stats.discovery_failures, 1); + assert_eq!(outcome.stats.skipped_inputs(), 1); + assert!(events.iter().any(|event| matches!( + event, + ScanProgress::InputsDiscovered { + total: 0, + skipped: 1 + } + ))); + + std::fs::remove_dir_all(cache_dir).unwrap(); } } diff --git a/docs/language-server.md b/docs/language-server.md index 384cdff..3af840b 100644 --- a/docs/language-server.md +++ b/docs/language-server.md @@ -186,16 +186,25 @@ different server executable inherently requires a new process. Indexing results are cached on disk so a restart reuses unchanged files. The cache lives in `%LOCALAPPDATA%\zerosyntax` on Windows and -`$XDG_CACHE_HOME/zerosyntax` (else the temp directory) elsewhere, as one -`index-v-.json` file per set of workspace and base roots. Every -cache-format bump, renamed workspace folder, or `baseIniRoots` change therefore -produces a new file. - -The server keeps that directory bounded: after each scan it deletes caches -written by an earlier cache version, caches unused for 30 days, and all but the -four most recently used current-version caches. The cache the running server -just wrote is always kept, and files it did not create are never touched. -Deleting the directory by hand is safe — the next scan rebuilds it. +`$XDG_CACHE_HOME/zerosyntax` (else the temp directory) elsewhere, in one +versioned SQLite database. Records belong to canonical physical inputs rather +than a workspace or configured root set. Consequently, changing workspaces, +adding or reordering `baseIniRoots`, or changing a root's base/workspace role +reuses every unchanged input and parses only new or modified files. A BIG +archive is one physical input whose cached payload contains its virtual files. + +Record identity includes the input fingerprint, the actual configured schema, +and the scanner/extractor format. SQLite WAL transactions allow editor windows +to share the database safely; cache failures degrade to ordinary parsing, a +corrupt payload invalidates only its input, and a corrupt database is rebuilt. +Failed scans are never cached. Clear/rebuild increments a database epoch so an +older in-flight scan cannot repopulate data after the clear completes. + +Unused records expire after 30 days and the logical payload budget is 1 GiB, +with least-recently-used records removed first. Older root-set JSON caches are +removed after the new store commits successfully. Deleting the cache directory +by hand is safe while the language server is stopped — the next scan rebuilds +it. ## Supported LSP features