diff --git a/unified/extractor/src/languages/swift/adapter.rs b/unified/extractor/src/languages/swift/adapter.rs index ae245eb21e80..f1ae56d6c4c2 100644 --- a/unified/extractor/src/languages/swift/adapter.rs +++ b/unified/extractor/src/languages/swift/adapter.rs @@ -61,10 +61,71 @@ const VARYING_TOKEN_KINDS: &[&str] = &[ fn is_metadata_key(key: &str) -> bool { matches!( key, - "kind" | "range" | "tokenKind" | "text" | "leadingTrivia" | "trailingTrivia" + "kind" + | "$pos" + | "$end" + | "$lineStarts" + | "tokenKind" + | "text" + | "leadingTrivia" + | "trailingTrivia" ) } +/// Converts compact UTF-8 byte offsets into tree-sitter-style points. +struct LocationTable { + line_starts: Vec, +} + +impl LocationTable { + fn from_root(root: &Value) -> Result { + let values = root + .get("$lineStarts") + .and_then(Value::as_array) + .ok_or("root node is missing an array `$lineStarts`")?; + let mut line_starts = Vec::with_capacity(values.len()); + for (index, value) in values.iter().enumerate() { + let offset = value + .as_u64() + .and_then(|offset| usize::try_from(offset).ok()) + .ok_or_else(|| format!("`$lineStarts[{index}]` is not a valid byte offset"))?; + line_starts.push(offset); + } + if line_starts.first() != Some(&0) { + return Err("`$lineStarts` must start with offset 0".to_string()); + } + if line_starts.windows(2).any(|pair| pair[0] >= pair[1]) { + return Err("`$lineStarts` offsets must be strictly increasing".to_string()); + } + Ok(Self { line_starts }) + } + + fn point(&self, offset: usize) -> Point { + let row = self + .line_starts + .partition_point(|line_start| *line_start <= offset) + - 1; + Point::new(row, offset - self.line_starts[row]) + } + + /// Parse a node's half-open UTF-8 byte range into a [`yeast::Range`]. + fn range(&self, node: &Value) -> Option { + let offset = |key: &str| { + node.get(key)? + .as_u64() + .and_then(|offset| usize::try_from(offset).ok()) + }; + let start_byte = offset("$pos")?; + let end_byte = offset("$end")?; + Some(Range { + start_byte, + end_byte, + start_point: self.point(start_byte), + end_point: self.point(end_byte), + }) + } +} + /// The classification of a JSON node into a yeast kind name and named-ness. struct KindInfo { /// The name under which the kind is registered in the schema. @@ -158,16 +219,21 @@ fn children_of(value: &Value) -> Vec<&Value> { /// comment/`unexpectedText` trivia carried by a token is harvested into /// `extras` (as [`ExtraToken`]s) during the same pass rather than embedded in /// the tree. -fn build(node: &Value, ast: &mut Ast, extras: &mut Vec) -> Result { +fn build( + node: &Value, + locations: &LocationTable, + ast: &mut Ast, + extras: &mut Vec, +) -> Result { let info = classify(node)?; - collect_extras(node, extras); + collect_extras(node, locations, extras); let mut fields: BTreeMap> = BTreeMap::new(); for (field, value) in field_entries(node) { let field_id = ast.register_field(field); let mut ids = Vec::new(); for child in children_of(value) { - ids.push(build(child, ast, extras)?); + ids.push(build(child, locations, ast, extras)?); } fields.insert(field_id, ids); } @@ -183,7 +249,7 @@ fn build(node: &Value, ast: &mut Ast, extras: &mut Vec) -> Result) -> Result) { +fn collect_extras(node: &Value, locations: &LocationTable, out: &mut Vec) { for key in ["leadingTrivia", "trailingTrivia"] { let Some(Value::Array(pieces)) = node.get(key) else { continue; @@ -199,7 +265,7 @@ fn collect_extras(node: &Value, out: &mut Vec) { for piece in pieces { let (Some(kind), Some(range)) = ( piece.get("kind").and_then(Value::as_str), - parse_range(piece), + locations.range(piece), ) else { continue; }; @@ -232,35 +298,6 @@ fn trivia_kind_id(kind: &str) -> usize { } } -/// Parse a node's `range` into a [`yeast::Range`]. -/// -/// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`, -/// a 1-based `line`, and a 1-based UTF-8 byte `column`. yeast (like tree-sitter) -/// uses byte offsets with 0-based rows/columns and an exclusive end, so the -/// line/column are shifted down by one. swift-syntax's end position is already -/// exclusive, so the byte offsets map across directly. -fn parse_range(node: &Value) -> Option { - let range = node.get("range")?; - let point = |key: &str| -> Option<(usize, Point)> { - let p = range.get(key)?; - let offset = p.get("offset")?.as_u64()? as usize; - let line = p.get("line")?.as_u64()? as usize; - let column = p.get("column")?.as_u64()? as usize; - Some(( - offset, - Point::new(line.saturating_sub(1), column.saturating_sub(1)), - )) - }; - let (start_byte, start_point) = point("start")?; - let (end_byte, end_point) = point("end")?; - Some(Range { - start_byte, - end_byte, - start_point, - end_point, - }) -} - /// The authoritative swift-syntax input node-types schema, generated from /// swift-syntax by `swift-syntax-rs/schemagen` (run /// `unified/scripts/regenerate-node-types.sh` to refresh it). @@ -276,10 +313,11 @@ const SWIFT_NODE_TYPES: &str = include_str!("../../../swift_node_types.yml"); /// ever consumes swift-syntax input, so the schema is not a parameter. pub fn json_to_ast(json: &str) -> Result { let root: Value = serde_json::from_str(json).map_err(|e| format!("invalid JSON: {e}"))?; + let locations = LocationTable::from_root(&root)?; let mut ast = Ast::with_schema(yeast::node_types_yaml::schema_from_yaml(SWIFT_NODE_TYPES)?); let mut extras = Vec::new(); - let root_id = build(&root, &mut ast, &mut extras)?; + let root_id = build(&root, &locations, &mut ast, &mut extras)?; ast.set_root(root_id); // Emit extras in source order (the traversal visits nodes bottom-up). @@ -297,23 +335,28 @@ mod tests { /// adapter is tested without needing the Swift toolchain. fn sample_json() -> &'static str { r#"{ + "$lineStarts": [0], + "$pos": 0, + "$end": 9, "kind": "sourceFile", - "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}}, "statements": [ { + "$pos": 0, + "$end": 9, "kind": "variableDecl", - "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":9,"line":1,"column":10}}, "bindingSpecifier": { + "$pos": 0, + "$end": 3, "kind": "token", "tokenKind": "keyword(SwiftSyntax.Keyword.let)", - "text": "let", - "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":3,"line":1,"column":4}} + "text": "let" }, "name": { + "$pos": 4, + "$end": 5, "kind": "token", "tokenKind": "identifier(\"x\")", - "text": "x", - "range": {"start":{"offset":4,"line":1,"column":5},"end":{"offset":5,"line":1,"column":6}} + "text": "x" } } ] @@ -377,30 +420,72 @@ mod tests { .iter() .find(|n| n.kind_name() == "identifier") .expect("identifier node exists"); - // `x` is at file offset 4..5, line 1, column 5 (1-based) in the JSON, - // which maps to 0-based row 0, column 4 and byte range 4..5. + // `x` is at UTF-8 byte range 4..5 on the first line. assert_eq!(ident.start_byte(), 4); assert_eq!(ident.end_byte(), 5); assert_eq!(ident.start_position(), Point::new(0, 4)); assert_eq!(ident.end_position(), Point::new(0, 5)); } + #[test] + fn maps_utf8_locations_across_swift_line_endings() { + // The implied source prefix is `// é😀\r\nlet `: the second line begins + // at UTF-8 byte 11 and `x` occupies bytes 15..16. + let json = r#"{ + "$lineStarts": [0, 11, 21, 31], + "$pos": 0, + "$end": 31, + "kind": "sourceFile", + "name": { + "$pos": 15, + "$end": 16, + "kind": "token", + "tokenKind": "identifier(\"x\")", + "text": "x" + } + }"#; + let ast = json_to_ast(json).expect("adapter should succeed").ast; + let ident = ast + .nodes() + .iter() + .find(|n| n.kind_name() == "identifier") + .expect("identifier node exists"); + assert_eq!(ident.start_byte(), 15); + assert_eq!(ident.end_byte(), 16); + assert_eq!(ident.start_position(), Point::new(1, 4)); + assert_eq!(ident.end_position(), Point::new(1, 5)); + } + + #[test] + fn rejects_invalid_line_starts() { + let json = r#"{"$lineStarts":[1],"$pos":0,"$end":0,"kind":"sourceFile"}"#; + let error = match json_to_ast(json) { + Ok(_) => panic!("invalid line starts should fail"), + Err(error) => error, + }; + assert!(error.contains("must start with offset 0"), "{error}"); + } + #[test] fn collects_extras_into_side_channel() { // A token carrying a trailing line comment in its trivia. let json = r#"{ + "$lineStarts": [0], + "$pos": 0, + "$end": 14, "kind": "sourceFile", - "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":14,"line":1,"column":15}}, "value": { + "$pos": 0, + "$end": 1, "kind": "token", "tokenKind": "integerLiteral(\"1\")", "text": "1", - "range": {"start":{"offset":0,"line":1,"column":1},"end":{"offset":1,"line":1,"column":2}}, "trailingTrivia": [ { + "$pos": 2, + "$end": 6, "kind": "lineComment", - "text": "// c", - "range": {"start":{"offset":2,"line":1,"column":3},"end":{"offset":6,"line":1,"column":7}} + "text": "// c" } ] } diff --git a/unified/extractor/tests/fixtures/let_x.swiftsyntax.json b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json index 6c3d18e2fef0..a0b911813fff 100644 --- a/unified/extractor/tests/fixtures/let_x.swiftsyntax.json +++ b/unified/extractor/tests/fixtures/let_x.swiftsyntax.json @@ -1,196 +1,80 @@ { + "$end": 10, + "$lineStarts": [ + 0, + 10 + ], + "$pos": 0, "endOfFileToken": { + "$end": 10, + "$pos": 10, "kind": "token", - "range": { - "end": { - "column": 1, - "line": 2, - "offset": 10 - }, - "start": { - "column": 1, - "line": 2, - "offset": 10 - } - }, "text": "", "tokenKind": "endOfFile" }, "kind": "sourceFile", - "range": { - "end": { - "column": 1, - "line": 2, - "offset": 10 - }, - "start": { - "column": 1, - "line": 1, - "offset": 0 - } - }, "statements": [ { + "$end": 9, + "$pos": 0, "item": { + "$end": 9, + "$pos": 0, "attributes": [], "bindings": [ { + "$end": 9, + "$pos": 4, "initializer": { + "$end": 9, + "$pos": 6, "equal": { + "$end": 7, + "$pos": 6, "kind": "token", - "range": { - "end": { - "column": 8, - "line": 1, - "offset": 7 - }, - "start": { - "column": 7, - "line": 1, - "offset": 6 - } - }, "text": "=", "tokenKind": "equal" }, "kind": "initializerClause", - "range": { - "end": { - "column": 10, - "line": 1, - "offset": 9 - }, - "start": { - "column": 7, - "line": 1, - "offset": 6 - } - }, "value": { + "$end": 9, + "$pos": 8, "kind": "integerLiteralExpr", "literal": { + "$end": 9, + "$pos": 8, "kind": "token", - "range": { - "end": { - "column": 10, - "line": 1, - "offset": 9 - }, - "start": { - "column": 9, - "line": 1, - "offset": 8 - } - }, "text": "1", "tokenKind": "integerLiteral(\"1\")" - }, - "range": { - "end": { - "column": 10, - "line": 1, - "offset": 9 - }, - "start": { - "column": 9, - "line": 1, - "offset": 8 - } } } }, "kind": "patternBinding", "pattern": { + "$end": 5, + "$pos": 4, "identifier": { + "$end": 5, + "$pos": 4, "kind": "token", - "range": { - "end": { - "column": 6, - "line": 1, - "offset": 5 - }, - "start": { - "column": 5, - "line": 1, - "offset": 4 - } - }, "text": "x", "tokenKind": "identifier(\"x\")" }, - "kind": "identifierPattern", - "range": { - "end": { - "column": 6, - "line": 1, - "offset": 5 - }, - "start": { - "column": 5, - "line": 1, - "offset": 4 - } - } - }, - "range": { - "end": { - "column": 10, - "line": 1, - "offset": 9 - }, - "start": { - "column": 5, - "line": 1, - "offset": 4 - } + "kind": "identifierPattern" } } ], "bindingSpecifier": { + "$end": 3, + "$pos": 0, "kind": "token", - "range": { - "end": { - "column": 4, - "line": 1, - "offset": 3 - }, - "start": { - "column": 1, - "line": 1, - "offset": 0 - } - }, "text": "let", "tokenKind": "keyword(SwiftSyntax.Keyword.let)" }, "kind": "variableDecl", - "modifiers": [], - "range": { - "end": { - "column": 10, - "line": 1, - "offset": 9 - }, - "start": { - "column": 1, - "line": 1, - "offset": 0 - } - } + "modifiers": [] }, - "kind": "codeBlockItem", - "range": { - "end": { - "column": 10, - "line": 1, - "offset": 9 - }, - "start": { - "column": 1, - "line": 1, - "offset": 0 - } - } + "kind": "codeBlockItem" } ] } diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index ee0e559820b4..32528952071d 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -10,69 +10,83 @@ builds that shim (via `build.rs`) and provides safe bindings on top of it. ## Output format The emitted JSON tree preserves the AST's named structure. Every node has a -`kind` and a `range` with `start`/`end` positions (UTF-8 `offset` plus 1-based -`line`/`column`). Beyond that: +`kind` and a half-open UTF-8 byte range encoded as 0-based `$pos`/`$end` +offsets. The root also has a `$lineStarts` array containing the UTF-8 byte +offset of every physical source line, so line/column positions can be +reconstructed without repeating them on every node. Beyond that: - **Tokens** carry `text`, `tokenKind`, and — only when non-empty — - `leadingTrivia`/`trailingTrivia` arrays of `{ kind, text }` pieces. + `leadingTrivia`/`trailingTrivia` arrays of `{ kind, text, $pos, $end }` + pieces. - **Layout nodes** (e.g. `functionDecl`) embed their children directly as members keyed by the child's name in the parent (`name`, `signature`, - `body`, …), alongside `kind`/`range`. Absent optional children are omitted. + `body`, …), alongside `kind`/`$pos`/`$end`. Absent optional children are + omitted. - **Collection nodes** (e.g. `codeBlockItemList`) are elided: a list-valued field is simply a JSON array of its elements (e.g. `parameters`, `statements`). - This drops the collection node's own `kind`/`range`. + This drops the collection node's own `kind`/location. Only meaningful trivia is kept — the four comment kinds (`lineComment`, `blockComment`, `docLineComment`, `docBlockComment`) and `unexpectedText` -(source the parser skipped). Whitespace trivia is dropped, since node ranges +(source the parser skipped). Whitespace trivia is dropped, since node offsets already encode positions. ### Example -Parsing `let x = 1 // c` produces the following (each `range` object is +Parsing `let x = 1 // c` produces the following (location offsets are abbreviated here as `…`): ```jsonc { + "$pos": 0, + "$end": …, + "$lineStarts": [0], "kind": "sourceFile", - "range": …, "statements": [ // collection node elided to an array { + "$pos": 0, + "$end": …, "kind": "codeBlockItem", - "range": …, "item": { + "$pos": 0, + "$end": …, "kind": "variableDecl", - "range": …, "attributes": [], // empty collection → empty array "modifiers": [], "bindingSpecifier": { // a token + "$pos": 0, + "$end": 3, "kind": "token", "text": "let", - "tokenKind": "keyword(SwiftSyntax.Keyword.let)", - "range": … + "tokenKind": "keyword(SwiftSyntax.Keyword.let)" }, "bindings": [ { + "$pos": …, + "$end": …, "kind": "patternBinding", - "range": …, "pattern": { + "$pos": …, + "$end": …, "kind": "identifierPattern", - "range": …, - "identifier": { "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")", "range": … } + "identifier": { "$pos": …, "$end": …, "kind": "token", "text": "x", "tokenKind": "identifier(\"x\")" } }, "initializer": { + "$pos": …, + "$end": …, "kind": "initializerClause", - "range": …, - "equal": { "kind": "token", "text": "=", "tokenKind": "equal", "range": … }, + "equal": { "$pos": …, "$end": …, "kind": "token", "text": "=", "tokenKind": "equal" }, "value": { + "$pos": …, + "$end": …, "kind": "integerLiteralExpr", - "range": …, "literal": { + "$pos": …, + "$end": …, "kind": "token", "text": "1", "tokenKind": "integerLiteral(\"1\")", - "range": …, - "trailingTrivia": [ { "kind": "lineComment", "text": "// c" } ] + "trailingTrivia": [ { "$pos": …, "$end": …, "kind": "lineComment", "text": "// c" } ] } } } @@ -81,7 +95,7 @@ abbreviated here as `…`): } } ], - "endOfFileToken": { "kind": "token", "text": "", "tokenKind": "endOfFile", "range": … } + "endOfFileToken": { "$pos": …, "$end": …, "kind": "token", "text": "", "tokenKind": "endOfFile" } } ``` diff --git a/unified/swift-syntax-rs/src/lib.rs b/unified/swift-syntax-rs/src/lib.rs index a87c817a8935..f209780f3eee 100644 --- a/unified/swift-syntax-rs/src/lib.rs +++ b/unified/swift-syntax-rs/src/lib.rs @@ -86,12 +86,18 @@ mod tests { "unexpected tree: {json}" ); assert!(json.contains("\"text\":\"x\""), "unexpected tree: {json}"); - // Source ranges are emitted for every node. - assert!(json.contains("\"range\""), "missing ranges: {json}"); + // Compact UTF-8 source ranges are emitted for every node, with one + // source-wide line-start table. assert!( - json.contains("\"line\"") && json.contains("\"column\"") && json.contains("\"offset\""), + json.contains("\"$pos\"") + && json.contains("\"$end\"") + && json.contains("\"$lineStarts\""), "missing location fields: {json}" ); + assert!( + !json.contains("\"range\""), + "unexpected verbose range: {json}" + ); } #[test] @@ -138,11 +144,32 @@ mod tests { "JSON string was not escaped correctly: {json}" ); assert!( - json.contains(r#""start":{"column":1,"line":1,"offset":0}"#), + json.starts_with(r#"{"$end":"#) && json.contains(r#","$lineStarts":[0,"#), "JSON object keys were not sorted: {json}" ); } + #[test] + fn emits_utf8_offsets_with_swift_syntax_line_boundaries() { + let source = "// é😀\r\nlet x = 1\rlet y = 2\n"; + let json = parse_to_json(source).expect("parsing should succeed"); + + // SwiftSyntax recognizes LF, CR, and CRLF as physical line breaks. The + // offsets are UTF-8 bytes, so the first CRLF ends at byte 11. + assert!( + json.contains(r#""$lineStarts":[0,11,21,31]"#), + "unexpected line starts: {json}" + ); + assert!( + json.contains(r#""$end":16,"$pos":15,"kind":"token","text":"x""#), + "unexpected UTF-8 token range: {json}" + ); + assert!( + json.contains(r#""$end":26,"$pos":25,"kind":"token","text":"y""#), + "unexpected UTF-8 token range: {json}" + ); + } + #[test] fn captures_trivia() { // A leading comment is kept as trivia on the token it precedes. diff --git a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift index e423e75cf33f..7f6af5898191 100644 --- a/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift +++ b/unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift @@ -11,25 +11,24 @@ import SwiftParser import Darwin #endif -/// Convert an absolute position into an `{ offset, line, column }` dictionary. +/// Return the UTF-8 byte offset of the start of every physical source line. /// -/// `offset` is a UTF-8 byte offset; `line`/`column` are 1-based. -private func location( - _ position: AbsolutePosition, - _ converter: SourceLocationConverter -) -> [String: Any] { - let loc = converter.location(for: position) - return [ - "offset": position.utf8Offset, - "line": loc.line, - "column": loc.column, - ] +/// Deriving this from `SourceLocationConverter.sourceLines` keeps newline +/// handling exactly aligned with swift-syntax (`\n`, `\r`, and `\r\n`). +private func lineStarts(_ converter: SourceLocationConverter) -> [Any] { + var result: [Any] = [] + var offset = 0 + for line in converter.sourceLines { + result.append(offset) + offset += line.utf8.count + } + return result } /// Trivia kinds worth preserving in the serialized tree. Comments carry /// developer intent (including doc comments), and `unexpectedText` flags source /// the parser had to skip. Whitespace and multi-line-string escape markers are -/// dropped: node ranges already encode positions, so they would only bloat the +/// dropped: node offsets already encode positions, so they would only bloat the /// output. private let keptTriviaKinds: Set = [ "lineComment", @@ -39,7 +38,7 @@ private let keptTriviaKinds: Set = [ "unexpectedText", ] -/// Serialize a trivia collection into an array of `{ kind, text, range }` +/// Serialize a trivia collection into an array of `{ kind, text, $pos, $end }` /// pieces, keeping only the kinds in `keptTriviaKinds`. /// /// `start` is the absolute position of the first piece (a token's leading @@ -48,8 +47,7 @@ private let keptTriviaKinds: Set = [ /// accumulating piece lengths, so kept pieces carry an exact source location. private func serializeTrivia( _ trivia: Trivia, - startingAt start: AbsolutePosition, - _ converter: SourceLocationConverter + startingAt start: AbsolutePosition ) -> [Any] { var result: [Any] = [] var offset = start.utf8Offset @@ -61,12 +59,10 @@ private func serializeTrivia( let kind = Mirror(reflecting: piece).children.first?.label ?? "\(piece)" if keptTriviaKinds.contains(kind) { result.append([ + "$pos": offset, + "$end": offset + length, "kind": kind, "text": Trivia(pieces: [piece]).description, - "range": [ - "start": location(AbsolutePosition(utf8Offset: offset), converter), - "end": location(AbsolutePosition(utf8Offset: offset + length), converter), - ], ]) } offset += length @@ -76,55 +72,50 @@ private func serializeTrivia( /// Recursively convert a SwiftSyntax node into a JSON-serializable value. /// -/// * Tokens carry `kind`, `tokenKind`, `text`, and `range`, plus +/// * Tokens carry `kind`, `tokenKind`, `text`, `$pos`, and `$end`, plus /// `leadingTrivia`/`trailingTrivia` — but only when non-empty (after /// filtering, most tokens have no trivia, so the keys are simply absent). -/// * Layout nodes (e.g. `functionDecl`) carry `kind` and source `range`, and -/// additionally embed their children directly as members keyed by the +/// * Layout nodes (e.g. `functionDecl`) carry `kind`, `$pos`, and `$end`, +/// and additionally embed their children directly as members keyed by the /// child's name in the parent (e.g. `name`, `signature`, `body`); absent -/// optional children are omitted. Field names never collide with -/// `kind`/`range`. +/// optional children are omitted. /// * Collection nodes (e.g. `codeBlockItemList`) are *elided*: they become a /// plain array of their serialized elements, taking the place of the /// collection node itself. A list-valued layout field (e.g. `parameters`) is /// therefore simply a JSON array. This drops the collection node's own -/// `kind`/`range`, which are unnamed and largely recoverable from the +/// `kind`/location, which are unnamed and largely recoverable from the /// elements. -private func serialize( - _ node: Syntax, - _ converter: SourceLocationConverter -) -> Any { +private func serialize(_ node: Syntax) -> Any { if node.kind.isSyntaxCollection { return node.children(viewMode: .sourceAccurate).map { - serialize($0, converter) + serialize($0) } } - // Source range covering the node's content, excluding surrounding trivia. - let range: [String: Any] = [ - "start": location(node.positionAfterSkippingLeadingTrivia, converter), - "end": location(node.endPositionBeforeTrailingTrivia, converter), - ] + // Half-open UTF-8 byte range covering the node's content, excluding + // surrounding trivia. + let start = node.positionAfterSkippingLeadingTrivia.utf8Offset + let end = node.endPositionBeforeTrailingTrivia.utf8Offset if let token = node.as(TokenSyntax.self) { var result: [String: Any] = [ + "$pos": start, + "$end": end, "kind": "token", "tokenKind": "\(token.tokenKind)", "text": token.text, - "range": range, ] // Only emit trivia when present; after filtering, most tokens have none. // Leading trivia starts at the token's own position; trailing trivia // starts just after the token's content. let leading = serializeTrivia( - token.leadingTrivia, startingAt: token.position, converter) + token.leadingTrivia, startingAt: token.position) if !leading.isEmpty { result["leadingTrivia"] = leading } let trailing = serializeTrivia( token.trailingTrivia, - startingAt: token.endPositionBeforeTrailingTrivia, - converter) + startingAt: token.endPositionBeforeTrailingTrivia) if !trailing.isEmpty { result["trailingTrivia"] = trailing } @@ -132,8 +123,9 @@ private func serialize( } var result: [String: Any] = [ + "$pos": start, + "$end": end, "kind": "\(node.kind)", - "range": range, ] var unnamed = 0 for child in node.children(viewMode: .sourceAccurate) { @@ -141,10 +133,10 @@ private func serialize( // parent (the same mechanism SwiftSyntax uses for its debug dump). A // child that is a collection serializes to an array (see above). if let keyPath = child.keyPathInParent, let name = childName(keyPath) { - result[name] = serialize(child, converter) + result[name] = serialize(child) } else { // Defensive fallback for any unnamed layout child. - result["child\(unnamed)"] = serialize(child, converter) + result["child\(unnamed)"] = serialize(child) unnamed += 1 } } @@ -315,7 +307,10 @@ public func ssr_parse_json(_ source: UnsafePointer?) -> UnsafeMutablePoin // converter built from the original tree maps the folded tree correctly. let folded = foldOperators(in: tree) let converter = SourceLocationConverter(fileName: "", tree: tree) - let json = serialize(folded, converter) + guard var json = serialize(folded) as? [String: Any] else { + return nil + } + json["$lineStarts"] = lineStarts(converter) var bytes: [UInt8] = [] do {