diff --git a/Cargo.lock b/Cargo.lock index 24479c721561..bebfc2da57cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,6 +449,7 @@ dependencies = [ "rayon", "regex", "serde_json", + "swift-syntax-rs", "tracing", "tracing-subscriber", "yeast", diff --git a/MODULE.bazel b/MODULE.bazel index 24260271ecad..8db349f73463 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -228,13 +228,8 @@ use_repo( # auto-registers `xcode_swift_toolchain` (host Xcode + OS-provided Swift # runtime), which is not hermetic. # -# The version is pinned as a literal rather than read from -# `unified/swift-syntax-rs/.swift-version` via `swift_version_file`: the latter -# makes the extension `module_ctx.read` a `//unified/...` label, which fails to -# resolve when this repo is consumed as a dependency module (`@@ql+`) whose -# `unified/swift-syntax-rs` package is not loadable in that context. Keep this -# in sync with `unified/swift-syntax-rs/.swift-version` (used by the `cargo` -# build) and the `swift-syntax` release in `swift/Package.swift`. +# This and the `swift-syntax` version above are the only pins: the extractor is +# built solely by Bazel, so there is no second Swift build to keep in step. swift = use_extension("@rules_swift//swift:extensions.bzl", "swift") swift.toolchain( name = "swift_toolchain", diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel index e4e959491d45..27d36c221ea4 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel @@ -541,18 +541,6 @@ alias( tags = ["manual"], ) -alias( - name = "swift-syntax-rs-0.1.0", - actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs", - tags = ["manual"], -) - -alias( - name = "swift-syntax-rs", - actual = "@vendor_ts__swift-syntax-rs-0.1.0//:swift_syntax_rs", - tags = ["manual"], -) - alias( name = "syn-2.0.106", actual = "@vendor_ts__syn-2.0.106//:syn", diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl index be869ecf7388..6f20001571cb 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl @@ -423,8 +423,6 @@ _NORMAL_DEPENDENCIES = { }, }, "unified/swift-syntax-rs": { - _COMMON_CONDITION: { - }, }, } @@ -469,8 +467,6 @@ _NORMAL_ALIASES = { }, }, "unified/swift-syntax-rs": { - _COMMON_CONDITION: { - }, }, } diff --git a/unified/AGENTS.md b/unified/AGENTS.md index 9c9bbbb534cb..ddc12f852774 100644 --- a/unified/AGENTS.md +++ b/unified/AGENTS.md @@ -4,13 +4,18 @@ This is a CodeQL extractor that maps a language's parse tree onto a shared AST using the `yeast` desugaring engine. Swift, the only language so far, is parsed by Apple's swift-syntax rather than by tree-sitter. +Everything is built and tested with Bazel, whose Swift toolchain is hermetic on +Linux, so nothing needs to be installed locally. `cargo build`/`cargo test` do +not work here: the extractor links a Swift FFI shim that only Bazel builds. +(`cargo check` still works, so rust-analyzer is unaffected.) + ## Building -- To build the extractor, run `scripts/create-extractor-pack.sh` +- To build the extractor pack, run `scripts/create-extractor-pack.sh`. ## Swift Parser -- Swift source is parsed by `swift-syntax-parse`, a small Swift/Rust binary in - `swift-syntax-rs` that wraps Apple's swift-syntax and emits the parse tree as - JSON. There is no grammar in this repository to edit. +- Swift source is parsed by the `swift-syntax-rs` crate, which wraps Apple's + swift-syntax and emits the parse tree as JSON. The extractor links it and + calls it in-process. There is no grammar in this repository to edit. - `extractor/src/languages/swift/adapter.rs` converts that JSON into a yeast AST. @@ -22,11 +27,7 @@ by Apple's swift-syntax rather than by tree-sitter. - The mapping from the parse tree to the target AST is found in `extractor/src/languages/swift/swift.rs` -- To run tests for the parser and mapping, run `cargo test` in the `extractor` - directory. The tests need the `swift-syntax-parse` binary: point - `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` at it, or put it on `PATH`. - Corpus tests skip themselves when it cannot be found, so check for skips - before concluding a change is clean. +- To run tests for the parser and mapping, run `bazel test //unified/extractor:all_tests`. - Extractor test cases are located at `extractor/tests/corpus/swift/*/*.swift`. diff --git a/unified/BUILD.bazel b/unified/BUILD.bazel index 405500d7d407..334c7da218d9 100644 --- a/unified/BUILD.bazel +++ b/unified/BUILD.bazel @@ -47,15 +47,13 @@ codeql_pkg_files( prefix = "tools/{CODEQL_PLATFORM}", ) -# The Swift front-end parser (wrapper + real binary + bundled Swift runtime), -# shipped next to the extractor. Only on platforms where swift-syntax builds -# (Linux/macOS); elsewhere the group is empty so the pack still builds (Swift -# extraction is simply unavailable there). -pkg_filegroup( - name = "swift-syntax-parse-arch", - srcs = select_os( +# The Swift runtime, which the extractor loads at startup. Linux only: macOS +# provides it with the OS. +codeql_pkg_files( + name = "swift-runtime-arch", + exes = select_os( + linux = ["//unified/swift-syntax-rs:swift_runtime_libs"], otherwise = [], - posix = ["//unified/swift-syntax-rs:swift-syntax-parse-pkg"], ), prefix = "tools/{CODEQL_PLATFORM}", ) @@ -66,7 +64,7 @@ codeql_pack( ":codeql-extractor-yml", ":dbscheme-group", ":extractor-arch", - ":swift-syntax-parse-arch", + ":swift-runtime-arch", "//unified/tools", ], ) diff --git a/unified/extractor/BUILD.bazel b/unified/extractor/BUILD.bazel index 58b32d9fc5c3..ff82bbb330be 100644 --- a/unified/extractor/BUILD.bazel +++ b/unified/extractor/BUILD.bazel @@ -1,8 +1,16 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") load("//misc/bazel:rust.bzl", "codeql_rust_binary") load("//misc/bazel/3rdparty/tree_sitter_extractors_deps:defs.bzl", "aliases", "all_crate_deps") exports_files(["Cargo.toml"]) +# swift-syntax builds on Linux and macOS only. +_SWIFT_SUPPORTED_PLATFORMS = select({ + "@platforms//os:linux": [], + "@platforms//os:macos": [], + "//conditions:default": ["@platforms//:incompatible"], +}) + codeql_rust_binary( name = "extractor", srcs = glob(["src/**/*.rs"]), @@ -11,14 +19,75 @@ codeql_rust_binary( "ast_types.yml", "swift_node_types.yml", ], + # Only for running from the build tree: there the Swift runtime is resolved + # through the toolchain-relative part of the runpath, which needs the + # libraries in the runfiles tree. An installed pack uses `$ORIGIN` instead. + data = select({ + "@platforms//os:linux": ["//unified/swift-syntax-rs:swift_runtime_libs"], + "//conditions:default": [], + }), proc_macro_deps = all_crate_deps( proc_macro = True, ), + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, visibility = ["//visibility:public"], deps = all_crate_deps( normal = True, ) + [ "//shared/tree-sitter-extractor", "//shared/yeast", + "//unified/swift-syntax-rs:swift_syntax_rs", ], ) + +# One target per file in `tests/`. Each pulls in `src/**` too, because the tests +# reach into the crate's modules with `#[path]`. +_TESTS = { + "corpus_tests": { + "data": glob(["tests/corpus/**"]), + "compile_data": [], + "size": "medium", + }, + # `include_str!`s a checked-in `parse_to_json` dump. + "swift_syntax_pipeline": { + "data": [], + "compile_data": glob(["tests/fixtures/**"]), + "size": "small", + }, +} + +[ + rust_test( + name = test_name, + size = spec["size"], + srcs = ["tests/%s.rs" % test_name] + glob(["src/**/*.rs"]), + aliases = aliases(), + compile_data = [ + "ast_types.yml", + "swift_node_types.yml", + ] + spec["compile_data"], + crate_root = "tests/%s.rs" % test_name, + data = spec["data"] + select({ + "@platforms//os:linux": ["//unified/swift-syntax-rs:swift_runtime_libs"], + "//conditions:default": [], + }), + edition = "2024", + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + deps = all_crate_deps( + normal = True, + ) + [ + "//shared/tree-sitter-extractor", + "//shared/yeast", + "//unified/swift-syntax-rs:swift_syntax_rs", + ], + ) + for test_name, spec in _TESTS.items() +] + +test_suite( + name = "all_tests", + tests = [":%s" % test_name for test_name in _TESTS], +) diff --git a/unified/extractor/Cargo.toml b/unified/extractor/Cargo.toml index be333298f1ad..de1cb6eb5429 100644 --- a/unified/extractor/Cargo.toml +++ b/unified/extractor/Cargo.toml @@ -18,3 +18,8 @@ serde_json = "1.0.145" codeql-extractor = { path = "../../shared/tree-sitter-extractor" } yeast = { path = "../../shared/yeast" } +# The Swift front-end links swift-syntax through this crate's FFI shim. Its +# Swift half is built by Bazel, so `cargo build`/`test` cannot link the +# extractor: use `bazel build //unified/extractor` and +# `bazel test //unified/extractor:all_tests`. +swift-syntax-rs = { path = "../swift-syntax-rs" } diff --git a/unified/extractor/src/languages/swift/parse.rs b/unified/extractor/src/languages/swift/parse.rs index c8633f1a27a5..2abc589332c8 100644 --- a/unified/extractor/src/languages/swift/parse.rs +++ b/unified/extractor/src/languages/swift/parse.rs @@ -1,33 +1,18 @@ -//! Swift front-end parser: shells out to the separate `swift-syntax-parse` -//! binary (which links swift-syntax) to obtain a JSON syntax tree, then adapts -//! that JSON into a `yeast::Ast` via the pure-Rust [`swift_adapter`] module. -//! -//! Running the parser in a separate process keeps the Swift toolchain out of -//! the extractor's own build: the extractor never links Swift, so working on -//! other (e.g. tree-sitter based) languages needs no Swift toolchain. Each call -//! spawns the parser afresh; a longer-lived parser process could be swapped in -//! behind this same seam later without touching the extraction pipeline. - -use std::io::Write; -use std::process::{Command, Stdio}; +//! Swift front-end parser: calls into the `swift-syntax-rs` crate (which links +//! swift-syntax) to obtain a JSON syntax tree, then adapts that JSON into a +//! `yeast::Ast` via the pure-Rust [`swift_adapter`] module. use codeql_extractor::extractor::ParsedTree; use super::swift_adapter; -/// Environment variable naming the `swift-syntax-parse` executable. When unset, -/// the parser is resolved next to the extractor executable, then on `PATH`. -const PARSE_BIN_ENV: &str = "CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE"; - -/// Base name of the `swift-syntax-parse` executable as shipped / looked up. -const PARSE_BIN_NAME: &str = "swift-syntax-parse"; - /// Parse Swift `source` into a [`ParsedTree`] (a raw `yeast::Ast` plus /// side-channel `extra` tokens), ready to be desugared via `run_from_ast`. pub fn parse(source: &[u8]) -> Result { let source = std::str::from_utf8(source).map_err(|e| format!("Swift source is not valid UTF-8: {e}"))?; - let json = run_parser(source)?; + let json = + swift_syntax_rs::parse_to_json(source).map_err(|e| format!("Swift parser failed: {e}"))?; let mut adapted = swift_adapter::json_to_ast(&json)?; adapted.ast.set_source(source.as_bytes().to_vec()); Ok(ParsedTree { @@ -35,87 +20,3 @@ pub fn parse(source: &[u8]) -> Result { extras: adapted.extras, }) } - -/// The `swift-syntax-parse` executable to invoke, resolved in priority order: -/// -/// 1. the `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` override, if set; -/// 2. a copy shipped next to the extractor executable — this is how the CodeQL -/// extractor pack lays it out (`tools//{extractor, -/// swift-syntax-parse}`), so a packaged extractor is self-contained with no -/// environment setup; -/// 3. a bare `swift-syntax-parse`, looked up on `PATH`. -fn parse_bin() -> String { - if let Ok(bin) = std::env::var(PARSE_BIN_ENV) { - if !bin.is_empty() { - return bin; - } - } - if let Ok(exe) = std::env::current_exe() { - if let Some(sibling) = exe.parent().map(|dir| dir.join(PARSE_BIN_NAME)) { - if sibling.is_file() { - return sibling.to_string_lossy().into_owned(); - } - } - } - PARSE_BIN_NAME.to_string() -} - -/// Whether the `swift-syntax-parse` executable can be launched at all. -/// -/// This reports availability of the *executable*, deliberately not whether -/// parsing succeeds: a binary that launches but then crashes or emits invalid -/// JSON is still "available", so callers run and surface the failure rather -/// than silently skipping. Only a genuinely missing/unlaunchable binary (e.g. -/// no Swift toolchain is installed) reports `false`. -pub fn binary_available() -> bool { - match Command::new(parse_bin()) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - { - Ok(mut child) => { - let _ = child.wait(); - true - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, - // Any other spawn failure (e.g. a permissions problem) is a genuine - // issue worth surfacing, so treat the parser as available and let the - // caller fail rather than masking it as "unavailable". - Err(_) => true, - } -} - -/// Run the external parser, feeding `source` on stdin and returning its JSON -/// stdout. -fn run_parser(source: &str) -> Result { - let bin = parse_bin(); - let mut child = Command::new(&bin) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| format!("failed to spawn Swift parser `{bin}`: {e}"))?; - - // The parser reads all of stdin before writing any stdout, so writing the - // whole source and then closing stdin (by dropping it) cannot deadlock. - child - .stdin - .take() - .expect("child stdin was piped") - .write_all(source.as_bytes()) - .map_err(|e| format!("failed to write source to Swift parser `{bin}`: {e}"))?; - - let output = child - .wait_with_output() - .map_err(|e| format!("failed to run Swift parser `{bin}`: {e}"))?; - if !output.status.success() { - return Err(format!( - "Swift parser `{bin}` failed ({}): {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - )); - } - String::from_utf8(output.stdout) - .map_err(|e| format!("Swift parser produced non-UTF-8 output: {e}")) -} diff --git a/unified/extractor/tests/corpus_tests.rs b/unified/extractor/tests/corpus_tests.rs index b2700b999b7c..9eeb2744e054 100644 --- a/unified/extractor/tests/corpus_tests.rs +++ b/unified/extractor/tests/corpus_tests.rs @@ -20,20 +20,6 @@ fn update_mode_enabled() -> bool { .unwrap_or(false) } -/// Whether the external swift-syntax parser is available. When the parser -/// binary genuinely cannot be found/launched (e.g. no Swift toolchain, and -/// neither `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` nor a `swift-syntax-parse` -/// on `PATH`), the corpus test is skipped rather than failed — it cannot run -/// without the Swift-backed parser. -/// -/// Crucially this checks only that the executable *launches*: a parser that is -/// present but crashes, emits invalid JSON, or otherwise regresses is -/// considered available, so the suite runs and fails (rather than silently -/// skipping the very failures CI needs to catch). -fn parser_available() -> bool { - languages::swift_parse::binary_available() -} - /// Parse a corpus `.output` file. The file holds a single test case made of /// three sections separated by `---` delimiter lines: /// @@ -110,19 +96,32 @@ fn collect_corpus_stems(dir: &Path, out: &mut Vec) { } } +/// The corpus root, in the runfiles tree. +/// +/// Bazel runs tests from the runfiles root, under a directory named after the +/// repository the test came from — `_main` when `github/codeql` is built +/// standalone, `ql+` when it is consumed as a dependency — so look for it +/// rather than hard-coding either name. +fn corpus_dir() -> std::path::PathBuf { + let srcdir = std::env::var_os("TEST_SRCDIR") + .expect("TEST_SRCDIR is unset; these tests are run with `bazel test`"); + let entries = fs::read_dir(&srcdir) + .unwrap_or_else(|e| panic!("failed to read TEST_SRCDIR {srcdir:?}: {e}")); + for entry in entries.flatten() { + let candidate = entry.path().join("unified/extractor/tests/corpus"); + if candidate.is_dir() { + return candidate; + } + } + panic!("no `unified/extractor/tests/corpus` under TEST_SRCDIR {srcdir:?}"); +} + #[test] fn test_corpus() { - if !parser_available() { - eprintln!( - "skipping test_corpus: the swift-syntax parser is unavailable \ - (set CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE or put \ - `swift-syntax-parse` on PATH)" - ); - return; - } let update_mode = update_mode_enabled(); let all_languages = languages::all_language_specs(); - let corpus_dir = Path::new("tests/corpus"); + let corpus_dir = corpus_dir(); + let mut tested = 0usize; for lang in all_languages { let output_schema = yeast::node_types_yaml::schema_from_yaml(languages::OUTPUT_AST_SCHEMA) @@ -139,6 +138,7 @@ fn test_corpus() { stems.dedup(); for stem in stems { + tested += 1; let swift_path = stem.with_extension("swift"); let output_path = stem.with_extension("output"); let mut failures = Vec::new(); @@ -265,4 +265,13 @@ fn test_corpus() { } } } + + // Every language whose corpus directory is missing is skipped silently + // above, which is right when a language simply has no corpus — but if that + // leaves nothing at all to check, the run is vacuous and must not pass. + assert!( + tested > 0, + "no corpus cases found under {}; the suite would have passed vacuously", + corpus_dir.display() + ); } diff --git a/unified/scripts/create-extractor-pack.sh b/unified/scripts/create-extractor-pack.sh index 7a41092e4a74..8fd4815ae36b 100755 --- a/unified/scripts/create-extractor-pack.sh +++ b/unified/scripts/create-extractor-pack.sh @@ -1,25 +1,21 @@ #!/bin/bash -set -eux -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - platform="linux64" -elif [[ "$OSTYPE" == "darwin"* ]]; then - platform="osx64" -else - echo "Unknown OS" - exit 1 -fi -cd "$(dirname "$0")/.." +# Build the extractor pack into `extractor-pack/`, ready for +# `codeql test run --search-path extractor-pack`. +# +# `unified.dbscheme` and `Ast.qll` are generated from `extractor/ast_types.yml`, +# so they are regenerated here before the pack is assembled. +set -euo pipefail +IFS=$'\n\t' -(cd extractor && cargo build --release) +cd "$(dirname "$0")/.." +root=$PWD -# we are in a cargo workspace rooted at the git checkout -BIN_DIR=../target/release -"$BIN_DIR/codeql-extractor-unified" generate --dbscheme ql/lib/unified.dbscheme --library ql/lib/codeql/unified/Ast.qll +# `bazel run` executes from the runfiles tree, so pass absolute output paths. +bazel run //unified/extractor -- generate \ + --dbscheme "$root/ql/lib/unified.dbscheme" \ + --library "$root/ql/lib/codeql/unified/Ast.qll" codeql query format -i ql/lib/codeql/unified/Ast.qll rm -rf extractor-pack -mkdir -p extractor-pack -cp -r codeql-extractor.yml tools ql/lib/unified.dbscheme ql/lib/unified.dbscheme.stats extractor-pack/ -mkdir -p extractor-pack/tools/${platform} -cp "$BIN_DIR/codeql-extractor-unified" extractor-pack/tools/${platform}/extractor +exec bazel run //unified:install "$@" diff --git a/unified/scripts/update-corpus.sh b/unified/scripts/update-corpus.sh index 2f3ebade8cb3..a49c3be5808f 100755 --- a/unified/scripts/update-corpus.sh +++ b/unified/scripts/update-corpus.sh @@ -1,8 +1,22 @@ #!/bin/bash +# Regenerate the extractor corpus: rerun the corpus tests with update mode on, +# so each `.output` file is rewritten from what the extractor currently +# produces. set -euo pipefail IFS=$'\n\t' cd "$(dirname "$0")/.." -cd extractor -UNIFIED_UPDATE_CORPUS=1 cargo test +# A sandboxed test cannot touch the source tree. `--strategy=TestRunner=local` +# drops the sandbox, and the corpus files in the runfiles tree are symlinks back +# to the real ones, so update mode writes through them. Setting this here rather +# than tagging the target keeps the ordinary test run sandboxed. +# +# `--nocache_test_results` because a cached PASS would skip the run, and the +# side effect is the point. +exec bazel test \ + --strategy=TestRunner=local \ + --test_env=UNIFIED_UPDATE_CORPUS=1 \ + --nocache_test_results \ + //unified/extractor:corpus_tests \ + "$@" diff --git a/unified/swift-syntax-rs/.gitignore b/unified/swift-syntax-rs/.gitignore index 0b6496865408..ea8c4bf7f35f 100644 --- a/unified/swift-syntax-rs/.gitignore +++ b/unified/swift-syntax-rs/.gitignore @@ -1,2 +1 @@ /target -/swift/.build diff --git a/unified/swift-syntax-rs/.swift-version b/unified/swift-syntax-rs/.swift-version deleted file mode 100644 index 91e4a9f26224..000000000000 --- a/unified/swift-syntax-rs/.swift-version +++ /dev/null @@ -1 +0,0 @@ -6.3.2 diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index db451e5b7b9a..4d7494d1150f 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -1,6 +1,5 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") -load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load("//misc/bazel:pkg.bzl", "codeql_pkg_runfiles") load(":swift_runtime.bzl", "swift_runtime_libs") load(":xcode_transition.bzl", "xcode_transition_swift_library") @@ -35,6 +34,24 @@ xcode_transition_swift_library( ], ) +# The Swift runtime has to be resolvable at load time, and Bazel links against +# it through the toolchain's own directory, which no installed pack has. The +# `$ORIGIN` runpath makes an executable look beside itself instead, where +# `//unified:swift-runtime-arch` puts the libraries. +# +# This must arrive through `CcInfo`, not `rustc_flags`: `.bazelrc` enables +# `experimental_use_cc_common_link` on Linux, making the link a +# `cc_common.link` action that rustc flags never reach. `:swift_syntax_rs` +# depends on it so that every binary linking swift-syntax inherits it. +cc_library( + name = "swift_runtime_rpath", + linkopts = select({ + "@platforms//os:linux": ["-Wl,-rpath,$$ORIGIN"], + "//conditions:default": [], + }), + target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, +) + # Safe Rust bindings on top of the C ABI. Under Bazel the Swift side comes # from `:swift_syntax_ffi`; `build.rs` is only used by the `cargo` workflow. rust_library( @@ -46,54 +63,27 @@ rust_library( edition = "2024", target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, deps = [ + ":swift_runtime_rpath", ":swift_syntax_ffi", ], ) -# The Swift front-end parser. We ship it like `//swift/extractor`: a small shell -# wrapper (`swift-syntax-parse`) sets `LD_LIBRARY_PATH`/`DYLD_LIBRARY_PATH` to its -# own directory and execs the real binary (`swift-syntax-parse.real`); the Swift -# runtime shared libraries are packaged alongside them. `parse.rs` resolves the -# wrapper as a sibling of the extractor executable. +# A debugging aid, for looking at the raw swift-syntax JSON for some input: +# +# echo 'let x = 1' | bazel run //unified/swift-syntax-rs:swift-syntax-parse rust_binary( - name = "swift-syntax-parse.real", + name = "swift-syntax-parse", srcs = ["src/main.rs"], - # Target name carries `.real` (invalid in a crate name), so set it explicitly. - crate_name = "swift_syntax_parse", - # On Linux, carry the toolchain's runtime shared libraries as runfiles so - # they get packaged next to the binary. On macOS the OS provides the Swift - # runtime, so nothing extra is bundled. + # See the note on `data` in `//unified/extractor:extractor`. data = select({ - "@platforms//os:macos": [], "@platforms//os:linux": [":swift_runtime_libs"], + "//conditions:default": [], }), edition = "2024", target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, deps = [":swift_syntax_rs"], ) -# `swift-syntax-parse` wrapper (see `swift-syntax-parse.sh`). Its runfiles carry -# the real binary and the runtime libraries; packaging flattens them into one -# directory. -sh_binary( - name = "swift-syntax-parse", - srcs = ["swift-syntax-parse.sh"], - data = [":swift-syntax-parse.real"], - target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, -) - -# Packaged form for the extractor pack: the wrapper (as `swift-syntax-parse`), -# the real binary, and the runtime libraries, flattened into one directory. -codeql_pkg_runfiles( - name = "swift-syntax-parse-pkg", - # The `.sh` source is shipped as `swift-syntax-parse` (the wrapper); drop the - # original filename. - excludes = ["swift-syntax-parse.sh"], - exes = [":swift-syntax-parse"], - target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, - visibility = ["//unified:__pkg__"], -) - rust_test( name = "swift_syntax_rs_test", size = "small", diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml index 6957fb0e50e9..79319fb7bc97 100644 --- a/unified/swift-syntax-rs/Cargo.toml +++ b/unified/swift-syntax-rs/Cargo.toml @@ -1,3 +1,6 @@ +# `BUILD.bazel` is the build. This file feeds `misc/bazel/3rdparty`, which +# resolves third-party crate versions from it, and `cargo check`; `cargo +# build`/`test` cannot link the Swift shim. [package] name = "swift-syntax-rs" description = "Rust wrapper around the swift-syntax package for parsing Swift source" diff --git a/unified/swift-syntax-rs/README.md b/unified/swift-syntax-rs/README.md index 7d15c6f2c23e..1529519143aa 100644 --- a/unified/swift-syntax-rs/README.md +++ b/unified/swift-syntax-rs/README.md @@ -4,8 +4,11 @@ A Rust wrapper around the [swift-syntax](https://github.com/swiftlang/swift-synt package, allowing Swift source code to be parsed from Rust. Parsing is delegated to a small Swift shim (in [`swift/`](swift/)) that links -against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. The Rust crate -builds that shim (via `build.rs`) and provides safe bindings on top of it. +against `SwiftSyntax`/`SwiftParser` and exposes a tiny C ABI. This crate provides +safe bindings on top of it. + +Bazel builds both halves; there is no `cargo` build (see +[Building & testing](#building--testing)). ## Output format @@ -122,47 +125,12 @@ an unparenthesised `a *** b + c` is a single flat sequence whose structure cannot be determined without knowing `***`'s precedence, so it is left flat in its entirety. -## Prerequisites - -The build does not depend on any particular version manager. You need: - -- **Rust** — pinned to `1.88` by the repo-root [`rust-toolchain.toml`](../../rust-toolchain.toml), - which `rustup` picks up automatically. -- **Swift** — pinned to the version in [`.swift-version`](.swift-version) - (currently `6.3.2`), used to build `swift-syntax` `603.0.2`. Install it any way - you like — [swift.org](https://www.swift.org/install/) or - [swiftly](https://www.swift.org/swiftly/) (which reads `.swift-version`), or a - system package. Just make sure `swift` is on your `PATH` (or point `build.rs` - at it with the `SWIFT` environment variable). - -On Debian/Ubuntu the Swift runtime also needs `libncurses6` (and related libs) -available on the system. - ## Building & testing -With `cargo` and `swift` on `PATH`: - -```sh -cargo build -cargo test -``` - -If your `swift`/`swiftc` are not on `PATH`, point the build at them explicitly: - -```sh -SWIFT=/path/to/swift SWIFTC=/path/to/swiftc cargo build -``` - -The first build compiles `swift-syntax` and can take several minutes. - -## Building with Bazel (CI) - -CI builds this crate hermetically with Bazel. A Swift toolchain is downloaded -from swift.org by the official `rules_swift` standalone toolchain extension -(wired up in the repo-root `MODULE.bazel`), `swift-syntax` is pulled from the -Bazel Central Registry, and the FFI shim is compiled as a `swift_library` that -the Rust targets link against. `build.rs` is not used under Bazel; it only -builds the Swift shim for the local `cargo` workflow. +Everything is built by Bazel, which downloads a Swift toolchain from swift.org +via the official `rules_swift` standalone toolchain extension (wired up in the +repo-root `MODULE.bazel`) and pulls `swift-syntax` from the Bazel Central +Registry. Nothing has to be installed locally on Linux: ```sh bazel build //unified/swift-syntax-rs:swift-syntax-parse @@ -170,6 +138,13 @@ bazel test //unified/swift-syntax-rs:swift_syntax_rs_test bazel run //unified/swift-syntax-rs:swift-syntax-parse < some.swift ``` +The first build compiles `swift-syntax` and can take several minutes. + +`cargo build`/`cargo test` do **not** work: the Swift shim is compiled by a +`swift_library` in [`BUILD.bazel`](BUILD.bazel), so a `cargo` link finds no +`ssr_*` symbols. `cargo check` does work — it does not link — which is all +rust-analyzer needs. + Requirements: - **`clang`** must be installed on the runner. `rules_swift` requires the Bazel @@ -186,20 +161,12 @@ Requirements: [`xcode_transition.bzl`](xcode_transition.bzl)), so other targets on macOS keep using Bazel's default CC toolchain. -The Swift compiler version is kept in sync across three places: the -[`.swift-version`](.swift-version) file (read by the local `cargo`/`swift build` -and by [swiftly](https://www.swift.org/swiftly/)), the literal `swift_version` -pinned on `swift.toolchain(...)` in the root `MODULE.bazel` (the hermetic -swift.org **Linux** Bazel toolchain), and the `swift-syntax` release in -`swift/Package.swift`. On **macOS** the version is *not* pinned by the Bazel -build: `rules_swift` auto-registers the host `xcode_swift_toolchain`, which uses -whichever Swift ships with the installed Xcode. So the pin governs Linux (and -local) builds, while the macOS compiler version depends on the host Xcode. - -(The Bazel toolchain pins a literal rather than reading `.swift-version` via -`swift_version_file`, because the latter makes the module extension read a -`//unified/...` label, which fails when this repo is consumed as a dependency -module.) +Versions are pinned in the root `MODULE.bazel` and nowhere else: the +`swift_version` literal on `swift.toolchain(...)` selects the hermetic swift.org +**Linux** toolchain, and `bazel_dep(name = "swift-syntax", ...)` selects the +`swift-syntax` release. On **macOS** the compiler version is *not* pinned: +`rules_swift` auto-registers the host `xcode_swift_toolchain`, so it follows +whichever Swift ships with the installed Xcode. ## Usage @@ -213,7 +180,7 @@ println!("{json}"); CLI (reads a file argument or stdin, prints the syntax tree as JSON): ```sh -echo 'let x = 1' | cargo run --bin swift-syntax-parse +echo 'let x = 1' | bazel run //unified/swift-syntax-rs:swift-syntax-parse ``` ## Converting to a yeast AST @@ -221,14 +188,12 @@ echo 'let x = 1' | cargo run --bin swift-syntax-parse The JSON tree is consumed by the CodeQL extractor, which converts it into a [`yeast::Ast`](../../shared/yeast) — the in-memory format its rewrite rules operate on. That adapter is a pure-Rust module living in the extractor -(`unified/extractor/src/languages/swift/adapter.rs`), so the extractor never -needs the Swift toolchain: it consumes the JSON produced out-of-process by this -crate's `parse_to_json` / the `swift-syntax-parse` binary. +(`unified/extractor/src/languages/swift/adapter.rs`); the extractor links this +crate and calls `parse_to_json` in-process. ## Layout -- `swift/` — Swift package exposing the `ssr_parse_json` / `ssr_string_free` C ABI. -- `build.rs` — builds the Swift package and emits link/rpath flags (local `cargo` only). -- `BUILD.bazel` — Bazel targets for the hermetic CI build (swift_library + rust targets). +- `swift/` — Swift sources exposing the `ssr_parse_json` / `ssr_string_free` C ABI. +- `BUILD.bazel` — the build (swift_library + rust targets). - `src/lib.rs` — safe Rust bindings (`parse_to_json`). - `src/main.rs` — demo CLI. diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs deleted file mode 100644 index 6a58ddb4d8da..000000000000 --- a/unified/swift-syntax-rs/build.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::env; -use std::path::PathBuf; -use std::process::Command; - -fn main() { - let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - let swift_dir = manifest_dir.join("swift"); - - // Emitting any `rerun-if-changed` disables Cargo's default whole-package - // scan, so we must list every input to the `swift build` below. Watch the - // package manifests, the pinned dependency lockfile, the pinned compiler - // version, and the entire Swift sources tree (a directory is scanned - // recursively). `.build/` is intentionally *not* watched (it is this - // build's output; watching it would cause perpetual rebuilds). - for input in [ - swift_dir.join("Package.swift"), - swift_dir.join("Package.resolved"), - swift_dir.join("Sources"), - manifest_dir.join(".swift-version"), - ] { - println!("cargo:rerun-if-changed={}", input.display()); - } - println!("cargo:rerun-if-env-changed=SWIFT"); - println!("cargo:rerun-if-env-changed=SWIFTC"); - - // Build the Swift FFI package as a release dynamic library. - // - // Degrade gracefully when there is no runnable Swift toolchain. This crate - // is a workspace member, so a plain `cargo check`/`fmt`/`clippy` at the repo - // root runs this build script; if `swift build` cannot even be spawned we - // emit a warning and skip the link directives rather than panicking, so - // those Swift-free workflows keep working. Only `cargo build`/`cargo test` - // then fail — at link time, which is fair: they genuinely need Swift (and CI - // builds go through Bazel anyway). A Swift toolchain that *is* present but - // whose build fails is still surfaced as a hard error below. - let mut command = Command::new(swift_bin()); - command - .args(["build", "-c", "release"]) - .current_dir(&swift_dir); - apply_bare_repository_workaround(&mut command); - let status = match command.status() { - Ok(status) => status, - Err(e) => { - println!( - "cargo:warning=skipping the Swift FFI build: failed to run `{swift} build`: {e}. \ - Install a Swift toolchain (see https://www.swift.org/install/, e.g. via swiftly) \ - and ensure `swift` is on PATH, or set the `SWIFT` environment variable, to build \ - or test this crate. `cargo check`/`fmt`/`clippy` work without it. The pinned \ - version is in `.swift-version`.", - swift = swift_bin(), - ); - return; - } - }; - assert!(status.success(), "`swift build` failed"); - - // Link against the freshly built dynamic library. - let build_dir = swift_dir.join(".build/release"); - println!("cargo:rustc-link-search=native={}", build_dir.display()); - println!("cargo:rustc-link-lib=dylib=SwiftSyntaxFFI"); - println!("cargo:rustc-link-arg=-Wl,-rpath,{}", build_dir.display()); - - // The executable also needs to find the Swift runtime libraries at run time. - if let Some(runtime) = swift_runtime_dir() { - println!("cargo:rustc-link-search=native={}", runtime.display()); - println!("cargo:rustc-link-arg=-Wl,-rpath,{}", runtime.display()); - } -} - -/// Query the active Swift toolchain for the directory containing its runtime -/// shared libraries (e.g. `libswiftCore.so`). -fn swift_runtime_dir() -> Option { - let output = Command::new(swiftc_bin()) - .arg("-print-target-info") - .output() - .ok()?; - if !output.status.success() { - return None; - } - let info = String::from_utf8_lossy(&output.stdout); - - // Extract the value of `"runtimeResourcePath": "..."` without pulling in a - // JSON dependency. - let key = "\"runtimeResourcePath\""; - let start = info.find(key)?; - let rest = &info[start + key.len()..]; - let colon = rest.find(':')?; - let after = &rest[colon + 1..]; - let open = after.find('"')?; - let value_start = &after[open + 1..]; - let close = value_start.find('"')?; - let resource_path = &value_start[..close]; - - Some(PathBuf::from(resource_path).join(if cfg!(target_os = "macos") { "macosx" } else { "linux" })) -} - -/// The `swift` driver to invoke: `$SWIFT` if set, otherwise `swift` from `PATH`. -/// This keeps the build tool-agnostic — any Swift install works; no particular -/// version manager is required. -fn swift_bin() -> String { - env::var("SWIFT").unwrap_or_else(|_| "swift".to_string()) -} - -/// The `swiftc` compiler to invoke: `$SWIFTC` if set, otherwise `swiftc` from -/// `PATH`. -fn swiftc_bin() -> String { - env::var("SWIFTC").unwrap_or_else(|_| "swiftc".to_string()) -} - -/// Some environments (notably GitHub Codespaces) inject -/// `GIT_CONFIG_KEY_0=safe.bareRepository` / `GIT_CONFIG_VALUE_0=explicit`, which -/// breaks the cached bare git repositories `swift build` uses. When exactly that -/// key is present, relax it to `all` for the `swift build` subprocess only -/// (rather than unconditionally, which could clobber an unrelated -/// `GIT_CONFIG_VALUE_0`). -fn apply_bare_repository_workaround(command: &mut Command) { - if env::var("GIT_CONFIG_KEY_0").as_deref() == Ok("safe.bareRepository") { - command.env("GIT_CONFIG_VALUE_0", "all"); - } -} diff --git a/unified/swift-syntax-rs/swift-syntax-parse.sh b/unified/swift-syntax-rs/swift-syntax-parse.sh deleted file mode 100755 index 811697c820c6..000000000000 --- a/unified/swift-syntax-rs/swift-syntax-parse.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Wrapper that lets the shipped `swift-syntax-parse` find its Swift runtime -# libraries, which are packaged in the same directory as this script (and the -# real binary). Mirrors `swift/extractor/extractor.sh`. -if [[ "$(uname)" == Darwin ]]; then - export DYLD_LIBRARY_PATH=$(dirname "$0") -else - export LD_LIBRARY_PATH=$(dirname "$0") -fi - -exec -a "$0" "$0.real" "$@" diff --git a/unified/swift-syntax-rs/swift/Package.resolved b/unified/swift-syntax-rs/swift/Package.resolved deleted file mode 100644 index dafd46e961b5..000000000000 --- a/unified/swift-syntax-rs/swift/Package.resolved +++ /dev/null @@ -1,15 +0,0 @@ -{ - "originHash" : "169957f8fbd882866f8f9f1c68cde813dca3560577939a8ffec366e3d77548e4", - "pins" : [ - { - "identity" : "swift-syntax", - "kind" : "remoteSourceControl", - "location" : "https://github.com/swiftlang/swift-syntax.git", - "state" : { - "revision" : "79e4b74a295b6eb74a8b585e3a39d29e70c1dbd1", - "version" : "603.0.2" - } - } - ], - "version" : 3 -} diff --git a/unified/swift-syntax-rs/swift/Package.swift b/unified/swift-syntax-rs/swift/Package.swift deleted file mode 100644 index 37fa5a1eed7b..000000000000 --- a/unified/swift-syntax-rs/swift/Package.swift +++ /dev/null @@ -1,36 +0,0 @@ -// swift-tools-version:6.0 -import PackageDescription - -let package = Package( - name: "SwiftSyntaxFFI", - platforms: [ - // swift-syntax 603 requires macOS 10.15; declare it explicitly - // rather than relying on the swift-tools-version default (10.13). - .macOS(.v10_15), - ], - products: [ - // Dynamic library so the produced .so records its dependency on the - // Swift runtime libraries, keeping the Rust link step simple. - .library( - name: "SwiftSyntaxFFI", - type: .dynamic, - targets: ["SwiftSyntaxFFI"] - ) - ], - dependencies: [ - .package( - url: "https://github.com/swiftlang/swift-syntax.git", - exact: "603.0.2" - ) - ], - targets: [ - .target( - name: "SwiftSyntaxFFI", - dependencies: [ - .product(name: "SwiftSyntax", package: "swift-syntax"), - .product(name: "SwiftParser", package: "swift-syntax"), - .product(name: "SwiftOperators", package: "swift-syntax"), - ] - ) - ] -)