diff --git a/scripts_bazel/BUILD b/scripts_bazel/BUILD index a9b4f4f36..650893318 100644 --- a/scripts_bazel/BUILD +++ b/scripts_bazel/BUILD @@ -22,7 +22,10 @@ filegroup( py_binary( name = "generate_sourcelinks", - srcs = ["generate_sourcelinks_cli.py"], + srcs = [ + "generate_sourcelinks_cli.py", + "source_code_link_parser.py", + ], main = "generate_sourcelinks_cli.py", visibility = ["//visibility:public"], deps = [ diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index e5b134a45..a4b45c17b 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -22,9 +22,7 @@ import sys from pathlib import Path -from src.extensions.score_source_code_linker.generate_source_code_links_json import ( - _extract_references_from_file, # pyright: ignore[reportPrivateUsage] TODO: move it out of the extension and into this script -) +from scripts_bazel.source_code_link_parser import extract_references_from_file from src.extensions.score_source_code_linker.helpers import parse_repo_name_from_path from src.extensions.score_source_code_linker.needlinks import ( DefaultMetaData, @@ -81,7 +79,7 @@ def main(): abs_file_path = file_path.resolve() assert abs_file_path.exists(), abs_file_path clean_path = clean_external_prefix(file_path) - references = _extract_references_from_file( + references = extract_references_from_file( abs_file_path.parent, Path(abs_file_path.name), clean_path ) all_need_references.extend(references) diff --git a/scripts_bazel/source_code_link_parser.py b/scripts_bazel/source_code_link_parser.py new file mode 100644 index 000000000..b1521e71a --- /dev/null +++ b/scripts_bazel/source_code_link_parser.py @@ -0,0 +1,78 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Parse source files for traceability tags used by the Bazel link generator.""" + +import logging +from pathlib import Path + +from src.extensions.score_source_code_linker.needlinks import NeedLink + +LOGGER = logging.getLogger(__name__) + +TAGS = [ + "# " + "req-traceability:", + "# " + "req-Id:", + "// " + "req-traceability:", + "// " + "req-Id:", +] + + +def _extract_references_from_line(line: str): + """Extract requirement IDs from a line containing a tag.""" + + for tag in TAGS: + tag_index = line.find(tag) + if tag_index >= 0: + line_after_tag = line[tag_index + len(tag) :].strip() + # Split by comma or space to get multiple requirements + for req in line_after_tag.replace(",", " ").split(): + yield tag, req.strip() + + +def extract_references_from_file( + root: Path, file_path_name: Path, file_path: Path +) -> list[NeedLink]: + """Scan a single file for traceability tags and return the findings. + + ``root / file_path_name`` identifies the file to read. ``file_path`` is + the path that should be recorded in the generated source-link data, which + may differ when the input file is located below Bazel's external prefix. + """ + + assert root.is_absolute(), "Root path must be absolute" + assert not file_path_name.is_absolute(), "File path must be relative to the root" + assert (root / file_path_name).exists(), ( + f"File {file_path_name} does not exist in root {root}." + ) + + findings: list[NeedLink] = [] + + try: + with open(root / file_path_name, encoding="utf-8", errors="ignore") as f: + for line_num, line in enumerate(f, 1): + for tag, req in _extract_references_from_line(line): + findings.append( + NeedLink( + file=file_path, + line=line_num, + tag=tag, + need=req, + full_line=line.strip(), + ) + ) + except (UnicodeDecodeError, PermissionError, OSError) as e: + # Skip files that can't be read as text + LOGGER.debug(f"Error reading file to parse for linked needs: \n{e}") + + return findings diff --git a/src/extensions/docs/source_code_linker.md b/src/extensions/docs/source_code_linker.md index b7d5f7eeb..9871cc10f 100644 --- a/src/extensions/docs/source_code_linker.md +++ b/src/extensions/docs/source_code_linker.md @@ -362,6 +362,7 @@ The bazel part: scripts_bazel/ ├── BUILD # Declare libraries and filegroups needed for bazel ├── generate_sourcelinks_cli.py # Bazel step 1 => Parses sourcefiles for tags +├── source_code_link_parser.py # Source-file parser used by the generator ├── merge_sourcelinks.py └── tests │ └── ... @@ -372,7 +373,6 @@ The Sphinx extension ```text score_source_code_linker/ ├── __init__.py # Main Sphinx extension; combines CodeLinks + TestLinks -├── generate_source_code_links_json.py # Most functionality moved to 'scripts_bazel/generate_sourcelinks_cli' ├── need_source_links.py # Data model for combined links ├── repo_source_links.py # Data model for Repo combined links (Final output JSON) ├── helpers.py # Misc. functions used throughout SCL diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index c887dffd6..2de69f1f4 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -31,9 +31,6 @@ from sphinx_needs.logging import get_logger from sphinx_needs.need_item import NeedItem -from src.extensions.score_source_code_linker.generate_source_code_links_json import ( - generate_source_code_links_json, -) from src.extensions.score_source_code_linker.helpers import get_github_link from src.extensions.score_source_code_linker.need_source_links import ( group_by_need, @@ -63,10 +60,7 @@ construct_and_add_need, run_xml_parser, ) -from src.helper_lib import ( - find_git_root, - find_ws_root, -) +from src.helper_lib import find_ws_root LOGGER = get_logger(__name__) # Uncomment this to enable more verbose logging @@ -87,29 +81,36 @@ def get_cache_filename(build_dir: Path, filename: str) -> Path: return build_dir / filename -def build_and_save_combined_file(outdir: Path): +def build_and_save_combined_file(outdir: Path, app: Sphinx | None = None): """ Reads the saved partial caches of codelink & testlink Builds the combined JSON cache & saves it """ - source_code_links_json = os.environ.get("SCORE_SOURCELINKS") - if not source_code_links_json: - # Fallback to the obsolete way of doing source code links, - # just in case someone is not using the docs(sourcelinks=...) attribute. - # TODO: Remove this once backwards compatibility is not needed anymore. - source_code_links_json = get_cache_filename( - outdir, "score_source_code_linker_cache.json" - ) + source_code_links_path = os.environ.get("SCORE_SOURCELINKS") + if not source_code_links_path and app is not None: + source_code_links_path = str( + getattr(app.config, "score_sourcelinks_json", "") or "" + ).strip() + if source_code_links_path: + source_code_links_json = Path(source_code_links_path) + try: + source_code_links = load_source_code_links_json(source_code_links_json) + except FileNotFoundError as exc: + raise FileNotFoundError( + "Pre-generated source-code links file does not exist: " + f"{source_code_links_json}. Check SCORE_SOURCELINKS or " + "score_sourcelinks_json." + ) from exc + except AssertionError: + source_code_links = load_source_code_links_with_metadata_json( + source_code_links_json + ) else: - source_code_links_json = Path(source_code_links_json) - - # This isn't pretty will think of a better solution later, for now this should work - try: - source_code_links = load_source_code_links_json(source_code_links_json) - except AssertionError: - source_code_links = load_source_code_links_with_metadata_json( - source_code_links_json + LOGGER.debug( + "No pre-generated source-code links provided. Continuing without code links.", + type="score_source_code_linker", ) + source_code_links = [] test_cache = get_cache_filename(outdir, "score_xml_parser_cache.json") if test_cache.exists(): test_code_links = load_test_xml_parsed_json(test_cache) @@ -130,7 +131,7 @@ def build_and_save_combined_file(outdir: Path): # ╰──────────────────────────────────────╯ -def setup_source_code_linker(app: Sphinx, ws_root: Path | None): +def setup_source_code_linker(app: Sphinx): """ Setting up source_code_linker with all needed options. Allows us to only have this run once during live_preview & esbonio @@ -164,42 +165,6 @@ def setup_source_code_linker(app: Sphinx, ws_root: Path | None): }, ) - score_sourcelinks_json = os.environ.get("SCORE_SOURCELINKS") - if not score_sourcelinks_json: - score_sourcelinks_json = str( - getattr(app.config, "score_sourcelinks_json", "") - ).strip() - if score_sourcelinks_json: - # Reuse existing code paths that expect this env var. - os.environ["SCORE_SOURCELINKS"] = score_sourcelinks_json - if score_sourcelinks_json: - # No need to generate the JSON file if this env var is set - # because it points to an existing file with the needed data. - return - - if ws_root is None: - LOGGER.info( - "No workspace root found and no SCORE_SOURCELINKS provided. " - "Skipping source-code-link scan.", - type="score_source_code_linker", - ) - return - - scl_cache_json = get_cache_filename( - app.outdir, "score_source_code_linker_cache.json" - ) - - if ( - not scl_cache_json.exists() - or not app.config.skip_rescanning_via_source_code_linker - ): - LOGGER.debug( - "INFO: Generating source code links JSON file.", - type="score_source_code_linker", - ) - - generate_source_code_links_json(ws_root, scl_cache_json) - def register_test_code_linker(app: Sphinx): # Connects function to sphinx to ensure correct execution order @@ -269,7 +234,7 @@ def setup_combined_linker(app: Sphinx, _: BuildEnvironment): "Did not find combined json 'score_scl_grouped_cache.json' in _build." "Generating new one" ) - build_and_save_combined_file(app.outdir) + build_and_save_combined_file(app.outdir, app) def register_repo_linker(app: Sphinx): @@ -308,26 +273,12 @@ def setup_once(app: Sphinx): # might be the only way to solve this? if "skip_rescanning_via_source_code_linker" in app.config: return - LOGGER.debug(f"DEBUG: Workspace root is {find_ws_root()}") - LOGGER.debug( - f"DEBUG: Current working directory is {Path('.')} = {Path('.').resolve()}" - ) - LOGGER.debug(f"DEBUG: Git root is {find_git_root()}") - - # Run for local files if possible. In Bazel sandbox builds, ws_root may be - # unavailable; in that case we can still operate when SCORE_SOURCELINKS - # (or score_sourcelinks_json config) is provided. - ws_root = find_ws_root() - if ws_root: - # When BUILD_WORKSPACE_DIRECTORY is set, we are inside a git repository. - assert find_git_root() - # Register & Run (if needed) parsing & saving of JSON caches # Note: This extension now runs on both internal and external needs_json invocations. # Both modes aggregate links from local sources and external dependencies, enabling # unified traceability reporting in integration repositories. Impact on external needs # invocations is minimal since they typically don't have local test logs or source code. - setup_source_code_linker(app, ws_root) + setup_source_code_linker(app) register_test_code_linker(app) register_combined_linker(app) register_repo_linker(app) diff --git a/src/extensions/score_source_code_linker/generate_source_code_links_json.py b/src/extensions/score_source_code_linker/generate_source_code_links_json.py deleted file mode 100644 index bdeddeaf7..000000000 --- a/src/extensions/score_source_code_linker/generate_source_code_links_json.py +++ /dev/null @@ -1,160 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -""" -This file is used by incremental.py to generate a JSON file with all source code links -for the needs. It's split this way, so that the live_preview action does not need to -parse everything on every run. -""" - -import os -from pathlib import Path - -from sphinx_needs.logging import get_logger - -from src.extensions.score_source_code_linker.needlinks import ( - NeedLink, - store_source_code_links_json, -) - -LOGGER = get_logger(__name__) - -TAGS = [ - "# " + "req-traceability:", - "# " + "req-Id:", - "// " + "req-traceability:", - "// " + "req-Id:", -] - - -def _extract_references_from_line(line: str): - """Extract requirement IDs from a line containing a tag.""" - - for tag in TAGS: - tag_index = line.find(tag) - if tag_index >= 0: - line_after_tag = line[tag_index + len(tag) :].strip() - # Split by comma or space to get multiple requirements - for req in line_after_tag.replace(",", " ").split(): - yield tag, req.strip() - - -def _extract_references_from_file( - root: Path, file_path_name: Path, file_path: Path -) -> list[NeedLink]: - """Scan a single file for template strings and return findings. - Examples: - # ROOT: /docs-as-code/src/extensions/score_source_code_linker - #FILE PATH: - external/score_docs_as_code+/src/extensions/score_source_code_linker/testlink.py - #FILE PATH NAME: testlink.py - """ - assert root.is_absolute(), "Root path must be absolute" - assert not file_path_name.is_absolute(), "File path must be relative to the root" - # assert file_path.is_relative_to(root), ( - # f"File path ({file_path}) must be relative to the root ({root})" - # ) - assert (root / file_path_name).exists(), ( - f"File {file_path_name} does not exist in root {root}." - ) - - findings: list[NeedLink] = [] - - try: - with open(root / file_path_name, encoding="utf-8", errors="ignore") as f: - for line_num, line in enumerate(f, 1): - for tag, req in _extract_references_from_line(line): - findings.append( - NeedLink( - file=file_path, - line=line_num, - tag=tag, - need=req, - full_line=line.strip(), - ) - ) - except (UnicodeDecodeError, PermissionError, OSError) as e: - # Skip files that can't be read as text - LOGGER.debug(f"Error reading file to parse for linked needs: \n{e}") - pass - - return findings - - -def iterate_files_recursively(search_path: Path): - def _should_skip_file(file_path: Path) -> bool: - """Check if a file should be skipped during scanning.""" - # TODO: consider using .gitignore - if file_path.is_dir(): - return True - if file_path.suffix in [".pyc", ".so", ".exe", ".bin"]: - return True # skip binaries - if file_path.suffix in [".rst", ".md"]: - return True # skip documentation - return file_path.name.startswith((".", "_")) - - for root, dirs, files in os.walk(search_path): - root_path = Path(root) - - # Skip directories that start with '.' or '_' by modifying dirs in-place - # This prevents os.walk from descending into these directories - dirs[:] = [d for d in dirs if not d.startswith((".", "_", "bazel-"))] - - for file in files: - f = root_path / file - if not _should_skip_file(f): - yield f.relative_to(search_path) - - -def find_all_need_references(search_path: Path) -> list[NeedLink]: - """ - Find all need references in all files in git root. - Search for any appearance of TAGS and collect line numbers and referenced - requirements. - - Returns: - list[FileFindings]: List of FileFindings objects containing all findings - for each file that contains template strings. - """ - start_time = os.times().elapsed - - all_need_references: list[NeedLink] = [] - - # Use os.walk to have better control over directory traversal - for file in iterate_files_recursively(search_path): - LOGGER.debug( - f"Scanning file by the name of: {file.name} " - f"in path: {search_path} with the file being: {file}" - ) - # print("Search_path: ", search_path) - # print("File.name: ", file.name) - # print("File: ", file) - references = _extract_references_from_file(search_path, Path(file), file) - all_need_references.extend(references) - - elapsed_time = os.times().elapsed - start_time - LOGGER.debug( - f"Found {len(all_need_references)} need references " - f"in {elapsed_time:.2f} seconds" - ) - - return all_need_references - - -def generate_source_code_links_json(search_path: Path, file: Path): - """ - Generate a JSON file with all source code links for the needs. - This is used to link the needs to the source code in the documentation. - """ - needlinks = find_all_need_references(search_path) - store_source_code_links_json(file, needlinks) diff --git a/src/extensions/score_source_code_linker/tests/test_codelink.py b/src/extensions/score_source_code_linker/tests/test_codelink.py index 5f2e9fed2..8b3d9bf61 100644 --- a/src/extensions/score_source_code_linker/tests/test_codelink.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -35,6 +35,7 @@ ) from src.extensions.score_source_code_linker import ( + build_and_save_combined_file, find_need, get_cache_filename, group_by_need, @@ -267,10 +268,10 @@ def sample_needs() -> dict[str, dict[str, str]]: def test_get_cache_filename(): - """Test cache filename generation.""" + """Test that cache paths are resolved relative to the build directory.""" build_dir = Path("/tmp/build") - expected = build_dir / "score_source_code_linker_cache.json" - result = get_cache_filename(build_dir, "score_source_code_linker_cache.json") + expected = build_dir / "source_links.json" + result = get_cache_filename(build_dir, "source_links.json") assert result == expected @@ -357,6 +358,34 @@ def test_cache_file_operations( assert loaded_links[3].line == 2 +def test_combining_without_source_links_continues_with_empty_code_links( + temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A build without a pre-generated source-link input must not scan or fail.""" + monkeypatch.delenv("SCORE_SOURCELINKS", raising=False) + + build_and_save_combined_file(temp_dir) + + grouped_cache = temp_dir / "score_scl_grouped_cache.json" + assert json.loads(grouped_cache.read_text(encoding="utf-8")) == [] + + +def test_combining_with_missing_source_links_reports_configured_path( + temp_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Report the configured source-link file when it cannot be found.""" + missing_file = temp_dir / "missing_source_links.json" + monkeypatch.setenv("SCORE_SOURCELINKS", str(missing_file)) + + with pytest.raises(FileNotFoundError) as exc_info: + build_and_save_combined_file(temp_dir) + + assert str(exc_info.value) == ( + "Pre-generated source-code links file does not exist: " + f"{missing_file}. Check SCORE_SOURCELINKS or score_sourcelinks_json." + ) + + def test_cache_file_with_encoded_comments(temp_dir: Path) -> None: """Test that cache file properly handles encoded comments.""" # Create needlinks with spaces in tags and full_line @@ -453,7 +482,7 @@ def another_function(): ) # Create needlinks manually - # (simulating what generate_source_code_links_json would do) + # Simulate the source-link generator's pre-generated input. needlinks = [ NeedLink( file=Path("src/implementation1.py"), diff --git a/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py b/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py index 687115aef..80f2e9818 100644 --- a/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_repo_source_link_integration.py @@ -94,6 +94,53 @@ def create_demo_files(sphinx_base_dir: Path, git_repo_setup: Path): curr_dir / "expected_repo_grouped.json", repo_path / ".expected_repo_grouped.json", ) + (repo_path / "source_links.json").write_text( + json.dumps( + [ + { + "file": "src/repo_a_impl.py", + "line": 3, + "tag": "#" + " req-Id:", + "need": "MOD_REQ_1", + "full_line": "#" + " req-Id: MOD_REQ_1", + "repo_name": "local_repo", + "hash": "", + "url": "", + }, + { + "file": "src/repo_a_impl.py", + "line": 7, + "tag": "#" + " req-Id:", + "need": "MOD_REQ_2", + "full_line": "#" + " req-Id: MOD_REQ_2", + "repo_name": "local_repo", + "hash": "", + "url": "", + }, + { + "file": "src/repo_b_impl.py", + "line": 3, + "tag": "#" + " req-Id:", + "need": "MOD_REQ_1", + "full_line": "#" + " req-Id: MOD_REQ_1", + "repo_name": "local_repo", + "hash": "", + "url": "", + }, + { + "file": "src/repo_b_impl.py", + "line": 7, + "tag": "#" + " req-Id:", + "need": "MOD_REQ_3", + "full_line": "#" + " req-Id: MOD_REQ_3", + "repo_name": "local_repo", + "hash": "", + "url": "", + }, + ] + ), + encoding="utf-8", + ) # Commit everything _ = subprocess.run(["git", "add", "."], cwd=repo_path, check=True) @@ -218,6 +265,10 @@ def sphinx_app_setup( git_repo_setup: Path, monkeypatch: pytest.MonkeyPatch, ) -> Callable[[], SphinxTestApp]: + # Source links are generated before Sphinx starts, matching the Bazel build + # contract used by the extension in production. + monkeypatch.setenv("SCORE_SOURCELINKS", str(sphinx_base_dir / "source_links.json")) + def _create_app(): base_dir = sphinx_base_dir docs_dir = base_dir / "docs" diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py index 2100bc459..d2f000fc3 100644 --- a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -30,13 +30,9 @@ DataForTestLink, DataForTestLink_JSON_Decoder, ) -from src.extensions.score_source_code_linker.tests.test_codelink import ( - needlink_test_decoder, -) from src.extensions.score_source_code_linker.tests.test_need_source_links import ( SourceCodeLinks_TEST_JSON_Decoder, ) -from src.helper_lib import find_ws_root @pytest.fixture(scope="module") @@ -217,6 +213,12 @@ def sphinx_app_setup( git_repo_setup: Path, monkeypatch: pytest.MonkeyPatch, ) -> Callable[[], SphinxTestApp]: + # Source links are generated before Sphinx starts, matching the Bazel build + # contract used by the extension in production. + monkeypatch.setenv( + "SCORE_SOURCELINKS", str(sphinx_base_dir / ".expected_codelink.json") + ) + def _create_app(): base_dir = sphinx_base_dir docs_dir = base_dir / "docs" @@ -524,15 +526,8 @@ def test_source_link_integration_ok( app = sphinx_app_setup() try: app.build() - ws_root = find_ws_root() - assert ws_root is not None Needs_Data = SphinxNeedsData(app.env) needs_data = {x["id"]: x for x in Needs_Data.get_needs_view().values()} - compare_json_files( - app.outdir / "score_source_code_linker_cache.json", - sphinx_base_dir / ".expected_codelink.json", - needlink_test_decoder, - ) compare_json_files( app.outdir / "score_xml_parser_cache.json", sphinx_base_dir / ".expected_testlink.json", diff --git a/src/incremental.py b/src/incremental.py index ef90614f4..d687e218b 100644 --- a/src/incremental.py +++ b/src/incremental.py @@ -237,7 +237,6 @@ def add_watch_dir(path: Path) -> None: action = get_env("ACTION") if action == "live_preview": - (build_dir / "score_source_code_linker_cache.json").unlink(missing_ok=True) mounts_manifest = os.environ.get("MOUNTS_MANIFEST", "") watch_arguments: list[str] = [] if mounts_manifest: