diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a427a18646b..58b686a911f 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -222,8 +222,8 @@ jobs: run: | python -m pip install uv if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt; fi - if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/operator-requirements.txt; fi - if [ -f amber/dev-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/dev-requirements.txt; fi + if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -c amber/requirements.txt -r amber/operator-requirements.txt; fi + if [ -f amber/dev-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -c amber/requirements.txt -r amber/dev-requirements.txt; fi - name: Install protoc run: | PROTOC_VERSION=$(cat bin/protoc-version.txt) @@ -304,8 +304,8 @@ jobs: # (sbt recompiles main's Scala automatically when run-benchmarks.sh # invokes it below; only the pip deps need an explicit re-sync.) if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt || { echo "::warning::main requirements install failed; skipping main baseline."; exit 0; }; fi - if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/operator-requirements.txt || { echo "::warning::main operator-requirements install failed; skipping main baseline."; exit 0; }; fi - if [ -f amber/dev-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/dev-requirements.txt || { echo "::warning::main dev-requirements install failed; skipping main baseline."; exit 0; }; fi + if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -c amber/requirements.txt -r amber/operator-requirements.txt || { echo "::warning::main operator-requirements install failed; skipping main baseline."; exit 0; }; fi + if [ -f amber/dev-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -c amber/requirements.txt -r amber/dev-requirements.txt || { echo "::warning::main dev-requirements install failed; skipping main baseline."; exit 0; }; fi # Regenerate proto bindings against main's protos, then re-bench. bash bin/python-proto-gen.sh || { echo "::warning::main proto-gen failed; skipping main baseline."; exit 0; } if bash bin/run-benchmarks.sh && [ -f bench-results/arrow-flight-e2e.csv ]; then diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0107997b39..39d68f500f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -435,8 +435,8 @@ jobs: run: | python -m pip install uv if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt; fi - if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/operator-requirements.txt; fi - if [ -f amber/dev-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/dev-requirements.txt; fi + if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -c amber/requirements.txt -r amber/operator-requirements.txt; fi + if [ -f amber/dev-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -c amber/requirements.txt -r amber/dev-requirements.txt; fi - name: Install protoc # Linux pins protoc to the version in bin/protoc-version.txt via # the upstream release zip. macOS uses brew's arm64-native @@ -1102,7 +1102,7 @@ jobs: install="uv pip install --system --index-strategy unsafe-best-match" fi if [ -f amber/requirements.txt ]; then $install -r amber/requirements.txt; fi - if [ -f amber/operator-requirements.txt ]; then $install -r amber/operator-requirements.txt; fi + if [ -f amber/operator-requirements.txt ]; then $install -c amber/requirements.txt -r amber/operator-requirements.txt; fi - name: Generate pip-licenses manifest if: matrix.python-version == '3.12' run: pip-licenses --format=csv --ignore-packages pip-licenses prettytable wcwidth > /tmp/pip-licenses.csv @@ -1120,7 +1120,7 @@ jobs: # is safe here regardless of leg because it runs post-snapshot. run: | python -m pip install uv - if [ -f amber/dev-requirements.txt ]; then uv pip install --system -r amber/dev-requirements.txt; fi + if [ -f amber/dev-requirements.txt ]; then uv pip install --system -c amber/requirements.txt -r amber/dev-requirements.txt; fi - name: Lint with Ruff # ruff comes from dev-requirements.txt, so this must run after the # dev-dependency install above. @@ -1287,7 +1287,7 @@ jobs: # instructions ends up with the TUI working. run: | python -m pip install --upgrade pip - python -m pip install -r amber/dev-requirements.txt + python -m pip install -c amber/requirements.txt -r amber/dev-requirements.txt - name: Run shell smoke tests # Discover + run every `test_*.sh` under bin/ and .github/scripts/. New # shell test suites picked up automatically — no edits to this workflow. diff --git a/amber/requirements.txt b/amber/requirements.txt index 25c30339eaa..08015b6f784 100644 --- a/amber/requirements.txt +++ b/amber/requirements.txt @@ -22,7 +22,7 @@ pyarrow==23.0.1 betterproto==2.0.0b7 pampy==0.3.0 overrides==7.7.0 -typing_extensions==4.14.1 +typing_extensions==4.16.0 bidict==0.22.0 cached_property==2.0.1 psutil==7.2.2 diff --git a/bin/dockerfiles/computing-unit-master.dockerfile b/bin/dockerfiles/computing-unit-master.dockerfile index df9928323ce..5c1a34312da 100644 --- a/bin/dockerfiles/computing-unit-master.dockerfile +++ b/bin/dockerfiles/computing-unit-master.dockerfile @@ -105,7 +105,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages RUN pip3 install --upgrade pip setuptools wheel && \ pip3 install -r /tmp/requirements.txt && \ - pip3 install -r /tmp/operator-requirements.txt + pip3 install -c /tmp/requirements.txt -r /tmp/operator-requirements.txt # Copy the built texera binary from the build phase COPY --from=build /texera/.git /texera/amber/.git diff --git a/bin/dockerfiles/computing-unit-worker.dockerfile b/bin/dockerfiles/computing-unit-worker.dockerfile index fc89c0f352d..eb242cc438b 100644 --- a/bin/dockerfiles/computing-unit-worker.dockerfile +++ b/bin/dockerfiles/computing-unit-worker.dockerfile @@ -104,7 +104,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages RUN pip3 install --upgrade pip setuptools wheel && \ pip3 install -r /tmp/requirements.txt && \ - (pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r /tmp/operator-requirements.txt || \ + (pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -c /tmp/requirements.txt -r /tmp/operator-requirements.txt || \ pip3 install --no-cache-dir wordcloud==1.9.2) # Copy the built texera binary from the build phase @@ -128,4 +128,4 @@ USER texera CMD ["bin/computing-unit-worker"] -EXPOSE 8085 \ No newline at end of file +EXPOSE 8085 diff --git a/bin/licensing/test_python_requirement_constraints.py b/bin/licensing/test_python_requirement_constraints.py new file mode 100644 index 00000000000..e128730ddb5 --- /dev/null +++ b/bin/licensing/test_python_requirement_constraints.py @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +INSTALL_FILES = ( + ROOT / ".github/workflows/build.yml", + ROOT / ".github/workflows/benchmarks.yml", + ROOT / "bin/dockerfiles/computing-unit-master.dockerfile", + ROOT / "bin/dockerfiles/computing-unit-worker.dockerfile", +) + + +def follow_up_installs(text: str) -> list[str]: + return [ + line.strip() + for line in text.splitlines() + if re.search(r"-r (?:amber/|/tmp/)?(?:operator-|dev-)requirements\.txt", line) + ] + + +class PythonRequirementConstraintTest(unittest.TestCase): + def test_every_follow_up_install_constrains_runtime_requirements(self) -> None: + installs = [ + (path.relative_to(ROOT), line) + for path in INSTALL_FILES + for line in follow_up_installs(path.read_text(encoding="utf-8")) + ] + self.assertTrue(installs, "the audit must discover follow-up requirement installs") + unconstrained = [ + f"{path}: {line}" + for path, line in installs + if not re.search(r"-c (?:amber/|/tmp/)?requirements\.txt", line) + ] + self.assertFalse( + unconstrained, "unconstrained follow-up installs:\n" + "\n".join(unconstrained) + ) + + def test_declared_typing_extensions_pin_matches_binary_manifest(self) -> None: + requirements = (ROOT / "amber/requirements.txt").read_text(encoding="utf-8") + license_binary = (ROOT / "amber/LICENSE-binary-python").read_text(encoding="utf-8") + declared = re.search(r"^typing_extensions==([^\s]+)$", requirements, re.MULTILINE) + recorded = re.search( + r"^\s*- typing-extensions==([^\s]+)$", license_binary, re.MULTILINE + ) + self.assertIsNotNone(declared) + self.assertIsNotNone(recorded) + self.assertEqual(declared.group(1), recorded.group(1))