diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 8dc328aab3..e68907175d 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -19,15 +19,18 @@ UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" -# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME. +# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME. Put +# our binaries (uv, just, ...) in a task-local dir we control, independent of the +# drivers-tools tree. if [ "${CI:-}" == "true" ]; then - PYMONGO_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-} -# We want to use a path that's already on PATH on spawn hosts. + PYMONGO_BIN_DIR="${TMPDIR:-/tmp}/pymongo-bin" +# On non-CI hosts (spawn hosts, VMs such as GCP/Azure, and local dev), use the +# conventional ~/.local/bin which tools on the PATH (or the shell rc) can find. else - PYMONGO_BIN_DIR=$HOME/cli_bin + PYMONGO_BIN_DIR=$HOME/.local/bin fi -PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH" +PATH_EXT="$MONGODB_BINARIES:$PYMONGO_BIN_DIR:$DRIVERS_TOOLS_BINARIES:\$PATH" # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin @@ -38,7 +41,9 @@ if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR") DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES") MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES") - PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR") + # Keep PYMONGO_BIN_DIR in cygwin form so bash can search it on PATH; native + # uv gets the Windows form (via cygpath -m) inside install-dependencies.sh. + PYMONGO_BIN_DIR=$(cygpath -u "$PYMONGO_BIN_DIR") fi SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" @@ -64,7 +69,8 @@ export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export CARGO_HOME="$CARGO_HOME" export UV_TOOL_DIR="$UV_TOOL_DIR" export UV_CACHE_DIR="$UV_CACHE_DIR" -export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES" +# Send uv tool installs into our own bin dir, alongside the pinned uv/just. +export UV_TOOL_BIN_DIR="$PYMONGO_BIN_DIR" export PYMONGO_BIN_DIR="$PYMONGO_BIN_DIR" export PATH="$PATH_EXT" # shellcheck disable=SC2154 @@ -90,25 +96,3 @@ cat < expansion.yml DRIVERS_TOOLS: "$DRIVERS_TOOLS" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" EOT - -# If the toolchain is available, symlink binaries to the bin dir. This has to be done -# after drivers-tools is cloned, since we might be using its binary dir. -_bin_path="" -if [ "Windows_NT" == "${OS:-}" ]; then - _bin_path="/cygdrive/c/Python/Current/Scripts" -elif [ "$(uname -s)" == "Darwin" ]; then - _bin_path="/Library/Frameworks/Python.Framework/Versions/Current/bin" -else - _bin_path="/opt/python/Current/bin" -fi -if [ -d "${_bin_path}" ]; then - _suffix="" - if [ "Windows_NT" == "${OS:-}" ]; then - _suffix=".exe" - fi - echo "Symlinking binaries from toolchain" - mkdir -p $PYMONGO_BIN_DIR - ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just${_suffix} - ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv${_suffix} - ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx${_suffix} -fi diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 7f943d7d00..ac6b54dc7c 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -1,6 +1,6 @@ #!/bin/bash # Install the necessary dependencies. -set -eu +set -euo pipefail HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" @@ -15,23 +15,87 @@ fi if [ -z "${PYMONGO_BIN_DIR:-}" ]; then PYMONGO_BIN_DIR="$HOME/.local/bin" fi +# uv.exe on Windows needs Windows-style paths, while bash uses the cygwin form. +# Keep PYMONGO_BIN_DIR in the form PATH uses and give uv the Windows form. +if [ "Windows_NT" = "${OS:-}" ]; then + _uv_tool_bin="$(cygpath -m "$PYMONGO_BIN_DIR")" + _uv_tool_dir="$(cygpath -m "${UV_TOOL_DIR:-$(dirname "$PYMONGO_BIN_DIR")/uv-tools}")" + export UV_TOOL_BIN_DIR="$_uv_tool_bin" + export UV_TOOL_DIR="$_uv_tool_dir" +else + export UV_TOOL_BIN_DIR="$PYMONGO_BIN_DIR" +fi +mkdir -p "$PYMONGO_BIN_DIR" + +# Locate the Python toolchain's binary dir, so we can prefer its uv and just. +_toolchain_bin="" +if [ "Windows_NT" = "${OS:-}" ]; then + _toolchain_bin="/cygdrive/c/Python/Current/Scripts" +elif [ "$(uname -s)" = "Darwin" ]; then + _toolchain_bin="/Library/Frameworks/Python.Framework/Versions/Current/bin" +else + _toolchain_bin="/opt/python/Current/bin" +fi + +# Prefer the toolchain's uv as a bootstrap when uv is not already on PATH, so we +# do not fall back to installing from astral. +if ! command -v uv &>/dev/null; then + if [ -x "$_toolchain_bin/uv" ] || [ -x "$_toolchain_bin/uv.exe" ]; then + echo "Found uv in the toolchain at $_toolchain_bin" + export PATH="$_toolchain_bin:$PATH" + fi +fi -# Ensure uv is installed. +# Ensure uv is available (bootstrap if absent). if ! command -v uv &>/dev/null; then _BIN_DIR=$PYMONGO_BIN_DIR mkdir -p ${_BIN_DIR} - echo "Installing uv..." + echo "uv not found on PATH; installing the latest uv from astral..." curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh if [ "Windows_NT" = "${OS:-}" ]; then chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe" fi export PATH="$PYMONGO_BIN_DIR:$PATH" - echo "Installing uv... done." fi -# Ensure just is installed. -if ! command -v just &>/dev/null; then - uv tool install rust-just +# Pin the uv binary to the version in pyproject.toml's [tool.uv] required-version. +# Run the current uv directly: it writes into PYMONGO_BIN_DIR (a different +# location), so nothing running is overwritten. If the running uv is already our +# pinned bin-dir uv, skip the install to avoid overwriting it (Windows refuses to +# overwrite a running executable); otherwise install so the pin lands in the bin +# dir even when the discovered uv already matches. +_uv_bin="$(command -v uv 2>/dev/null || true)" +if [ -n "$_uv_bin" ]; then + _uv_pin="$(awk -F'"' '/^[[:space:]]*required-version[[:space:]]*=/{print $2}' pyproject.toml)" + case "$_uv_bin" in + "$PYMONGO_BIN_DIR"/*) + _uv_vers="$(uv --version 2>/dev/null | head -1 | awk '{print $2}' | sed 's/^v//')" + if [ "uv${_uv_pin}" != "uv==${_uv_vers}" ]; then + uv tool install -q --force --from "uv${_uv_pin}" uv + echo "Using uv at $PYMONGO_BIN_DIR/uv ($("$PYMONGO_BIN_DIR/uv" --version 2>/dev/null | head -1 | awk '{print $2}'))" + fi + ;; + *) + uv tool install -q --force --from "uv${_uv_pin}" uv + echo "Using uv at $PYMONGO_BIN_DIR/uv ($("$PYMONGO_BIN_DIR/uv" --version 2>/dev/null | head -1 | awk '{print $2}'))" + ;; + esac +fi + +# Use just from the toolchain if available, otherwise install it. It must live in +# our bin dir to be on PATH for callers; copying it keeps the toolchain's just. +if [ ! -x "$PYMONGO_BIN_DIR/just" ] && [ ! -x "$PYMONGO_BIN_DIR/just.exe" ]; then + if [ -x "$_toolchain_bin/just" ]; then + echo "Using just from the toolchain" + cp "$_toolchain_bin/just" "$PYMONGO_BIN_DIR/just" + chmod +x "$PYMONGO_BIN_DIR/just" + elif [ -x "$_toolchain_bin/just.exe" ]; then + echo "Using just from the toolchain" + cp "$_toolchain_bin/just.exe" "$PYMONGO_BIN_DIR/just.exe" + chmod +x "$PYMONGO_BIN_DIR/just.exe" + else + uv tool install rust-just + fi fi popd > /dev/null diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index e58d6210fe..5a1794ee71 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -16,23 +16,24 @@ if [ -f $HERE/test-env.sh ]; then . $HERE/test-env.sh fi -# Handle the value for UV_PYTHON. -. $HERE/setup-uv-python.sh - # Ensure dependencies are installed. bash $HERE/install-dependencies.sh -# Re-source env.sh: install-dependencies.sh may have appended to it, e.g. when it -# had to install Python on an image that lacks a toolchain. +# Re-source env.sh in case a dependency install updated it, e.g. on a host +# without a toolchain where uv was installed into a shared bin dir. if [ -f $HERE/env.sh ]; then . $HERE/env.sh fi -# Add the default install path to the path if needed. +# Add the default install path to the path before configuring uv: setup-uv-python.sh +# calls uv, and on a machine without env.sh the tool dir must already be on PATH. if [ -z "${PYMONGO_BIN_DIR:-}" ]; then export PATH="$PATH:$HOME/.local/bin" fi +# Handle the value for UV_PYTHON. +. $HERE/setup-uv-python.sh + # Only run the next part if not running on CI. if [ -z "${CI:-}" ]; then # Set up venv, making sure c extensions build unless disabled. diff --git a/.evergreen/scripts/setup-system.sh b/.evergreen/scripts/setup-system.sh index bd7e2dd4bc..eca8375a53 100755 --- a/.evergreen/scripts/setup-system.sh +++ b/.evergreen/scripts/setup-system.sh @@ -15,6 +15,16 @@ if [ -z "${CI:-}" ]; then bash $HERE/setup-dev-env.sh fi +# On non-CI hosts (spawn hosts, VMs such as GCP/Azure, and local dev) the pinned +# uv and just live in ~/.local/bin, so make sure that is on PATH, adding it to +# .bashrc if it is not already. +if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then + case ":$PATH:" in + *":$HOME/.local/bin:"*) ;; + *) echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$HOME/.bashrc" ;; + esac +fi + # Enable core dumps if enabled on the machine # Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml if [ -f /proc/self/coredump_filter ]; then diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ef90224c2..e9608aaeb4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -199,6 +199,10 @@ the pages will re-render and the browser will automatically refresh. - Run `just install` to set a local virtual environment, or you can manually create a virtual environment and run `pytest` directly. If you want to use a specific version of Python, set `UV_PYTHON` before running `just install`. + + `just install` installs the pinned version of `uv` (from `[tool.uv] required-version`) into `$HOME/.local/bin`, + so make sure that directory is on your `PATH` (it usually is). If a project `uv` command (e.g. `just test`) runs + with a different `uv` version, `uv` fails fast and tells you how to update. - Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args to set up the server. All given options will be passed to [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h` @@ -419,6 +423,10 @@ tasks are host-agnostic. supported version of Python and use that. This ensures a consistent behavior across host types that do not have the Python toolchain (e.g. Azure VMs), by having a known version of Python with the build headers (`Python.h`) needed to build the C extensions. + - The uv binary version is pinned once in `[tool.uv] required-version` in `pyproject.toml`. + `.evergreen/scripts/install-dependencies.sh` installs it with `uv tool install`, uv enforces it locally, and + `astral-sh/setup-uv` reads it on GitHub. Bump it manually when a newer uv is needed. If uv cannot find the + requested Python, it installs it; if that fails, the task fails. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. diff --git a/pyproject.toml b/pyproject.toml index 346df2f427..0323e044c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,11 @@ Source = "https://github.com/mongodb/mongo-python-driver" Tracker = "https://jira.mongodb.org/projects/PYTHON/issues" [tool.uv] +# Pin the uv binary version across local dev, GitHub Actions, and Evergreen. +# uv enforces this locally, astral-sh/setup-uv reads it on GitHub, and +# install-dependencies.sh installs it in Evergreen. Bump manually when a newer uv +# is needed. +required-version = "==0.12.12" # boto3 dropped Python 3.9 support in 1.43, so the universal lock forks at # 3.10. Without a floor the pre-3.10 fork back-solves to boto3 1.7.84 (2018), # whose vendored six and invalid escape sequences break test collection. No