PYTHON-6091 Pin a consistent uv binary version in CI and locally

.evergreen/scripts/configure-env.sh

@@ -14,31 +14,45 @@ fi
 PROJECT_DIRECTORY="$(pwd)"
 DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
 CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
-UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools
-UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache
 DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
 MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
 
-# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME.
+# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME or
+# have binaries shared across tasks, so use a TMPDIR. On non-CI hosts
+# (spawn hosts, VMs such as GCP/Azure, and local dev), use the conventional
+# ~/.local/bin which tools on the PATH (or the shell rc) can find.
 if [ "${CI:-}" == "true" ]; then
-  PYMONGO_BIN_DIR=${DRIVERS_TOOLS_BINARIES:-}
-# We want to use a path that's already on PATH on spawn hosts.
+  PYMONGO_BIN_DIR="${TMPDIR:-/tmp}"/pymongo_bin
 else
-  PYMONGO_BIN_DIR=$HOME/cli_bin
+  PYMONGO_BIN_DIR=$HOME/.local/bin
 fi
 
-PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH"
+# Add the latest MongoDB toolchain bin dir to PATH if it exists, so that hosts
+# with an old system Python (e.g. RHEL8's 3.6) still get a modern interpreter
+# for tool installs like `uv tool install rust-just`. It goes after
+# PYMONGO_BIN_DIR so the pinned uv (installed there by setup-uv.py) takes
+# precedence over the toolchain's uv.
+if [ "Windows_NT" = "${OS:-}" ]; then
+  _toolchain_bin="/cygdrive/c/Python/Current/Scripts"
+elif [ "$(uname -s)" == "Darwin" ]; then
+  _toolchain_bin="/Library/Frameworks/Python.Framework/Versions/Current/bin"
+else
+  _toolchain_bin="/opt/python/Current/bin"
+fi
+if [ -d "$_toolchain_bin" ]; then
+  PATH_EXT="$MONGODB_BINARIES:$PYMONGO_BIN_DIR:$_toolchain_bin:$DRIVERS_TOOLS_BINARIES:\$PATH"
+else
+  PATH_EXT="$MONGODB_BINARIES:$PYMONGO_BIN_DIR:$DRIVERS_TOOLS_BINARIES:\$PATH"
+fi
 
 # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
 if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
     DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
     PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
     CARGO_HOME=$(cygpath -m $CARGO_HOME)
-    UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR")
-    UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR")
     DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES")
     MONGODB_BINARIES=$(cygpath -m "$MONGODB_BINARIES")
-    PYMONGO_BIN_DIR=$(cygpath -m "$PYMONGO_BIN_DIR")
+    PYMONGO_BIN_DIR=$(cygpath -u "$PYMONGO_BIN_DIR")
 fi
 
 SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
@@ -62,9 +76,6 @@ export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES"
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
 
 export CARGO_HOME="$CARGO_HOME"
-export UV_TOOL_DIR="$UV_TOOL_DIR"
-export UV_CACHE_DIR="$UV_CACHE_DIR"
-export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES"
 export PYMONGO_BIN_DIR="$PYMONGO_BIN_DIR"
 export PATH="$PATH_EXT"
 # shellcheck disable=SC2154
@@ -90,25 +101,3 @@ cat <<EOT > expansion.yml
 DRIVERS_TOOLS: "$DRIVERS_TOOLS"
 PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
 EOT
-
-# If the toolchain is available, symlink binaries to the bin dir.  This has to be done
-# after drivers-tools is cloned, since we might be using its binary dir.
-_bin_path=""
-if [ "Windows_NT" == "${OS:-}" ]; then
-  _bin_path="/cygdrive/c/Python/Current/Scripts"
-elif [ "$(uname -s)" == "Darwin" ]; then
-  _bin_path="/Library/Frameworks/Python.Framework/Versions/Current/bin"
-else
-  _bin_path="/opt/python/Current/bin"
-fi
-if [ -d "${_bin_path}" ]; then
-  _suffix=""
-  if [ "Windows_NT" == "${OS:-}" ]; then
-    _suffix=".exe"
-  fi
-  echo "Symlinking binaries from toolchain"
-  mkdir -p $PYMONGO_BIN_DIR
-  ln -s ${_bin_path}/just${_suffix} $PYMONGO_BIN_DIR/just${_suffix}
-  ln -s ${_bin_path}/uv${_suffix} $PYMONGO_BIN_DIR/uv${_suffix}
-  ln -s ${_bin_path}/uvx${_suffix} $PYMONGO_BIN_DIR/uvx${_suffix}
-fi

.evergreen/scripts/install-dependencies.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # Install the necessary dependencies.
-set -eu
+set -euo pipefail
 
 HERE=$(dirname ${BASH_SOURCE:-$0})
 HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
@@ -11,27 +11,61 @@ if [ -f $HERE/env.sh ]; then
   . $HERE/env.sh
 fi
 
-# Set up the default bin directory.
-if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
-  PYMONGO_BIN_DIR="$HOME/.local/bin"
+# PYMONGO_BIN_DIR is set by setup-system.sh/env.sh (or setup-dev-env.sh); default
+# it for robustness. UV_TOOL_BIN_DIR is uv's name for the same dir (setup-uv.py
+# reads both). UV_TOOL_DIR is left to ensure_uv.sh.
+export PYMONGO_BIN_DIR="${PYMONGO_BIN_DIR:-$HOME/.local/bin}"
+export UV_TOOL_BIN_DIR="${UV_TOOL_BIN_DIR:-$PYMONGO_BIN_DIR}"
+# uv is a native Windows binary: give it a Windows path on cygwin.
+if [ "Windows_NT" = "${OS:-}" ]; then
+  _uv_tool_bin="$(cygpath -m "$PYMONGO_BIN_DIR")"
+  export UV_TOOL_BIN_DIR="$_uv_tool_bin"
 fi
 
-# Ensure uv is installed.
-if ! command -v uv &>/dev/null; then
-  _BIN_DIR=$PYMONGO_BIN_DIR
-  mkdir -p ${_BIN_DIR}
-  echo "Installing uv..."
-  curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$_BIN_DIR" INSTALLER_NO_MODIFY_PATH=1 sh
+# If uv is on PATH, check it via `uv sync`, which fails fast if it is not the
+# pinned version (from pyproject.toml's [tool.uv] required-version). If that
+# succeeds, the environment is already correct and there is nothing to set up;
+# otherwise fall through to the setup below.
+#
+# On CI we also require UV_CACHE_DIR to be set: ensure_uv.sh scopes uv's cache
+# to a task-local dir, so an unset UV_CACHE_DIR means the uv setup has not run
+# yet in this task and we must do the setup phase.
+_need_setup=1
+if command -v uv >/dev/null 2>&1 && uv sync >/dev/null 2>&1; then
+  if [ "${CI:-}" != "true" ] || [ -n "${UV_CACHE_DIR:-}" ]; then
+    echo "uv is already set up; skipping uv setup."
+    _need_setup=0
+  fi
+fi
+
+# Set up uv if needed.
+if [ "$_need_setup" = "1" ]; then
+  # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env.
+  if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then
+    . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh"
+    ensure_uv || exit 1
+  fi
+
+  # Do the uv setup (bin dir, pinning, env.sh). Uses the toolchain python3
+  # (added to PATH by configure-env.sh) so no project .venv is created here,
+  # and no required-version check is triggered. On Windows the script path must
+  # be a native Windows path for python3.
+  _uv_setup_script="$HERE/setup-uv.py"
   if [ "Windows_NT" = "${OS:-}" ]; then
-    chmod +x "$(cygpath -u $_BIN_DIR)/uv.exe"
+    _uv_setup_script="$(cygpath -m "$_uv_setup_script")"
+  fi
+  python3 "$_uv_setup_script"
+
+  # Re-source env.sh so the values setup-uv.py wrote are available.
+  if [ -f $HERE/env.sh ]; then
+    . $HERE/env.sh
   fi
-  export PATH="$PYMONGO_BIN_DIR:$PATH"
-  echo "Installing uv... done."
 fi
 
-# Ensure just is installed.
-if ! command -v just &>/dev/null; then
-  uv tool install rust-just
+# Make just available. It has no version constraint, so if it is already on PATH
+# there is nothing to do; otherwise install it into the bin dir via uv.
+if ! command -v just >/dev/null 2>&1; then
+  uv tool install --no-config rust-just
 fi
 
 popd > /dev/null

.evergreen/scripts/setup-dev-env.sh

@@ -16,22 +16,36 @@ if [ -f $HERE/test-env.sh ]; then
   . $HERE/test-env.sh
 fi
 
-# Handle the value for UV_PYTHON.
-. $HERE/setup-uv-python.sh
+# The bin dir for the pinned uv/just. setup-system.sh sets it on evergreen hosts;
+# default it here so local dev (without setup-system.sh) also has a usable value.
+export PYMONGO_BIN_DIR="${PYMONGO_BIN_DIR:-$HOME/.local/bin}"
+
+# Make sure a login shell can find the bin dir by adding it to the rc file, so
+# local dev (which may never run setup-system.sh) still has it on PATH. env.sh's
+# PATH does not persist past this session. Prefer .zshrc when the shell is zsh.
+if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then
+  if [ -f "$HOME/.zshrc" ]; then
+    _rc="$HOME/.zshrc"
+  else
+    _rc="$HOME/.bashrc"
+  fi
+  if [ -f "$_rc" ]; then
+    grep -qF 'export PATH="'"$PYMONGO_BIN_DIR"':$PATH"' "$_rc" 2>/dev/null || \
+      printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR" >> "$_rc"
+  fi
+fi
 
 # Ensure dependencies are installed.
 bash $HERE/install-dependencies.sh
 
-# Re-source env.sh: install-dependencies.sh may have appended to it, e.g. when it
-# had to install Python on an image that lacks a toolchain.
+# Re-source env.sh in case a dependency install updated it, e.g. on a host
+# without a toolchain where uv was installed into a shared bin dir.
 if [ -f $HERE/env.sh ]; then
   . $HERE/env.sh
 fi
 
-# Add the default install path to the path if needed.
-if [ -z "${PYMONGO_BIN_DIR:-}" ]; then
-  export PATH="$PATH:$HOME/.local/bin"
-fi
+# Handle the value for UV_PYTHON.
+. $HERE/setup-uv-python.sh
 
 # Only run the next part if not running on CI.
 if [ -z "${CI:-}" ]; then

.evergreen/scripts/setup-uv.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""Bootstrap the pinned uv/just for the test environment.
+
+install-dependencies.sh bails out if the pinned uv is already on PATH, sources
+ensure-uv.sh (which finds or installs uv), then runs this script for the rest.
+Only the standard library is used.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent.parent
+ENV_SH = HERE / "env.sh"
+ASTRAL_INSTALL_URL = "https://astral.sh/uv/install.sh"
+
+
+def required_uv_pin() -> str:
+    """Return [tool.uv] required-version, e.g. '==0.12.12' ('' if absent)."""
+    pattern = re.compile(r"required-version\s*=\s*['\"]?([^'\"\s]+)['\"]?")
+    in_uv = False
+    for line in (ROOT / "pyproject.toml").read_text().splitlines():
+        stripped = line.strip()
+        if stripped.startswith("["):
+            in_uv = stripped == "[tool.uv]"
+            continue
+        if in_uv:
+            match = pattern.search(stripped)
+            if match:
+                return match.group(1)
+    return ""
+
+
+def _add_path(dir_: str) -> None:
+    os.environ["PATH"] = dir_ + os.pathsep + os.environ.get("PATH", "")
+
+
+def _install_uv_astral() -> None:
+    """Install uv from astral when ensure-uv.sh was not available.
+
+    UV_TOOL_BIN_DIR is set (in native form on Windows) by install-dependencies.sh.
+    """
+    print("uv not found; installing the latest uv from astral...")
+    env = {
+        **os.environ,
+        "UV_INSTALL_DIR": os.environ["UV_TOOL_BIN_DIR"],
+        "INSTALLER_NO_MODIFY_PATH": "1",
+    }
+    curl = shutil.which("curl")
+    sh = shutil.which("sh")
+    proc = subprocess.run(  # noqa: S603
+        [curl, "-LsSf", ASTRAL_INSTALL_URL], capture_output=True, env=env, check=True
+    )
+    subprocess.run([sh], input=proc.stdout, env=env, check=True)  # noqa: S603
+    _add_path(os.environ["UV_TOOL_BIN_DIR"])
+
+
+def _pin_uv(uv_pin: str) -> None:
+    """Install the pinned uv via uv tool install --force.
+
+    UV_TOOL_BIN_DIR and UV_TOOL_DIR are inherited from the environment (set by
+    install-dependencies.sh / ensure-uv.sh, in native form on Windows). uv tool
+    install writes the binary into UV_TOOL_BIN_DIR and the tool venv into
+    UV_TOOL_DIR; --force lets it overwrite an existing install.
+
+    Windows will not overwrite a running executable, so when the current uv is
+    already the target bin dir, run the install from a copy of the binary.
+    """
+    uv_path = shutil.which("uv")
+    tool = uv_path
+    tmp_uv = None
+    if os.name == "nt":
+        tmp_uv = Path(tempfile.mkdtemp()) / Path(uv_path).name
+        shutil.copy2(uv_path, tmp_uv)
+        tool = str(tmp_uv)
+    try:
+        subprocess.run(  # noqa: S603
+            [
+                tool,
+                "tool",
+                "install",
+                "--no-config",
+                "-q",
+                "--force",
+                "--from",
+                f"uv{uv_pin}",
+                "uv",
+            ],
+            check=True,
+        )
+    finally:
+        if tmp_uv:
+            shutil.rmtree(tmp_uv.parent, ignore_errors=True)
+
+
+def _write_env() -> None:
+    """Write every UV_* env var into env.sh, replacing existing UV_* entries."""
+    values = {k: v for k, v in os.environ.items() if k.startswith("UV_")}
+    if not values:
+        return
+    existing = ENV_SH.read_text() if ENV_SH.exists() else ""
+    keep = []
+    for line in existing.splitlines():
+        stripped = line.strip()
+        if stripped.startswith("export "):
+            var, _, _ = stripped[len("export ") :].partition("=")
+            if var.startswith("UV_"):
+                continue
+        keep.append(line)
+    keep.append("")
+    keep.extend(f'export {name}="{value}"' for name, value in sorted(values.items()))
+    # Write LF bytes directly: on Windows text mode translates \n to \r\n, which
+    # breaks bash sourcing env.sh, and `newline=` isn't available on all Pythons.
+    ENV_SH.write_bytes(("\n".join(keep) + "\n").encode())
+
+
+def main() -> int:
+    bin_dir = os.environ["UV_TOOL_BIN_DIR"]
+    Path(bin_dir).mkdir(parents=True, exist_ok=True)
+    _add_path(bin_dir)
+
+    # Bootstrap a uv from astral if ensure-uv.sh didn't run and uv isn't on PATH.
+    if shutil.which("uv") is None:
+        _install_uv_astral()
+
+    uv_pin = required_uv_pin()
+    if uv_pin:
+        _pin_uv(uv_pin)
+
+    _write_env()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/codeql.yml

@@ -37,7 +37,7 @@ jobs:
         - language: actions
           build-mode: none
     steps:
-    - uses: mongodb-labs/drivers-github-tools/codeql@89904229eb55a063655af02f5838bd7fe76505e5 # v3
+    - uses: mongodb-labs/drivers-github-tools/codeql@1a2d86bd5fed4b0090453bcd510e4c2ba3b92356  # v3.0.4
       with:
         language: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}

.github/workflows/create-release-branch.yml

@@ -57,15 +57,15 @@ jobs:
         run: |
           auth_header=$(printf 'x-access-token:%s' "${GH_TOKEN}" | base64 -w0)
           git config --local http.https://github.com/.extraheader "AUTHORIZATION: basic ${auth_header}"
-      - uses: mongodb-labs/drivers-github-tools/setup@v3
+      - uses: mongodb-labs/drivers-github-tools/setup@1a2d86bd5fed4b0090453bcd510e4c2ba3b92356  # v3.0.4
         with:
           aws_role_arn: ${{ secrets.AWS_ROLE_ARN }}
           aws_region_name: ${{ vars.AWS_REGION_NAME }}
           aws_secret_id: ${{ secrets.AWS_SECRET_ID }}
           artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }}
       - name: Get hatch
         run: pip install hatch
-      - uses: mongodb-labs/drivers-github-tools/create-branch@v3
+      - uses: mongodb-labs/drivers-github-tools/create-branch@1a2d86bd5fed4b0090453bcd510e4c2ba3b92356  # v3.0.4
         id: create-branch
         with:
           branch_name: ${{ inputs.branch_name }}