Skip to content

Fix CUDA CI: HF writes fail on read-only /mnt/hf_cache (HF_TOKEN + writable HF_HOME) #20782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Gasoonjia merged 4 commits into from
Jul 8, 2026
+41 −8
Merged

Fix CUDA CI: HF writes fail on read-only /mnt/hf_cache (HF_TOKEN + writable HF_HOME) #20782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
415786e
Fix CUDA CI: use HF_TOKEN env instead of hf auth login (read-only HF …
Gasoonjia Jul 8, 2026
5976cf4
Redirect HF_HOME to a writable dir for CUDA CI
Gasoonjia Jul 8, 2026
6836c95
Also redirect HF_HOME in the CUDA e2e/benchmark jobs
Gasoonjia Jul 8, 2026
3d7cf15
Fall back to /tmp when RUNNER_TEMP isn't writable for HF_HOME
Gasoonjia Jul 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions .github/workflows/cuda-perf.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,13 @@ jobs:
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
script: |
set -eux
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
mkdir -p "${HF_HOME}"
echo "::group::Setup ExecuTorch"
# OSDC runners can't reach the public PyPI CDN that download.pytorch.org's
# transitive deps resolve to. Pre-install torch's pure-python deps from the
Expand All @@ -160,8 +167,7 @@ jobs:
echo "::group::Setup Huggingface"
pip install -U "huggingface_hub[cli]>=1.2.1,<2.0" accelerate "optimum~=2.0.0" "transformers==5.0.0rc1"
HF_AUTH_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
hf auth login --token "$HF_AUTH_TOKEN"
export HF_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
pip install --no-deps git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
echo "::endgroup::"
Expand Down Expand Up @@ -219,6 +225,13 @@ jobs:
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
script: |
set -eux
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
mkdir -p "${HF_HOME}"
echo "::group::Setup environment"
# OSDC runners can't reach the public PyPI CDN that download.pytorch.org's
# transitive deps resolve to. Pre-install torch's pure-python deps from the
Expand Down
9 changes: 7 additions & 2 deletions .github/workflows/cuda-windows.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,12 @@ jobs:
script: |
set -eux

# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
mkdir -p "${HF_HOME}"

echo "::group::Fix libstdc++ GLIBCXX version"
# The executorch pybindings require GLIBCXX_3.4.30 which conda's libstdc++ doesn't have.
# Replace conda's libstdc++ with the system version to fix ImportError.
Expand Down Expand Up @@ -126,8 +132,7 @@ jobs:
if [ "${{ matrix.model_name }}" != "dinov2-small-imagenet1k-1-layer" ]; then
echo "::group::Setup Huggingface"
pip install -U "huggingface_hub[cli]>=1.2.1,<2.0" accelerate "optimum~=2.0.0" "transformers==5.0.0rc1"
HF_AUTH_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
hf auth login --token "$HF_AUTH_TOKEN"
export HF_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
pip install --no-deps git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
echo "::endgroup::"
Expand Down
23 changes: 19 additions & 4 deletions .github/workflows/cuda.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,12 @@ jobs:
script: |
set -eux

# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
mkdir -p "${HF_HOME}"

echo "::group::Setup ExecuTorch"
# OSDC runners can't reach the public PyPI CDN that download.pytorch.org's
# transitive deps resolve to. Pre-install torch's pure-python deps from the
Expand All @@ -406,8 +412,7 @@ jobs:
if [ "${{ matrix.model.name }}" != "parakeet-tdt" ] && [ "${{ matrix.model.name }}" != "dinov2-small-imagenet1k-1-layer" ]; then
echo "::group::Setup Huggingface"
pip install -U "huggingface_hub[cli]>=1.2.1,<2.0" accelerate "optimum~=2.0.0" "transformers==5.0.0rc1"
HF_AUTH_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
hf auth login --token "$HF_AUTH_TOKEN"
export HF_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
pip install --no-deps git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
echo "::endgroup::"
Expand Down Expand Up @@ -554,6 +559,11 @@ jobs:
# fsspec is pinned to satisfy datasets' fsspec[http]<=2025.3.0 so the later
# examples install doesn't try to downgrade it from the public CDN.
pip install filelock typing-extensions "setuptools<82" sympy networkx jinja2 "fsspec[http]<=2025.3.0" numpy pillow
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
mkdir -p "${HF_HOME}"
source .ci/scripts/test_model_e2e.sh cuda "${{ matrix.model.repo }}/${{ matrix.model.name }}" "${{ matrix.quant }}" "${RUNNER_ARTIFACT_DIR}"

test-cuda-pybind:
Expand Down Expand Up @@ -605,6 +615,12 @@ jobs:
script: |
set -eux

# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
mkdir -p "${HF_HOME}"

echo "::group::Setup ExecuTorch"
# Disable MKL to avoid duplicate target error when conda has multiple MKL installations
export USE_MKL=OFF
Expand All @@ -626,8 +642,7 @@ jobs:

echo "::group::Setup Huggingface"
pip install -U "huggingface_hub[cli]>=1.2.1,<2.0"
HF_AUTH_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
hf auth login --token "$HF_AUTH_TOKEN"
export HF_TOKEN="$(printf '%s' "$SECRET_EXECUTORCH_HF_TOKEN" | tr -d '\r\n')"
echo "::endgroup::"

echo "::group::Install optimum-executorch"
Expand Down
Loading