From 5348b24d4b7b9ccd901cf62841d4c7336424fa30 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 19 May 2026 17:30:21 +0530 Subject: [PATCH 1/2] ci: use uv overrides to make sure tokenizers install from <=0.23.0 under subs --- .github/workflows/nightly_tests.yml | 17 ++++++++++------- .github/workflows/pr_modular_tests.yml | 5 ++++- .github/workflows/pr_tests.yml | 10 ++++++---- .github/workflows/pr_tests_gpu.yml | 10 +++++++--- .github/workflows/push_tests.yml | 12 +++++++++--- .github/workflows/push_tests_mps.yml | 5 ++++- .github/workflows/release_tests_fast.yml | 17 ++++++++++------- 7 files changed, 50 insertions(+), 26 deletions(-) diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index 0d113d677040..6cbdc9425e6d 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -19,6 +19,9 @@ env: PIPELINE_USAGE_CUTOFF: 0 SLACK_API_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} CONSOLIDATED_REPORT_PATH: consolidated_test_report.md + # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: setup_torch_cuda_pipeline_matrix: @@ -74,9 +77,9 @@ jobs: run: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip install pytest-reportlog - name: Environment @@ -128,9 +131,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip install pytest-reportlog @@ -196,9 +199,9 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -238,9 +241,9 @@ jobs: run: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip install pytest-reportlog @@ -289,9 +292,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git @@ -365,6 +368,7 @@ jobs: run: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install -U ${{ matrix.config.backend }} if [ "${{ join(matrix.config.additional_deps, ' ') }}" != "" ]; then @@ -372,7 +376,6 @@ jobs: fi uv pip install pytest-reportlog uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -418,10 +421,10 @@ jobs: run: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install -U bitsandbytes optimum_quanto uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install pytest-reportlog - name: Environment run: | diff --git a/.github/workflows/pr_modular_tests.yml b/.github/workflows/pr_modular_tests.yml index 91a471748bc4..3b674b0dd508 100644 --- a/.github/workflows/pr_modular_tests.yml +++ b/.github/workflows/pr_modular_tests.yml @@ -34,6 +34,9 @@ env: OMP_NUM_THREADS: 4 MKL_NUM_THREADS: 4 PYTEST_TIMEOUT: 60 + # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: check_code_quality: @@ -120,9 +123,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps - name: Environment diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index f2282dc12bf9..0ec6fbb0a218 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -29,6 +29,9 @@ env: OMP_NUM_THREADS: 4 MKL_NUM_THREADS: 4 PYTEST_TIMEOUT: 60 + # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: check_code_quality: @@ -116,9 +119,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps - name: Environment @@ -193,9 +196,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -244,13 +247,12 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" # TODO (sayakpaul, DN6): revisit `--no-deps` uv pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps - uv pip install -U tokenizers uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diff --git a/.github/workflows/pr_tests_gpu.yml b/.github/workflows/pr_tests_gpu.yml index 96e018562f4c..0e83dd2a0271 100644 --- a/.github/workflows/pr_tests_gpu.yml +++ b/.github/workflows/pr_tests_gpu.yml @@ -30,6 +30,9 @@ env: HF_XET_HIGH_PERFORMANCE: 1 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 1000000000 # set high cutoff so that only always-test pipelines run + # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: check_code_quality: @@ -91,6 +94,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" - name: Environment run: | @@ -132,10 +136,10 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -202,11 +206,11 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -267,8 +271,8 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install -e ".[quality,training]" - name: Environment diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index ee49ab41bad6..b9ff7be0819d 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -20,6 +20,9 @@ env: HF_XET_HIGH_PERFORMANCE: 1 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 50000 + # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: setup_torch_cuda_pipeline_matrix: @@ -37,6 +40,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" - name: Environment run: | @@ -77,10 +81,10 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -129,11 +133,11 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -184,9 +188,9 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -228,6 +232,7 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" - name: Environment run: | @@ -268,6 +273,7 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" - name: Environment diff --git a/.github/workflows/push_tests_mps.yml b/.github/workflows/push_tests_mps.yml index 6a6825713e33..4f7b2d84dadd 100644 --- a/.github/workflows/push_tests_mps.yml +++ b/.github/workflows/push_tests_mps.yml @@ -14,6 +14,9 @@ env: HF_XET_HIGH_PERFORMANCE: 1 PYTEST_TIMEOUT: 600 RUN_SLOW: no + # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -43,12 +46,12 @@ jobs: - name: Install dependencies shell: arch -arch arm64 bash {0} run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" ${CONDA_RUN} python -m pip install --upgrade pip uv ${CONDA_RUN} python -m uv pip install -e ".[quality]" ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git ${CONDA_RUN} python -m uv pip install transformers --upgrade - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment shell: arch -arch arm64 bash {0} diff --git a/.github/workflows/release_tests_fast.yml b/.github/workflows/release_tests_fast.yml index 51709ba834f7..59f006de6baf 100644 --- a/.github/workflows/release_tests_fast.yml +++ b/.github/workflows/release_tests_fast.yml @@ -19,6 +19,9 @@ env: MKL_NUM_THREADS: 8 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 50000 + # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: setup_torch_cuda_pipeline_matrix: @@ -36,9 +39,9 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -78,10 +81,10 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -130,11 +133,11 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -182,11 +185,11 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -243,9 +246,9 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -287,9 +290,9 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -331,9 +334,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | From 0681ff6106d9035619ea92c1dbac1da81ab360ec Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 19 May 2026 18:19:41 +0530 Subject: [PATCH 2/2] up --- .github/workflows/nightly_tests.yml | 16 ++++++++-------- .github/workflows/pr_modular_tests.yml | 4 ++-- .github/workflows/pr_tests.yml | 8 ++++---- .github/workflows/pr_tests_gpu.yml | 10 +++++----- .github/workflows/push_tests.yml | 14 +++++++------- .github/workflows/push_tests_mps.yml | 4 ++-- .github/workflows/release_tests_fast.yml | 16 ++++++++-------- 7 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index 6cbdc9425e6d..1e31f6589ac0 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -19,7 +19,7 @@ env: PIPELINE_USAGE_CUTOFF: 0 SLACK_API_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} CONSOLIDATED_REPORT_PATH: consolidated_test_report.md - # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, # even when transformers@main declares a higher lower-bound. UV_OVERRIDE: /tmp/uv-overrides.txt @@ -77,7 +77,7 @@ jobs: run: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git @@ -131,7 +131,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git uv pip install peft@git+https://github.com/huggingface/peft.git @@ -199,7 +199,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - name: Environment @@ -241,7 +241,7 @@ jobs: run: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git uv pip install peft@git+https://github.com/huggingface/peft.git @@ -292,7 +292,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git uv pip install peft@git+https://github.com/huggingface/peft.git @@ -368,7 +368,7 @@ jobs: run: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install -U ${{ matrix.config.backend }} if [ "${{ join(matrix.config.additional_deps, ' ') }}" != "" ]; then @@ -421,7 +421,7 @@ jobs: run: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install -U bitsandbytes optimum_quanto uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git diff --git a/.github/workflows/pr_modular_tests.yml b/.github/workflows/pr_modular_tests.yml index 3b674b0dd508..e17fe31c4c91 100644 --- a/.github/workflows/pr_modular_tests.yml +++ b/.github/workflows/pr_modular_tests.yml @@ -34,7 +34,7 @@ env: OMP_NUM_THREADS: 4 MKL_NUM_THREADS: 4 PYTEST_TIMEOUT: 60 - # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, # even when transformers@main declares a higher lower-bound. UV_OVERRIDE: /tmp/uv-overrides.txt @@ -123,7 +123,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 0ec6fbb0a218..898c20dc418f 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -29,7 +29,7 @@ env: OMP_NUM_THREADS: 4 MKL_NUM_THREADS: 4 PYTEST_TIMEOUT: 60 - # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, # even when transformers@main declares a higher lower-bound. UV_OVERRIDE: /tmp/uv-overrides.txt @@ -119,7 +119,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps @@ -196,7 +196,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git @@ -247,7 +247,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" # TODO (sayakpaul, DN6): revisit `--no-deps` uv pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps diff --git a/.github/workflows/pr_tests_gpu.yml b/.github/workflows/pr_tests_gpu.yml index 0e83dd2a0271..32b7e040f63d 100644 --- a/.github/workflows/pr_tests_gpu.yml +++ b/.github/workflows/pr_tests_gpu.yml @@ -30,7 +30,7 @@ env: HF_XET_HIGH_PERFORMANCE: 1 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 1000000000 # set high cutoff so that only always-test pipelines run - # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, # even when transformers@main declares a higher lower-bound. UV_OVERRIDE: /tmp/uv-overrides.txt @@ -94,7 +94,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" - name: Environment run: | @@ -136,7 +136,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git @@ -206,7 +206,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git @@ -271,7 +271,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git uv pip install -e ".[quality,training]" diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index b9ff7be0819d..15876eb4c44a 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -20,7 +20,7 @@ env: HF_XET_HIGH_PERFORMANCE: 1 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 50000 - # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, # even when transformers@main declares a higher lower-bound. UV_OVERRIDE: /tmp/uv-overrides.txt @@ -40,7 +40,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" - name: Environment run: | @@ -81,7 +81,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git @@ -133,7 +133,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git @@ -188,7 +188,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - name: Environment @@ -232,7 +232,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" - name: Environment run: | @@ -273,7 +273,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" - name: Environment diff --git a/.github/workflows/push_tests_mps.yml b/.github/workflows/push_tests_mps.yml index 4f7b2d84dadd..386737bbe42e 100644 --- a/.github/workflows/push_tests_mps.yml +++ b/.github/workflows/push_tests_mps.yml @@ -14,7 +14,7 @@ env: HF_XET_HIGH_PERFORMANCE: 1 PYTEST_TIMEOUT: 600 RUN_SLOW: no - # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, # even when transformers@main declares a higher lower-bound. UV_OVERRIDE: /tmp/uv-overrides.txt @@ -46,7 +46,7 @@ jobs: - name: Install dependencies shell: arch -arch arm64 bash {0} run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" ${CONDA_RUN} python -m pip install --upgrade pip uv ${CONDA_RUN} python -m uv pip install -e ".[quality]" ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio diff --git a/.github/workflows/release_tests_fast.yml b/.github/workflows/release_tests_fast.yml index 59f006de6baf..2eb4bba55c18 100644 --- a/.github/workflows/release_tests_fast.yml +++ b/.github/workflows/release_tests_fast.yml @@ -19,7 +19,7 @@ env: MKL_NUM_THREADS: 8 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 50000 - # Force tokenizers<=0.23.0 across every `uv pip install` in this workflow, + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, # even when transformers@main declares a higher lower-bound. UV_OVERRIDE: /tmp/uv-overrides.txt @@ -39,7 +39,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - name: Environment @@ -81,7 +81,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git @@ -133,7 +133,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git @@ -185,7 +185,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git @@ -246,7 +246,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - name: Environment @@ -290,7 +290,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - name: Environment @@ -334,7 +334,7 @@ jobs: - name: Install dependencies run: | - echo 'tokenizers<=0.23.0' > "$UV_OVERRIDE" + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git