ci: use uv overrides to make sure tokenizers install from <=0.23.0 under subs

.github/workflows/nightly_tests.yml

@@ -19,6 +19,9 @@ env:
   PIPELINE_USAGE_CUTOFF: 0
   SLACK_API_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
   CONSOLIDATED_REPORT_PATH: consolidated_test_report.md
+  # Force tokenizers<0.23.0 across every `uv pip install` in this workflow,
+  # even when transformers@main declares a higher lower-bound.
+  UV_OVERRIDE: /tmp/uv-overrides.txt
 
 jobs:
   setup_torch_cuda_pipeline_matrix:
@@ -74,9 +77,9 @@ jobs:
         run: nvidia-smi
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
           uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
           uv pip install pytest-reportlog
       - name: Environment
@@ -128,9 +131,9 @@ jobs:
 
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality]"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
         uv pip install peft@git+https://github.com/huggingface/peft.git
         uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
         uv pip install pytest-reportlog
@@ -196,9 +199,9 @@ jobs:
         nvidia-smi
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality,training]"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
     - name: Environment
       run: |
         python utils/print_env.py
@@ -238,9 +241,9 @@ jobs:
         run: nvidia-smi
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
           uv pip install peft@git+https://github.com/huggingface/peft.git
           uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
           uv pip install pytest-reportlog
@@ -289,9 +292,9 @@ jobs:
 
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
           uv pip install peft@git+https://github.com/huggingface/peft.git
           uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
 
@@ -365,14 +368,14 @@ jobs:
         run: nvidia-smi
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip install -U ${{ matrix.config.backend }}
           if [ "${{ join(matrix.config.additional_deps, ' ') }}" != "" ]; then
               uv pip install ${{ join(matrix.config.additional_deps, ' ') }}
           fi
           uv pip install pytest-reportlog
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
       - name: Environment
         run: |
           python utils/print_env.py
@@ -418,10 +421,10 @@ jobs:
         run: nvidia-smi
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip install -U bitsandbytes optimum_quanto
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
           uv pip install pytest-reportlog
       - name: Environment
         run: |

.github/workflows/pr_modular_tests.yml

@@ -34,6 +34,9 @@ env:
   OMP_NUM_THREADS: 4
   MKL_NUM_THREADS: 4
   PYTEST_TIMEOUT: 60
+  # Force tokenizers<0.23.0 across every `uv pip install` in this workflow,
+  # even when transformers@main declares a higher lower-bound.
+  UV_OVERRIDE: /tmp/uv-overrides.txt
 
 jobs:
   check_code_quality:
@@ -120,9 +123,9 @@ jobs:
 
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality]"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
         uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
 
     - name: Environment

.github/workflows/pr_tests.yml

@@ -29,6 +29,9 @@ env:
   OMP_NUM_THREADS: 4
   MKL_NUM_THREADS: 4
   PYTEST_TIMEOUT: 60
+  # Force tokenizers<0.23.0 across every `uv pip install` in this workflow,
+  # even when transformers@main declares a higher lower-bound.
+  UV_OVERRIDE: /tmp/uv-overrides.txt
 
 jobs:
   check_code_quality:
@@ -116,9 +119,9 @@ jobs:
 
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality]"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
         uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
 
     - name: Environment
@@ -193,9 +196,9 @@ jobs:
 
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality]"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
 
     - name: Environment
       run: |
@@ -244,13 +247,12 @@ jobs:
 
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality]"
         # TODO (sayakpaul, DN6): revisit `--no-deps`
         uv pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps
-        uv pip install -U tokenizers
         uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
 
     - name: Environment
       run: |

.github/workflows/pr_tests_gpu.yml

@@ -30,6 +30,9 @@ env:
   HF_XET_HIGH_PERFORMANCE: 1
   PYTEST_TIMEOUT: 600
   PIPELINE_USAGE_CUTOFF: 1000000000 # set high cutoff so that only always-test pipelines run
+  # Force tokenizers<0.23.0 across every `uv pip install` in this workflow,
+  # even when transformers@main declares a higher lower-bound.
+  UV_OVERRIDE: /tmp/uv-overrides.txt
 
 jobs:
   check_code_quality:
@@ -91,6 +94,7 @@ jobs:
           fetch-depth: 2
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
       - name: Environment
         run: |
@@ -132,10 +136,10 @@ jobs:
           nvidia-smi
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
 
       - name: Environment
         run: |
@@ -202,11 +206,11 @@ jobs:
 
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality]"
         uv pip install peft@git+https://github.com/huggingface/peft.git
         uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
 
     - name: Environment
       run: |
@@ -267,8 +271,8 @@ jobs:
         nvidia-smi
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
         uv pip install -e ".[quality,training]"
 
     - name: Environment

.github/workflows/push_tests.yml

@@ -20,6 +20,9 @@ env:
   HF_XET_HIGH_PERFORMANCE: 1
   PYTEST_TIMEOUT: 600
   PIPELINE_USAGE_CUTOFF: 50000
+  # Force tokenizers<0.23.0 across every `uv pip install` in this workflow,
+  # even when transformers@main declares a higher lower-bound.
+  UV_OVERRIDE: /tmp/uv-overrides.txt
 
 jobs:
   setup_torch_cuda_pipeline_matrix:
@@ -37,6 +40,7 @@ jobs:
           fetch-depth: 2
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
       - name: Environment
         run: |
@@ -77,10 +81,10 @@ jobs:
           nvidia-smi
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
       - name: Environment
         run: |
           python utils/print_env.py
@@ -129,11 +133,11 @@ jobs:
 
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality]"
         uv pip install peft@git+https://github.com/huggingface/peft.git
         uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
 
     - name: Environment
       run: |
@@ -184,9 +188,9 @@ jobs:
         nvidia-smi
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality,training]"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
     - name: Environment
       run: |
         python utils/print_env.py
@@ -228,6 +232,7 @@ jobs:
         nvidia-smi
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality,training]"
     - name: Environment
       run: |
@@ -268,6 +273,7 @@ jobs:
         nvidia-smi
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality,training]"
 
     - name: Environment

.github/workflows/push_tests_mps.yml

@@ -14,6 +14,9 @@ env:
   HF_XET_HIGH_PERFORMANCE: 1
   PYTEST_TIMEOUT: 600
   RUN_SLOW: no
+  # Force tokenizers<0.23.0 across every `uv pip install` in this workflow,
+  # even when transformers@main declares a higher lower-bound.
+  UV_OVERRIDE: /tmp/uv-overrides.txt
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -43,12 +46,12 @@ jobs:
     - name: Install dependencies
       shell: arch -arch arm64 bash {0}
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         ${CONDA_RUN} python -m pip install --upgrade pip uv
         ${CONDA_RUN} python -m uv pip install -e ".[quality]"
         ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio
         ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git
         ${CONDA_RUN} python -m uv pip install transformers --upgrade
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
 
     - name: Environment
       shell: arch -arch arm64 bash {0}

.github/workflows/release_tests_fast.yml

@@ -19,6 +19,9 @@ env:
   MKL_NUM_THREADS: 8
   PYTEST_TIMEOUT: 600
   PIPELINE_USAGE_CUTOFF: 50000
+  # Force tokenizers<0.23.0 across every `uv pip install` in this workflow,
+  # even when transformers@main declares a higher lower-bound.
+  UV_OVERRIDE: /tmp/uv-overrides.txt
 
 jobs:
   setup_torch_cuda_pipeline_matrix:
@@ -36,9 +39,9 @@ jobs:
           fetch-depth: 2
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
       - name: Environment
         run: |
           python utils/print_env.py
@@ -78,10 +81,10 @@ jobs:
           nvidia-smi
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
       - name: Environment
         run: |
           python utils/print_env.py
@@ -130,11 +133,11 @@ jobs:
 
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality]"
         uv pip install peft@git+https://github.com/huggingface/peft.git
         uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
 
     - name: Environment
       run: |
@@ -182,11 +185,11 @@ jobs:
 
       - name: Install dependencies
         run: |
+          echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
           uv pip install -e ".[quality]"
           uv pip install peft@git+https://github.com/huggingface/peft.git
           uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
           uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-          uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
 
       - name: Environment
         run: |
@@ -243,9 +246,9 @@ jobs:
         nvidia-smi
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality,training]"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
     - name: Environment
       run: |
         python utils/print_env.py
@@ -287,9 +290,9 @@ jobs:
         nvidia-smi
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality,training]"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
     - name: Environment
       run: |
         python utils/print_env.py
@@ -331,9 +334,9 @@ jobs:
 
     - name: Install dependencies
       run: |
+        echo 'tokenizers<0.23.0' > "$UV_OVERRIDE"
         uv pip install -e ".[quality,training]"
         uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
-        uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0"
 
     - name: Environment
       run: |