diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index 94474a7359eb..fd19d0e20997 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -19,6 +19,9 @@ env: PIPELINE_USAGE_CUTOFF: 0 SLACK_API_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} CONSOLIDATED_REPORT_PATH: consolidated_test_report.md + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: setup_torch_cuda_pipeline_matrix: @@ -74,9 +77,9 @@ jobs: run: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip install pytest-reportlog - name: Environment @@ -128,9 +131,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip install pytest-reportlog @@ -196,9 +199,9 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diffusers-cli env @@ -238,9 +241,9 @@ jobs: run: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip install pytest-reportlog @@ -289,9 +292,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git @@ -365,6 +368,7 @@ jobs: run: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install -U ${{ matrix.config.backend }} if [ "${{ join(matrix.config.additional_deps, ' ') }}" != "" ]; then @@ -372,7 +376,6 @@ jobs: fi uv pip install pytest-reportlog uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diffusers-cli env @@ -418,10 +421,10 @@ jobs: run: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install -U bitsandbytes optimum_quanto uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install pytest-reportlog - name: Environment run: | diff --git a/.github/workflows/pr_modular_tests.yml b/.github/workflows/pr_modular_tests.yml index a64ecb7229dc..32b63e75bab9 100644 --- a/.github/workflows/pr_modular_tests.yml +++ b/.github/workflows/pr_modular_tests.yml @@ -34,6 +34,9 @@ env: OMP_NUM_THREADS: 4 MKL_NUM_THREADS: 4 PYTEST_TIMEOUT: 60 + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: check_code_quality: @@ -121,9 +124,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps - name: Environment diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 668b4ca33008..543f44418568 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -29,6 +29,9 @@ env: OMP_NUM_THREADS: 4 MKL_NUM_THREADS: 4 PYTEST_TIMEOUT: 60 + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: check_code_quality: @@ -117,9 +120,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps - name: Environment @@ -194,9 +197,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -245,13 +248,12 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" # TODO (sayakpaul, DN6): revisit `--no-deps` uv pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps - uv pip install -U tokenizers uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diff --git a/.github/workflows/pr_tests_gpu.yml b/.github/workflows/pr_tests_gpu.yml index ddd7d551f2de..a2f99fa2b7db 100644 --- a/.github/workflows/pr_tests_gpu.yml +++ b/.github/workflows/pr_tests_gpu.yml @@ -30,6 +30,9 @@ env: HF_XET_HIGH_PERFORMANCE: 1 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 1000000000 # set high cutoff so that only always-test pipelines run + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: check_code_quality: @@ -92,6 +95,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" - name: Environment run: | @@ -133,10 +137,10 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -203,11 +207,11 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -268,8 +272,8 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install -e ".[quality,training]" - name: Environment diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index caff08545a6e..25d97a9e57cd 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -20,6 +20,9 @@ env: HF_XET_HIGH_PERFORMANCE: 1 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 50000 + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: setup_torch_cuda_pipeline_matrix: @@ -37,6 +40,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" - name: Environment run: | @@ -77,10 +81,10 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diffusers-cli env @@ -129,11 +133,11 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -184,9 +188,9 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diffusers-cli env @@ -228,6 +232,7 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" - name: Environment run: | @@ -268,6 +273,7 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" - name: Environment diff --git a/.github/workflows/push_tests_mps.yml b/.github/workflows/push_tests_mps.yml index f3b59dcda5ef..984a81e8cb22 100644 --- a/.github/workflows/push_tests_mps.yml +++ b/.github/workflows/push_tests_mps.yml @@ -14,6 +14,9 @@ env: HF_XET_HIGH_PERFORMANCE: 1 PYTEST_TIMEOUT: 600 RUN_SLOW: no + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -43,12 +46,12 @@ jobs: - name: Install dependencies shell: arch -arch arm64 bash {0} run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" ${CONDA_RUN} python -m pip install --upgrade pip uv ${CONDA_RUN} python -m uv pip install -e ".[quality]" ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git ${CONDA_RUN} python -m uv pip install transformers --upgrade - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment shell: arch -arch arm64 bash {0} diff --git a/.github/workflows/release_tests_fast.yml b/.github/workflows/release_tests_fast.yml index 2c0c984ace6e..dcc8df755cc6 100644 --- a/.github/workflows/release_tests_fast.yml +++ b/.github/workflows/release_tests_fast.yml @@ -19,6 +19,9 @@ env: MKL_NUM_THREADS: 8 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 50000 + # Force tokenizers<0.23.0 across every `uv pip install` in this workflow, + # even when transformers@main declares a higher lower-bound. + UV_OVERRIDE: /tmp/uv-overrides.txt jobs: setup_torch_cuda_pipeline_matrix: @@ -36,9 +39,9 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diffusers-cli env @@ -78,10 +81,10 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diffusers-cli env @@ -130,11 +133,11 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -182,11 +185,11 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality]" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -243,9 +246,9 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diffusers-cli env @@ -287,9 +290,9 @@ jobs: nvidia-smi - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | diffusers-cli env @@ -331,9 +334,9 @@ jobs: - name: Install dependencies run: | + echo 'tokenizers<0.23.0' > "$UV_OVERRIDE" uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: |