on:
  workflow_call:
    inputs:
      runs-on_value:
        required: true
        type: string
      artifacts_version:
        required: true
        type: string
      catboost_package_name:
        required: false
        type: string
        default: catboost_dev
      catboost_package_version:
        required: true
        type: string
      artifact_retention_days:
        required: false
        type: number
        default: 3

jobs:
  build:
    runs-on: ${{ inputs.runs-on_value }}
    env:
      CATBOOST_PACKAGE_NAME: ${{ inputs.catboost_package_name }}
      CATBOOST_PACKAGE_VERSION: ${{ inputs.catboost_package_version }}
      CMAKE_BUILD_CACHE_DIR: cmake_build_cache
    steps:
      - if: inputs.runs-on_value == 'ubuntu-latest'
        name: Maximize build space
        uses: easimon/maximize-build-space@master
        with:
          root-reserve-mb: 36000  # needed for docker image
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
          remove-codeql: 'true'

      - if: inputs.runs-on_value == 'macos-13'
        name: Get python_3.7_macos_x86_64 from cache
        uses: actions/cache/restore@v4
        with:
          enableCrossOsArchive: true
          fail-on-cache-miss: true
          path: python_3.7_macos_x86_64.zip
          key: python_3.7_macos_x86_64-${{ github.run_id }} # Can use time based key as well
          restore-keys: |
            python_3.7_macos_x86_64

      - if: inputs.runs-on_value == 'macos-13'
        name: Get python_versions_macos_arm64 from cache
        uses: actions/cache/restore@v4
        with:
          enableCrossOsArchive: true
          fail-on-cache-miss: true
          path: python_versions_macos_arm64.zip
          key: python_versions_macos_arm64-${{ github.run_id }} # Can use time based key as well
          restore-keys: |
            python_versions_macos_arm64

      - if: inputs.runs-on_value == 'macos-13'
        name: Env setup on macOS
        run: |
          python3 -m pip install -U conan==1.62.0 setuptools wheel jupyterlab==3.0.6 build

          brew update
          brew install ninja

          npm install --global rimraf

          export CMAKE_BUILD_ENV_ROOT=${{ github.workspace }}/cmake_build_env_root

          mkdir -p $CMAKE_BUILD_ENV_ROOT/darwin-x86_64

          # JDK
          cd $CMAKE_BUILD_ENV_ROOT/darwin-x86_64
          mkdir -p Library/Java/JavaVirtualMachines
          cd Library/Java/JavaVirtualMachines
          ln -s /Library/Java/JavaVirtualMachines/Temurin-Hotspot-8.jdk ./jdk-8

          # Python
          mkdir -p $HOME/.pyenv/versions
          cd $HOME/.pyenv/versions
          unzip ${{ github.workspace }}/python_3.7_macos_x86_64.zip

          cd $CMAKE_BUILD_ENV_ROOT/darwin-x86_64
          mkdir -p Python
          cd Python
          ln -s $HOME/.pyenv/versions/3.7 ./3.7
          ln -s $RUNNER_TOOL_CACHE/Python/3.8.18/x64 ./3.8
          ln -s $RUNNER_TOOL_CACHE/Python/3.9.19/x64 ./3.9
          ln -s $RUNNER_TOOL_CACHE/Python/3.10.14/x64 ./3.10
          ln -s $RUNNER_TOOL_CACHE/Python/3.11.9/x64 ./3.11
          ln -s $RUNNER_TOOL_CACHE/Python/3.12.4/x64 ./3.12

          for PY_VER in 3.7 3.8 3.9 3.10 3.11 3.12
          do
              $CMAKE_BUILD_ENV_ROOT/darwin-x86_64/Python/$PY_VER/bin/python -m pip install --upgrade pip setuptools wheel
          done

          mkdir -p $CMAKE_BUILD_ENV_ROOT/darwin-arm64

          # JDK
          cd $CMAKE_BUILD_ENV_ROOT/darwin-arm64
          mkdir -p Library/Java/JavaVirtualMachines
          cd Library/Java/JavaVirtualMachines
          wget https://cdn.azul.com/zulu/bin/zulu8.78.0.19-ca-jdk8.0.412-macosx_aarch64.tar.gz
          tar xzvf zulu8.78.0.19-ca-jdk8.0.412-macosx_aarch64.tar.gz
          ln -s ./zulu8.78.0.19-ca-jdk8.0.412-macosx_aarch64/zulu-8.jdk ./jdk-8

          # Python
          cd $CMAKE_BUILD_ENV_ROOT/darwin-arm64
          mkdir -p Python
          cd Python
          unzip ${{ github.workspace }}/python_versions_macos_arm64

      - if: inputs.runs-on_value == 'windows-2022'
        name: Install CUDA on Windows
        uses: Jimver/cuda-toolkit@v0.2.15
        id: cuda-toolkit
        with:
          cuda: '11.8.0'
          method: 'local'

      - if: inputs.runs-on_value == 'windows-2022'
        name: Env setup on Windows
        run: |
          python -m pip install -U conan==1.62.0 setuptools wheel jupyterlab==3.0.6 build

          choco install ninja

          npm install --global rimraf

          $CMAKE_BUILD_ENV_ROOT="${{ github.workspace }}\cmake_build_env_root"

          New-Item $CMAKE_BUILD_ENV_ROOT\windows-x86_64 -ItemType Directory

          # JDK
          Set-Location -Path $CMAKE_BUILD_ENV_ROOT\windows-x86_64
          New-Item -Path .\jdk-8 -ItemType SymbolicLink -Value $Env:JAVA_HOME_8_X64

          # Python
          Set-Location -Path $CMAKE_BUILD_ENV_ROOT\windows-x86_64
          New-Item -Path Python -ItemType Directory
          Set-Location -Path .\Python
          New-Item -Path .\3.7 -ItemType SymbolicLink -Value "$Env:RUNNER_TOOL_CACHE\Python\3.7.9\x64"
          New-Item -Path .\3.8 -ItemType SymbolicLink -Value "$Env:RUNNER_TOOL_CACHE\Python\3.8.10\x64"
          New-Item -Path .\3.9 -ItemType SymbolicLink -Value "$Env:RUNNER_TOOL_CACHE\Python\3.9.13\x64"
          New-Item -Path .\3.10 -ItemType SymbolicLink -Value "$Env:RUNNER_TOOL_CACHE\Python\3.10.11\x64"
          New-Item -Path .\3.11 -ItemType SymbolicLink -Value "$Env:RUNNER_TOOL_CACHE\Python\3.11.9\x64"
          New-Item -Path .\3.12 -ItemType SymbolicLink -Value "$Env:RUNNER_TOOL_CACHE\Python\3.12.3\x64"

          foreach($PY_VER in '3.7','3.8','3.9','3.10','3.11','3.12') {
            & "$CMAKE_BUILD_ENV_ROOT\windows-x86_64\Python\$PY_VER\python.exe" -m pip install --upgrade pip setuptools wheel
          }

          # CUDA
          Set-Location -Path $CMAKE_BUILD_ENV_ROOT\windows-x86_64
          New-Item -Path CUDA -ItemType Directory
          New-Item -Path .\CUDA\v11.8 -ItemType SymbolicLink -Value "${{steps.cuda-toolkit.outputs.CUDA_PATH}}"

          # MSVC toolset
          $VS_DIR="C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise"
          & "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vs_installer.exe" modify --installPath "$VS_DIR" --channelId VisualStudio.16.Release --add Microsoft.VisualStudio.Component.VC.14.28.16.9.x86.x64 --downloadThenInstall --quiet --norestart --nocache

      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: catboost
          fetch-depth: 0

      - name: Update CMake build cache on every commit
        uses: actions/cache@v4
        with:
          path: cmake_build_cache
          key: cmake_build_cache-${{ runner.os }}-${{ github.run_id }} # Can use time based key as well
          restore-keys: |
            cmake_build_cache-${{ runner.os }}

      - name: Prepare CMake build cache
        shell: pwsh
        run: |
          if (Test-Path cmake_build_cache) {
            if (Test-Path cmake_build_cache/git_commit_sha) {
              # Set timestamps such as that only files changed after the commit in cache are newer than its files
              Set-Location -LiteralPath ${{ github.workspace }}/catboost
              git checkout (Get-Content ${{ github.workspace }}/cmake_build_cache/git_commit_sha)
              Start-Sleep -Seconds 2  # to make sure timestamps are different
              Set-Location -LiteralPath ${{ github.workspace }}/cmake_build_cache
              dir -R | foreach { $_.LastWriteTime = [System.DateTime]::Now }
              Start-Sleep -Seconds 2  # to make sure timestamps are different
              Set-Location -LiteralPath ${{ github.workspace }}/catboost
              git checkout ${{ github.sha }}
            }
          } else {
            New-Item -Path cmake_build_cache -ItemType Directory
          }

      - name: Fixup python package version
        shell: pwsh
        run: |
          Set-Location -Path catboost
          python ./ci/fixup_python_version.py

      - if: inputs.runs-on_value == 'ubuntu-latest'
        name: Build all in container
        run: |
          mkdir -p ${{ github.workspace }}/catboost_build_home
          rm -fr ${{ github.workspace }}/catboost_build_home/*
          docker run --rm --name=catboost_build --user $(id -u):$(id -g) -v /etc/passwd:/etc/passwd -v ${{ github.workspace }}/catboost_build_home:$HOME -v ${{ github.workspace }}/catboost:/src/catboost -v ${{ github.workspace }}/$CMAKE_BUILD_CACHE_DIR:/cmake_build_cache -e CATBOOST_PACKAGE_NAME -e CATBOOST_PACKAGE_VERSION ghcr.io/catboost/manylinux2014_x86_64_and_cross_arm64_for_catboost_ci:latest bash -c ". /work/setup_env/build_x86_64.sh && export CMAKE_BUILD_CACHE_DIR=/cmake_build_cache/ && ln -sf /opt/cmake_build_env_root $HOME/ && rsync -rptl /opt/.conan $HOME/  && cd /src/catboost && ./ci/build_all.py"

      - if: inputs.runs-on_value != 'ubuntu-latest'
        name: Build all without container
        shell: pwsh
        run: |
          $Env:CMAKE_BUILD_ENV_ROOT = Join-Path ${{ github.workspace }} cmake_build_env_root
          $Env:CMAKE_BUILD_CACHE_DIR = Join-Path ${{ github.workspace }} cmake_build_cache
          Set-Location -Path catboost
          python ./ci/build_all.py --build-test-tools

      - name: Save commit id to CMake build cache
        shell: pwsh
        run: |
          Set-Content -Path cmake_build_cache/git_commit_sha -Value ${{ github.sha }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-cli-linux-x86_64-${{ inputs.artifacts_version }}
          path: catboost/build_native_root/have_cuda/linux-x86_64/catboost/app/catboost
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-cli-linux-aarch64-${{ inputs.artifacts_version }}
          path: catboost/build_native_root/have_cuda/linux-aarch64/catboost/app/catboost
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-python-package-dist-linux-${{ inputs.artifacts_version }}
          path: catboost/catboost/python-package/dist/*.whl
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-python-package-dist-src-${{ inputs.artifacts_version }}
          path: catboost/catboost/python-package/dist/*.tar.gz
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-R-package-linux-x86_64-${{ inputs.artifacts_version }}
          path: catboost/catboost/R-package/catboost-R-linux-x86_64.tgz
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-R-package-linux-aarch64-${{ inputs.artifacts_version }}
          path: catboost/catboost/R-package/catboost-R-linux-aarch64.tgz
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-model-interface-linux-x86_64-${{ inputs.artifacts_version }}
          path: catboost/build_native_root/have_cuda/linux-x86_64/catboost/libs/model_interface/libcatboostmodel.*
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-model-interface-linux-aarch64-${{ inputs.artifacts_version }}
          path: catboost/build_native_root/have_cuda/linux-aarch64/catboost/libs/model_interface/libcatboostmodel.*
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost4j-spark_core_resources_classes-${{ inputs.artifacts_version }}
          path: catboost/catboost/spark/catboost4j-spark/core/src/main/resources/ru
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost4j-spark_core_resources-linux-${{ inputs.artifacts_version }}
          path: catboost/catboost/spark/catboost4j-spark/core/src/main/resources/linux*
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost4j-prediction_resources-linux-${{ inputs.artifacts_version }}
          path: catboost/catboost/jvm-packages/catboost4j-prediction/src/main/resources/*
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-test-tools-linux-x86_64-${{ inputs.artifacts_version }}
          path: |
            catboost/build_native_root/no_cuda/linux-x86_64/catboost/tools/limited_precision_dsv_diff/limited_precision_dsv_diff
            catboost/build_native_root/no_cuda/linux-x86_64/catboost/tools/limited_precision_json_diff/limited_precision_json_diff
            catboost/build_native_root/no_cuda/linux-x86_64/catboost/tools/model_comparator/model_comparator
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'ubuntu-latest'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-test-tools-linux-aarch64-${{ inputs.artifacts_version }}
          path: |
            catboost/build_native_root/no_cuda/linux-aarch64/catboost/tools/limited_precision_dsv_diff/limited_precision_dsv_diff
            catboost/build_native_root/no_cuda/linux-aarch64/catboost/tools/limited_precision_json_diff/limited_precision_json_diff
            catboost/build_native_root/no_cuda/linux-aarch64/catboost/tools/model_comparator/model_comparator
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'macos-13'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-cli-darwin-universal2-${{ inputs.artifacts_version }}
          path: catboost/build_native_root/no_cuda/darwin-universal2/catboost/app/catboost
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'macos-13'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-python-package-dist-darwin-${{ inputs.artifacts_version }}
          path: catboost/catboost/python-package/dist/*.whl
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'macos-13'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-R-package-darwin-universal2-${{ inputs.artifacts_version }}
          path: catboost/catboost/R-package/catboost-R-darwin-universal2.tgz
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'macos-13'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-model-interface-darwin-universal2-${{ inputs.artifacts_version }}
          path: catboost/build_native_root/no_cuda/darwin-universal2/catboost/libs/model_interface/libcatboostmodel.*
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'macos-13'
        uses: actions/upload-artifact@v4
        with:
          name: catboost4j-spark_core_resources-darwin-${{ inputs.artifacts_version }}
          path: catboost/catboost/spark/catboost4j-spark/core/src/main/resources/darwin*
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'macos-13'
        uses: actions/upload-artifact@v4
        with:
          name: catboost4j-prediction_resources-darwin-${{ inputs.artifacts_version }}
          path: catboost/catboost/jvm-packages/catboost4j-prediction/src/main/resources/*
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'macos-13'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-test-tools-darwin-universal2-${{ inputs.artifacts_version }}
          path: |
            catboost/build_native_root/no_cuda/darwin-universal2/catboost/tools/limited_precision_dsv_diff/limited_precision_dsv_diff
            catboost/build_native_root/no_cuda/darwin-universal2/catboost/tools/limited_precision_json_diff/limited_precision_json_diff
            catboost/build_native_root/no_cuda/darwin-universal2/catboost/tools/model_comparator/model_comparator
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'windows-2022'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-cli-windows-x86_64-${{ inputs.artifacts_version }}
          path: catboost/build_native_root/have_cuda/windows-x86_64/catboost/app/catboost.exe
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'windows-2022'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-python-package-dist-windows-${{ inputs.artifacts_version }}
          path: catboost/catboost/python-package/dist/*.whl
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'windows-2022'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-R-package-windows-x86_64-${{ inputs.artifacts_version }}
          path: catboost/catboost/R-package/catboost-R-windows-x86_64.tgz
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'windows-2022'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-model-interface-windows-x86_64-${{ inputs.artifacts_version }}
          path: |
            catboost/build_native_root/have_cuda/windows-x86_64/catboost/libs/model_interface/catboostmodel.dll
            catboost/build_native_root/have_cuda/windows-x86_64/catboost/libs/model_interface/catboostmodel.lib
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'windows-2022'
        uses: actions/upload-artifact@v4
        with:
          name: catboost4j-spark_core_resources-windows-${{ inputs.artifacts_version }}
          path: catboost/catboost/spark/catboost4j-spark/core/src/main/resources/win*
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'windows-2022'
        uses: actions/upload-artifact@v4
        with:
          name: catboost4j-prediction_resources-windows-${{ inputs.artifacts_version }}
          path: catboost/catboost/jvm-packages/catboost4j-prediction/src/main/resources/*
          retention-days: ${{ inputs.artifact_retention_days }}

      - if: inputs.runs-on_value == 'windows-2022'
        uses: actions/upload-artifact@v4
        with:
          name: catboost-test-tools-windows-x86_64-${{ inputs.artifacts_version }}
          path: |
            catboost/build_native_root/no_cuda/windows-x86_64/catboost/tools/limited_precision_dsv_diff/limited_precision_dsv_diff.exe
            catboost/build_native_root/no_cuda/windows-x86_64/catboost/tools/limited_precision_json_diff/limited_precision_json_diff.exe
            catboost/build_native_root/no_cuda/windows-x86_64/catboost/tools/model_comparator/model_comparator.exe
          retention-days: ${{ inputs.artifact_retention_days }}


on:
  push:
    branches:
      - master

jobs:
  check:
    name: Check per OS
    strategy:
      matrix:
        os: [ubuntu-latest, macos-13, windows-2022]
      fail-fast: false
    with:
      runs-on_value: ${{ matrix.os }}
      artifacts_version: ${{ github.sha }}
      catboost_package_version: 1.2.5
    uses: ./.github/workflows/check_per_os.yaml


on:
  workflow_call:
    inputs:
      runs-on_value:
        required: true
        type: string
      artifacts_version:
        required: true
        type: string
      catboost_package_version:
        required: true
        type: string

jobs:
  build:
    name: Build
    uses: ./.github/workflows/build.yaml
    with:
      runs-on_value: ${{ inputs.runs-on_value }}
      artifacts_version: ${{ inputs.artifacts_version }}
      catboost_package_version: ${{ inputs.catboost_package_version }}

  #test:
  #  name: Test
  #  runs-on: ${{ inputs.runs-on_value }}
  #  uses: ./.github/workflows/test.yaml
  #  needs:
  #    - build
  #  with:
  #    runs-on_value: ${{ inputs.runs-on_value }}
  #    artifacts_version: ${{ inputs.artifacts_version }}
  #    catboost_package_version: ${{ inputs.catboost_package_version }}


on:
  workflow_call:
    inputs:
      runs-on_value:
        required: true
        type: string
      artifacts_version:
        required: true
        type: string
      catboost_package_version:
        required: true
        type: string

jobs:
  test:
    runs-on: ${{ inputs.runs-on_value }}
    defaults:
      run:
        shell: bash
    steps:
    - name: Run CMake-based tests
      run: cd build_native_root/have_cuda/linux-x86_64/ && ninja test

    - name: Run pytest
      run: cd catboost/pytest && python3.11 -m pytest

    - name: Run python package tests
      run: cd catboost/python-package/ut/medium && python3.11 -m pytest


Self-hosted runner (push)

name: Self-hosted runner (push)

on:
  workflow_run:
    workflows: ["Self-hosted runner (push-caller)"]
    branches: ["main"]
    types: [completed]
  push:
    branches:
      - ci_*
      - ci-*
    paths:
      - "src/**"
      - "tests/**"
      - ".github/**"
      - "templates/**"
      - "utils/**"
  repository_dispatch:

env:
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  OMP_NUM_THREADS: 8
  MKL_NUM_THREADS: 8
  PYTEST_TIMEOUT: 60
  TF_FORCE_GPU_ALLOW_GROWTH: true
  RUN_PT_TF_CROSS_TESTS: 1
  CUDA_VISIBLE_DEVICES: 0,1

jobs:
  setup:
    name: Setup
    strategy:
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, push-ci]
    container:
      image: huggingface/transformers-all-latest-gpu-push-ci
      options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      test_map: ${{ steps.set-matrix.outputs.test_map }}
    steps:
      # Necessary to get the correct branch name and commit SHA for `workflow_run` event
      # We also take into account the `push` event (we might want to test some changes in a branch)
      - name: Prepare custom environment variables
        shell: bash
        # `CI_BRANCH_PUSH`: The branch name from the push event
        # `CI_BRANCH_WORKFLOW_RUN`: The name of the branch on which this workflow is triggered by `workflow_run` event
        # `CI_BRANCH`: The non-empty branch name from the above two (one and only one of them is empty)
        # `CI_SHA_PUSH`: The commit SHA from the push event
        # `CI_SHA_WORKFLOW_RUN`: The commit SHA that triggers this workflow by `workflow_run` event
        # `CI_SHA`: The non-empty commit SHA from the above two (one and only one of them is empty)
        run: |
          CI_BRANCH_PUSH=${{ github.event.ref }}
          CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''}
          CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }}
          CI_SHA_PUSH=${{ github.event.head_commit.id }}
          CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }}
          echo $CI_BRANCH_PUSH
          echo $CI_BRANCH_WORKFLOW_RUN
          echo $CI_SHA_PUSH
          echo $CI_SHA_WORKFLOW_RUN
          [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV
          [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV

      - name: print environment variables
        run: |
          echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}"
          echo "env.CI_SHA = ${{ env.CI_SHA }}"

      - name: Update clone using environment variables
        working-directory: /transformers
        run: |
          echo "original branch = $(git branch --show-current)"
          git fetch && git checkout ${{ env.CI_BRANCH }}
          echo "updated branch = $(git branch --show-current)"
          git checkout ${{ env.CI_SHA }}
          echo "log = $(git log -n 1)"

      - name: Cleanup
        working-directory: /transformers
        run: |
          rm -rf tests/__pycache__
          rm -rf tests/models/__pycache__
          rm -rf reports

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Fetch the tests to run
        working-directory: /transformers
        # TODO: add `git-python` in the docker images
        run: |
          pip install --upgrade git-python
          python3 utils/tests_fetcher.py --diff_with_last_commit | tee test_preparation.txt

      - name: Report fetched tests
        uses: actions/upload-artifact@v4
        with:
          name: test_fetched
          path: /transformers/test_preparation.txt

      - id: set-matrix
        name: Organize tests into models
        working-directory: /transformers
        # The `keys` is used as GitHub actions matrix for jobs, i.e. `models/bert`, `tokenization`, `pipeline`, etc.
        # The `test_map` is used to get the actual identified test files under each key.
        # If no test to run (so no `test_map.json` file), create a dummy map (empty matrix will fail)
        run: |
          if [ -f test_map.json ]; then
              keys=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); d = list(test_map.keys()); print(d)')
              test_map=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); print(test_map)')
          else
              keys=$(python3 -c 'keys = ["dummy"]; print(keys)')
              test_map=$(python3 -c 'test_map = {"dummy": []}; print(test_map)')
          fi
          echo $keys
          echo $test_map
          echo "matrix=$keys" >> $GITHUB_OUTPUT
          echo "test_map=$test_map" >> $GITHUB_OUTPUT

  run_tests_single_gpu:
    name: Model tests
    needs: setup
    # `dummy` means there is no test to run
    if: contains(fromJson(needs.setup.outputs.matrix), 'dummy') != true
    strategy:
      fail-fast: false
      matrix:
        folders: ${{ fromJson(needs.setup.outputs.matrix) }}
        machine_type: [single-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, push-ci]
    container:
      image: huggingface/transformers-all-latest-gpu-push-ci
      options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      # Necessary to get the correct branch name and commit SHA for `workflow_run` event
      # We also take into account the `push` event (we might want to test some changes in a branch)
      - name: Prepare custom environment variables
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          CI_BRANCH_PUSH=${{ github.event.ref }}
          CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''}
          CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }}
          CI_SHA_PUSH=${{ github.event.head_commit.id }}
          CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }}
          echo $CI_BRANCH_PUSH
          echo $CI_BRANCH_WORKFLOW_RUN
          echo $CI_SHA_PUSH
          echo $CI_SHA_WORKFLOW_RUN
          [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV
          [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV

      - name: print environment variables
        run: |
          echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}"
          echo "env.CI_SHA = ${{ env.CI_SHA }}"

      - name: Update clone using environment variables
        working-directory: /transformers
        run: |
          echo "original branch = $(git branch --show-current)"
          git fetch && git checkout ${{ env.CI_BRANCH }}
          echo "updated branch = $(git branch --show-current)"
          git checkout ${{ env.CI_SHA }}
          echo "log = $(git log -n 1)"

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Echo folder ${{ matrix.folders }}
        shell: bash
        # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
        # set the artifact folder names (because the character `/` is not allowed).
        run: |
          echo "${{ matrix.folders }}"
          echo "${{ fromJson(needs.setup.outputs.test_map)[matrix.folders] }}"
          matrix_folders=${{ matrix.folders }}
          matrix_folders=${matrix_folders/'models/'/'models_'}
          echo "$matrix_folders"
          echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all non-slow selected tests on GPU
        working-directory: /transformers
        run: |
          python3 -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} ${{ fromJson(needs.setup.outputs.test_map)[matrix.folders] }}

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}

  run_tests_multi_gpu:
    name: Model tests
    needs: setup
    # `dummy` means there is no test to run
    if: contains(fromJson(needs.setup.outputs.matrix), 'dummy') != true
    strategy:
      fail-fast: false
      matrix:
        folders: ${{ fromJson(needs.setup.outputs.matrix) }}
        machine_type: [multi-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, push-ci]
    container:
      image: huggingface/transformers-all-latest-gpu-push-ci
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      # Necessary to get the correct branch name and commit SHA for `workflow_run` event
      # We also take into account the `push` event (we might want to test some changes in a branch)
      - name: Prepare custom environment variables
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          CI_BRANCH_PUSH=${{ github.event.ref }}
          CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''}
          CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }}
          CI_SHA_PUSH=${{ github.event.head_commit.id }}
          CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }}
          echo $CI_BRANCH_PUSH
          echo $CI_BRANCH_WORKFLOW_RUN
          echo $CI_SHA_PUSH
          echo $CI_SHA_WORKFLOW_RUN
          [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV
          [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV

      - name: print environment variables
        run: |
          echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}"
          echo "env.CI_SHA = ${{ env.CI_SHA }}"

      - name: Update clone using environment variables
        working-directory: /transformers
        run: |
          echo "original branch = $(git branch --show-current)"
          git fetch && git checkout ${{ env.CI_BRANCH }}
          echo "updated branch = $(git branch --show-current)"
          git checkout ${{ env.CI_SHA }}
          echo "log = $(git log -n 1)"

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Echo folder ${{ matrix.folders }}
        shell: bash
        # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
        # set the artifact folder names (because the character `/` is not allowed).
        run: |
          echo "${{ matrix.folders }}"
          echo "${{ fromJson(needs.setup.outputs.test_map)[matrix.folders] }}"
          matrix_folders=${{ matrix.folders }}
          matrix_folders=${matrix_folders/'models/'/'models_'}
          echo "$matrix_folders"
          echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all non-slow selected tests on GPU
        env:
          MKL_SERVICE_FORCE_INTEL: 1
        working-directory: /transformers
        run: |
          python3 -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} ${{ fromJson(needs.setup.outputs.test_map)[matrix.folders] }}

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}

  run_tests_torch_cuda_extensions_single_gpu:
    name: Torch CUDA extension tests
    needs: setup
    if: contains(fromJson(needs.setup.outputs.matrix), 'deepspeed') || contains(fromJson(needs.setup.outputs.matrix), 'extended')
    strategy:
      fail-fast: false
      matrix:
        machine_type: [single-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, push-ci]
    container:
      image: huggingface/transformers-pytorch-deepspeed-latest-gpu-push-ci
      options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      # Necessary to get the correct branch name and commit SHA for `workflow_run` event
      # We also take into account the `push` event (we might want to test some changes in a branch)
      - name: Prepare custom environment variables
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          CI_BRANCH_PUSH=${{ github.event.ref }}
          CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''}
          CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }}
          CI_SHA_PUSH=${{ github.event.head_commit.id }}
          CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }}
          echo $CI_BRANCH_PUSH
          echo $CI_BRANCH_WORKFLOW_RUN
          echo $CI_SHA_PUSH
          echo $CI_SHA_WORKFLOW_RUN
          [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV
          [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV

      - name: print environment variables
        run: |
          echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}"
          echo "env.CI_SHA = ${{ env.CI_SHA }}"

      - name: Update clone using environment variables
        working-directory: /workspace/transformers
        run: |
          echo "original branch = $(git branch --show-current)"
          git fetch && git checkout ${{ env.CI_BRANCH }}
          echo "updated branch = $(git branch --show-current)"
          git checkout ${{ env.CI_SHA }}
          echo "log = $(git log -n 1)"

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /workspace/transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Remove cached torch extensions
        run: rm -rf /github/home/.cache/torch_extensions/

      # To avoid unknown test failures
      - name: Pre build DeepSpeed *again*
        working-directory: /workspace
        run: |
          python3 -m pip uninstall -y deepspeed
          DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install deepspeed --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /workspace/transformers
        run: |
          python utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /workspace/transformers
        run: pip freeze

      - name: Run all non-slow selected tests on GPU
        working-directory: /workspace/transformers
        # TODO: Here we pass all tests in the 2 folders for simplicity. It's better to pass only the identified tests.
        run: |
          python -m pytest -n 1 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
          path: /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports

  run_tests_torch_cuda_extensions_multi_gpu:
    name: Torch CUDA extension tests
    needs: setup
    if: contains(fromJson(needs.setup.outputs.matrix), 'deepspeed') || contains(fromJson(needs.setup.outputs.matrix), 'extended')
    strategy:
      fail-fast: false
      matrix:
        machine_type: [multi-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, push-ci]
    container:
      image: huggingface/transformers-pytorch-deepspeed-latest-gpu-push-ci
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      # Necessary to get the correct branch name and commit SHA for `workflow_run` event
      # We also take into account the `push` event (we might want to test some changes in a branch)
      - name: Prepare custom environment variables
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          CI_BRANCH_PUSH=${{ github.event.ref }}
          CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''}
          CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }}
          CI_SHA_PUSH=${{ github.event.head_commit.id }}
          CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }}
          echo $CI_BRANCH_PUSH
          echo $CI_BRANCH_WORKFLOW_RUN
          echo $CI_SHA_PUSH
          echo $CI_SHA_WORKFLOW_RUN
          [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV
          [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV

      - name: print environment variables
        run: |
          echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}"
          echo "env.CI_SHA = ${{ env.CI_SHA }}"

      - name: Update clone using environment variables
        working-directory: /workspace/transformers
        run: |
          echo "original branch = $(git branch --show-current)"
          git fetch && git checkout ${{ env.CI_BRANCH }}
          echo "updated branch = $(git branch --show-current)"
          git checkout ${{ env.CI_SHA }}
          echo "log = $(git log -n 1)"

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /workspace/transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Remove cached torch extensions
        run: rm -rf /github/home/.cache/torch_extensions/

      # To avoid unknown test failures
      - name: Pre build DeepSpeed *again*
        working-directory: /workspace
        run: |
          python3 -m pip uninstall -y deepspeed
          DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install deepspeed --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /workspace/transformers
        run: |
          python utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /workspace/transformers
        run: pip freeze

      - name: Run all non-slow selected tests on GPU
        working-directory: /workspace/transformers
        # TODO: Here we pass all tests in the 2 folders for simplicity. It's better to pass only the identified tests.
        run: |
          python -m pytest -n 1 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
          path: /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports

  send_results:
    name: Send results to webhook
    runs-on: ubuntu-22.04
    if: always()
    needs: [
        setup,
        run_tests_single_gpu,
        run_tests_multi_gpu,
        run_tests_torch_cuda_extensions_single_gpu,
        run_tests_torch_cuda_extensions_multi_gpu
    ]
    steps:
      - name: Preliminary job status
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          echo "Setup status: ${{ needs.setup.result }}"

      # Necessary to get the correct branch name and commit SHA for `workflow_run` event
      # We also take into account the `push` event (we might want to test some changes in a branch)
      - name: Prepare custom environment variables
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          CI_BRANCH_PUSH=${{ github.event.ref }}
          CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''}
          CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }}
          CI_SHA_PUSH=${{ github.event.head_commit.id }}
          CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }}
          echo $CI_BRANCH_PUSH
          echo $CI_BRANCH_WORKFLOW_RUN
          echo $CI_SHA_PUSH
          echo $CI_SHA_WORKFLOW_RUN
          [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV
          [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV

      - name: print environment variables
        run: |
          echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}"
          echo "env.CI_SHA = ${{ env.CI_SHA }}"

      - uses: actions/checkout@v4
        # To avoid failure when multiple commits are merged into `main` in a short period of time.
        # Checking out to an old commit beyond the fetch depth will get an error `fatal: reference is not a tree: ...
        # (Only required for `workflow_run` event, where we get the latest HEAD on `main` instead of the event commit)
        with:
          fetch-depth: 20

      - name: Update clone using environment variables
        run: |
          echo "original branch = $(git branch --show-current)"
          git fetch && git checkout ${{ env.CI_BRANCH }}
          echo "updated branch = $(git branch --show-current)"
          git checkout ${{ env.CI_SHA }}
          echo "log = $(git log -n 1)"

      - uses: actions/download-artifact@v4
      - name: Send message to Slack
        env:
          CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
          CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }}
          CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }}
          CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }}
          CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }}
          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
          CI_EVENT: push
          CI_TITLE_PUSH: ${{ github.event.head_commit.message }}
          CI_TITLE_WORKFLOW_RUN: ${{ github.event.workflow_run.head_commit.message }}
          CI_SHA: ${{ env.CI_SHA }}
          SETUP_STATUS: ${{ needs.setup.result }}

        # We pass `needs.setup.outputs.matrix` as the argument. A processing in `notification_service.py` to change
        # `models/bert` to `models_bert` is required, as the artifact names use `_` instead of `/`.
        run: |
          pip install slack_sdk
          pip show slack_sdk
          python utils/notification_service.py "${{ needs.setup.outputs.matrix }}"


Self-hosted runner (scheduled)

name: Self-hosted runner (scheduled)

# Note that each job's dependencies go into a corresponding docker file.
#
# For example for `run_torch_cuda_extensions_gpu` the docker image is
# `huggingface/transformers-pytorch-deepspeed-latest-gpu`, which can be found at
# `docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile`

on:
  workflow_call:
    inputs:
      job:
        required: true
        type: string
      slack_report_channel:
        required: true
        type: string
      runner:
        required: true
        type: string
      docker:
        required: true
        type: string
      ci_event:
        required: true
        type: string
      working-directory-prefix:
        default: ''
        required: false
        type: string

env:
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  OMP_NUM_THREADS: 8
  MKL_NUM_THREADS: 8
  RUN_SLOW: yes
  # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access.
  # This token is created under the bot `hf-transformers-bot`.
  HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
  SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
  TF_FORCE_GPU_ALLOW_GROWTH: true
  RUN_PT_TF_CROSS_TESTS: 1
  CUDA_VISIBLE_DEVICES: 0,1
  NUM_SLICES: 2

jobs:
  setup:
    if: contains(fromJSON('["run_models_gpu", "run_quantization_torch_gpu"]'), inputs.job)
    name: Setup
    strategy:
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, '${{ inputs.runner }}']
    container:
      image: huggingface/transformers-all-latest-gpu
      options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    outputs:
      folder_slices: ${{ steps.set-matrix.outputs.folder_slices }}
      slice_ids: ${{ steps.set-matrix.outputs.slice_ids }}
      quantization_matrix: ${{ steps.set-matrix-quantization.outputs.quantization_matrix }}
    steps:
      - name: Update clone
        working-directory: /transformers
        run: |
          git fetch && git checkout ${{ github.sha }}

      - name: Cleanup
        working-directory: /transformers
        run: |
          rm -rf tests/__pycache__
          rm -rf tests/models/__pycache__
          rm -rf reports

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - id: set-matrix
        if: ${{ inputs.job == 'run_models_gpu' }}
        name: Identify models to test
        working-directory: /transformers/tests
        run: |
          echo "folder_slices=$(python3 ../utils/split_model_tests.py --num_splits ${{ env.NUM_SLICES }})" >> $GITHUB_OUTPUT
          echo "slice_ids=$(python3 -c 'd = list(range(${{ env.NUM_SLICES }})); print(d)')" >> $GITHUB_OUTPUT

      - id: set-matrix-quantization
        if: ${{ inputs.job == 'run_quantization_torch_gpu' }}
        name: Identify quantization method to test
        working-directory: /transformers/tests
        run: |
          echo "quantization_matrix=$(python3 -c 'import os; tests = os.getcwd(); quantization_tests = os.listdir(os.path.join(tests, "quantization")); d = sorted(list(filter(os.path.isdir, [f"quantization/{x}" for x in quantization_tests]))) ;  print(d)')" >> $GITHUB_OUTPUT

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

  run_models_gpu:
    if: ${{ inputs.job == 'run_models_gpu' }}
    name: " "
    needs: setup
    strategy:
      fail-fast: false
      matrix:
        machine_type: [single-gpu, multi-gpu]
        slice_id: ${{ fromJSON(needs.setup.outputs.slice_ids) }}
    uses: ./.github/workflows/model_jobs.yml
    with:
      folder_slices: ${{ needs.setup.outputs.folder_slices }}
      machine_type: ${{ matrix.machine_type }}
      slice_id: ${{ matrix.slice_id }}
      runner: ${{ inputs.runner }}
      docker: ${{ inputs.docker }}
    secrets: inherit

  run_pipelines_torch_gpu:
    if: ${{ inputs.job == 'run_pipelines_torch_gpu' }}
    name: PyTorch pipelines
    strategy:
      fail-fast: false
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, '${{ inputs.runner }}']
    container:
      image: huggingface/transformers-pytorch-gpu
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all pipeline tests on GPU
        working-directory: /transformers
        run: |
          python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports tests/pipelines

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports

  run_pipelines_tf_gpu:
    if: ${{ inputs.job == 'run_pipelines_tf_gpu' }}
    name: TensorFlow pipelines
    strategy:
      fail-fast: false
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, '${{ inputs.runner }}']
    container:
      image: huggingface/transformers-tensorflow-gpu
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Update clone
        working-directory: /transformers
        run: |
          git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all pipeline tests on GPU
        working-directory: /transformers
        run: |
          python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports tests/pipelines

      - name: Failure short reports
        if: ${{ always() }}
        run: |
          cat /transformers/reports/${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports

  run_examples_gpu:
    if: ${{ inputs.job == 'run_examples_gpu' }}
    name: Examples directory
    strategy:
      fail-fast: false
      matrix:
        machine_type: [single-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, '${{ inputs.runner }}']
    container:
      image: huggingface/transformers-all-latest-gpu
      options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run examples tests on GPU
        working-directory: /transformers
        run: |
          pip install -r examples/pytorch/_tests_requirements.txt
          python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_examples_gpu_test_reports examples/pytorch

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_examples_gpu_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_examples_gpu_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_examples_gpu_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_examples_gpu_test_reports

  run_torch_cuda_extensions_gpu:
    if: ${{ inputs.job == 'run_torch_cuda_extensions_gpu' }}
    name: Torch CUDA extension tests
    strategy:
      fail-fast: false
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, '${{ inputs.runner }}']
    container:
      image: ${{ inputs.docker }}
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Update clone
        working-directory: ${{ inputs.working-directory-prefix }}/transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: ${{ inputs.working-directory-prefix }}/transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Update / Install some packages (for Past CI)
        if: ${{ contains(inputs.docker, '-past-') && contains(inputs.docker, '-pytorch-') }}
        working-directory: ${{ inputs.working-directory-prefix }}/transformers
        run: |
          python3 -m pip install -U datasets
          python3 -m pip install --no-cache-dir git+https://github.com/huggingface/accelerate@main#egg=accelerate

      - name: Remove cached torch extensions
        run: rm -rf /github/home/.cache/torch_extensions/

      # To avoid unknown test failures
      - name: Pre build DeepSpeed *again* (for daily CI)
        if: ${{ contains(inputs.ci_event, 'Daily CI') }}
        working-directory: ${{ inputs.working-directory-prefix }}/
        run: |
          python3 -m pip uninstall -y deepspeed
          DS_DISABLE_NINJA=1 DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install deepspeed --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check

      # To avoid unknown test failures
      - name: Pre build DeepSpeed *again* (for nightly & Past CI)
        if: ${{ contains(inputs.ci_event, 'Nightly CI') || contains(inputs.ci_event, 'Past CI') }}
        working-directory: ${{ inputs.working-directory-prefix }}/
        run: |
          python3 -m pip uninstall -y deepspeed
          rm -rf DeepSpeed
          git clone https://github.com/microsoft/DeepSpeed && cd DeepSpeed && rm -rf build
          DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install . --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: ${{ inputs.working-directory-prefix }}/transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: ${{ inputs.working-directory-prefix }}/transformers
        run: pip freeze

      - name: Run all tests on GPU
        working-directory: ${{ inputs.working-directory-prefix }}/transformers
        run: |
          python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat ${{ inputs.working-directory-prefix }}/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
          path: ${{ inputs.working-directory-prefix }}/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports

  run_quantization_torch_gpu:
    if: ${{ inputs.job == 'run_quantization_torch_gpu' }}
    name: " "
    needs: setup
    strategy:
      max-parallel: 4
      fail-fast: false
      matrix:
        folders: ${{ fromJson(needs.setup.outputs.quantization_matrix) }}
        machine_type: [single-gpu, multi-gpu]
    runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, '${{ inputs.runner }}']
    container:
      image: huggingface/transformers-quantization-latest-gpu
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Echo folder ${{ matrix.folders }}
        shell: bash
        run: |
          echo "${{ matrix.folders }}"
          matrix_folders=${{ matrix.folders }}
          matrix_folders=${matrix_folders/'quantization/'/'quantization_'}
          echo "$matrix_folders"
          echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV

      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run quantization tests on GPU
        working-directory: /transformers
        run: |
          python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ matrix.folders }}_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ env.matrix_folders }}_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ env.matrix_folders }}_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ matrix.folders }}_test_reports

  run_extract_warnings:
    # Let's only do this for the job `run_models_gpu` to simplify the (already complex) logic.
    if: ${{ always() && inputs.job == 'run_models_gpu' }}
    name: Extract warnings in CI artifacts
    runs-on: ubuntu-22.04
    needs: [setup, run_models_gpu]
    steps:
      - name: Checkout transformers
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Install transformers
        run: pip install transformers

      - name: Show installed libraries and their versions
        run: pip freeze

      - name: Create output directory
        run: mkdir warnings_in_ci

      - uses: actions/download-artifact@v4
        with:
          path: warnings_in_ci

      - name: Show artifacts
        run: echo "$(python3 -c 'import os; d = os.listdir(); print(d)')"
        working-directory: warnings_in_ci

      - name: Extract warnings in CI artifacts
        run: |
          python3 utils/extract_warnings.py --workflow_run_id ${{ github.run_id }} --output_dir warnings_in_ci --token ${{ secrets.ACCESS_REPO_INFO_TOKEN }} --from_gh
          echo "$(python3 -c 'import os; import json; fp = open("warnings_in_ci/selected_warnings.json"); d = json.load(fp); d = "\n".join(d) ;print(d)')"

      - name: Upload artifact
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: warnings_in_ci
          path: warnings_in_ci/selected_warnings.json

  send_results:
    name: Slack Report
    needs: [
      setup,
      run_models_gpu,
      run_pipelines_torch_gpu,
      run_pipelines_tf_gpu,
      run_examples_gpu,
      run_torch_cuda_extensions_gpu,
      run_quantization_torch_gpu,
      run_extract_warnings
    ]
    if: ${{ always() }}
    uses: ./.github/workflows/slack-report.yml
    with:
      job: ${{ inputs.job }}
      # This would be `skipped` if `setup` is skipped.
      setup_status: ${{ needs.setup.result }}
      slack_report_channel: ${{ inputs.slack_report_channel }}
      # This would be an empty string if `setup` is skipped.
      folder_slices: ${{ needs.setup.outputs.folder_slices }}
      quantization_matrix: ${{ needs.setup.outputs.quantization_matrix }}
      ci_event: ${{ inputs.ci_event }}

    secrets: inherit


Release - Conda

name: Release - Conda

on:
  push:
    tags:
      - v*
    branches:
      - conda_*

env:
  ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}

jobs:
  build_and_package:
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash -l {0}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v1

      - name: Install miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          auto-update-conda: true
          auto-activate-base: false
          python-version: 3.8
          activate-environment: "build-transformers"
          channels: huggingface

      - name: Setup conda env
        run: |
          conda install -c defaults anaconda-client conda-build

      - name: Extract version
        run: echo "TRANSFORMERS_VERSION=`python setup.py --version`" >> $GITHUB_ENV

      - name: Build conda packages
        run: |
          conda info
          conda list
          conda-build .github/conda

      - name: Upload to Anaconda
        run: anaconda upload `conda-build .github/conda --output` --force


Stale Bot

name: Stale Bot

on:
  schedule:
    - cron: "0 8 * * *"

jobs:
  close_stale_issues:
    name: Close Stale Issues
    if: github.repository == 'huggingface/transformers'
    runs-on: ubuntu-22.04
    env:
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
    - uses: actions/checkout@v4

    - name: Setup Python
      uses: actions/setup-python@v4
      with:
        python-version: 3.8

    - name: Install requirements
      run: |
        pip install PyGithub
    - name: Close stale issues
      run: |
        python scripts/stale.py


Doctests

name: Doctests

on:
  push:
    branches:
      - run_doctest*
  repository_dispatch:
  schedule:
    - cron: "17 2 * * *"

env:
  NUM_SLICES: 3

jobs:
  setup:
    name: Setup
    runs-on: [single-gpu, nvidia-gpu, t4, ci]
    container:
      image: huggingface/transformers-all-latest-gpu
      options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    outputs:
      job_splits: ${{ steps.set-matrix.outputs.job_splits }}
      split_keys: ${{ steps.set-matrix.outputs.split_keys }}
    steps:
      - name: Update clone
        working-directory: /transformers
        run: |
          git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Check values for matrix
        working-directory: /transformers
        run: |
          python3 utils/split_doctest_jobs.py
          python3 utils/split_doctest_jobs.py --only_return_keys --num_splits ${{ env.NUM_SLICES }}

      - id: set-matrix
        working-directory: /transformers
        name: Set values for matrix
        run: |
          echo "job_splits=$(python3 utils/split_doctest_jobs.py)" >> $GITHUB_OUTPUT
          echo "split_keys=$(python3 utils/split_doctest_jobs.py --only_return_keys --num_splits ${{ env.NUM_SLICES }})" >> $GITHUB_OUTPUT

  call_doctest_job:
    name: "Call doctest jobs"
    needs: setup
    strategy:
      max-parallel: 1  # 1 split at a time (in `doctest_job.yml`, we set `8` to run 8 jobs at the same time)
      fail-fast: false
      matrix:
        split_keys: ${{ fromJson(needs.setup.outputs.split_keys) }}
    uses: ./.github/workflows/doctest_job.yml
    with:
      job_splits: ${{ needs.setup.outputs.job_splits }}
      split_keys: ${{ toJson(matrix.split_keys) }}
    secrets: inherit

  send_results:
    name: Send results to webhook
    runs-on: ubuntu-22.04
    if: always()
    needs: [call_doctest_job]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
      - name: Send message to Slack
        env:
          CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
          # Use `CI_SLACK_CHANNEL_DUMMY_TESTS` when doing experimentation
          SLACK_REPORT_CHANNEL: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY_DOCS }}
        run: |
          pip install slack_sdk
          python utils/notification_service_doc_tests.py

      - name: "Upload results"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: doc_test_results
          path: doc_test_results


Build documentation

name: Build documentation

on:
  push:
    branches:
      - main
      - doc-builder*
      - v*-release
      - use_templates

jobs:
   build:
    uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main
    with:
      commit_sha: ${{ github.sha }}
      package: transformers
      notebook_folder: transformers_doc
      languages: de en es fr hi it ko pt tr zh ja te
      custom_container: huggingface/transformers-doc-builder
    secrets:
      token: ${{ secrets.HUGGINGFACE_PUSH }}
      hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}


Update Transformers metadata

name: Update Transformers metadata

on:
  push:
    branches:
      - main
      - update_transformers_metadata*

jobs:
  build_and_package:
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash -l {0}

    steps:
      - uses: actions/checkout@v4

      - name: Setup environment
        run: |
          pip install --upgrade pip
          pip install datasets pandas==2.0.3
          pip install .[torch,tf,flax]

      - name: Update metadata
        run: |
          python utils/update_metadata.py --token ${{ secrets.LYSANDRE_HF_TOKEN }} --commit_sha ${{ github.sha }}


Add model like runner

name: Add model like runner

on:
  push:
    branches:
      - none # put main here when this is fixed
  #pull_request:
  #  paths:
  #    - "src/**"
  #    - "tests/**"
  #    - ".github/**"
  #  types: [opened, synchronize, reopened]

jobs:
  run_tests_templates_like:
    name: "Add new model like template tests"
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4

      - name: Install dependencies
        run: |
          sudo apt -y update && sudo apt install -y libsndfile1-dev

      - name: Load cached virtual environment
        uses: actions/cache@v2
        id: cache
        with:
          path: ~/venv/
          key: v4-tests_model_like-${{ hashFiles('setup.py') }}

      - name: Create virtual environment on cache miss
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m venv ~/venv && . ~/venv/bin/activate
          pip install --upgrade pip!=21.3
          pip install -e .[dev]

      - name: Check transformers location
        # make `transformers` available as package (required since we use `-e` flag) and check it's indeed from the repo.
        run: |
          . ~/venv/bin/activate
          python setup.py develop
          transformers_install=$(pip list -e | grep transformers)
          transformers_install_array=($transformers_install)
          transformers_loc=${transformers_install_array[-1]}
          transformers_repo_loc=$(pwd .)
          if [ "$transformers_loc" != "$transformers_repo_loc" ]; then
              echo "transformers is from $transformers_loc but it shoud be from $transformers_repo_loc/src."
              echo "A fix is required. Stop testing."
              exit 1
          fi

      - name: Create model files
        run: |
          . ~/venv/bin/activate
          transformers-cli add-new-model-like --config_file tests/fixtures/add_distilbert_like_config.json --path_to_repo .
          make style
          make fix-copies

      - name: Run all PyTorch modeling test
        run: |
          . ~/venv/bin/activate
          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_new_models tests/bert_new/test_modeling_bert_new.py

      - name: Run style changes
        run: |
          . ~/venv/bin/activate
          make style && make quality && make repo-consistency

      - name: Failure short reports
        if: ${{ always() }}
        run: cat reports/tests_new_models/failures_short.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: run_all_tests_new_models_test_reports
          path: reports/tests_new_models


Build docker images (scheduled)

name: Build docker images (scheduled)

on:
  push:
    branches:
      - build_ci_docker_image*
  repository_dispatch:
  workflow_call:
    inputs:
      image_postfix:
        required: true
        type: string
  schedule:
    - cron: "17 0 * * *"

concurrency:
  group: docker-images-builds
  cancel-in-progress: false

jobs:
  latest-docker:
    name: "Latest PyTorch + TensorFlow [dev]"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-all-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-all-latest-gpu${{ inputs.image_postfix }}
      # Push CI images still need to be re-built daily
      -
        name: Build and push (for Push CI) in a daily basis
        # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
        # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
        if: inputs.image_postfix != '-push-ci'
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-all-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-all-latest-gpu-push-ci

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.CI_SLACK_CHANNEL_DOCKER }}
          title: 🤗 Results of the transformers-all-latest-gpu-push-ci docker build
          status: ${{ job.status }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

  latest-torch-deepspeed-docker:
    name: "Latest PyTorch + DeepSpeed"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-deepspeed-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-deepspeed-latest-gpu${{ inputs.image_postfix }}

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.CI_SLACK_CHANNEL_DOCKER}}
          title: 🤗 Results of the transformers-pytorch-deepspeed-latest-gpu docker build
          status: ${{ job.status }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

  # Can't build 2 images in a single job `latest-torch-deepspeed-docker` (for `nvcr.io/nvidia`)
  latest-torch-deepspeed-docker-for-push-ci-daily-build:
    name: "Latest PyTorch + DeepSpeed (Push CI - Daily Build)"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      # Push CI images still need to be re-built daily
      -
        name: Build and push (for Push CI) in a daily basis
        # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
        # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
        if: inputs.image_postfix != '-push-ci'
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-deepspeed-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-deepspeed-latest-gpu-push-ci

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.CI_SLACK_CHANNEL_DOCKER }}
          title: 🤗 Results of the transformers-pytorch-deepspeed-latest-gpu-push-ci docker build
          status: ${{ job.status }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

  doc-builder:
    name: "Doc builder"
    # Push CI doesn't need this image
    if: inputs.image_postfix != '-push-ci'
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-doc-builder
          push: true
          tags: huggingface/transformers-doc-builder

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.CI_SLACK_CHANNEL_DOCKER }}
          title: 🤗 Results of the huggingface/transformers-doc-builder docker build
          status: ${{ job.status }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

  latest-pytorch:
    name: "Latest PyTorch [dev]"
    # Push CI doesn't need this image
    if: inputs.image_postfix != '-push-ci'
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-gpu

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.CI_SLACK_CHANNEL_DOCKER }}
          title: 🤗 Results of the huggingface/transformers-pytorch-gpudocker build
          status: ${{ job.status }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

  latest-pytorch-amd:
    name: "Latest PyTorch (AMD) [dev]"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-amd-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-amd-gpu${{ inputs.image_postfix }}
      # Push CI images still need to be re-built daily
      -
        name: Build and push (for Push CI) in a daily basis
        # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
        # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
        if: inputs.image_postfix != '-push-ci'
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-amd-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-amd-gpu-push-ci

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.CI_SLACK_CHANNEL_DOCKER }}
          title: 🤗 Results of the huggingface/transformers-pytorch-amd-gpu-push-ci build
          status: ${{ job.status }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

  latest-tensorflow:
    name: "Latest TensorFlow [dev]"
    # Push CI doesn't need this image
    if: inputs.image_postfix != '-push-ci'
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-tensorflow-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-tensorflow-gpu

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.CI_SLACK_CHANNEL_DOCKER }}
          title: 🤗 Results of the huggingface/transformers-tensorflow-gpu build
          status: ${{ job.status }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

  latest-pytorch-deepspeed-amd:
    name: "PyTorch + DeepSpeed (AMD) [dev]"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-deepspeed-amd-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-deepspeed-amd-gpu${{ inputs.image_postfix }}
      # Push CI images still need to be re-built daily
      -
        name: Build and push (for Push CI) in a daily basis
        # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
        # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
        if: inputs.image_postfix != '-push-ci'
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-deepspeed-amd-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-deepspeed-amd-gpu-push-ci

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.CI_SLACK_CHANNEL_DOCKER }}
          title: 🤗 Results of the transformers-pytorch-deepspeed-amd-gpu build
          status: ${{ job.status }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

  latest-quantization-torch-docker:
    name: "Latest Pytorch + Quantization [dev]"
     # Push CI doesn't need this image
    if: inputs.image_postfix != '-push-ci'
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-quantization-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-quantization-latest-gpu${{ inputs.image_postfix }}

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ secrets.CI_SLACK_CHANNEL_DOCKER }}
          title: 🤗 Results of the transformers-quantization-latest-gpu build
          status: ${{ job.status }}
          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}


Build PR Documentation

name: Build PR Documentation

on:
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  build:
    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main
    with:
      commit_sha: ${{ github.event.pull_request.head.sha }}
      pr_number: ${{ github.event.number }}
      package: transformers
      languages: de en es fr hi it ko pt tr zh ja te
      custom_container: huggingface/transformers-doc-builder


Self-hosted runner (push-caller)

# Used to trigger self-push CI
name: Self-hosted runner (push-caller)

on:
  push:
    branches:
      - main
    paths:
      - "src/**"
      - "tests/**"
      - ".github/**"
      - "templates/**"
      - "utils/**"

jobs:
  check-for-setup:
      runs-on: ubuntu-22.04
      name: Check if setup was changed
      outputs:
        changed: ${{ steps.was_changed.outputs.changed }}
      steps:
        - uses: actions/checkout@v4
          with:
            fetch-depth: "2"

        - name: Get changed files
          id: changed-files
          uses: tj-actions/changed-files@v41

        - name: Was setup changed
          id: was_changed
          run: |
            for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
              if [ `basename "${file}"` = "setup.py" ]; then
                echo "changed=1" >> $GITHUB_OUTPUT
              fi
            done

  build-docker-containers:
    needs: check-for-setup
    if: (github.event_name == 'push') && (needs.check-for-setup.outputs.changed == '1')
    uses: ./.github/workflows/build-docker-images.yml
    with:
      image_postfix: "-push-ci"
    secrets: inherit

  run_push_ci:
    name: Trigger Push CI
    runs-on: ubuntu-22.04
    if: ${{ always() }}
    needs: build-docker-containers
    steps:
      - name: Trigger push CI via workflow_run
        run: echo "Trigger push CI via workflow_run"


Build docker images (Past CI)

name: Build docker images (Past CI)

on:
  push:
    branches:
      - build_past_ci_docker_image*

concurrency:
  group: docker-images-builds
  cancel-in-progress: false

jobs:
  past-pytorch-docker:
    name: "Past PyTorch Docker"
    strategy:
      fail-fast: false
      matrix:
        version: ["1.13", "1.12", "1.11"]
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        id: get-base-image
        name: Get Base Image
        env:
          framework_version: ${{ matrix.version }}
        run: |
          echo "base_image=$(python3 -c 'import os; from utils.past_ci_versions import past_versions_testing; base_image = past_versions_testing["pytorch"][os.environ["framework_version"]]["base_image"]; print(base_image)')" >> $GITHUB_OUTPUT
      -
        name: Print Base Image
        run: |
          echo ${{ steps.get-base-image.outputs.base_image }}
      -
        name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v3
        with:
          context: ./docker/transformers-past-gpu
          build-args: |
            REF=main
            BASE_DOCKER_IMAGE=${{ steps.get-base-image.outputs.base_image }}
            FRAMEWORK=pytorch
            VERSION=${{ matrix.version }}
          push: true
          tags: huggingface/transformers-pytorch-past-${{ matrix.version }}-gpu

  past-tensorflow-docker:
    name: "Past TensorFlow Docker"
    strategy:
      fail-fast: false
      matrix:
        version: ["2.11", "2.10", "2.9", "2.8", "2.7", "2.6", "2.5"]
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        id: get-base-image
        name: Get Base Image
        env:
          framework_version: ${{ matrix.version }}
        run: |
          echo "base_image=$(python3 -c 'import os; from utils.past_ci_versions import past_versions_testing; base_image = past_versions_testing["tensorflow"][os.environ["framework_version"]]["base_image"]; print(base_image)')" >> $GITHUB_OUTPUT
      -
        name: Print Base Image
        run: |
          echo ${{ steps.get-base-image.outputs.base_image }}
      -
        name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v3
        with:
          context: ./docker/transformers-past-gpu
          build-args: |
            REF=main
            BASE_DOCKER_IMAGE=${{ steps.get-base-image.outputs.base_image }}
            FRAMEWORK=tensorflow
            VERSION=${{ matrix.version }}
          push: true
          tags: huggingface/transformers-tensorflow-past-${{ matrix.version }}-gpu


Self-hosted runner (past-ci)

name: Self-hosted runner (past-ci)

on:
  workflow_call:
    inputs:
      framework:
        required: true
        type: string
      version:
        required: true
        type: string
      # Use this to control the commit to test against
      sha:
        default: 'main'
        required: false
        type: string

jobs:
  model-ci:
    name: Model CI
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_models_gpu
      slack_report_channel: "#transformers-ci-past-future"
      runner: past-ci
      docker: huggingface/transformers-${{ inputs.framework }}-past-${{ inputs.version }}-gpu
      ci_event: Past CI - ${{ inputs.framework }}-${{ inputs.version }}
    secrets: inherit

  deepspeed-ci:
    name: DeepSpeed CI
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_torch_cuda_extensions_gpu
      slack_report_channel: "#transformers-ci-past-future"
      runner: past-ci
      docker: huggingface/transformers-${{ inputs.framework }}-past-${{ inputs.version }}-gpu
      ci_event: Past CI - ${{ inputs.framework }}-${{ inputs.version }}
    secrets: inherit


name: Self-hosted runner (nightly-past-ci-caller)

on:
  schedule:
    - cron: "17 2,14 * * *"
  push:
    branches:
      - run_past_ci*

jobs:
  get_number:
    name: Get number
    runs-on: ubuntu-22.04
    outputs:
      run_number: ${{ steps.get_number.outputs.run_number }}
    steps:
      - name: Get number
        id: get_number
        run: |
          echo "${{ github.run_number }}"
          echo "$(python3 -c 'print(int(${{ github.run_number }}) % 10)')"
          echo "run_number=$(python3 -c 'print(int(${{ github.run_number }}) % 10)')" >> $GITHUB_OUTPUT

  run_past_ci_pytorch_1-13:
    name: PyTorch 1.13
    needs: get_number
    if: needs.get_number.outputs.run_number == 0 && (cancelled() != true) && ((github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: pytorch
      version: "1.13"
      sha: ${{ github.sha }}
    secrets: inherit

  run_past_ci_pytorch_1-12:
    name: PyTorch 1.12
    needs: get_number
    if: needs.get_number.outputs.run_number == 1 && (cancelled() != true) && ((github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: pytorch
      version: "1.12"
      sha: ${{ github.sha }}
    secrets: inherit

  run_past_ci_pytorch_1-11:
    name: PyTorch 1.11
    needs: get_number
    if: needs.get_number.outputs.run_number == 2 && (cancelled() != true) && ((github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: pytorch
      version: "1.11"
      sha: ${{ github.sha }}
    secrets: inherit

  run_past_ci_tensorflow_2-11:
    name: TensorFlow 2.11
    needs: get_number
    if: needs.get_number.outputs.run_number == 3 && (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: tensorflow
      version: "2.11"
      sha: ${{ github.sha }}
    secrets: inherit

  run_past_ci_tensorflow_2-10:
    name: TensorFlow 2.10
    needs: get_number
    if: needs.get_number.outputs.run_number == 4 && (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: tensorflow
      version: "2.10"
      sha: ${{ github.sha }}
    secrets: inherit

  run_past_ci_tensorflow_2-9:
    name: TensorFlow 2.9
    needs: get_number
    if: needs.get_number.outputs.run_number == 5 && (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: tensorflow
      version: "2.9"
      sha: ${{ github.sha }}
    secrets: inherit

  run_past_ci_tensorflow_2-8:
    name: TensorFlow 2.8
    needs: get_number
    if: needs.get_number.outputs.run_number == 6 && (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: tensorflow
      version: "2.8"
      sha: ${{ github.sha }}
    secrets: inherit

  run_past_ci_tensorflow_2-7:
    name: TensorFlow 2.7
    needs: get_number
    if: needs.get_number.outputs.run_number == 7 && (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: tensorflow
      version: "2.7"
      sha: ${{ github.sha }}
    secrets: inherit

  run_past_ci_tensorflow_2-6:
    name: TensorFlow 2.6
    needs: get_number
    if: needs.get_number.outputs.run_number == 8 && (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: tensorflow
      version: "2.6"
      sha: ${{ github.sha }}
    secrets: inherit

  run_past_ci_tensorflow_2-5:
    name: TensorFlow 2.5
    needs: get_number
    if: needs.get_number.outputs.run_number == 9 &&  (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))
    uses: ./.github/workflows/self-past-caller.yml
    with:
      framework: tensorflow
      version: "2.5"
      sha: ${{ github.sha }}
    secrets: inherit