Build docker images (Nightly CI)

name: Build docker images (Nightly CI)

on:
  workflow_call:
  push:
    branches:
      - build_nightly_ci_docker_image*

concurrency:
  group: docker-images-builds
  cancel-in-progress: false

jobs:
  latest-with-torch-nightly-docker:
    name: "Nightly PyTorch + Stable TensorFlow"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v3
        with:
          context: ./docker/transformers-all-latest-gpu
          build-args: |
            REF=main
            PYTORCH=pre
          push: true
          tags: huggingface/transformers-all-latest-torch-nightly-gpu

  nightly-torch-deepspeed-docker:
    name: "Nightly PyTorch + DeepSpeed"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v3
        with:
          context: ./docker/transformers-pytorch-deepspeed-nightly-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-deepspeed-nightly-gpu


Check Tiny Models

name: Check Tiny Models

on:
  push:
    branches:
      - check_tiny_models*
  repository_dispatch:
  schedule:
    - cron: "0 2 * * *"

env:
  TOKEN: ${{ secrets.TRANSFORMERS_HUB_BOT_HF_TOKEN }}

jobs:
  check_tiny_models:
    name: Check tiny models
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout transformers
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - uses: actions/checkout@v4
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          # Semantic version range syntax or exact version of a Python version
          python-version: '3.8'
          # Optional - x64 or x86 architecture, defaults to x64
          architecture: 'x64'

      - name: Install
        run: |
          sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake
          pip install --upgrade pip
          python -m pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video,tf-cpu]
          pip install tensorflow_probability
          python -m pip install -U 'natten<0.15.0'

      - name: Create all tiny models (locally)
        run: |
          python utils/create_dummy_models.py tiny_local_models --all --num_workers 2

      - name: Local tiny model reports artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: tiny_local_model_creation_reports
          path: tiny_local_models/reports

      # GitHub-hosted runners have 2-core CPUs
      - name: Run pipeline tests against all new (local) tiny models
        run: |
          OMP_NUM_THREADS=1 TRANSFORMERS_TINY_MODEL_PATH=tiny_local_models python -m pytest --max-worker-restart=0 -n 2 --dist=loadfile -s -rA --make-reports=tests_pipelines tests/models -m is_pipeline_test -k "test_pipeline_" | tee tests_output.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: tiny_local_model_creation_reports
          path: reports/tests_pipelines

      - name: Create + Upload tiny models for new model architecture(s)
        run: |
          python utils/update_tiny_models.py --num_workers 2

      - name: Full report
        run: cat tiny_models/reports/tiny_model_creation_report.json

      - name: Failure report
        run: cat tiny_models/reports/simple_failed_report.txt

      - name: Summary report
        run: cat tiny_models/reports/tiny_model_summary.json

      - name: New tiny model creation reports artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: tiny_model_creation_reports
          path: tiny_models/reports


Upload PR Documentation

name: Upload PR Documentation

on:
  workflow_run:
    workflows: ["Build PR Documentation"]
    types:
      - completed

jobs:
  build:
    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main
    with:
      package_name: transformers
    secrets:
      hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}
      comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }}


Self-hosted runner AMD GPU (push)

name: Self-hosted runner AMD GPU (push)

on:
  workflow_call:
    inputs:
      gpu_flavor:
        required: true
        type: string

env:
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  OMP_NUM_THREADS: 8
  MKL_NUM_THREADS: 8
  PYTEST_TIMEOUT: 60
  TF_FORCE_GPU_ALLOW_GROWTH: true
  RUN_PT_TF_CROSS_TESTS: 1
  HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}

jobs:
  check_runner_status:
    name: Check Runner Status
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout transformers
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Check Runner Status
        run: python utils/check_self_hosted_runner.py --target_runners amd-mi210-single-gpu-ci-runner-docker --token ${{ secrets.ACCESS_REPO_INFO_TOKEN }}

  check_runners:
    name: Check Runners
    needs: check_runner_status
    strategy:
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    container:
      image: huggingface/transformers-pytorch-amd-gpu-push-ci  # <--- We test only for PyTorch for now
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: ROCM-SMI
        run: |
          rocm-smi
      - name: ROCM-INFO
        run: |
          rocminfo  | grep "Agent" -A 14
      - name: Show ROCR environment
        run: |
          echo "ROCR: $ROCR_VISIBLE_DEVICES"

  setup_gpu:
    name: Setup
    needs: check_runners
    strategy:
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    container:
      image: huggingface/transformers-pytorch-amd-gpu-push-ci  # <--- We test only for PyTorch for now
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      test_map: ${{ steps.set-matrix.outputs.test_map }}
    steps:
      # Necessary to get the correct branch name and commit SHA for `workflow_run` event
      # We also take into account the `push` event (we might want to test some changes in a branch)
      - name: Prepare custom environment variables
        shell: bash
        # `CI_BRANCH_PUSH`: The branch name from the push event
        # `CI_BRANCH_WORKFLOW_RUN`: The name of the branch on which this workflow is triggered by `workflow_run` event
        # `CI_BRANCH`: The non-empty branch name from the above two (one and only one of them is empty)
        # `CI_SHA_PUSH`: The commit SHA from the push event
        # `CI_SHA_WORKFLOW_RUN`: The commit SHA that triggers this workflow by `workflow_run` event
        # `CI_SHA`: The non-empty commit SHA from the above two (one and only one of them is empty)
        run: |
          CI_BRANCH_PUSH=${{ github.event.ref }}
          CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''}
          CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }}
          CI_SHA_PUSH=${{ github.event.head_commit.id }}
          CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }}
          echo $CI_BRANCH_PUSH
          echo $CI_BRANCH_WORKFLOW_RUN
          echo $CI_SHA_PUSH
          echo $CI_SHA_WORKFLOW_RUN
          [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV
          [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV

      - name: print environment variables
        run: |
          echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}"
          echo "env.CI_SHA = ${{ env.CI_SHA }}"

      - name: Update clone using environment variables
        working-directory: /transformers
        run: |
          echo "original branch = $(git branch --show-current)"
          git fetch && git checkout ${{ env.CI_BRANCH }}
          echo "updated branch = $(git branch --show-current)"
          git checkout ${{ env.CI_SHA }}
          echo "log = $(git log -n 1)"

      - name: Cleanup
        working-directory: /transformers
        run: |
          rm -rf tests/__pycache__
          rm -rf tests/models/__pycache__
          rm -rf reports

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Fetch the tests to run
        working-directory: /transformers
        # TODO: add `git-python` in the docker images
        run: |
          pip install --upgrade git-python
          python3 utils/tests_fetcher.py --diff_with_last_commit | tee test_preparation.txt

      - name: Report fetched tests
        uses: actions/upload-artifact@v4
        with:
          name: test_fetched
          path: /transformers/test_preparation.txt

      - id: set-matrix
        name: Organize tests into models
        working-directory: /transformers
        # The `keys` is used as GitHub actions matrix for jobs, i.e. `models/bert`, `tokenization`, `pipeline`, etc.
        # The `test_map` is used to get the actual identified test files under each key.
        # If no test to run (so no `test_map.json` file), create a dummy map (empty matrix will fail)
        run: |
          if [ -f test_map.json ]; then
              keys=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); d = list(test_map.keys()); print(d)')
              test_map=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); print(test_map)')
          else
              keys=$(python3 -c 'keys = ["dummy"]; print(keys)')
              test_map=$(python3 -c 'test_map = {"dummy": []}; print(test_map)')
          fi
          echo $keys
          echo $test_map
          echo "matrix=$keys" >> $GITHUB_OUTPUT
          echo "test_map=$test_map" >> $GITHUB_OUTPUT

  run_models_gpu:
    name: Model tests
    needs: setup_gpu
    # `dummy` means there is no test to run
    if: contains(fromJson(needs.setup_gpu.outputs.matrix), 'dummy') != true
    strategy:
      fail-fast: false
      matrix:
        folders: ${{ fromJson(needs.setup_gpu.outputs.matrix) }}
        machine_type: [single-gpu, multi-gpu]
    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    container:
      image: huggingface/transformers-pytorch-amd-gpu-push-ci  # <--- We test only for PyTorch for now
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      # Necessary to get the correct branch name and commit SHA for `workflow_run` event
      # We also take into account the `push` event (we might want to test some changes in a branch)
      - name: Prepare custom environment variables
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          CI_BRANCH_PUSH=${{ github.event.ref }}
          CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''}
          CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }}
          CI_SHA_PUSH=${{ github.event.head_commit.id }}
          CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }}
          echo $CI_BRANCH_PUSH
          echo $CI_BRANCH_WORKFLOW_RUN
          echo $CI_SHA_PUSH
          echo $CI_SHA_WORKFLOW_RUN
          [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV
          [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV

      - name: print environment variables
        run: |
          echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}"
          echo "env.CI_SHA = ${{ env.CI_SHA }}"

      - name: Update clone using environment variables
        working-directory: /transformers
        run: |
          echo "original branch = $(git branch --show-current)"
          git fetch && git checkout ${{ env.CI_BRANCH }}
          echo "updated branch = $(git branch --show-current)"
          git checkout ${{ env.CI_SHA }}
          echo "log = $(git log -n 1)"

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Echo folder ${{ matrix.folders }}
        shell: bash
        # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
        # set the artifact folder names (because the character `/` is not allowed).
        run: |
          echo "${{ matrix.folders }}"
          echo "${{ fromJson(needs.setup_gpu.outputs.test_map)[matrix.folders] }}"
          matrix_folders=${{ matrix.folders }}
          matrix_folders=${matrix_folders/'models/'/'models_'}
          echo "$matrix_folders"
          echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV

      - name: ROCM-SMI
        run: |
          rocm-smi
      - name: ROCM-INFO
        run: |
          rocminfo  | grep "Agent" -A 14
      - name: Show ROCR environment
        run: |
          echo "ROCR: $ROCR_VISIBLE_DEVICES"

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all non-slow selected tests on GPU
        working-directory: /transformers
        run: |
          python3 -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports ${{ fromJson(needs.setup_gpu.outputs.test_map)[matrix.folders] }} -m "not not_device_test"

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports

  send_results:
    name: Send results to webhook
    runs-on: ubuntu-22.04
    if: always()
    needs: [
        check_runner_status,
        check_runners,
        setup_gpu,
        run_models_gpu,
#        run_tests_torch_cuda_extensions_single_gpu,
#        run_tests_torch_cuda_extensions_multi_gpu
    ]
    steps:
      - name: Preliminary job status
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          echo "Runner availability: ${{ needs.check_runner_status.result }}"
          echo "Setup status: ${{ needs.setup_gpu.result }}"
          echo "Runner status: ${{ needs.check_runners.result }}"

      # Necessary to get the correct branch name and commit SHA for `workflow_run` event
      # We also take into account the `push` event (we might want to test some changes in a branch)
      - name: Prepare custom environment variables
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          CI_BRANCH_PUSH=${{ github.event.ref }}
          CI_BRANCH_PUSH=${CI_BRANCH_PUSH/'refs/heads/'/''}
          CI_BRANCH_WORKFLOW_RUN=${{ github.event.workflow_run.head_branch }}
          CI_SHA_PUSH=${{ github.event.head_commit.id }}
          CI_SHA_WORKFLOW_RUN=${{ github.event.workflow_run.head_sha }}
          echo $CI_BRANCH_PUSH
          echo $CI_BRANCH_WORKFLOW_RUN
          echo $CI_SHA_PUSH
          echo $CI_SHA_WORKFLOW_RUN
          [[ ! -z "$CI_BRANCH_PUSH" ]] && echo "CI_BRANCH=$CI_BRANCH_PUSH" >> $GITHUB_ENV || echo "CI_BRANCH=$CI_BRANCH_WORKFLOW_RUN" >> $GITHUB_ENV
          [[ ! -z "$CI_SHA_PUSH" ]] && echo "CI_SHA=$CI_SHA_PUSH" >> $GITHUB_ENV || echo "CI_SHA=$CI_SHA_WORKFLOW_RUN" >> $GITHUB_ENV

      - name: print environment variables
        run: |
          echo "env.CI_BRANCH = ${{ env.CI_BRANCH }}"
          echo "env.CI_SHA = ${{ env.CI_SHA }}"

      - uses: actions/checkout@v4
        # To avoid failure when multiple commits are merged into `main` in a short period of time.
        # Checking out to an old commit beyond the fetch depth will get an error `fatal: reference is not a tree: ...
        # (Only required for `workflow_run` event, where we get the latest HEAD on `main` instead of the event commit)
        with:
          fetch-depth: 20

      - name: Update clone using environment variables
        run: |
          echo "original branch = $(git branch --show-current)"
          git fetch && git checkout ${{ env.CI_BRANCH }}
          echo "updated branch = $(git branch --show-current)"
          git checkout ${{ env.CI_SHA }}
          echo "log = $(git log -n 1)"

      - uses: actions/download-artifact@v4
      - name: Send message to Slack
        env:
          CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
          CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }}
          CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }}
          CI_SLACK_CHANNEL_ID_AMD: ${{ secrets.CI_SLACK_CHANNEL_ID_AMD }}
          CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }}
          CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_AMD }}
          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
          CI_EVENT: Push CI (AMD) - ${{ inputs.gpu_flavor }}
          CI_TITLE_PUSH: ${{ github.event.head_commit.message }}
          CI_TITLE_WORKFLOW_RUN: ${{ github.event.workflow_run.head_commit.message }}
          CI_SHA: ${{ env.CI_SHA }}
          RUNNER_STATUS: ${{ needs.check_runner_status.result }}
          RUNNER_ENV_STATUS: ${{ needs.check_runners.result }}
          SETUP_STATUS: ${{ needs.setup_gpu.result }}

        # We pass `needs.setup_gpu.outputs.matrix` as the argument. A processing in `notification_service.py` to change
        # `models/bert` to `models_bert` is required, as the artifact names use `_` instead of `/`.
        run: |
          pip install slack_sdk
          pip show slack_sdk
          python utils/notification_service.py "${{ needs.setup_gpu.outputs.matrix }}"


name: Self-hosted runner (AMD mi210 CI caller)

on:
  workflow_run:
    workflows: ["Self-hosted runner (push-caller)"]
    branches: ["main"]
    types: [completed]
  push:
    branches:
      - run_amd_push_ci_caller*
    paths:
      - "src/**"
      - "tests/**"
      - ".github/**"
      - "templates/**"
      - "utils/**"

jobs:
  run_amd_ci:
    name: AMD mi210
    if: (cancelled() != true) && ((github.event_name == 'workflow_run') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_amd_push_ci_caller')))
    uses: ./.github/workflows/self-push-amd.yml
    with:
      gpu_flavor: mi210
    secrets: inherit


name: Self-hosted runner (AMD mi250 CI caller)

on:
  workflow_run:
    workflows: ["Self-hosted runner (push-caller)"]
    branches: ["main"]
    types: [completed]
  push:
    branches:
      - run_amd_push_ci_caller*
    paths:
      - "src/**"
      - "tests/**"
      - ".github/**"
      - "templates/**"
      - "utils/**"

jobs:
  run_amd_ci:
    name: AMD mi250
    if: (cancelled() != true) && ((github.event_name == 'workflow_run') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_amd_push_ci_caller')))
    uses: ./.github/workflows/self-push-amd.yml
    with:
      gpu_flavor: mi250
    secrets: inherit


name: Self-hosted runner (AMD scheduled CI caller)

on:
  schedule:
    - cron: "17 2 * * *"

jobs:
  run_scheduled_amd_ci:
    name: Trigger Scheduled AMD CI
    runs-on: ubuntu-22.04
    if: ${{ always() }}
    steps:
      - name: Trigger scheduled AMD CI via workflow_run
        run: echo "Trigger scheduled AMD CI via workflow_run"


Self-hosted runner (scheduled-amd)

name: Self-hosted runner (scheduled-amd)

# Note: For the AMD CI, we rely on a caller workflow and on the workflow_call event to trigger the
# CI in order to run it on both MI210 and MI250, without having to use matrix here which pushes
# us towards the limit of allowed jobs on GitHub Actions.
on:
  workflow_call:
    inputs:
      gpu_flavor:
        required: true
        type: string

env:
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  OMP_NUM_THREADS: 8
  MKL_NUM_THREADS: 8
  RUN_SLOW: yes
  HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
  SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}

# Important note: each job (run_tests_single_gpu, run_tests_multi_gpu, run_examples_gpu, run_pipelines_torch_gpu) requires all the previous jobs before running.
# This is done so that we avoid parallelizing the scheduled tests, to leave available
# runners for the push CI that is running on the same machine.
jobs:
  check_runner_status:
    name: Check Runner Status
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout transformers
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Check Runner Status
        run: python utils/check_self_hosted_runner.py --target_runners hf-amd-mi210-ci-1gpu-1,hf-amd-mi250-ci-1gpu-1,hf-amd-mi300-ci-1gpu-1 --token ${{ secrets.ACCESS_REPO_INFO_TOKEN }}

  check_runners:
    name: Check Runners
    needs: check_runner_status
    strategy:
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    container:
      image: huggingface/transformers-pytorch-amd-gpu
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: ROCM-SMI
        run: |
          rocm-smi
      - name: ROCM-INFO
        run: |
          rocminfo  | grep "Agent" -A 14
      - name: Show ROCR environment
        run: |
          echo "ROCR: $ROCR_VISIBLE_DEVICES"

  setup:
    name: Setup
    needs: check_runners
    strategy:
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    container:
      image: huggingface/transformers-pytorch-amd-gpu
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Update clone
        working-directory: /transformers
        run: |
          git fetch && git checkout ${{ github.sha }}

      - name: Cleanup
        working-directory: /transformers
        run: |
          rm -rf tests/__pycache__
          rm -rf tests/models/__pycache__
          rm -rf reports

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - id: set-matrix
        name: Identify models to test
        working-directory: /transformers/tests
        run: |
          echo "matrix=$(python3 -c 'import os; tests = os.getcwd(); model_tests = os.listdir(os.path.join(tests, "models")); d1 = sorted(list(filter(os.path.isdir, os.listdir(tests)))); d2 = sorted(list(filter(os.path.isdir, [f"models/{x}" for x in model_tests]))); d1.remove("models"); d = d2 + d1; print(d)')" >> $GITHUB_OUTPUT

      - name: ROCM-SMI
        run: |
          rocm-smi

      - name: ROCM-INFO
        run: |
          rocminfo  | grep "Agent" -A 14
      - name: Show ROCR environment
        run: |
          echo "ROCR: $ROCR_VISIBLE_DEVICES"

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

  run_models_gpu_single_gpu:
    name: Single GPU tests
    strategy:
      max-parallel: 1  # For now, not to parallelize. Can change later if it works well.
      fail-fast: false
      matrix:
        folders: ${{ fromJson(needs.setup.outputs.matrix) }}
        machine_type: [single-gpu]
    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    container:
      image: huggingface/transformers-pytorch-amd-gpu
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    needs: setup
    steps:
      - name: Echo folder ${{ matrix.folders }}
        shell: bash
        # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
        # set the artifact folder names (because the character `/` is not allowed).
        run: |
          echo "${{ matrix.folders }}"
          matrix_folders=${{ matrix.folders }}
          matrix_folders=${matrix_folders/'models/'/'models_'}
          echo "$matrix_folders"
          echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV

      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: ROCM-SMI
        run: |
          rocm-smi
      - name: ROCM-INFO
        run: |
          rocminfo  | grep "Agent" -A 14
      - name: Show ROCR environment
        run: |
          echo "ROCR: $ROCR_VISIBLE_DEVICES"

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all tests on GPU
        working-directory: /transformers
        run: python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }} -m "not not_device_test"

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports

  run_models_gpu_multi_gpu:
    name: Multi GPU tests
    strategy:
      max-parallel: 1
      fail-fast: false
      matrix:
        folders: ${{ fromJson(needs.setup.outputs.matrix) }}
        machine_type: [multi-gpu]
    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    container:
      image: huggingface/transformers-pytorch-amd-gpu
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    needs: setup
    steps:
      - name: Echo folder ${{ matrix.folders }}
        shell: bash
        # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
        # set the artifact folder names (because the character `/` is not allowed).
        run: |
          echo "${{ matrix.folders }}"
          matrix_folders=${{ matrix.folders }}
          matrix_folders=${matrix_folders/'models/'/'models_'}
          echo "$matrix_folders"
          echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV

      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: ROCM-SMI
        run: |
          rocm-smi
      - name: ROCM-INFO
        run: |
          rocminfo  | grep "Agent" -A 14
      - name: Show ROCR environment
        run: |
          echo "ROCR: $ROCR_VISIBLE_DEVICES"

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all tests on GPU
        working-directory: /transformers
        run: python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }} -m "not not_device_test"

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports

  run_examples_gpu:
    name: Examples tests
    strategy:
      fail-fast: false
      matrix:
        machine_type: [single-gpu]
    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    container:
      image: huggingface/transformers-pytorch-amd-gpu
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    needs: setup
    steps:
      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: ROCM-SMI
        run: |
          rocm-smi
      - name: ROCM-INFO
        run: |
          rocminfo  | grep "Agent" -A 14
      - name: Show ROCR environment
        run: |
          echo "ROCR: $ROCR_VISIBLE_DEVICES"

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run examples tests on GPU
        working-directory: /transformers
        run: |
          pip install -r examples/pytorch/_tests_requirements.txt
          python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_examples_gpu_test_reports examples/pytorch -m "not not_device_test"

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_examples_gpu_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_examples_gpu_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_examples_gpu_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_examples_gpu_test_reports

  run_pipelines_torch_gpu:
    name: PyTorch pipelines tests
    strategy:
      fail-fast: false
      matrix:
        machine_type: [single-gpu, multi-gpu]
    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    container:
      image: huggingface/transformers-pytorch-amd-gpu
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    needs: setup
    steps:
      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: ROCM-SMI
        run: |
          rocm-smi
      - name: ROCM-INFO
        run: |
          rocminfo  | grep "Agent" -A 14
      - name: Show ROCR environment
        run: |
          echo "ROCR: $ROCR_VISIBLE_DEVICES"

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all pipeline tests on GPU
        working-directory: /transformers
        run: |
          python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports tests/pipelines -m "not not_device_test"

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports

  run_torch_cuda_extensions_gpu:
    name: Torch ROCm deepspeed tests
    strategy:
      fail-fast: false
      matrix:
        machine_type: [single-gpu, multi-gpu]

    runs-on: [self-hosted, amd-gpu, '${{ matrix.machine_type }}', '${{ inputs.gpu_flavor }}']
    needs: setup
    container:
      image: huggingface/transformers-pytorch-deepspeed-amd-gpu
      options: --device /dev/kfd --device /dev/dri --env ROCR_VISIBLE_DEVICES --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: ROCM-SMI
        run: |
          rocm-smi
      - name: ROCM-INFO
        run: |
          rocminfo  | grep "Agent" -A 14

      - name: Show ROCR environment
        run: |
          echo "ROCR: $ROCR_VISIBLE_DEVICES"

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all tests on GPU
        working-directory: /transformers
        run: python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended -m "not not_device_test"

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports

  run_extract_warnings:
    name: Extract warnings in CI artifacts
    runs-on: ubuntu-22.04
    if: always()
    needs: [
      check_runner_status,
      check_runners,
      setup,
      run_models_gpu_single_gpu,
      run_models_gpu_multi_gpu,
      run_examples_gpu,
      run_pipelines_torch_gpu,
      run_torch_cuda_extensions_gpu
    ]
    steps:
      - name: Checkout transformers
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Install transformers
        run: pip install transformers

      - name: Show installed libraries and their versions
        run: pip freeze

      - name: Create output directory
        run: mkdir warnings_in_ci

      - uses: actions/download-artifact@v4
        with:
          path: warnings_in_ci

      - name: Show artifacts
        run: echo "$(python3 -c 'import os; d = os.listdir(); print(d)')"
        working-directory: warnings_in_ci

      - name: Extract warnings in CI artifacts
        run: |
          python3 utils/extract_warnings.py --workflow_run_id ${{ github.run_id }} --output_dir warnings_in_ci --token ${{ secrets.ACCESS_REPO_INFO_TOKEN }} --from_gh
          echo "$(python3 -c 'import os; import json; fp = open("warnings_in_ci/selected_warnings.json"); d = json.load(fp); d = "\n".join(d) ;print(d)')"

      - name: Upload artifact
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: warnings_in_ci
          path: warnings_in_ci/selected_warnings.json

  send_results:
    name: Send results to webhook
    runs-on: ubuntu-22.04
    if: always()
    needs: [
      check_runner_status,
      check_runners,
      setup,
      run_models_gpu_single_gpu,
      run_models_gpu_multi_gpu,
      run_examples_gpu,
      run_pipelines_torch_gpu,
      run_torch_cuda_extensions_gpu,
      run_extract_warnings
    ]
    steps:
      - name: Preliminary job status
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          echo "Runner availability: ${{ needs.check_runner_status.result }}"
          echo "Runner status: ${{ needs.check_runners.result }}"
          echo "Setup status: ${{ needs.setup.result }}"

      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
      - name: Send message to Slack
        env:
          CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
          CI_SLACK_CHANNEL_ID_DAILY_AMD: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY_AMD }}
          CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }}
          CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY_AMD }}
          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
          CI_EVENT: Scheduled CI (AMD) - ${{ inputs.gpu_flavor }}
          CI_SHA: ${{ github.sha }}
          CI_WORKFLOW_REF: ${{ github.workflow_ref }}
          RUNNER_STATUS: ${{ needs.check_runner_status.result }}
          RUNNER_ENV_STATUS: ${{ needs.check_runners.result }}
          SETUP_STATUS: ${{ needs.setup.result }}
        # We pass `needs.setup.outputs.matrix` as the argument. A processing in `notification_service.py` to change
        # `models/bert` to `models_bert` is required, as the artifact names use `_` instead of `/`.
        run: |
          sudo apt-get install -y curl
          pip install slack_sdk
          pip show slack_sdk
          python utils/notification_service.py "${{ needs.setup.outputs.matrix }}"

      # Upload complete failure tables, as they might be big and only truncated versions could be sent to Slack.
      - name: Failure table artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: test_failure_tables
          path: test_failure_tables


name: Self-hosted runner (AMD mi250 scheduled CI caller)

on:
  workflow_run:
    workflows: ["Self-hosted runner (AMD scheduled CI caller)"]
    branches: ["main"]
    types: [completed]
  push:
    branches:
      - run_amd_scheduled_ci_caller*

jobs:
  run_amd_ci:
    name: AMD mi250
    if: (cancelled() != true) && ((github.event_name == 'workflow_run') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_amd_scheduled_ci_caller')))
    uses: ./.github/workflows/self-scheduled-amd.yml
    with:
      gpu_flavor: mi250
      slack_report_channel: "#transformers-ci-daily-amd"
    secrets: inherit


name: Self-hosted runner (AMD mi210 scheduled CI caller)

on:
  workflow_run:
    workflows: ["Self-hosted runner (AMD scheduled CI caller)"]
    branches: ["main"]
    types: [completed]
  push:
    branches:
      - run_amd_scheduled_ci_caller*

jobs:
  run_amd_ci:
    name: AMD mi210
    if: (cancelled() != true) && ((github.event_name == 'workflow_run') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_amd_scheduled_ci_caller')))
    uses: ./.github/workflows/self-scheduled-amd.yml
    with:
      gpu_flavor: mi210
      slack_report_channel: "#transformers-ci-daily-amd"
    secrets: inherit


model jobs

name: model jobs

on:
  workflow_call:
    inputs:
      folder_slices:
        required: true
        type: string
      machine_type:
        required: true
        type: string
      slice_id:
        required: true
        type: number
      runner:
        required: true
        type: string
      docker:
        required: true
        type: string

env:
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  OMP_NUM_THREADS: 8
  MKL_NUM_THREADS: 8
  RUN_SLOW: yes
  # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access.
  # This token is created under the bot `hf-transformers-bot`.
  HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
  SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
  TF_FORCE_GPU_ALLOW_GROWTH: true
  RUN_PT_TF_CROSS_TESTS: 1
  CUDA_VISIBLE_DEVICES: 0,1

jobs:
  run_models_gpu:
    name: " "
    strategy:
      max-parallel: 8
      fail-fast: false
      matrix:
        folders: ${{ fromJson(inputs.folder_slices)[inputs.slice_id] }}
    runs-on: ['${{ inputs.machine_type }}', nvidia-gpu, t4, '${{ inputs.runner }}']
    container:
      image: ${{ inputs.docker }}
      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Echo input and matrix info
        shell: bash
        run: |
          echo "${{ inputs.folder_slices }}"
          echo "${{ matrix.folders }}"
          echo "${{ toJson(fromJson(inputs.folder_slices)[inputs.slice_id]) }}"

      - name: Echo folder ${{ matrix.folders }}
        shell: bash
        # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
        # set the artifact folder names (because the character `/` is not allowed).
        run: |
          echo "${{ matrix.folders }}"
          matrix_folders=${{ matrix.folders }}
          matrix_folders=${matrix_folders/'models/'/'models_'}
          echo "$matrix_folders"
          echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV

      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Update / Install some packages (for Past CI)
        if: ${{ contains(inputs.docker, '-past-') }}
        working-directory: /transformers
        run: |
          python3 -m pip install -U datasets

      - name: Update / Install some packages (for Past CI)
        if: ${{ contains(inputs.docker, '-past-') && contains(inputs.docker, '-pytorch-') }}
        working-directory: /transformers
        run: |
          python3 -m pip install --no-cache-dir git+https://github.com/huggingface/accelerate@main#egg=accelerate

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all tests on GPU
        working-directory: /transformers
        run: python3 -m pytest -rsfE -v --make-reports=${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt

      - name: Run test
        shell: bash
        run: |
          mkdir -p /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
          echo "hello" > /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
          echo "${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"

      - name: "Test suite reports artifacts: ${{ inputs.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ inputs.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
          path: /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports


Self-hosted runner - SLOW integration tests (DeepSpeed, PEFT, FA2, etc.) (scheduled)


Self-hosted runner - SLOW Example tests (scheduled)


name: Slow tests on important models (on Push - A10)

on:
  push:
    branches: [ main ]

env:
  OUTPUT_SLACK_CHANNEL_ID: "C06L2SGMEEA"
  HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  OMP_NUM_THREADS: 8
  MKL_NUM_THREADS: 8
  RUN_SLOW: yes # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access. # This token is created under the bot `hf-transformers-bot`.
  SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
  TF_FORCE_GPU_ALLOW_GROWTH: true
  RUN_PT_TF_CROSS_TESTS: 1

jobs:
  get_modified_models:
    name: "Get all modified files"
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@3f54ebb830831fc121d3263c1857cfbdc310cdb9 #v42
        with:
          files: src/transformers/models/**

      - name: Run step if only the files listed above change
        if: steps.changed-files.outputs.any_changed == 'true'
        id: set-matrix
        env:
          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
        run: |
            model_arrays=()
            for file in $ALL_CHANGED_FILES; do
                model_path="${file#*models/}"
                model_path="models/${model_path%%/*}"
                if grep -qFx "$model_path" utils/important_models.txt; then
                    # Append the file to the matrix string
                    model_arrays+=("$model_path")
                fi
            done
            matrix_string=$(printf '"%s", ' "${model_arrays[@]}" | sed 's/, $//')
            echo "matrix=[$matrix_string]" >> $GITHUB_OUTPUT
  test_modified_files:
    needs: get_modified_models
    name: Slow & FA2 tests
    runs-on: [single-gpu, nvidia-gpu, a10, ci]
    container:
      image: huggingface/transformers-all-latest-gpu
      options: --gpus all --privileged --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    if: ${{ needs.get_modified_models.outputs.matrix != '[]' && needs.get_modified_models.outputs.matrix != '' && fromJson(needs.get_modified_models.outputs.matrix)[0] != null }}
    strategy:
      fail-fast: false
      matrix:
        model-name: ${{ fromJson(needs.get_modified_models.outputs.matrix) }}

    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Install locally transformers & other libs
        run: |
          apt install sudo
          sudo -H pip install --upgrade pip
          sudo -H pip uninstall -y transformers
          sudo -H pip install -U -e ".[testing]"
          MAX_JOBS=4 pip install flash-attn --no-build-isolation
          pip install bitsandbytes

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Show installed libraries and their versions
        run: pip freeze

      - name: Run FA2 tests
        id: run_fa2_tests
        run:
          pytest -rsfE -m "flash_attn_test" --make-reports=${{ matrix.model-name }}_fa2_tests/ tests/${{ matrix.model-name }}/test_modeling_*

      - name: "Test suite reports artifacts: ${{ matrix.model-name }}_fa2_tests"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.model-name }}_fa2_tests
          path: /transformers/reports/${{ matrix.model-name }}_fa2_tests

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ env.OUTPUT_SLACK_CHANNEL_ID }}
          title: 🤗 Results of the FA2 tests - ${{ matrix.model-name }}
          status: ${{ steps.run_fa2_tests.conclusion}}
          slack_token: ${{ secrets.CI_SLACK_BOT_TOKEN }}

      - name: Run integration tests
        id: run_integration_tests
        if: always()
        run:
          pytest -rsfE -k "IntegrationTest"  --make-reports=tests_integration_${{ matrix.model-name }} tests/${{ matrix.model-name }}/test_modeling_*

      - name: "Test suite reports artifacts: tests_integration_${{ matrix.model-name }}"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: tests_integration_${{ matrix.model-name }}
          path: /transformers/reports/tests_integration_${{ matrix.model-name }}

      - name: Post to Slack
        if: always()
        uses: huggingface/hf-workflows/.github/actions/post-slack@main
        with:
          slack_channel: ${{ env.OUTPUT_SLACK_CHANNEL_ID }}
          title: 🤗 Results of the Integration tests - ${{ matrix.model-name }}
          status: ${{ steps.run_integration_tests.conclusion}}
          slack_token: ${{ secrets.CI_SLACK_BOT_TOKEN }}

      - name: Tailscale # In order to be able to SSH when a test fails
        if: ${{ runner.debug == '1'}}
        uses: huggingface/tailscale-action@v1
        with:
          authkey: ${{ secrets.TAILSCALE_SSH_AUTHKEY }}
          slackChannel: ${{ secrets.SLACK_CIFEEDBACK_CHANNEL }}
          slackToken: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
          waitForSSH: true

  benchmark:
    name: Benchmark workflow
    needs: get_modified_models
    if: ${{ needs.get_modified_models.outputs.matrix != '[]' && needs.get_modified_models.outputs.matrix != '' && fromJson(needs.get_modified_models.outputs.matrix)[0] != null }}
    uses: ./.github/workflows/benchmark.yml
    secrets: inherit


Self-hosted runner - SLOW Generation tests (scheduled)


Self-hosted runner - SLOW pipeline tests (scheduled)


Self-hosted runner (scheduled)

name: Self-hosted runner (scheduled)

on:
  repository_dispatch:
  schedule:
    - cron: "17 2 * * *"
  push:
    branches:
      - run_scheduled_ci*

jobs:
  model-ci:
    name: Model CI
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_models_gpu
      slack_report_channel: "#transformers-ci-daily-models"
      runner: daily-ci
      docker: huggingface/transformers-all-latest-gpu
      ci_event: Daily CI
    secrets: inherit

  torch-pipeline:
    name: Torch pipeline CI
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_pipelines_torch_gpu
      slack_report_channel: "#transformers-ci-daily-pipeline-torch"
      runner: daily-ci
      docker: huggingface/transformers-pytorch-gpu
      ci_event: Daily CI
    secrets: inherit

  tf-pipeline:
    name: TF pipeline CI
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_pipelines_tf_gpu
      slack_report_channel: "#transformers-ci-daily-pipeline-tf"
      runner: daily-ci
      docker: huggingface/transformers-tensorflow-gpu
      ci_event: Daily CI
    secrets: inherit

  example-ci:
    name: Example CI
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_examples_gpu
      slack_report_channel: "#transformers-ci-daily-examples"
      runner: daily-ci
      docker: huggingface/transformers-all-latest-gpu
      ci_event: Daily CI
    secrets: inherit

  deepspeed-ci:
    name: DeepSpeed CI
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_torch_cuda_extensions_gpu
      slack_report_channel: "#transformers-ci-daily-deepspeed"
      runner: daily-ci
      docker: huggingface/transformers-pytorch-deepspeed-latest-gpu
      ci_event: Daily CI
      working-directory-prefix: /workspace
    secrets: inherit

  quantization-ci:
    name: Quantization CI
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_quantization_torch_gpu
      slack_report_channel: "#transformers-ci-daily-quantization"
      runner: daily-ci
      docker: huggingface/transformers-quantization-latest-gpu
      ci_event: Daily CI
    secrets: inherit


CI slack report

name: CI slack report

on:
  workflow_call:
    inputs:
      job:
        required: true
        type: string
      slack_report_channel:
        required: true
        type: string
      setup_status:
        required: true
        type: string
      folder_slices:
        required: true
        type: string
      quantization_matrix:
        required: true
        type: string
      ci_event:
        required: true
        type: string

env:
  TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }}

jobs:
  send_results:
    name: Send results to webhook
    runs-on: ubuntu-22.04
    if: always()
    steps:
      - name: Preliminary job status
        shell: bash
        # For the meaning of these environment variables, see the job `Setup`
        run: |
          echo "Setup status: ${{ inputs.setup_status }}"

      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
      - name: Send message to Slack
        if: ${{ inputs.job != 'run_quantization_torch_gpu' }}
        env:
          CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
          CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }}
          CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }}
          CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }}
          SLACK_REPORT_CHANNEL: ${{ inputs.slack_report_channel }}
          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
          CI_EVENT: ${{ inputs.ci_event }}
          CI_SHA: ${{ github.sha }}
          CI_WORKFLOW_REF: ${{ github.workflow_ref }}
          CI_TEST_JOB: ${{ inputs.job }}
          SETUP_STATUS: ${{ inputs.setup_status }}
        # We pass `needs.setup.outputs.matrix` as the argument. A processing in `notification_service.py` to change
        # `models/bert` to `models_bert` is required, as the artifact names use `_` instead of `/`.
        # For a job that doesn't depend on (i.e. `needs`) `setup`, the value for `inputs.folder_slices` would be an
        # empty string, and the called script still get one argument (which is the emtpy string).
        run: |
          sudo apt-get install -y curl
          pip install huggingface_hub
          pip install slack_sdk
          pip show slack_sdk
          python utils/notification_service.py "${{ inputs.folder_slices }}"

      # Upload complete failure tables, as they might be big and only truncated versions could be sent to Slack.
      - name: Failure table artifacts
        uses: actions/upload-artifact@v4
        with:
          name: ci_results_${{ inputs.job }}
          path: ci_results_${{ inputs.job }}

      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
      - name: Send message to Slack for quantization workflow
        if: ${{ inputs.job == 'run_quantization_torch_gpu' }}
        env:
          CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
          ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}
          SLACK_REPORT_CHANNEL: ${{ inputs.slack_report_channel }}
          CI_EVENT: ${{ inputs.ci_event }}
          CI_SHA: ${{ github.sha }}
          CI_TEST_JOB: ${{ inputs.job }}
          SETUP_STATUS: ${{ inputs.setup_status }}
        # We pass `needs.setup.outputs.quantization_matrix` as the argument. A processing in `notification_service_quantization.py` to change
        # `quantization/bnb` to `quantization_bnb` is required, as the artifact names use `_` instead of `/`.
        run: |
          sudo apt-get install -y curl
          pip install huggingface_hub
          pip install slack_sdk
          pip show slack_sdk
          python utils/notification_service_quantization.py "${{ inputs.quantization_matrix }}"

      # Upload complete failure tables, as they might be big and only truncated versions could be sent to Slack.
      - name: Failure table artifacts
        if: ${{ inputs.job == 'run_quantization_torch_gpu' }}
        uses: actions/upload-artifact@v4
        with:
          name: ci_results_${{ inputs.job }}
          path: ci_results_${{ inputs.job }}


Build pr ci-docker

name: Build pr ci-docker

on:
  push:
    branches:
      - push-ci-image # for now let's only build on this branch
  repository_dispatch:
  workflow_call:
    inputs:
      image_postfix:
        required: true
        type: string
  schedule:
    - cron: "6 0 * * *"

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  build:
    runs-on: ubuntu-22.04

    if: ${{ contains(github.event.head_commit.message, '[build-ci-image]') || contains(github.event.head_commit.message, '[push-ci-image]') && '!cancelled()' || github.event_name == 'schedule' }}

    strategy:
      matrix:
        file: ["quality", "consistency", "custom-tokenizers", "torch-light", "tf-light", "exotic-models", "torch-tf-light", "torch-jax-light", "jax-light", "examples-torch",  "examples-tf"]
    continue-on-error: true

    steps:
      -
        name: Set tag
        run: |
              if ${{contains(github.event.head_commit.message, '[build-ci-image]')}}; then
                  echo "TAG=huggingface/transformers-${{ matrix.file }}:dev" >> "$GITHUB_ENV"
                  echo "setting it to DEV!"
              else
                  echo "TAG=huggingface/transformers-${{ matrix.file }}" >> "$GITHUB_ENV"

              fi
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build ${{ matrix.file }}.dockerfile
        uses: docker/build-push-action@v5
        with:
          context: ./docker
          build-args: |
            REF=${{ github.sha }}
          file: "./docker/${{ matrix.file }}.dockerfile"
          push: ${{ contains(github.event.head_commit.message, 'ci-image]') ||  github.event_name == 'schedule' }}
          tags: ${{ env.TAG }}


Doctest job

name: Doctest job

on:
  workflow_call:
    inputs:
      job_splits:
        required: true
        type: string
      split_keys:
        required: true
        type: string

env:
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  RUN_SLOW: yes
  OMP_NUM_THREADS: 16
  MKL_NUM_THREADS: 16
  SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
  TF_FORCE_GPU_ALLOW_GROWTH: true

jobs:
  run_doctests:
    name: " "
    strategy:
      max-parallel: 8  # 8 jobs at a time
      fail-fast: false
      matrix:
        split_keys: ${{ fromJson(inputs.split_keys) }}
    runs-on: [single-gpu, nvidia-gpu, t4, ci]
    container:
      image: huggingface/transformers-all-latest-gpu
      options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Update clone
        working-directory: /transformers
        run: git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .[flax]

      - name: GPU visibility
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        run: pip freeze

      - name: Get doctest files
        working-directory: /transformers
        run: |
          echo "${{ toJson(fromJson(inputs.job_splits)[matrix.split_keys]) }}" > doc_tests.txt
          cat doc_tests.txt

      - name: Set `split_keys`
        shell: bash
        run: |
          echo "${{ matrix.split_keys }}"
          split_keys=${{ matrix.split_keys }}
          split_keys=${split_keys//'/'/'_'}
          echo "split_keys"
          echo "split_keys=$split_keys" >> $GITHUB_ENV

      - name: Run doctests
        working-directory: /transformers
        run: |
          cat doc_tests.txt
          python3 -m pytest -v --make-reports doc_tests_gpu_${{ env.split_keys }} --doctest-modules $(cat doc_tests.txt) -sv --doctest-continue-on-failure --doctest-glob="*.md"

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/doc_tests_gpu_${{ env.split_keys }}/failures_short.txt

      - name: "Test suite reports artifacts: doc_tests_gpu_test_reports_${{ env.split_keys }}"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: doc_tests_gpu_test_reports_${{ env.split_keys }}
          path: /transformers/reports/doc_tests_gpu_${{ env.split_keys }}


SSH into our runners

name: SSH into our runners

on:
  workflow_dispatch:
    inputs:
      runner_type:
        description: 'Type of runner to test (a10 or t4)'
        required: true
      docker_image:
        description: 'Name of the Docker image'
        required: true
      num_gpus:
        description: 'Type of the number of gpus to use (`single` or `multi`)'
        required: true

env:
  HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  OMP_NUM_THREADS: 8
  MKL_NUM_THREADS: 8
  RUN_SLOW: yes # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access. # This token is created under the bot `hf-transformers-bot`.
  SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
  TF_FORCE_GPU_ALLOW_GROWTH: true
  CUDA_VISIBLE_DEVICES: 0,1
  RUN_PT_TF_CROSS_TESTS: 1

jobs:
  ssh_runner:
    name: "SSH"
    runs-on: ["${{ github.event.inputs.num_gpus }}-gpu", nvidia-gpu, "${{ github.event.inputs.runner_type }}", ci]
    container:
      image: ${{ github.event.inputs.docker_image }}
      options: --gpus all --privileged --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/

    steps:
      - name: Update clone
        working-directory: /transformers
        run: |
          git fetch && git checkout ${{ github.sha }}

      - name: Cleanup
        working-directory: /transformers
        run: |
          rm -rf tests/__pycache__
          rm -rf tests/models/__pycache__
          rm -rf reports

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Tailscale # In order to be able to SSH when a test fails
        uses: huggingface/tailscale-action@main
        with:
          authkey: ${{ secrets.TAILSCALE_SSH_AUTHKEY }}
          slackChannel: ${{ secrets.SLACK_CIFEEDBACK_CHANNEL }}
          slackToken: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
          waitForSSH: true


name: Self-hosted runner (AMD mi300 CI caller)

on:
  workflow_run:
    workflows: ["Self-hosted runner (push-caller)"]
    branches: ["main"]
    types: [completed]
  push:
    branches:
      - run_amd_push_ci_caller*
    paths:
      - "src/**"
      - "tests/**"
      - ".github/**"
      - "templates/**"
      - "utils/**"

jobs:
  run_amd_ci:
    name: AMD mi300
    if: (cancelled() != true) && ((github.event_name == 'workflow_run') || ((github.event_name == 'push') && (startsWith(github.ref_name, 'run_amd_push_ci_caller') || startsWith(github.ref_name, 'mi300-ci'))))
    uses: ./.github/workflows/self-push-amd.yml
    with:
      gpu_flavor: mi300
    secrets: inherit


name: Self-hosted runner (AMD mi300 scheduled CI caller)

on:
  workflow_run:
    workflows: ["Self-hosted runner (AMD scheduled CI caller)"]
    branches: ["main"]
    types: [completed]
  push:
    branches:
      - run_amd_scheduled_ci_caller*

jobs:
  run_amd_ci:
    name: AMD mi300
    needs: build-docker-containers
    if: (cancelled() != true) && ((github.event_name == 'workflow_run') || ((github.event_name == 'push') && (startsWith(github.ref_name, 'run_amd_push_ci_caller') || startsWith(github.ref_name, 'mi300-ci'))))
    uses: ./.github/workflows/self-scheduled-amd.yml
    with:
      gpu_flavor: mi300
      slack_report_channel: "#transformers-ci-daily-amd"
    secrets: inherit


PR slow CI

name: PR slow CI

on:
  pull_request:
    paths:
      - "src/transformers/models/*/modeling_*.py"
      - "tests/models/*/test_*.py"

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  HF_HOME: /mnt/cache
  TRANSFORMERS_IS_CI: yes
  OMP_NUM_THREADS: 8
  MKL_NUM_THREADS: 8
  RUN_SLOW: yes
  # For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access.
  # This token is created under the bot `hf-transformers-bot`.
  HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
  SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
  TF_FORCE_GPU_ALLOW_GROWTH: true
  RUN_PT_TF_CROSS_TESTS: 1
  CUDA_VISIBLE_DEVICES: 0,1

jobs:
  find_models_to_run:
      runs-on: ubuntu-22.04
      name: Find models to run slow tests
      # Triggered only if the required label `run-slow` is added
      if: ${{ contains(github.event.pull_request.labels.*.name, 'run-slow') }}
      outputs:
        models: ${{ steps.models_to_run.outputs.models }}
      steps:
        - uses: actions/checkout@v4
          with:
            fetch-depth: "0"
            ref: ${{ github.event.pull_request.head.sha }}

        - name: Get commit message
          run: |
            echo "commit_message=$(git show -s --format=%s)" >> $GITHUB_ENV

        - name: Get models to run slow tests
          run: |
            echo "${{ env.commit_message }}"
            python -m pip install GitPython
            python utils/pr_slow_ci_models.py --commit_message "${{ env.commit_message }}" | tee output.txt
            echo "models=$(tail -n 1 output.txt)" >> $GITHUB_ENV

        - name: Models to run slow tests
          id: models_to_run
          run: |
            echo "${{ env.models }}"
            echo "models=${{ env.models }}" >> $GITHUB_OUTPUT

  run_models_gpu:
      name: Run all tests for the model
      # Triggered only `find_models_to_run` is triggered (label `run-slow` is added) which gives the models to run
      # (either a new model PR or via a commit message)
      if: ${{ needs.find_models_to_run.outputs.models != '[]' }}
      needs: find_models_to_run
      strategy:
        fail-fast: false
        matrix:
          folders: ${{ fromJson(needs.find_models_to_run.outputs.models) }}
          machine_type: [single-gpu, multi-gpu]
      runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, ci]
      container:
        image: huggingface/transformers-all-latest-gpu
        options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
      steps:
      - name: Echo input and matrix info
        shell: bash
        run: |
          echo "${{ matrix.folders }}"

      - name: Echo folder ${{ matrix.folders }}
        shell: bash
        # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
        # set the artifact folder names (because the character `/` is not allowed).
        run: |
          echo "${{ matrix.folders }}"
          matrix_folders=${{ matrix.folders }}
          matrix_folders=${matrix_folders/'models/'/'models_'}
          echo "$matrix_folders"
          echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV

      - name: Update clone
        working-directory: /transformers
        run: git fetch && git fetch origin pull/${{ github.event.pull_request.number }}/head:pull/${{ github.event.pull_request.number }}/merge && git checkout pull/${{ github.event.pull_request.number }}/merge

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: NVIDIA-SMI
        run: |
          nvidia-smi

      - name: Environment
        working-directory: /transformers
        run: |
          python3 utils/print_env.py

      - name: Show installed libraries and their versions
        working-directory: /transformers
        run: pip freeze

      - name: Run all tests on GPU
        working-directory: /transformers
        run: |
          export CUDA_VISIBLE_DEVICES="$(python3 utils/set_cuda_devices_for_ci.py --test_folder ${{ matrix.folders }})"
          echo $CUDA_VISIBLE_DEVICES
          python3 -m pytest -v -rsfE --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}

      - name: Failure short reports
        if: ${{ failure() }}
        continue-on-error: true
        run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt

      - name: Make sure report directory exists
        shell: bash
        run: |
          mkdir -p /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
          echo "hello" > /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
          echo "${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"

      - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
          path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports


Self-hosted runner (nightly-ci)

name: Self-hosted runner (nightly-ci)

on:
  repository_dispatch:
  schedule:
    - cron: "17 2 * * *"
  push:
    branches:
      - run_nightly_ci*

jobs:
  build_nightly_ci_images:
    name: Build Nightly CI Docker Images
    if: (github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_nightly_ci'))
    uses: ./.github/workflows/build-nightly-ci-docker-images.yml
    secrets: inherit

  model-ci:
    name: Model CI
    needs: [build_nightly_ci_images]
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_models_gpu
      slack_report_channel: "#transformers-ci-past-future"
      runner: ci
      docker: huggingface/transformers-all-latest-torch-nightly-gpu
      ci_event: Nightly CI
    secrets: inherit

  deepspeed-ci:
    name: DeepSpeed CI
    needs: [build_nightly_ci_images]
    uses: ./.github/workflows/self-scheduled.yml
    with:
      job: run_torch_cuda_extensions_gpu
      slack_report_channel: "#transformers-ci-past-future"
      runner: ci
      # test deepspeed nightly build with the latest release torch
      docker: huggingface/transformers-pytorch-deepspeed-latest-gpu
      ci_event: Nightly CI
      working-directory-prefix: /workspace
    secrets: inherit


Self-hosted runner (benchmark)

name: Self-hosted runner (benchmark)

on:
  schedule:
    - cron: "17 2 * * *"
  workflow_call:

env:
  HF_HOME: /mnt/cache
  TF_FORCE_GPU_ALLOW_GROWTH: true

jobs:
  benchmark:
    name: Benchmark
    runs-on: [single-gpu, nvidia-gpu, a10, ci]
    container:
      image: huggingface/transformers-all-latest-gpu
      options: --gpus all --privileged --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Update clone
        working-directory: /transformers
        run: |
          git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Benchmark (daily)
        if: github.event_name == 'schedule'
        working-directory: /transformers
        run: |
          python3 -m pip install optimum-benchmark>=0.2.0
          HF_TOKEN=${{ secrets.TRANSFORMERS_BENCHMARK_TOKEN }} python3 benchmark/benchmark.py --repo_id hf-internal-testing/benchmark_results --path_in_repo $(date +'%Y-%m-%d') --config-dir benchmark/config --config-name generation --commit=${{ github.sha }} backend.model=google/gemma-2b backend.cache_implementation=null,static backend.torch_compile=false,true --multirun

      - name: Benchmark (merged to main event)
        if: github.event_name == 'push' && github.ref_name == 'main'
        working-directory: /transformers
        run: |
          python3 -m pip install optimum-benchmark>=0.2.0
          HF_TOKEN=${{ secrets.TRANSFORMERS_BENCHMARK_TOKEN }} python3 benchmark/benchmark.py --repo_id hf-internal-testing/benchmark_results_merge_event --path_in_repo $(date +'%Y-%m-%d') --config-dir benchmark/config --config-name generation --commit=${{ github.sha }} backend.model=google/gemma-2b backend.cache_implementation=null,static backend.torch_compile=false,true --multirun


Secret Leaks

on:
  push:

name: Secret Leaks

permissions:
  contents: read

jobs:
  trufflehog:
    runs-on: ubuntu-latest
    steps:
    - shell: bash
      run: |
        if [ "${{ github.event_name }}" == "push" ]; then
          echo "depth=$(($(jq length <<< '${{ toJson(github.event.commits) }}') + 2))" >> $GITHUB_ENV
          echo "branch=${{ github.ref_name }}" >> $GITHUB_ENV
        fi
        if [ "${{ github.event_name }}" == "pull_request" ]; then
          echo "depth=$((${{ github.event.pull_request.commits }}+2))" >> $GITHUB_ENV
          echo "branch=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV
        fi
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        ref: ${{env.branch}}
        fetch-depth: ${{env.depth}}
    - name: Secret Scanning
      uses: trufflesecurity/trufflehog@main


Tests

name: Tests
on:
  push:
    branches:
      - master
  pull_request_target:
    branches:
      - master
env:
  NODE_VERSION: 18
  YARN_VERSION: 1.22.22
jobs:
  backend-lint:
    runs-on: ubuntu-22.04
    steps:
      - if: github.event.pull_request.mergeable == 'false'
        name: Exit if PR is not mergeable
        run: exit 1
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: actions/setup-python@v5
        with:
          python-version: '3.8'
      - run: sudo pip install black==23.1.0 ruff==0.0.287
      - run: ruff check .
      - run: black --check .

  backend-unit-tests:
    runs-on: ubuntu-22.04
    needs: backend-lint
    env:
      COMPOSE_FILE: .ci/compose.ci.yaml
      COMPOSE_PROJECT_NAME: redash
      COMPOSE_DOCKER_CLI_BUILD: 1
      DOCKER_BUILDKIT: 1
    steps:
      - if: github.event.pull_request.mergeable == 'false'
        name: Exit if PR is not mergeable
        run: exit 1
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Build Docker Images
        run: |
          set -x
          docker compose build --build-arg install_groups="main,all_ds,dev" --build-arg skip_frontend_build=true
          docker compose up -d
          sleep 10
      - name: Create Test Database
        run: docker compose -p redash run --rm postgres psql -h postgres -U postgres -c "create database tests;"
      - name: List Enabled Query Runners
        run: docker compose -p redash run --rm redash manage ds list_types
      - name: Run Tests
        run: docker compose -p redash run --name tests redash tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/
      - name: Copy Test Results
        run: |
          mkdir -p /tmp/test-results/unit-tests
          docker cp tests:/app/coverage.xml ./coverage.xml
          docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v3
      - name: Store Test Results
        uses: actions/upload-artifact@v3
        with:
          name: test-results
          path: /tmp/test-results
      - name: Store Coverage Results
        uses: actions/upload-artifact@v3
        with:
          name: coverage
          path: coverage.xml

  frontend-lint:
    runs-on: ubuntu-22.04
    steps:
      - if: github.event.pull_request.mergeable == 'false'
        name: Exit if PR is not mergeable
        run: exit 1
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'yarn'
      - name: Install Dependencies
        run: |
          npm install --global --force yarn@$YARN_VERSION
          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
      - name: Run Lint
        run: yarn lint:ci
      - name: Store Test Results
        uses: actions/upload-artifact@v3
        with:
          name: test-results
          path: /tmp/test-results

  frontend-unit-tests:
    runs-on: ubuntu-22.04
    needs: frontend-lint
    steps:
      - if: github.event.pull_request.mergeable == 'false'
        name: Exit if PR is not mergeable
        run: exit 1
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'yarn'
      - name: Install Dependencies
        run: |
          npm install --global --force yarn@$YARN_VERSION
          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
      - name: Run App Tests
        run: yarn test
      - name: Run Visualizations Tests
        run: cd viz-lib && yarn test
      - run: yarn lint

  frontend-e2e-tests:
    runs-on: ubuntu-22.04
    needs: frontend-lint
    env:
      COMPOSE_FILE: .ci/compose.cypress.yaml
      COMPOSE_PROJECT_NAME: cypress
      CYPRESS_INSTALL_BINARY: 0
      PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1
      PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
      CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
      CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
    steps:
      - if: github.event.pull_request.mergeable == 'false'
        name: Exit if PR is not mergeable
        run: exit 1
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'yarn'
      - name: Enable Code Coverage Report For Master Branch
        if: endsWith(github.ref, '/master')
        run: |
          echo "CODE_COVERAGE=true" >> "$GITHUB_ENV"
      - name: Install Dependencies
        run: |
          npm install --global --force yarn@$YARN_VERSION
          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
      - name: Setup Redash Server
        run: |
          set -x
          yarn cypress build
          yarn cypress start -- --skip-db-seed
          docker compose run cypress yarn cypress db-seed
      - name: Execute Cypress Tests
        run: yarn cypress run-ci
      - name: "Failure: output container logs to console"
        if: failure()
        run: docker compose logs
      - name: Copy Code Coverage Results
        run: docker cp cypress:/usr/src/app/coverage ./coverage || true
      - name: Store Coverage Results
        uses: actions/upload-artifact@v3
        with:
          name: coverage
          path: coverage


Periodic Snapshot

name: Periodic Snapshot

# 10 minutes after midnight on the first of every month
on:
  schedule:
    - cron: '10 0 1 * *'

permissions:
  contents: write

jobs:
  bump-version-and-tag:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ssh-key: ${{secrets.ACTION_PUSH_KEY}}
      - run: |
          # https://api.github.com/users/github-actions[bot]
          git config user.name 'github-actions[bot]'
          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'

          TAG_NAME="$(date +%y.%m).0-dev"
          gawk -i inplace -F: -v q=\" -v tag=${TAG_NAME} '/^  "version": / { print $1 FS, q tag q ","; next} { print }' package.json
          gawk -i inplace -F= -v q=\" -v tag=${TAG_NAME} '/^__version__ =/ { print $1 FS, q tag q; next} { print }' redash/__init__.py
          gawk -i inplace -F= -v q=\" -v tag=${TAG_NAME} '/^version =/ { print $1 FS, q tag q; next} { print }' pyproject.toml
          git add package.json redash/__init__.py pyproject.toml
          git commit -m "Snapshot: ${TAG_NAME}"
          git tag ${TAG_NAME}
          git push --atomic origin master refs/tags/${TAG_NAME}


Preview Image

name: Preview Image
on:
  push:
    tags:
      - '*-dev'

env:
  NODE_VERSION: 18

jobs:
  build-skip-check:
    runs-on: ubuntu-22.04
    outputs:
      skip: ${{ steps.skip-check.outputs.skip }}
    steps:
      - name: Skip?
        id: skip-check
        run: |
          if [[ "${{ vars.DOCKER_USER }}" == '' ]]; then
            echo 'Docker user is empty. Skipping build+push'
            echo skip=true >> "$GITHUB_OUTPUT"
          elif [[ "${{ secrets.DOCKER_PASS }}" == '' ]]; then
            echo 'Docker password is empty. Skipping build+push'
            echo skip=true >> "$GITHUB_OUTPUT"
          else
            echo 'Docker user and password are set and branch is `master`.'
            echo 'Building + pushing `preview` image.'
            echo skip=false >> "$GITHUB_OUTPUT"
          fi

  build-docker-image:
    runs-on: ubuntu-22.04
    needs:
      - build-skip-check
    if: needs.build-skip-check.outputs.skip == 'false'
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.event.push.after }}

      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'yarn'

      - name: Install Dependencies
        run: |
          npm install --global --force yarn@1.22.22
          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ vars.DOCKER_USER }}
          password: ${{ secrets.DOCKER_PASS }}

      - name: Set version
        id: version
        run: |
          set -x
          .ci/update_version
          VERSION_TAG=$(jq -r .version package.json)
          echo "VERSION_TAG=$VERSION_TAG" >> "$GITHUB_OUTPUT"

      - name: Build and push preview image to Docker Hub
        uses: docker/build-push-action@v4
        with:
          push: true
          tags: |
            redash/redash:preview
            redash/preview:${{ steps.version.outputs.VERSION_TAG }}
          context: .
          build-args: |
            test_all_deps=true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          platforms: linux/amd64
        env:
          DOCKER_CONTENT_TRUST: true

      - name: "Failure: output container logs to console"
        if: failure()
        run: docker compose logs


update-html

name: update-html
run-name: Update dash html components attributes
on:
  schedule:
    - cron: 10 0 1 * *
  workflow_dispatch:
    inputs:
      name:
        description: Name of the run
        required: false
        type: string

jobs:
  update-attributes:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          ref: dev
      - uses: actions/setup-node@v3
      - name: Extract elements
        working-directory: ./components/dash-html-components
        run: |
          npm ci
          npm run extract
      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v5
        with:
          commit-message: Update Dash HTML elements & attributes.
          body: Automated HTML attributes update.


deploy

name: deploy

on: push

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-n-publish:
    runs-on: ubuntu-latest
    if: startsWith(github.event.ref, 'refs/tags')
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - name: Install torch
        run: pip install torch
      - name: Install wheel
        run: pip install wheel
      - name: Build MMDetection
        run: python setup.py sdist bdist_wheel
      - name: Publish distribution to PyPI
        run: |
          pip install twine
          twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }}


CodeQL

name: 'CodeQL'

on:
  schedule:
    - cron: '0 8 * * 3'

permissions:
  contents: read # to fetch code (actions/checkout)

jobs:
  analyze:
    permissions:
      contents: read # to fetch code (actions/checkout)
      security-events: write # to upload SARIF results (github/codeql-action/analyze)

    name: Analyze
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
        language: ['javascript', 'python']

    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Initialize CodeQL
        uses: github/codeql-action/init@v1
        with:
          languages: ${{ matrix.language }}

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v1