protobuf cross tests

name: protobuf cross tests

on:
  pull_request:
    types:
      - opened
      - synchronize
      - reopened
      - ready_for_review
    paths:
      - "mlflow/protos/**"
  push:
    branches:
      - master
      - branch-[0-9]+.[0-9]+
    paths:
      - "mlflow/protos/**"

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

# Use `bash --noprofile --norc -exo pipefail` by default for all `run` steps in this workflow:
# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#defaultsrun
defaults:
  run:
    shell: bash --noprofile --norc -exo pipefail {0}

env:
  MLFLOW_HOME: /home/runner/work/mlflow/mlflow
  # Note miniconda is pre-installed in the virtual environments for GitHub Actions:
  # https://github.com/actions/virtual-environments/blob/main/images/linux/scripts/installers/miniconda.sh
  MLFLOW_CONDA_HOME: /usr/share/miniconda
  SPARK_LOCAL_IP: localhost
  PYTHONUTF8: "1"

jobs:
  core_tests:
    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        group: [1, 2]
        protobuf_major_version: [3, 4, 5]
        include:
          - splits: 2
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: ./.github/actions/untracked
      - uses: ./.github/actions/free-disk-space
      - uses: ./.github/actions/setup-python
      - uses: ./.github/actions/setup-pyenv
      - uses: ./.github/actions/setup-java
      - name: Install dependencies
        run: |
          pip install --no-dependencies tests/resources/mlflow-test-plugin
          pip install .[extras]
          pip install pyspark
          # Install Hugging Face datasets to test Hugging Face usage with MLflow dataset tracking
          pip install datasets
          # Install TensorFlow to test TensorFlow dataset usage with MLflow dataset tracking
          pip install tensorflow
          # Install torch and transformers to test metrics
          pip install torch transformers
          pip install -r requirements/test-requirements.txt
          # Test the latest minor version in protobuf_major_version
          pip install "protobuf==${{ matrix.protobuf_major_version }}.*"
      - uses: ./.github/actions/pipdeptree
      - name: Run tests
        run: |
          pytest --splits=${{ matrix.splits }} --group=${{ matrix.group }} \
            --ignore-flavors --ignore=tests/projects --ignore=tests/examples --ignore=tests/recipes --ignore=tests/evaluate --ignore tests/deployments/server \
            tests


CMake

name: CMake

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
  workflow_dispatch:

jobs:
  linux:
    strategy:
      matrix:
        include:
          - runner: linux.12xlarge
            gpu-arch-type: cpu
          - runner: linux.g5.4xlarge.nvidia.gpu
            gpu-arch-type: cuda
            gpu-arch-version: "11.8"
      fail-fast: false
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      runner: ${{ matrix.runner }}
      gpu-arch-type: ${{ matrix.gpu-arch-type }}
      gpu-arch-version: ${{ matrix.gpu-arch-version }}
      test-infra-ref: main
      script: |
        set -euo pipefail

        export PYTHON_VERSION=3.8
        export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
        export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}

        ./.github/scripts/cmake.sh

  macos:
    strategy:
      matrix:
        include:
          - runner: macos-m1-stable
      fail-fast: false
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    with:
      repository: pytorch/vision
      runner: ${{ matrix.runner }}
      test-infra-ref: main
      script: |
        set -euo pipefail

        export PYTHON_VERSION=3.8
        export GPU_ARCH_TYPE=cpu
        export GPU_ARCH_VERSION=''

        ${CONDA_RUN} ./.github/scripts/cmake.sh

  windows:
    strategy:
      matrix:
        include:
          - runner: windows.4xlarge
            gpu-arch-type: cpu
          - runner: windows.g5.4xlarge.nvidia.gpu
            gpu-arch-type: cuda
            gpu-arch-version: "11.8"
      fail-fast: false
    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
    with:
      repository: pytorch/vision
      runner: ${{ matrix.runner }}
      gpu-arch-type: ${{ matrix.gpu-arch-type }}
      gpu-arch-version: ${{ matrix.gpu-arch-version }}
      test-infra-ref: main
      script: |
        set -euo pipefail

        export PYTHON_VERSION=3.8
        export VC_YEAR=2022
        export VSDEVCMD_ARGS=""
        export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
        export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}

        ./.github/scripts/cmake.sh


Build Linux Conda

name: Build Linux Conda

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
        # NOTE: Binary build pipelines should only get triggered on release candidate builds
        # Release candidate tags look like: v1.11.0-rc1
        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: conda
      os: linux
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
  build:
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include:
          - repository: pytorch/vision
            pre-script: ""
            post-script: ""
            conda-package-directory: packaging/torchvision
            smoke-test-script: test/smoke_test.py
            package-name: torchvision
    name: ${{ matrix.repository }}
    uses: pytorch/test-infra/.github/workflows/build_conda_linux.yml@main
    with:
      conda-package-directory: ${{ matrix.conda-package-directory }}
      repository: ${{ matrix.repository }}
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
      pre-script: ${{ matrix.pre-script }}
      post-script: ${{ matrix.post-script }}
      package-name: ${{ matrix.package-name }}
      smoke-test-script: ${{ matrix.smoke-test-script }}
      trigger-event: ${{ github.event_name }}
    secrets:
      CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
      CONDA_PYTORCHBOT_TOKEN_TEST: ${{ secrets.CONDA_PYTORCHBOT_TOKEN_TEST }}


Build M1 Conda

name: Build M1 Conda

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
        # NOTE: Binary build pipelines should only get triggered on release candidate builds
        # Release candidate tags look like: v1.11.0-rc1
        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: conda
      os: macos-arm64
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
  build:
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include:
          - repository: pytorch/vision
            pre-script: ""
            post-script: ""
            conda-package-directory: packaging/torchvision
            smoke-test-script: test/smoke_test.py
            package-name: torchvision
    name: ${{ matrix.repository }}
    uses: pytorch/test-infra/.github/workflows/build_conda_macos.yml@main
    with:
      conda-package-directory: ${{ matrix.conda-package-directory }}
      repository: ${{ matrix.repository }}
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
      pre-script: ${{ matrix.pre-script }}
      post-script: ${{ matrix.post-script }}
      package-name: ${{ matrix.package-name }}
      smoke-test-script: ${{ matrix.smoke-test-script }}
      runner-type: macos-m1-stable
      trigger-event: ${{ github.event_name }}
    secrets:
      CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
      CONDA_PYTORCHBOT_TOKEN_TEST: ${{ secrets.CONDA_PYTORCHBOT_TOKEN_TEST }}


Build Windows Conda

name: Build Windows Conda

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
        # NOTE: Binary build pipelines should only get triggered on release candidate builds
        # Release candidate tags look like: v1.11.0-rc1
        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: conda
      os: windows
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
  build:
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include:
          - repository: pytorch/vision
            pre-script: packaging/pre_build_script.sh
            env-script: packaging/windows/internal/vc_env_helper.bat
            post-script: ""
            smoke-test-script: test/smoke_test.py
            package-name: torchvision
    name: ${{ matrix.repository }}
    uses: pytorch/test-infra/.github/workflows/build_conda_windows.yml@main
    with:
      conda-package-directory: ${{ matrix.conda-package-directory }}
      repository: ${{ matrix.repository }}
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
      pre-script: ${{ matrix.pre-script }}
      post-script: ${{ matrix.post-script }}
      package-name: ${{ matrix.package-name }}
      smoke-test-script: ${{ matrix.smoke-test-script }}
      trigger-event: ${{ github.event_name }}
    secrets:
      CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
      CONDA_PYTORCHBOT_TOKEN_TEST: ${{ secrets.CONDA_PYTORCHBOT_TOKEN_TEST }}


Build Aarch64 Linux Wheels

name: Build Aarch64 Linux Wheels

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
        # NOTE: Binary build pipelines should only get triggered on release candidate builds
        # Release candidate tags look like: v1.11.0-rc1
        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

permissions:
  id-token: write
  contents: read

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: wheel
      os: linux-aarch64
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      with-cuda: disable
  build:
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include:
          - repository: pytorch/vision
            pre-script: packaging/pre_build_script.sh
            post-script: packaging/post_build_script.sh
            smoke-test-script: test/smoke_test.py
            package-name: torchvision
    name: ${{ matrix.repository }}
    uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
    with:
      repository: ${{ matrix.repository }}
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
      pre-script: ${{ matrix.pre-script }}
      post-script: ${{ matrix.post-script }}
      package-name: ${{ matrix.package-name }}
      smoke-test-script: ${{ matrix.smoke-test-script }}
      trigger-event: ${{ github.event_name }}
      architecture: aarch64
      setup-miniconda: false


Build Linux Wheels

name: Build Linux Wheels

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
        # NOTE: Binary build pipelines should only get triggered on release candidate builds
        # Release candidate tags look like: v1.11.0-rc1
        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

permissions:
  id-token: write
  contents: read

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: wheel
      os: linux
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
  build:
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include:
          - repository: pytorch/vision
            pre-script: packaging/pre_build_script.sh
            post-script: packaging/post_build_script.sh
            smoke-test-script: test/smoke_test.py
            package-name: torchvision
    name: ${{ matrix.repository }}
    uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
    with:
      repository: ${{ matrix.repository }}
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
      pre-script: ${{ matrix.pre-script }}
      post-script: ${{ matrix.post-script }}
      package-name: ${{ matrix.package-name }}
      smoke-test-script: ${{ matrix.smoke-test-script }}
      trigger-event: ${{ github.event_name }}


Build M1 Wheels

name: Build M1 Wheels

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
        # NOTE: Binary build pipelines should only get triggered on release candidate builds
        # Release candidate tags look like: v1.11.0-rc1
        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

permissions:
  id-token: write
  contents: read

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: wheel
      os: macos-arm64
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
  build:
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include:
          - repository: pytorch/vision
            pre-script: packaging/pre_build_script.sh
            post-script: packaging/post_build_script.sh
            smoke-test-script: test/smoke_test.py
            package-name: torchvision
    name: ${{ matrix.repository }}
    uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main
    with:
      repository: ${{ matrix.repository }}
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
      pre-script: ${{ matrix.pre-script }}
      post-script: ${{ matrix.post-script }}
      package-name: ${{ matrix.package-name }}
      runner-type: macos-m1-stable
      smoke-test-script: ${{ matrix.smoke-test-script }}
      trigger-event: ${{ github.event_name }}


Build Windows Wheels

name: Build Windows Wheels

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
        # NOTE: Binary build pipelines should only get triggered on release candidate builds
        # Release candidate tags look like: v1.11.0-rc1
        - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

permissions:
  id-token: write
  contents: read

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: wheel
      os: windows
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
  build:
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include:
          - repository: pytorch/vision
            pre-script: packaging/pre_build_script.sh
            env-script: packaging/windows/internal/vc_env_helper.bat
            post-script: "python packaging/wheel/relocate.py"
            smoke-test-script: test/smoke_test.py
            package-name: torchvision
    name: ${{ matrix.repository }}
    uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main
    with:
      repository: ${{ matrix.repository }}
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
      pre-script: ${{ matrix.pre-script }}
      env-script: ${{ matrix.env-script }}
      post-script: ${{ matrix.post-script }}
      package-name: ${{ matrix.package-name }}
      smoke-test-script: ${{ matrix.smoke-test-script }}
      trigger-event: ${{ github.event_name }}


Docs

name: Docs

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
      - v[0-9]+.[0-9]+.[0-9]
      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

jobs:
  build:
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      upload-artifact: docs
      test-infra-ref: main
      script: |
        set -euo pipefail

        export PYTHON_VERSION=3.8
        export GPU_ARCH_TYPE=cpu
        export GPU_ARCH_VERSION=''
        ./.github/scripts/setup-env.sh

        # Prepare conda
        CONDA_PATH=$(which conda)
        eval "$(${CONDA_PATH} shell.bash hook)"
        conda activate ci
        # FIXME: not sure why we need this. `ldd torchvision/video_reader.so` shows that it
        #  already links against the one pulled from conda. However, at runtime it pulls from
        #  /lib64
        # Should we maybe always do this in `./.github/scripts/setup-env.sh` so that we don't
        # have to pay attention in all other workflows?
        export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}"

        cd docs

        echo '::group::Install doc requirements'
        pip install --progress-bar=off -r requirements.txt
        echo '::endgroup::'

        if [[ ${{ github.event_name }} == push && (${{ github.ref_type }} == tag || (${{ github.ref_type }} == branch && ${{ github.ref_name }} == release/*)) ]]; then
          echo '::group::Enable version string sanitization'
          # This environment variable just has to exist and must not be empty. The actual value is arbitrary.
          # See docs/source/conf.py for details
          export TORCHVISION_SANITIZE_VERSION_STR_IN_DOCS=1
          echo '::endgroup::'
        fi

        # The runner does not have sufficient memory to run with as many processes as there are
        # cores (`-j auto`). Thus, we limit to a single process (`-j 1`) here.
        sed -i -e 's/-j auto/-j 1/' Makefile
        make html

        # Below is an imperfect way for us to add "try on collab" links to all of our gallery examples.
        # sphinx-gallery will convert all gallery examples to .ipynb notebooks and stores them in
        # build/html/_downloads/<some_hash>/<example_name>.ipynb
        # We copy all those ipynb files in a more convenient folder so that we can more easily link to them.
        mkdir build/html/_generated_ipynb_notebooks
        for file in `find build/html/_downloads`; do
          if [[ $file == *.ipynb ]]; then
            cp $file build/html/_generated_ipynb_notebooks/
          fi
        done

        cp -r build/html "${RUNNER_ARTIFACT_DIR}"

        # On PRs we also want to upload the docs into our S3 bucket for preview.
        if [[ ${{ github.event_name == 'pull_request' }} ]]; then
          cp -r build/html/* "${RUNNER_DOCS_DIR}"
        fi

  upload:
    needs: build
    if: github.repository == 'pytorch/vision' && github.event_name == 'push' &&
        ((github.ref_type == 'branch' && github.ref_name == 'main') || github.ref_type == 'tag')
    permissions:
      contents: write
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      download-artifact: docs
      ref: gh-pages
      test-infra-ref: main
      script: |
        set -euo pipefail

        REF_TYPE=${{ github.ref_type }}
        REF_NAME=${{ github.ref_name }}

        if [[ "${REF_TYPE}" == branch ]]; then
          TARGET_FOLDER="${REF_NAME}"
        elif [[ "${REF_TYPE}" == tag ]]; then
          case "${REF_NAME}" in
            *-rc*)
              echo "Aborting upload since this is an RC tag: ${REF_NAME}"
              exit 0
              ;;
            *)
              # Strip the leading "v" as well as the trailing patch version. For example:
              # 'v0.15.2' -> '0.15'
              TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/')
              ;;
          esac
        fi
        echo "Target Folder: ${TARGET_FOLDER}"

        mkdir -p "${TARGET_FOLDER}"
        rm -rf "${TARGET_FOLDER}"/*
        mv "${RUNNER_ARTIFACT_DIR}"/html/* "${TARGET_FOLDER}"
        git add "${TARGET_FOLDER}" || true

        if [[ "${TARGET_FOLDER}" == main ]]; then
          mkdir -p _static
          rm -rf _static/*
          cp -r "${TARGET_FOLDER}"/_static/* _static
          git add _static || true
        fi

        git config user.name 'pytorchbot'
        git config user.email 'soumith+bot@pytorch.org'
        git config http.postBuffer 524288000
        git commit -m "auto-generating sphinx docs" || true
        git push


Lint

name: Lint

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
  workflow_dispatch:

jobs:
  python-source-and-configs:
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      test-infra-ref: main
      script: |
        set -euo pipefail

        echo '::group::Setup environment'
        CONDA_PATH=$(which conda)
        eval "$(${CONDA_PATH} shell.bash hook)"
        conda create --name ci --quiet --yes python=3.8 pip
        conda activate ci
        echo '::endgroup::'

        echo '::group::Install lint tools'
        pip install --progress-bar=off pre-commit
        echo '::endgroup::'

        set +e
        pre-commit run --all-files

        if [ $? -ne 0 ]; then
          git --no-pager diff
          exit 1
        fi

  c-source:
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      test-infra-ref: main
      script: |
        set -euo pipefail

        echo '::group::Setup environment'
        CONDA_PATH=$(which conda)
        eval "$(${CONDA_PATH} shell.bash hook)"
        # clang-format needs some shared libraries that conflict with the system ones. Thus, we install them from conda
        # and prepend the libraries to linker path to prioritize them. `ncurses=5` is only available on the conda-forge
        # channel. Since we are not building or testing here, this is fine.
        conda create --name ci --quiet --yes -c conda-forge python=3.8 ncurses=5 libgcc
        conda activate ci
        export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}"
        echo '::endgroup::'

        echo '::group::Install lint tools'
        curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o ./clang-format
        chmod +x ./clang-format
        echo '::endgroup::'

        echo '::group::Lint C source'
        set +e
        ./.github/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable ./clang-format --exclude "torchvision/csrc/io/image/cpu/giflib/*"

        if [ $? -ne 0 ]; then
          git --no-pager diff
          exit 1
        fi
        echo '::endgroup::'

  python-types:
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      test-infra-ref: main
      script: |
        set -euo pipefail

        export PYTHON_VERSION=3.8
        export GPU_ARCH_TYPE=cpu
        export GPU_ARCH_VERSION=''

        ./.github/scripts/setup-env.sh

        CONDA_PATH=$(which conda)
        eval "$(${CONDA_PATH} shell.bash hook)"
        conda activate ci

        echo '::group::Install lint tools'
        pip install --progress-bar=off mypy
        echo '::endgroup::'

        echo '::group::Lint Python types'
        mypy --install-types --non-interactive --config-file mypy.ini
        echo '::endgroup::'

  bc:
    if: github.event.pull_request
    runs-on: ubuntu-latest
    steps:
      - name: Run BC Lint Action
        uses: pytorch/test-infra/.github/actions/bc-lint@main
        with:
          repo: ${{ github.event.pull_request.head.repo.full_name }}
          base_sha: ${{ github.event.pull_request.base.sha }}
          head_sha: ${{ github.event.pull_request.head.sha }}


pr-labels

name: pr-labels

on:
  push:
    branches:
      - main

jobs:
  is-properly-labeled:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - name: Set up python
        uses: actions/setup-python@v5

      - name: Install requests
        run: pip install requests

      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Process commit and find merger responsible for labeling
        id: commit
        run: |
          MERGER=$(python .github/process_commit.py ${{ github.sha }})
          echo "merger=${MERGER}" | tee --append $GITHUB_OUTPUT

      - name: Ping merger responsible for labeling if necessary
        if: ${{ steps.commit.outputs.merger != '' }}
        uses: mshick/add-pr-comment@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          message: |
            Hey ${{ steps.commit.outputs.merger }}!

            You merged this PR, but no labels were added.
            The list of valid labels is available at https://github.com/pytorch/vision/blob/main/.github/process_commit.py


Prototype tests on Linux

name: Prototype tests on Linux

# IMPORTANT: This workflow has been manually disabled from the GitHub interface
# in June 2024. The file is kept for reference in case we ever put this back.

on:
  pull_request:

jobs:
  unittests-prototype:
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        runner: ["linux.12xlarge"]
        gpu-arch-type: ["cpu"]
        include:
          - python-version: "3.8"
            runner: linux.g5.4xlarge.nvidia.gpu
            gpu-arch-type: cuda
            gpu-arch-version: "11.8"
      fail-fast: false
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      runner: ${{ matrix.runner }}
      gpu-arch-type: ${{ matrix.gpu-arch-type }}
      gpu-arch-version: ${{ matrix.gpu-arch-version }}
      timeout: 120
      script: |
        set -euo pipefail

        export PYTHON_VERSION=${{ matrix.python-version }}
        export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
        export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}
        ./.github/scripts/setup-env.sh

        # Prepare conda
        CONDA_PATH=$(which conda)
        eval "$(${CONDA_PATH} shell.bash hook)"
        conda activate ci

        echo '::group::Install testing utilities'
        pip install --progress-bar=off pytest pytest-mock pytest-cov
        echo '::endgroup::'

        # We don't want to run the prototype datasets tests. Since the positional glob into `pytest`, i.e.
        # `test/test_prototype*.py` takes the highest priority, neither `--ignore` nor `--ignore-glob` can help us here.
        rm test/test_prototype_datasets*.py
        pytest \
          -v --durations=25 \
          --cov=torchvision/prototype --cov-report=term-missing \
          --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" \
          test/test_prototype_*.py


tests

name: tests

on:
  pull_request:
    paths:
      - "test/test_datasets_download.py"
      - ".github/failed_schedule_issue_template.md"
      - ".github/workflows/tests-schedule.yml"

  schedule:
    - cron: "0 9 * * *"

jobs:
  download:
    runs-on: ubuntu-latest

    steps:
      - name: Set up python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Upgrade system packages
        run: python -m pip install --upgrade pip setuptools wheel

      - name: SSL
        run: python -c 'import ssl; print(ssl.OPENSSL_VERSION)'

      - name: Checkout repository
        uses: actions/checkout@v2

      - name: TODO REMOVE THIS! Install non pre-release version of mpmath.
        run: pip install "mpmath<1.4"

      - name: Install torch nightly build
        run: pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html

      - name: Install torchvision
        run: pip install --no-build-isolation --editable .

      - name: Install all optional dataset requirements
        run: pip install scipy pycocotools lmdb gdown

      - name: Install tests requirements
        run: pip install pytest

      - name: Run tests
        run: pytest -ra -v test/test_datasets_download.py

      - uses: JasonEtco/create-an-issue@v2.4.0
        name: Create issue if download tests failed
        if: failure() && github.event_name == 'schedule'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REPO: ${{ github.repository }}
          WORKFLOW: ${{ github.workflow }}
          JOB: ${{ github.job }}
          ID: ${{ github.run_id }}
        with:
          filename: .github/failed_schedule_issue_template.md


Tests

name: Tests

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
  workflow_dispatch:

jobs:
  unittests-linux:
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        runner: ["linux.12xlarge"]
        gpu-arch-type: ["cpu"]
        include:
          - python-version: 3.8
            runner: linux.g5.4xlarge.nvidia.gpu
            gpu-arch-type: cuda
            gpu-arch-version: "11.8"
      fail-fast: false
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      runner: ${{ matrix.runner }}
      gpu-arch-type: ${{ matrix.gpu-arch-type }}
      gpu-arch-version: ${{ matrix.gpu-arch-version }}
      timeout: 120
      test-infra-ref: main
      script: |
        set -euo pipefail

        export PYTHON_VERSION=${{ matrix.python-version }}
        export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
        export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}

        ./.github/scripts/unittest.sh

  unittests-macos:
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        runner: ["macos-m1-stable"]
      fail-fast: false
    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
    with:
      repository: pytorch/vision
      timeout: 240
      runner: ${{ matrix.runner }}
      test-infra-ref: main
      script: |
        set -euo pipefail

        export PYTHON_VERSION=${{ matrix.python-version }}
        export GPU_ARCH_TYPE=cpu
        export GPU_ARCH_VERSION=''

        ${CONDA_RUN} ./.github/scripts/unittest.sh

  unittests-windows:
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        runner: ["windows.4xlarge"]
        gpu-arch-type: ["cpu"]
        include:
          - python-version: "3.8"
            runner: windows.g5.4xlarge.nvidia.gpu
            gpu-arch-type: cuda
            gpu-arch-version: "11.8"
      fail-fast: false
    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
    with:
      repository: pytorch/vision
      runner: ${{ matrix.runner }}
      gpu-arch-type: ${{ matrix.gpu-arch-type }}
      gpu-arch-version: ${{ matrix.gpu-arch-version }}
      timeout: 120
      test-infra-ref: main
      script: |
        set -euxo pipefail

        export PYTHON_VERSION=${{ matrix.python-version }}
        export VC_YEAR=2019
        export VSDEVCMD_ARGS=""
        export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
        export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}

        ./.github/scripts/unittest.sh

  onnx:
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      test-infra-ref: main
      script: |
        set -euo pipefail

        export PYTHON_VERSION=3.8
        export GPU_ARCH_TYPE=cpu
        export GPU_ARCH_VERSION=''

        ./.github/scripts/setup-env.sh

        # Prepare conda
        CONDA_PATH=$(which conda)
        eval "$(${CONDA_PATH} shell.bash hook)"
        conda activate ci

        echo '::group::Install ONNX'
        pip install --progress-bar=off onnx onnxruntime
        echo '::endgroup::'

        echo '::group::Install testing utilities'
        pip install --progress-bar=off pytest
        echo '::endgroup::'

        echo '::group::Run ONNX tests'
        pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 test/test_onnx.py
        echo '::endgroup::'

  unittests-extended:
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      repository: pytorch/vision
      test-infra-ref: main
      script: |
        set -euo pipefail

        export PYTHON_VERSION=3.8
        export GPU_ARCH_TYPE=cpu
        export GPU_ARCH_VERSION=''

        ./.github/scripts/setup-env.sh

        # Prepare conda
        CONDA_PATH=$(which conda)
        eval "$(${CONDA_PATH} shell.bash hook)"
        conda activate ci

        echo '::group::Pre-download model weights'
        pip install --progress-bar=off aiohttp aiofiles tqdm
        python scripts/download_model_urls.py
        echo '::endgroup::'

        echo '::group::Install testing utilities'
        # TODO: remove the <8 constraint on pytest when https://github.com/pytorch/vision/issues/8238 is closed
        pip install --progress-bar=off "pytest<8"
        echo '::endgroup::'

        echo '::group::Run extended unittests'
        export PYTORCH_TEST_WITH_EXTENDED=1
        pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25 test/test_extended_*.py
        echo '::endgroup::'


Update viable/strict

name: Update viable/strict

on:
  pull_request:
    paths:
      - .github/workflows/update-viablestrict.yml
  schedule:
    - cron: 10,40 * * * *
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: false

jobs:
  do_update_viablestrict:
    uses: pytorch/test-infra/.github/workflows/update-viablestrict.yml@main
    with:
      repository: pytorch/vision
      required_checks: "Build Linux,Build M1,Build Macos,Build Windows,Tests,CMake,Lint,Docs"
      test-infra-ref: main
    secrets:
      ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }}
      GITHUB_DEPLOY_KEY : ${{ secrets.VISION_GITHUB_DEPLOY_KEY }}


Build and upload package

name: Build and upload package
on:
  push:
    branches:
    - main
  release:
    types:
    - published
  workflow_dispatch:

env:
  FORCE_COLOR: 1

permissions:
  contents: read

jobs:
  build:
    name: Build & Verify package
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v5
      with:
        python-version: 3.12

    - run: python -m pip install --upgrade pip setuptools_scm build twine

    - name: Force version for Test PyPI upload
      if: ${{ !startsWith(github.ref, 'refs/tags') }}
      run: echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV

    - run: echo 'PKG = "pip"'>dvc/_build.py
    - run: python -m build
      env:
        SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DVC: ${{ env.version }}
    - run: twine check --strict dist/*

    - uses: actions/upload-artifact@v4
      with:
        name: Packages
        path: dist/

  test-pypi-publish:
    name: Publish dev package to test.pypi.org
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.action == 'published' || (github.repository == 'iterative/dvc' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}

    environment:
      name: test-pypi
      url: https://test.pypi.org/p/dvc

    permissions:
      id-token: write

    steps:
    - uses: actions/download-artifact@v4
      with:
        name: Packages
        path: dist

    - name: Upload package to Test PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        repository-url: https://test.pypi.org/legacy/
        skip-existing: true

  pypi-publish:
    name: Publish released package to pypi.org
    runs-on: ubuntu-latest
    needs: build
    if: github.event.action == 'published'

    environment:
      name: pypi
      url: https://pypi.org/p/dvc

    permissions:
      id-token: write

    steps:
    - uses: actions/download-artifact@v4
      with:
        name: Packages
        path: dist

    - name: Upload package to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1


Tests

name: Tests

on:
  push:
    branches: [main]
  pull_request:
  schedule:
    - cron: '5 1 * * *'  # every day at 01:05
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  tests:
    timeout-minutes: 50
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-20.04, windows-latest, macos-latest]
        pyv: ["3.9", "3.10", "3.11", "3.12"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.pyv }}
        cache: pip
        cache-dependency-path: pyproject.toml
    - name: install
      run: |
        pip install --upgrade pip wheel
        pip install -e ".[dev]"
    - uses: pre-commit/action@v3.0.1
    - name: run tests
      timeout-minutes: 40
      env:
        PYTHONUTF8: 1
      run: >
        pytest -n=logical --timeout=300 --durations=100
        --cov --cov-report=xml --cov-report=term
    - name: upload coverage report
      uses: codecov/codecov-action@v3
      with:
        file: ./coverage.xml
        fail_ci_if_error: false
  check:
    if: always()
    needs: [tests]
    runs-on: ubuntu-latest
    steps:
      - uses: re-actors/alls-green@release/v1
        with:
          jobs: ${{ toJSON(needs) }}
  notify:
    if: github.ref == 'refs/heads/main' && failure()
    needs: [tests]
    runs-on: ubuntu-latest
    steps:
    - name: Slack Notification
      uses: rtCamp/action-slack-notify@v2.3.0
      env:
        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
        SLACK_COLOR: ${{ job.status }}
        SLACK_MESSAGE: 'CI Failed on main :boom:'
        SLACK_TITLE: CI Status
        SLACK_USERNAME: DVC-CI


benchmarks

name: benchmarks
on: [pull_request, workflow_dispatch]

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  bench:
    name: run benchmarks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-python@v5
        with:
            python-version: "3.11"
      - uses: iterative/dvc-bench@main
        with:
            pytest_options: "-k 'test_init or test_help'"


CodeQL

name: "CodeQL"

on:
  push:
    branches: ["main"]
  pull_request:
    branches: ["main"]
  schedule:
    - cron: '32 19 * * 2'
  workflow_dispatch:

permissions:
  contents: read

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: ['python']

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        queries: security-extended

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3


Remote Plugin Tests

name: Remote Plugin Tests

on:
  pull_request:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  tests:
    timeout-minutes: 45
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-20.04]
        pyv: ["3.10"]
        plugin: ["dvc-s3"]

    steps:
    - uses: actions/checkout@v4
      with:
        path: dvc

    - uses: actions/checkout@v4
      with:
        repository: iterative/${{ matrix.plugin }}
        ref: main
        path: ${{ matrix.plugin }}

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.pyv }}
        cache: pip
        cache-dependency-path: |
          dvc/pyproject.toml
          ${{ matrix.plugin }}/setup.cfg

    - name: Install plugin + DVC@PR
      run: |
        pip install --upgrade pip wheel
        pip install "./dvc[testing]"
        pip install -e "./${{ matrix.plugin }}[tests]"

    - name: Run plugin tests
      timeout-minutes: 15
      working-directory: ${{ matrix.plugin }}
      run: pytest -v -n=auto


PR Tests - Linting

name: PR Tests - Linting

on:
  workflow_call:

  pull_request:
    branches:
      - "**"

concurrency:
  group: linting-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.workflow, github.event.pull_request.number) || github.workflow_ref }}
  cancel-in-progress: true

jobs:
  pr-tests-linting:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 1
      matrix:
        python-version: ["3.12"]

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install pip packages
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      # TODO: change cache key from setup.cfg to something more general
      - name: pip cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}-

      - name: Install Tox
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - uses: pre-commit/action@v3.0.1

      - name: Check Protocol Version
        run: |
          tox -e syft.protocol.check


PR Tests - Stack

name: PR Tests - Stack

on:
  # schedule:
  #   - cron: "0 22 * * *" # 7pm Brazil, 10pm UTC, 8am AEST
  workflow_call:

  pull_request:
    branches:
      - dev
      - main
      - "0.8"

  workflow_dispatch:
    inputs:
      none:
        description: "Run Version Tests Manually"
        required: false

concurrency:
  group: stack-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.workflow, github.event.pull_request.number) || github.workflow_ref }}
  cancel-in-progress: true

jobs:
  pr-syft-image-test:
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]
      fail-fast: false

    runs-on: ${{matrix.os}}

    steps:
      - uses: actions/checkout@v4

      - name: Check for file changes
        uses: dorny/paths-filter@v3
        id: changes
        with:
          base: ${{ github.ref }}
          token: ${{ github.token }}
          filters: .github/file-filters.yml

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        if: steps.changes.outputs.stack == 'true'
        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        if: steps.changes.outputs.stack == 'true'
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        if: steps.changes.outputs.stack == 'true'
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        if: steps.changes.outputs.stack == 'true'
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}

      - name: Install tox
        if: steps.changes.outputs.stack == 'true'
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - name: Run syft backend base image building test
        if: steps.changes.outputs.stack == 'true'
        timeout-minutes: 60
        run: |
          tox -e backend.test.basecpu

  pr-tests-syft-integration:
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]
        pytest-modules: ["local_node"]
      fail-fast: false

    runs-on: ${{matrix.os}}

    steps:
      - uses: actions/checkout@v4

      - name: Check for file changes
        uses: dorny/paths-filter@v3
        id: changes
        with:
          base: ${{ github.ref }}
          token: ${{ github.token }}
          filters: .github/file-filters.yml

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        if: steps.changes.outputs.stack == 'true'
        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        if: steps.changes.outputs.stack == 'true'
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        if: steps.changes.outputs.stack == 'true'
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        if: steps.changes.outputs.stack == 'true'
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}

      - name: Install tox
        if: steps.changes.outputs.stack == 'true'
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - name: Run Syft Integration Tests
        if: steps.changes.outputs.stack == 'true'
        timeout-minutes: 60
        env:
          PYTEST_MODULES: "${{ matrix.pytest-modules }}"
          GITHUB_CI: true
        shell: bash
        run: |
          tox -e syft.test.integration

  pr-tests-integration-k8s:
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]
        pytest-modules: ["frontend network container_workload"]
      fail-fast: false

    runs-on: ${{matrix.os}}

    steps:
      - name: Permission to home directory
        run: |
          sudo chown -R $USER:$USER $HOME
      - uses: actions/checkout@v4
      - name: Check for file changes
        uses: dorny/paths-filter@v3
        id: changes
        with:
          base: ${{ github.ref }}
          token: ${{ github.token }}
          filters: .github/file-filters.yml

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        if: steps.changes.outputs.stack == 'true'
        with:
          python-version: ${{ matrix.python-version }}

      - name: Add K3d Registry
        run: |
          sudo python ./scripts/patch_hosts.py --add-k3d-registry

      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          large-packages: false

      # free 10GB of space
      - name: Remove unnecessary files
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          docker image prune --all --force
          docker builder prune --all --force
          docker system prune --all --force

      - name: Upgrade pip
        if: steps.changes.outputs.stack == 'true'
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        if: steps.changes.outputs.stack == 'true'
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        if: steps.changes.outputs.stack == 'true'
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}

      - name: Install tox
        if: steps.changes.outputs.stack == 'true'
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - name: Install kubectl
        if: steps.changes.outputs.stack == 'true'
        run: |
          # cleanup apt version
          sudo apt remove kubectl || true
          # install kubectl 1.27
          curl -LO https://dl.k8s.io/release/v1.27.2/bin/linux/amd64/kubectl
          chmod +x kubectl
          sudo install kubectl /usr/local/bin;

      - name: Install helm
        if: steps.changes.outputs.stack == 'true'
        run: |
          # install helm
          curl  -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
          chmod 700 get_helm.sh
          ./get_helm.sh

      - name: Run K8s & Helm integration tests
        if: steps.changes.outputs.stack == 'true'
        timeout-minutes: 60
        env:
          PYTEST_MODULES: "${{ matrix.pytest-modules }}"
          GITHUB_CI: true
          AZURE_BLOB_STORAGE_KEY: "${{ secrets.AZURE_BLOB_STORAGE_KEY }}"
        shell: bash
        run: |
          K3D_VERSION=v5.6.3
          DEVSPACE_VERSION=v6.3.12
          # install k3d
          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
          mv k3d-linux-amd64 k3d
          chmod +x k3d
          export PATH=`pwd`:$PATH
          k3d version
          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
          chmod +x devspace
          devspace version

          tox -e stack.test.integration.k8s
          tox -e syft.build.helm
          tox -e syft.package.helm
          # tox -e syft.test.helm

      - name: Get current timestamp
        id: date
        if: failure()
        shell: bash
        run: echo "date=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Collect logs from k3d
        if: steps.changes.outputs.stack == 'true' && failure()
        shell: bash
        run: |
          mkdir -p ./k8s-logs
          kubectl describe all -A --context k3d-test-gateway-1 --namespace syft > ./k8s-logs/test-gateway-1-desc-${{ steps.date.outputs.date }}.txt
          kubectl describe all -A --context k3d-test-domain-1 --namespace syft > ./k8s-logs/test-domain-1-desc-${{ steps.date.outputs.date }}.txt
          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-gateway-1 --namespace syft > ./k8s-logs/test-gateway-1-logs-${{ steps.date.outputs.date }}.txt
          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-domain-1 --namespace syft > ./k8s-logs/test-domain-1-logs-${{ steps.date.outputs.date }}.txt
          ls -la ./k8s-logs

      - name: Upload logs to GitHub
        uses: actions/upload-artifact@master
        if: steps.changes.outputs.stack == 'true' && failure()
        with:
          name: k8s-logs-integration-${{ matrix.os }}-${{ steps.date.outputs.date }}
          path: ./k8s-logs/

      - name: Cleanup k3d
        if: steps.changes.outputs.stack == 'true' && failure()
        shell: bash
        run: |
          export PATH=`pwd`:$PATH
          k3d cluster delete test-gateway-1 || true
          k3d cluster delete test-domain-1 || true
          k3d registry delete k3d-registry.localhost || true

  pr-tests-notebook-k8s:
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]
      fail-fast: false

    runs-on: ${{matrix.os}}

    steps:
      - name: Permission to home directory
        run: |
          sudo chown -R $USER:$USER $HOME
      - uses: actions/checkout@v4
      - name: Check for file changes
        uses: dorny/paths-filter@v3
        id: changes
        with:
          base: ${{ github.ref }}
          token: ${{ github.token }}
          filters: .github/file-filters.yml

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        if: steps.changes.outputs.stack == 'true'
        with:
          python-version: ${{ matrix.python-version }}

      - name: Add K3d Registry
        run: |
          sudo python ./scripts/patch_hosts.py --add-k3d-registry

      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          large-packages: false

      # free 10GB of space
      - name: Remove unnecessary files
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          docker image prune --all --force
          docker builder prune --all --force
          docker system prune --all --force

      - name: Upgrade pip
        if: steps.changes.outputs.stack == 'true'
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        if: steps.changes.outputs.stack == 'true'
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        if: steps.changes.outputs.stack == 'true'
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}

      - name: Install tox
        if: steps.changes.outputs.stack == 'true'
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - name: Install kubectl
        if: steps.changes.outputs.stack == 'true'
        run: |
          # cleanup apt version
          sudo apt remove kubectl || true
          # install kubectl 1.27
          curl -LO https://dl.k8s.io/release/v1.27.2/bin/linux/amd64/kubectl
          chmod +x kubectl
          sudo install kubectl /usr/local/bin;

      - name: Install helm
        if: steps.changes.outputs.stack == 'true'
        run: |
          # install helm
          curl  -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
          chmod 700 get_helm.sh
          ./get_helm.sh

      - name: Run Notebooks Tests
        if: steps.changes.outputs.stack == 'true'
        timeout-minutes: 60
        env:
          GITHUB_CI: true
        shell: bash
        run: |
          K3D_VERSION=v5.6.3
          DEVSPACE_VERSION=v6.3.12
          # install k3d
          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
          mv k3d-linux-amd64 k3d
          chmod +x k3d
          export PATH=`pwd`:$PATH
          k3d version
          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
          chmod +x devspace
          devspace version
          tox -e stack.test.notebook.k8s

      - name: Get current timestamp
        id: date
        if: failure()
        shell: bash
        run: echo "date=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Collect logs from k3d
        if: steps.changes.outputs.stack == 'true' && failure()
        shell: bash
        run: |
          mkdir -p ./k8s-logs
          kubectl describe all -A --context k3d-test-gateway-1 --namespace syft > ./k8s-logs/test-gateway-1-desc-${{ steps.date.outputs.date }}.txt
          kubectl describe all -A --context k3d-test-domain-1 --namespace syft > ./k8s-logs/test-domain-1-desc-${{ steps.date.outputs.date }}.txt
          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-gateway-1 --namespace syft > ./k8s-logs/test-gateway-1-logs-${{ steps.date.outputs.date }}.txt
          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-domain-1 --namespace syft > ./k8s-logs/test-domain-1-logs-${{ steps.date.outputs.date }}.txt
          ls -la ./k8s-logs

      - name: Upload logs to GitHub
        uses: actions/upload-artifact@master
        if: steps.changes.outputs.stack == 'true' && failure()
        with:
          name: k8s-logs-notebook-${{ matrix.os }}-${{ steps.date.outputs.date }}
          path: ./k8s-logs/

      - name: Cleanup k3d
        if: steps.changes.outputs.stack == 'true' && failure()
        shell: bash
        run: |
          export PATH=`pwd`:$PATH
          k3d cluster delete test-gateway-1 || true
          k3d cluster delete test-domain-1 || true
          k3d registry delete k3d-registry.localhost || true


PR Tests - Syft

name: PR Tests - Syft

on:
  workflow_call:

  pull_request:
    branches:
      - dev
      - main
      - "0.8"

  workflow_dispatch:
    inputs:
      none:
        description: "Run Tests Manually"
        required: false

concurrency:
  group: syft-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.workflow, github.event.pull_request.number) || github.workflow_ref }}
  cancel-in-progress: true

jobs:
  pr-tests-syft-unit:
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.12"]
        include:
          - python-version: "3.11"
            os: "ubuntu-latest"
          - python-version: "3.10"
            os: "ubuntu-latest"

    runs-on: ${{ matrix.os }}
    steps:
      # - name: Permission to home directory
      #   if: matrix.os == 'ubuntu-latest'
      #   run: |
      #     sudo chown -R $USER:$USER $HOME
      - name: "clean .git/config"
        if: matrix.os == 'windows-latest'
        continue-on-error: true
        shell: bash
        run: |
          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
          rm ${GITHUB_WORKSPACE}/.git/config

      - uses: actions/checkout@v4

      - name: Check for file changes
        uses: dorny/paths-filter@v3
        id: changes
        with:
          base: ${{ github.ref }}
          token: ${{ github.token }}
          filters: .github/file-filters.yml

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        if: steps.changes.outputs.syft == 'true'
        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        if: steps.changes.outputs.syft == 'true'
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        id: pip-cache
        if: steps.changes.outputs.syft == 'true'
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        if: steps.changes.outputs.syft == 'true'
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}-

      # - name: Docker on MacOS
      #   if: steps.changes.outputs.syft == 'true' && matrix.os == 'macos-latest'
      #   uses: crazy-max/ghaction-setup-docker@v3.3.0
      #   with:
      #     set-host: true

      - name: Install Dependencies
        if: steps.changes.outputs.syft == 'true'
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - name: Run unit tests
        if: steps.changes.outputs.syft == 'true'
        run: |
          tox -e syft.test.unit
          tox -e seaweedfs.test.unit

  pr-tests-syft-notebook-python:
    strategy:
      max-parallel: 99
      matrix:
        # Disable on windows until its flakyness is reduced.
        # os: [ubuntu-latest, macos-latest, windows-latest]
        os: [ubuntu-latest, macos-latest]
        python-version: ["3.12"]
        deployment-type: ["python"]
        notebook-paths: ["tutorials"]
        bump-version: ["False"]
        include:
          - python-version: "3.11"
            os: "ubuntu-latest"
            deployment-type: "python"
            notebook-paths: "tutorials"
          - python-version: "3.10"
            os: "ubuntu-latest"
            deployment-type: "python"
            notebook-paths: "tutorials"
          - python-version: "3.12"
            os: "ubuntu-latest"
            deployment-type: "python"
            notebook-paths: "tutorials"
            bump-version: "True"

    runs-on: ${{ matrix.os }}
    steps:
      # - name: Permission to home directory
      #   if: matrix.os == 'ubuntu-latest'
      #   run: |
      #     sudo chown -R $USER:$USER $HOME
      - name: "clean .git/config"
        if: matrix.os == 'windows-latest'
        continue-on-error: true
        shell: bash
        run: |
          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
          rm ${GITHUB_WORKSPACE}/.git/config

      - uses: actions/checkout@v4

      - name: Check for file changes
        uses: dorny/paths-filter@v3
        id: changes
        with:
          base: ${{ github.ref }}
          token: ${{ github.token }}
          filters: .github/file-filters.yml

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        id: pip-cache
        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}-

      - name: Install Dependencies
        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - name: Run notebook tests
        uses: nick-fields/retry@v3
        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks == 'true'
        env:
          ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
          TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}"
          BUMP_VERSION: "${{ matrix.bump-version }}"
        with:
          timeout_seconds: 2400
          max_attempts: 3
          command: tox -e syft.test.notebook

  pr-tests-syft-notebook-single-container:
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10", "3.11", "3.12"]
        deployment-type: ["remote"]
        notebook-paths: ["api/0.8"]
      fail-fast: false

    runs-on: ${{matrix.os}}

    steps:
      # - name: Permission to home directory
      #   if: matrix.os == 'ubuntu-latest'
      #   run: |
      #     sudo chown -R $USER:$USER $HOME
      - uses: actions/checkout@v4

      # free 10GB of space
      - name: Remove unnecessary files
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          docker image prune --all --force
          docker builder prune --all --force
          docker system prune --all --force

      - name: Check for file changes
        uses: dorny/paths-filter@v3
        id: changes
        with:
          base: ${{ github.ref }}
          token: ${{ github.token }}
          filters: .github/file-filters.yml

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        id: pip-cache
        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}-

      - name: Install Dependencies
        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - name: Docker Compose on Linux
        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'ubuntu-latest'
        shell: bash
        run: |
          mkdir -p ~/.docker/cli-plugins
          DOCKER_COMPOSE_VERSION=v2.21.0
          curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
          chmod +x ~/.docker/cli-plugins/docker-compose
          docker compose version

      - name: Docker on MacOS
        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest'
        uses: crazy-max/ghaction-setup-docker@v3.3.0

      - name: Docker Compose on MacOS
        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest'
        shell: bash
        run: |
          brew install docker-compose
          mkdir -p ~/.docker/cli-plugins
          ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose || true
          docker compose version

      - name: Docker on Windows
        if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'windows-latest'
        shell: pwsh
        run: |
          [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
          Start-BitsTransfer -Source "https://download.docker.com/win/static/stable/x86_64/docker-23.0.1.zip"
          Expand-Archive docker-23.0.1.zip -DestinationPath $Env:ProgramFiles
          &$Env:ProgramFiles\Docker\dockerd --register-service
          Start-Service docker
          docker version
          docker compose version

      - name: Run unit tests
        if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true'
        env:
          ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
          DEV_MODE: "True" # force orchestra --build
          TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}"
        run: |
          tox -e single_container.launch
          tox -e stack.test.notebook
          tox -e single_container.destroy

  pr-tests-syft-security:
    strategy:
      max-parallel: 1
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]

    runs-on: ${{ matrix.os }}
    steps:
      # - name: Permission to home directory
      #   if: matrix.os == 'ubuntu-latest'
      #   run: |
      #     sudo chown -R $USER:$USER $HOME
      - uses: actions/checkout@v4
      - name: Check for file changes
        uses: dorny/paths-filter@v3
        id: changes
        with:
          base: ${{ github.ref }}
          token: ${{ github.token }}
          filters: .github/file-filters.yml

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        if: steps.changes.outputs.syft == 'true'
        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        if: steps.changes.outputs.syft == 'true'
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        if: steps.changes.outputs.syft == 'true'
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        if: steps.changes.outputs.syft == 'true'
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}-

      - name: Install Dependencies
        if: steps.changes.outputs.syft == 'true'
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - name: Scan for security issues
        if: steps.changes.outputs.syft == 'true'
        run: |
          tox -e syft.test.security


CD - Docs

name: CD - Docs
on:
  workflow_dispatch:
    inputs:
      none:
        description: "Deploy Syft Documentation"
        required: false
  # pull_request:
  #   branches: [dev]
  #   paths: [docs/]
  # push:
  #   branches: [dev]
  #   paths: [docs/]

jobs:
  cd-docs:
    strategy:
      matrix:
        python-version: ["3.12"]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install tox
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0
          uv --version

      - name: Build the docs
        run: |
          tox -e syft.docs

      - name: Deploy the docs
        uses: s0/git-publish-subdir-action@develop
        env:
          REPO: self
          BRANCH: gh-pages
          FOLDER: docs/build/html
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


CD - Syft

name: CD - Syft

on:
  schedule:
    - cron: "0 12 * * 0" # At 12:00 UTC on every Sunday

  workflow_dispatch:
    inputs:
      skip_tests:
        description: "If true, skip pre-release tests"
        required: false
        default: "false"
        type: choice
        options:
          - false
          - true

      release_platform:
        description: "Release Platform"
        required: true
        default: "REAL_PYPI"
        type: choice
        options:
          - REAL_PYPI
          - TEST_PYPI
          - REAL_AND_TEST_PYPI

# Prevents concurrent runs of the same workflow
# while the previous run is still in progress
concurrency:
  group: "CD - Syft"
  cancel-in-progress: false

jobs:
  release-checks:
    if: github.repository == 'OpenMined/PySyft' # don't run on forks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      # The steps ensure that the cron job is able to run only for
      # for beta releases and not for stable releases
      # For stable release, we always have to manually run the workflow
      - name: Check if the release is a beta release
        run: |
          if [[ $(python packages/grid/VERSION) != *"beta"* && ${{github.event_name}} == 'schedule'  ]]; then
            echo "Cron Job could only be run for beta releases"
            exit 1
          fi

      # Print the github user triggering the workflow
      - name: Github User triggering the workflow
        run: |
          echo "Github User: ${{ github.actor }}"

  call-pr-tests-linting:
    needs: [release-checks]
    if: github.repository == 'OpenMined/PySyft' && needs.release-checks.result == 'success' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
    uses: OpenMined/PySyft/.github/workflows/pr-tests-linting.yml@dev

  call-pr-tests-syft:
    needs: [release-checks]
    if: github.repository == 'OpenMined/PySyft' && needs.release-checks.result == 'success' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
    uses: OpenMined/PySyft/.github/workflows/pr-tests-syft.yml@dev

  call-pr-tests-stack:
    needs: [release-checks]
    if: github.repository == 'OpenMined/PySyft' && needs.release-checks.result == 'success' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
    uses: OpenMined/PySyft/.github/workflows/pr-tests-stack.yml@dev
    secrets: inherit

  build-and-push-docker-images:
    needs:
      [
        call-pr-tests-linting,
        call-pr-tests-syft,
        call-pr-tests-stack,
        release-checks,
      ]
    if: always() && needs.release-checks.result == 'success' && (needs.call-pr-tests-linting.result == 'success' && needs.call-pr-tests-syft.result == 'success' && needs.call-pr-tests-stack.result == 'success' || github.event.inputs.skip_tests == 'true')

    strategy:
      matrix:
        runner: [sh-arc-linux-x64, sh-arc-linux-arm64]
    runs-on: ${{ matrix.runner }}

    outputs:
      release_tag: ${{ steps.get_release_tag.outputs.release_tag }}
      grid_version: ${{ steps.release_metadata.outputs.grid_version }}

    steps:
      - uses: actions/checkout@v4

      # actions/setup-python doesn't yet support ARM
      - name: Setup Python on x64
        if: ${{ !endsWith(matrix.runner, '-arm64') }}
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      # Currently psutil package requires gcc to be installed on arm
      # for building psutil from source
      # as linux/aarch64 wheels not avaialble for psutil
      # We could remove once we have aarch64 wheels
      # https://github.com/giampaolo/psutil/issues/1972
      - name: Install Metadata packages for arm64
        if: ${{ endsWith(matrix.runner, '-arm64') }}
        run: |
          sudo apt update -y
          sudo apt install software-properties-common -y
          sudo apt install gcc -y

      - name: Setup Python on arm64
        if: ${{ endsWith(matrix.runner, '-arm64') }}
        uses: deadsnakes/action@v3.1.0
        with:
          python-version: "3.12"

      - name: Install Git
        run: |
          sudo apt-get update
          sudo apt-get install git -y

      - name: Check python version
        run: |
          python --version
          python3 --version
          which python

      - name: Install dependencies
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13 bump2version tox tox-uv==1.9.0
          uv --version

      - name: Get Release tag
        id: get_release_tag
        run: |
          if [[ $(python packages/grid/VERSION) == *"beta"* ]]; then
          echo "release_tag=beta" >> $GITHUB_OUTPUT
          else
            echo "release_tag=latest" >> $GITHUB_OUTPUT
          fi

      - name: Bump the Version
        if: steps.get_release_tag.outputs.release_tag == 'beta'
        run: |
          ls **/VERSION | xargs -I {} python {}
          cat packages/grid/devspace.yaml | grep '0\.'
          bump2version prenum --allow-dirty --no-commit
          ls **/VERSION | xargs -I {} python {}
          cat packages/grid/devspace.yaml | grep '0\.'

      - name: Generate Release Metadata
        id: release_metadata
        run: |
          if [[ ${{matrix.runner}} == *"x64"* ]]; then
            echo "release_platform=linux/amd64" >> $GITHUB_OUTPUT
            echo "short_release_platform=amd64" >> $GITHUB_OUTPUT
          else
            echo "release_platform=linux/arm64" >> $GITHUB_OUTPUT
            echo "short_release_platform=arm64" >> $GITHUB_OUTPUT
          fi
          echo "grid_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT

      # TODO: Optimize redundant bump protocol version checks
      - name: Check and Bump Protocol Version
        run: |
          if [[ "${{ steps.get_release_tag.outputs.release_tag }}" == "latest" ]]; then
            export BUMP=True
          fi
          tox -e syft.protocol.check

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_LOGIN }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Build and push `grid-backend` image to DockerHub
        id: grid-backend-build
        uses: docker/build-push-action@v5
        with:
          context: ./packages
          file: ./packages/grid/backend/backend.dockerfile
          platforms: ${{ steps.release_metadata.outputs.release_platform }}
          target: backend
          outputs: type=image,name=openmined/grid-backend,push-by-digest=true,name-canonical=true,push=true
          cache-from: type=registry,ref=openmined/grid-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }}
          cache-to: type=registry,ref=openmined/grid-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max

      - name: Export digest for grid-backend
        run: |
          mkdir -p /tmp/digests/grid-backend
          digest="${{ steps.grid-backend-build.outputs.digest }}"
          touch "/tmp/digests/grid-backend/${digest#sha256:}"

      - name: Build and push `grid-frontend` image to DockerHub
        id: grid-frontend-build
        uses: docker/build-push-action@v5
        with:
          context: ./packages/grid/frontend
          file: ./packages/grid/frontend/frontend.dockerfile
          platforms: ${{ steps.release_metadata.outputs.release_platform }}
          outputs: type=image,name=openmined/grid-frontend,push-by-digest=true,name-canonical=true,push=true
          target: grid-ui-development
          cache-from: type=registry,ref=openmined/grid-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform }}
          cache-to: type=registry,ref=openmined/grid-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max

      - name: Export digest for grid-frontend
        run: |
          mkdir -p /tmp/digests/grid-frontend
          digest="${{ steps.grid-frontend-build.outputs.digest }}"
          touch "/tmp/digests/grid-frontend/${digest#sha256:}"

      - name: Build and push `grid-seaweedfs` image to DockerHub
        id: grid-seaweedfs-build
        uses: docker/build-push-action@v5
        with:
          context: ./packages/grid/seaweedfs
          file: ./packages/grid/seaweedfs/seaweedfs.dockerfile
          platforms: ${{ steps.release_metadata.outputs.release_platform }}
          outputs: type=image,name=openmined/grid-seaweedfs,push-by-digest=true,name-canonical=true,push=true
          cache-from: type=registry,ref=openmined/grid-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform }}
          cache-to: type=registry,ref=openmined/grid-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max

      - name: Export digest for grid-seaweedfs
        run: |
          mkdir -p /tmp/digests/grid-seaweedfs
          digest="${{ steps.grid-seaweedfs-build.outputs.digest }}"
          touch "/tmp/digests/grid-seaweedfs/${digest#sha256:}"

      # Some of the dependencies of grid-enclave-attestation are not available for arm64
      # Hence, we are building grid-enclave-attestation only for x64 (see the `if` conditional)
      - name: Build and push `grid-enclave-attestation` image to DockerHub
        if: ${{ endsWith(matrix.runner, '-x64') }}
        id: grid-enclave-attestation-build
        uses: docker/build-push-action@v5
        with:
          context: ./packages/grid/enclave/attestation
          file: ./packages/grid/enclave/attestation/attestation.dockerfile
          platforms: ${{ steps.release_metadata.outputs.release_platform }}
          outputs: type=image,name=openmined/grid-enclave-attestation,push-by-digest=true,name-canonical=true,push=true
          cache-from: type=registry,ref=openmined/grid-enclave-attestation:cache-${{ steps.release_metadata.outputs.short_release_platform }}
          cache-to: type=registry,ref=openmined/grid-enclave-attestation:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max

      - name: Export digest for grid-enclave-attestation
        if: ${{ endsWith(matrix.runner, '-x64') }}
        run: |
          mkdir -p /tmp/digests/grid-enclave-attestation
          digest="${{ steps.grid-enclave-attestation-build.outputs.digest }}"
          touch "/tmp/digests/grid-enclave-attestation/${digest#sha256:}"

      - name: Build and push `syft` image to registry
        id: syft-build
        uses: docker/build-push-action@v5
        with:
          context: ./packages/
          file: ./packages/grid/syft-client/syft.Dockerfile
          outputs: type=image,name=openmined/syft-client,push-by-digest=true,name-canonical=true,push=true
          platforms: ${{ steps.release_metadata.outputs.release_platform }}
          cache-from: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }}
          cache-to: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max

      - name: Export digest for `syft` image
        run: |
          mkdir -p /tmp/digests/syft
          digest="${{ steps.syft-build.outputs.digest }}"
          touch "/tmp/digests/syft/${digest#sha256:}"

      - name: Upload digests
        uses: actions/upload-artifact@v4
        with:
          name: digests-${{ steps.release_metadata.outputs.grid_version }}-${{ steps.release_metadata.outputs.short_release_platform }}
          path: /tmp/digests/*
          if-no-files-found: error
          retention-days: 1

  #Used to merge x64 and arm64 into one docker image
  merge-docker-images:
    needs: [build-and-push-docker-images]
    if: always() && (needs.build-and-push-docker-images.result == 'success')

    runs-on: sh-arc-linux-x64

    outputs:
      release_tag: ${{ needs.build-and-push-docker-images.outputs.release_tag }}

    steps:
      - name: Download digests
        uses: actions/download-artifact@v4
        with:
          path: /tmp/digests
          pattern: digests-${{ needs.build-and-push-docker-images.outputs.grid_version }}-*
          merge-multiple: true

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_LOGIN }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Create manifest list and push for grid-backend
        working-directory: /tmp/digests/grid-backend
        run: |
          docker buildx imagetools create  \
            -t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
            -t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
            $(printf 'openmined/grid-backend@sha256:%s ' *)

      - name: Create manifest list and push for grid-frontend
        working-directory: /tmp/digests/grid-frontend
        run: |
          docker buildx imagetools create  \
            -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
            -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
            $(printf 'openmined/grid-frontend@sha256:%s ' *)

      - name: Create manifest list and push for grid-seaweedfs
        working-directory: /tmp/digests/grid-seaweedfs
        run: |
          docker buildx imagetools create  \
            -t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
            -t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
            $(printf 'openmined/grid-seaweedfs@sha256:%s ' *)

      - name: Create manifest list and push for grid-enclave-attestation
        working-directory: /tmp/digests/grid-enclave-attestation
        run: |
          docker buildx imagetools create  \
            -t openmined/grid-enclave-attestation:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
            -t openmined/grid-enclave-attestation:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
            $(printf 'openmined/grid-enclave-attestation@sha256:%s ' *)

      - name: Create manifest list and push for syft
        working-directory: /tmp/digests/syft
        run: |
          docker buildx imagetools create  \
            -t openmined/syft-client:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
            -t openmined/syft-client:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
            $(printf 'openmined/syft-client@sha256:%s ' *)

  deploy-syft:
    needs: [merge-docker-images]
    if: always() &&  needs.merge-docker-images.result == 'success'

    runs-on: ubuntu-latest
    permissions:
      contents: write # For tag and release notes.

    outputs:
      syft_version: ${{ steps.release_checks.outputs.syft_version }}
      release_platform: ${{ github.event.inputs.release_platform }}

    steps:
      - name: Permission to home directory
        run: |
          sudo chown -R $USER:$USER $HOME

      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}

      # free 10GB of space
      - name: Remove unnecessary files
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          docker image prune --all --force
          docker builder prune --all --force
          docker system prune --all --force

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install dependencies
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0 setuptools wheel twine bump2version PyYAML
          uv --version

      - name: Bump the Version
        if: needs.merge-docker-images.outputs.release_tag == 'beta'
        run: |
          ls **/VERSION | xargs -I {} python {}
          cat packages/grid/devspace.yaml | grep '0\.'
          bump2version prenum --allow-dirty --no-commit
          ls **/VERSION | xargs -I {} python {}
          cat packages/grid/devspace.yaml | grep '0\.'

      - name: Generate Release Metadata
        id: release_checks
        run: |
          echo "github_release_version=$(python packages/grid/VERSION |  sed 's/-beta./b/')" >> $GITHUB_OUTPUT
          echo "future_stable_version=$(python packages/grid/VERSION |   sed 's/-beta.*//')" >> $GITHUB_OUTPUT
          echo "syft_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
          if [[ ${{ needs.merge-docker-images.outputs.release_tag}} == "beta" ]]; then
            echo "is_beta_release=true" >> $GITHUB_OUTPUT
          else
            echo "is_beta_release=false" >> $GITHUB_OUTPUT
          fi

      - name: Check Protocol Version
        run: |
          tox -e syft.protocol.check

      - name: Build Helm Chart
        shell: bash
        run: |
          # install k3d
          K3D_VERSION=v5.6.3
          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
          mv k3d-linux-amd64 k3d
          chmod +x k3d
          export PATH=`pwd`:$PATH
          k3d version

          #Install Devspace
          DEVSPACE_VERSION=v6.3.12
          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
          chmod +x devspace
          devspace version

          # Install helm
          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
          helm version

          tox -e syft.build.helm
          tox -e syft.package.helm

      - name: Check and Bump Protocol Version
        run: |
          if [[ "${{ needs.merge-docker-images.outputs.release_tag }}" == "latest" ]]; then
            export BUMP=True
          fi
          tox -e syft.protocol.check

      - name: Update PyPI Readme
        run: |
          python scripts/convert_to_pypi_readme.py --input-file  packages/syft/README.md --output-file packages/syft/PYPI.md --version ${{ steps.release_checks.outputs.future_stable_version }}

      - name: Linting
        run: |
          tox -e lint || true

      - name: Changes to commit to Syft Repo during beta release
        if: needs.merge-docker-images.outputs.release_tag == 'beta'
        uses: EndBug/add-and-commit@v9
        with:
          author_name: ${{ secrets.OM_BOT_NAME }}
          author_email: ${{ secrets.OM_BOT_EMAIL }}
          message: "[syft]bump version"
          add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo',  'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/', 'packages/grid/backend/grid/images/worker_cpu.dockerfile','packages/grid/helm/syft/values.yaml','packages/grid/helm/syft']"

      - name: Changes to commit to Syft Repo during stable release
        if: needs.merge-docker-images.outputs.release_tag == 'latest'
        uses: EndBug/add-and-commit@v9
        with:
          author_name: ${{ secrets.OM_BOT_NAME }}
          author_email: ${{ secrets.OM_BOT_EMAIL }}
          message: "[syft] bump protocol version"
          add: "['packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/','packages/syft/PYPI.md','packages/grid/helm/repo']"

      - name: Scheduled Build and Publish
        if: github.event_name == 'schedule'
        run: |
          tox -e syft.publish
          twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*

      - name: Manual Build and Publish
        if: github.event_name != 'schedule'
        run: |
          tox -e syft.publish
          if [[ "${{ github.event.inputs.release_platform }}" == "REAL_PYPI" ]]; then
            twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
          fi
          if [[ "${{ github.event.inputs.release_platform }}" == "TEST_PYPI" ]]; then
            twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
          fi
          if [[ "${{ github.event.inputs.release_platform }}" == "REAL_AND_TEST_PYPI" ]]; then
            twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
            twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
          fi

      # Checkout Infra repo (nested)
      - name: Checkout Infra Repo
        uses: actions/checkout@v4
        with:
          repository: ${{ secrets.INFRA_REPO }}
          ref: "main"
          token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
          path: infrastructure

      # This step will copy the generated K8s manifest files to the correct directory in Infra repo
      - name: Copy files to Infra Repo
        run: |
          if [[ "${{ needs.merge-docker-images.outputs.release_tag }}" == "beta" ]]; then
            rm -rf infrastructure/gitops/environments/pre_release/
            mkdir -p infrastructure/gitops/environments/pre_release/
            cp -R packages/grid/helm/syft/. infrastructure/gitops/environments/pre_release/
          else
            rm -rf infrastructure/gitops/environments/stable/
            mkdir -p infrastructure/gitops/environments/stable/
            cp -R packages/grid/helm/syft/. infrastructure/gitops/environments/stable/
          fi

      - name: Commit changes to Infra Repo
        uses: EndBug/add-and-commit@v9
        with:
          author_name: ${{ secrets.OM_BOT_NAME }}
          author_email: ${{ secrets.OM_BOT_EMAIL }}
          message: "Update K8s Manifests from Syft Repo"
          add: "."
          push: "origin main"
          cwd: "./infrastructure/"

      - name: Create SyftCLI Config assets
        run: |
          pip install pyyaml
          python scripts/create_syftcli_config.py

      - name: GitHub Release
        if: github.event.inputs.release_platform  != 'TEST_PYPI'
        uses: softprops/action-gh-release@v2
        with:
          name: v${{ steps.release_checks.outputs.github_release_version }}
          generate_release_notes: true
          prerelease: ${{ steps.release_checks.outputs.is_beta_release }}
          files: |
            ./packages/syftcli/manifest.yml
            ./build/syftcli-config/*
          tag_name: v${{ steps.release_checks.outputs.github_release_version }}

      # Checkout to gh-pages and update helm repo
      - name: Checkout to gh-pages
        uses: actions/checkout@v4
        with:
          ref: gh-pages
          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
          path: ghpages

      - name: Copy helm repo files from Syft Repo
        run: |
          rm -rf ghpages/helm/*
          cp -R packages/grid/helm/repo/. ghpages/helm/

      - name: Commit changes to gh-pages
        uses: EndBug/add-and-commit@v9
        with:
          author_name: ${{ secrets.OM_BOT_NAME }}
          author_email: ${{ secrets.OM_BOT_EMAIL }}
          message: "Update Helm package from Syft Repo"
          add: "helm/"
          push: "origin gh-pages"
          cwd: "./ghpages/"

      # Since the post-release tests are dependent on the release being available on PyPI
      # we need to wait for PyPI to update before running the post-release tests
      - name: Wait for PyPI to update
        run: |
          sleep 60

  # Can we remove the always flag from the below job?
  call-cd-post-release-tests:
    needs: [deploy-syft]
    if: always() && github.repository == 'OpenMined/PySyft' && needs.deploy-syft.result == 'success' # don't run on forks
    uses: OpenMined/PySyft/.github/workflows/cd-post-release-tests.yml@dev
    with:
      syft_version: ${{ needs.deploy-syft.outputs.syft_version }}
      release_platform: ${{ github.event.inputs.release_platform}}


Nightlies

name: Nightlies

on:
  schedule:
    - cron: "0 00 * * *" # 12am UTC, 5:30pm Indian, 9pm Brazil, 11am AEDT
  workflow_dispatch:
    inputs:
      none:
        description: "Run Nightlies Tests Manually"
        required: false

jobs:
  call-pr-tests-linting:
    if: github.repository == 'OpenMined/PySyft' # don't run on forks
    uses: OpenMined/PySyft/.github/workflows/pr-tests-linting.yml@dev

  call-pr-tests-syft:
    if: github.repository == 'OpenMined/PySyft' # don't run on forks
    uses: OpenMined/PySyft/.github/workflows/pr-tests-syft.yml@dev

  call-pr-tests-stack:
    if: github.repository == 'OpenMined/PySyft' # don't run on forks
    uses: OpenMined/PySyft/.github/workflows/pr-tests-stack.yml@dev
    secrets: inherit

  call-container-scan:
    if: github.repository == 'OpenMined/PySyft' # don't run on forks
    uses: OpenMined/PySyft/.github/workflows/container-scan.yml@dev
    secrets: inherit


PR Tests - Frontend

name: PR Tests - Frontend

on:
  workflow_call:

  pull_request:
    branches:
      - dev
      - main
      - "0.8"

concurrency:
  group: frontend-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.workflow, github.event.pull_request.number) || github.workflow_ref }}
  cancel-in-progress: true

defaults:
  run:
    working-directory: ./packages/grid/frontend

jobs:
  pr-tests-frontend-unit:
    strategy:
      max-parallel: 3
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]

    runs-on: ubuntu-20.04 # ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - name: Check for file changes
        uses: dorny/paths-filter@v3
        id: changes
        with:
          base: ${{ github.ref }}
          token: ${{ github.token }}
          filters: .github/file-filters.yml

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        if: steps.changes.outputs.frontend == 'true'
        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        if: steps.changes.outputs.frontend == 'true'
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13
          uv --version

      - name: Get pip cache dir
        id: pip-cache
        if: steps.changes.outputs.frontend == 'true'
        shell: bash
        run: |
          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        if: steps.changes.outputs.frontend == 'true'
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-frontend
          restore-keys: |
            ${{ runner.os }}-uv-py${{ matrix.python-version }}-

      - name: Docker on MacOS
        if: steps.changes.outputs.frontend == 'true' && matrix.os == 'macos-latest'
        uses: crazy-max/ghaction-setup-docker@v3.3.0

      - name: Install Tox
        if: steps.changes.outputs.frontend == 'true'
        run: |
          pip install --upgrade tox tox-uv==1.9.0

      - name: Remove existing containers
        if: steps.changes.outputs.frontend == 'true'
        continue-on-error: true
        shell: bash
        run: |
          docker rm $(docker ps -aq) --force || true
          docker volume prune -f || true
          docker buildx use default || true

      - name: Run Frontend Unit Tests
        if: steps.changes.outputs.frontend == 'true'
        env:
          DOCKER: true
        run: |
          tox -e frontend.test.unit