Build MQ Broker Docker Image

name: Build MQ Broker Docker Image

on:
  push:
    branches:
      - 'main'
    paths:
      - 'dev/mq/**'
      - '.github/workflows/build-mq-broker.yml'

  workflow_dispatch:

env:
  RABBITMQ_VERSION: 3.12-alpine

jobs:
  publish:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write

    steps:
    - uses: actions/checkout@v4

    - name: Set up QEMU
      uses: docker/setup-qemu-action@v3

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3

    - name: Login to GitHub Container Registry
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Docker Build & Push
      uses: docker/build-push-action@v6
      env:
        DOCKER_BUILD_NO_SUMMARY: true
      with:
        context: .
        file: dev/mq/Dockerfile
        platforms: linux/amd64,linux/arm64
        push: true
        build-args: RABBITMQ_VERSION=${{ env.RABBITMQ_VERSION }}
        tags: |
          ghcr.io/ietf-tools/datatracker-mq:${{ env.RABBITMQ_VERSION }}
          ghcr.io/ietf-tools/datatracker-mq:latest


Dependency Review

# Dependency Review Action
#
# This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging.
#
# Source repository: https://github.com/actions/dependency-review-action
# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement
name: 'Dependency Review'
on: [pull_request]

permissions:
  contents: read

jobs:
  dependency-review:
    runs-on: ubuntu-latest
    steps:
      - name: 'Checkout Repository'
        uses: actions/checkout@v4
      - name: 'Dependency Review'
        uses: actions/dependency-review-action@v4
        with:
          vulnerability-check: false


Nightly Dev Shared Assets Sync

# GITHUB ACTIONS - WORKFLOW

# RSync the assets in the shared assets volume

name: Nightly Dev Shared Assets Sync

# Controls when the workflow will run
on:
  # Run every night
  schedule:
    - cron: '0 1 * * *'

  # Run on app-rsync-extras.sh changes
  push:
    branches:
      - main
    paths:
      - 'docker/scripts/app-rsync-extras.sh'

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  build:
    name: Build Docker Image
    runs-on: ubuntu-latest
    if: ${{ github.event_name != 'schedule' }}
    permissions:
      contents: read
      packages: write
    steps:
      - uses: actions/checkout@v4

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Docker Build & Push
        uses: docker/build-push-action@v6
        env:
          DOCKER_BUILD_NO_SUMMARY: true
        with:
          context: .
          file: dev/shared-assets-sync/Dockerfile
          push: true
          tags: ghcr.io/ietf-tools/datatracker-rsync-assets:latest

  sync:
    name: Run assets rsync
    if: ${{ always() }}
    runs-on: [self-hosted, dev-server]
    needs: [build]
    steps:
    - name: Run rsync
      env:
        DEBIAN_FRONTEND: noninteractive
      run: |
        docker pull ghcr.io/ietf-tools/datatracker-rsync-assets:latest
        docker run --rm -v dt-assets:/assets ghcr.io/ietf-tools/datatracker-rsync-assets:latest
        docker image prune -a -f


Sandbox Refresh

name: Sandbox Refresh

on:
  # Run every night
  schedule:
   - cron: '0 9 * * *'

  workflow_dispatch:

jobs:
  main:
    name: Refresh DBs
    runs-on: [self-hosted, dev-server]
    permissions:
      contents: read

    steps:
    - uses: actions/checkout@v4

    - name: Refresh DBs
      env:
        DEBIAN_FRONTEND: noninteractive
      run: |
        echo "Install Deploy to Container CLI dependencies..."
        cd dev/deploy-to-container
        npm ci
        cd ../..
        echo "Start Refresh..."
        node ./dev/deploy-to-container/refresh.js

    - name: Cleanup old docker resources
      env:
        DEBIAN_FRONTEND: noninteractive
      run: |
        docker image prune -a -f


Reusable Tests Workflow

name: Reusable Tests Workflow

on:
  workflow_call:
    inputs:
      ignoreLowerCoverage:
        description: 'Ignore Lower Coverage'
        default: false
        required: true
        type: boolean

jobs:
  tests-python:
    name: Python Tests
    runs-on: ubuntu-latest
    container: ghcr.io/ietf-tools/datatracker-app-base:latest

    services:
      db:
        image: ghcr.io/ietf-tools/datatracker-db:latest

    steps:
    - uses: actions/checkout@v4

    - name: Prepare for tests
      run: |
        chmod +x ./dev/tests/prepare.sh
        sh ./dev/tests/prepare.sh

    - name: Ensure DB is ready
      run: |
        /usr/local/bin/wait-for db:5432 -- echo "DB ready"

    - name: Run all tests
      shell: bash
      run: |
        echo "Running checks..."
        ./ietf/manage.py check
        ./ietf/manage.py migrate --fake-initial
        echo "Validating migrations..."
        if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then
          echo "Model changes without migrations found."
          exit 1
        fi
        echo "Running tests..."
        if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then
          echo "Lower coverage failures will be ignored."
          HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage
        else
          HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test
        fi
        coverage xml

    - name: Upload geckodriver.log
      uses: actions/upload-artifact@v4
      if: ${{ failure() }}
      with:
        name: geckodriverlog
        path: geckodriver.log

    - name: Upload Coverage Results to Codecov
      uses: codecov/codecov-action@v4.5.0
      with:
        files: coverage.xml

    - name: Convert Coverage Results
      if: ${{ always() }}
      run: |
        mv latest-coverage.json coverage.json

    - name: Upload Coverage Results as Build Artifact
      uses: actions/upload-artifact@v4
      if: ${{ always() }}
      with:
        name: coverage
        path: coverage.json

  tests-playwright:
    name: Playwright Tests
    runs-on: macos-latest
    strategy:
      fail-fast: false
      matrix:
        project: [chromium, firefox]

    steps:
    - uses: actions/checkout@v4

    - uses: actions/setup-node@v4
      with:
        node-version: '18'

    - name: Run all tests
      run: |
        echo "Installing dependencies..."
        yarn
        echo "Installing Playwright..."
        cd playwright
        mkdir test-results
        npm ci
        npx playwright install --with-deps ${{ matrix.project }}
        echo "Running tests..."
        npx playwright test --project=${{ matrix.project }}

    - name: Upload Report
      uses: actions/upload-artifact@v4
      if: ${{ always() }}
      continue-on-error: true
      with:
        name: playwright-results-${{ matrix.project }}
        path: playwright/test-results/
        if-no-files-found: ignore

  tests-playwright-legacy:
    name: Playwright Legacy Tests
    runs-on: ubuntu-latest
    container: ghcr.io/ietf-tools/datatracker-app-base:latest
    strategy:
      fail-fast: false
      matrix:
        project: [chromium, firefox]

    services:
      db:
        image: ghcr.io/ietf-tools/datatracker-db:latest

    steps:
    - uses: actions/checkout@v4

    - name: Prepare for tests
      run: |
        chmod +x ./dev/tests/prepare.sh
        sh ./dev/tests/prepare.sh

    - name: Ensure DB is ready
      run: |
        /usr/local/bin/wait-for db:5432 -- echo "DB ready"

    - name: Start Datatracker
      run: |
        echo "Running checks..."
        ./ietf/manage.py check
        ./ietf/manage.py migrate --fake-initial
        echo "Starting datatracker..."
        ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local &
        echo "Waiting for datatracker to be ready..."
        /usr/local/bin/wait-for localhost:8000 -- echo "Datatracker ready"

    - name: Run all tests
      env:
        # Required to get firefox to run as root:
        HOME: ""
      run: |
        echo "Installing dependencies..."
        yarn
        echo "Installing Playwright..."
        cd playwright
        mkdir test-results
        npm ci
        npx playwright install --with-deps ${{ matrix.project }}
        echo "Running tests..."
        npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js

    - name: Upload Report
      uses: actions/upload-artifact@v4
      if: ${{ always() }}
      continue-on-error: true
      with:
        name: playwright-legacy-results-${{ matrix.project }}
        path: playwright/test-results/
        if-no-files-found: ignore


Tests (Azure Test)

name: Tests (Azure Test)

on:
  workflow_dispatch:

jobs:
  main:
    name: Run Tests on Azure temp VM
    runs-on: ubuntu-latest

    permissions:
      contents: read

    steps:
    - name: Launch VM on Azure
      id: azlaunch
      run: |
        echo "Authenticating to Azure..."
        az login --service-principal -u ${{ secrets.AZ_TESTS_APP_ID }} -p ${{ secrets.AZ_TESTS_PWD }} --tenant ${{ secrets.AZ_TESTS_TENANT_ID }}
        echo "Creating VM..."
        vminfo=$(az vm create \
          --resource-group ghaDatatrackerTests \
          --name tmpGhaVM2 \
          --image Ubuntu2204 \
          --admin-username azureuser \
          --generate-ssh-keys \
          --priority Spot \
          --size Standard_D4as_v5 \
          --max-price -1 \
          --os-disk-size-gb 30 \
          --eviction-policy Delete \
          --nic-delete-option Delete \
          --output tsv \
          --query "publicIpAddress")
        echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT"
        echo "VM Public IP: $vminfo"
        cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key
        ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts

    - name: Remote SSH into VM
      uses: appleboy/ssh-action@029f5b4aeeeb58fdfe1410a5d17f967dacf36262
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        host: ${{ steps.azlaunch.outputs.ipaddr }}
        port: 22
        username: azureuser
        command_timeout: 60m
        key_path: ${{ github.workspace }}/prvkey.key
        envs: GITHUB_TOKEN
        script_stop: true
        script: |
          export DEBIAN_FRONTEND=noninteractive
          lsb_release -a
          sudo apt-get update
          sudo apt-get upgrade -y

          echo "Installing Docker..."
          curl -fsSL https://get.docker.com -o get-docker.sh
          sudo sh get-docker.sh

          echo "Starting Containers..."
          sudo docker network create dtnet
          sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest &
          sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity &
          wait

          echo "Cloning datatracker repo..."
          sudo docker exec app git clone --depth=1 https://github.com/ietf-tools/datatracker.git .
          echo "Prepare tests..."
          sudo docker exec app chmod +x ./dev/tests/prepare.sh
          sudo docker exec app sh ./dev/tests/prepare.sh
          echo "Running checks..."
          sudo docker exec app ietf/manage.py check
          sudo docker exec app ietf/manage.py migrate --fake-initial
          echo "Running tests..."
          sudo docker exec app ietf/manage.py test -v2 --validate-html-harder --settings=settings_test

    - name: Destroy VM + resources
      if: always()
      shell: pwsh
      run: |
        echo "Destroying VM..."
        az vm delete -g ghaDatatrackerTests -n tmpGhaVM2 --yes --force-deletion true

        $resourceOrderRemovalOrder = [ordered]@{
            "Microsoft.Compute/virtualMachines" = 0
            "Microsoft.Compute/disks" = 1
            "Microsoft.Network/networkInterfaces" = 2
            "Microsoft.Network/publicIpAddresses" = 3
            "Microsoft.Network/networkSecurityGroups" = 4
            "Microsoft.Network/virtualNetworks" = 5
        }
        echo "Fetching remaining resources..."
        $resources = az resource list --resource-group ghaDatatrackerTests | ConvertFrom-Json

        $orderedResources = $resources
            | Sort-Object @{
                Expression = {$resourceOrderRemovalOrder[$_.type]}
                Descending = $False
            }

        echo "Deleting remaining resources..."
        $orderedResources | ForEach-Object {
            az resource delete --resource-group ghaDatatrackerTests --ids $_.id --verbose
        }

        echo "Logout from Azure..."
        az logout


test

name: test

on:
  push:
    branches: ["master"]
  pull_request:
    branches: ["master", "release/*"]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
      fail-fast: false

    steps:
    - uses: actions/checkout@v2

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies
      run: |
        pip install -r requirements.txt

    - name: Black fmt
      run: |
        black . --check

    - name: Unittest
      run: |
        python -m unittest discover -s tests

    - name: Coverage
      run: |
        coverage run -m unittest discover
        coverage report

  test-speed:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2

    - name: Set up Python 3.11
      uses: actions/setup-python@v2
      with:
        python-version: 3.11

    - name: Install dependencies
      run: |
        pip install -r requirements.txt
        pip install -r requirements-speed.txt

    - name: Unittest
      run: |
        python -m unittest discover -s tests

  deploy:
    needs: [test, test-speed]
    runs-on: ubuntu-latest
    environment:
      name: deploy
    if: github.ref == 'refs/heads/master'

    steps:
    - uses: actions/checkout@v2

    - name: Set up Python 3.8
      uses: actions/setup-python@v2
      with:
        python-version: 3.8

    - name: Install dependencies
      run: |
        pip install -r requirements.txt

    - name: Build package and docs
      run: |
        python -m build
        twine check dist/*
        mkdocs build

    - name: Publish
      run: |
        git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
        git config --local user.name "GitHub Action"
        twine upload dist/*
        mkdocs gh-deploy --force
      env:
        TWINE_USERNAME: __token__
        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}


Documentation

name: Documentation

on:
  workflow_dispatch:
  push:
    branches: [master, dev]

env:
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}

jobs:
  Documentation:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@v4
      with:
        cache: 'pip'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install '.[dev]'
    - name: Set up Git
      run: |
        git config user.name ${{ github.actor }}
        git config user.email ${{ github.actor }}@users.noreply.github.com
    - name: Build documentation
      run: |
        git fetch origin gh-pages
        mike delete $BRANCH_NAME
        mike deploy --push $BRANCH_NAME


Upload Python Package

# This workflow will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.

name: Upload Python Package

on:
  workflow_dispatch:
  release:
    types: [published]

jobs:
  build_wheels:
    name: Build wheels on ${{ matrix.os }}/${{ matrix.arch || '*' }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]

    steps:
      - uses: actions/checkout@v2

      - name: Build wheels
        uses: pypa/cibuildwheel@v2.16.5
        env:
          CIBW_ARCHS_MACOS: "x86_64 arm64"

      - uses: actions/upload-artifact@v2
        with:
          path: ./wheelhouse/*.whl

  build_sdist:
    name: Build source distribution
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2

      - name: Build sdist
        run: pipx run build --sdist

      - uses: actions/upload-artifact@v2
        with:
          path: dist/*.tar.gz

  PyPI:
    name: Upload to PyPI

    needs: [build_wheels, build_sdist]
    runs-on: ubuntu-latest

    steps:
    - uses: actions/download-artifact@v2
      with:
        name: artifact
        path: dist
    - name: Publish package
      uses: pypa/gh-action-pypi-publish@v1.5.0
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}

        # password: ${{ secrets.TEST_PYPI_API_TOKEN }}
        # repository_url: https://test.pypi.org/legacy/

  Documentation:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: "3.9"
        cache: 'pip'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install '.[dev]'
    - name: Set up Git
      run: |
        git config user.name ${{ github.actor }}
        git config user.email ${{ github.actor }}@users.noreply.github.com
    - name: Build documentation
      run: |
        git fetch origin gh-pages
        mike deploy --push --no-redirect --update-aliases $GITHUB_REF_NAME latest


Test Build

# This tries to build packages, and tests the packages.
# It runs on every push to branches following the pattern v*.*.*.
# It makes sure that everything will run when the version is released.

name: Test Build

on:
  workflow_dispatch:
  pull_request:
    branches:
    - v*.*.*
    - build-*

jobs:
  build_wheels:
    name: Build wheels on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]

    steps:
      - uses: actions/checkout@v2

      - name: Build wheels
        # 2.4 is too low (can't build for macos, 2.16 is too high (OpenSSL issues)
        uses: pypa/cibuildwheel@v2.16.5
        env:
          CIBW_ARCHS_MACOS: "x86_64 arm64"

  build_sdist:
    name: Build source distribution
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2

      - name: Build sdist
        run: pipx run build --sdist


Tests and Linting

name: Tests and Linting

on:
  workflow_dispatch:
  pull_request:
  push:
    branches: [master]

jobs:
  linting:
    name: Linting
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          # requites to grab the history of the PR
          fetch-depth: 0
      - uses: actions/setup-python@v4
      - uses: pre-commit/action@v3.0.0
        with:
          extra_args: --color=always --from-ref ${{ github.event.pull_request.base.sha }} --to-ref ${{ github.event.pull_request.head.sha }}

  pytest:
    name: Pytest
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        python-version: ["3.7", "3.8", "3.9"]
    steps:
      - uses: actions/checkout@v2

      - name: Cache downloaded resources
        uses: actions/cache@v3
        with:
          path: ~/.data/
          key: resources

      # - name: Cache pip
      #   uses: actions/cache@v3
      #   with:
      #     path: ~/.cache/pip
      #     key: ${{ runner.os }}-python-${{ matrix.python-version }}-pip

      - run: echo WEEK=$(date +%V) >>$GITHUB_ENV
        shell: bash

      - uses: hynek/setup-cached-uv@v1
        with:
          cache-suffix: -tests-${{ matrix.python-version }}-${{ env.WEEK }}

      - name: Set up Java
        uses: actions/setup-java@v2
        with:
          distribution: "temurin" # See 'Supported distributions' for available options
          java-version: "8"

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          uv venv
          source .venv/bin/activate
          uv pip install -e '.[dev,setup]' pytest-xdist poetry pip

      - name: Test with Pytest on Python ${{ matrix.python-version }}
        env:
          UMLS_API_KEY: ${{ secrets.UMLS_API_KEY }}
        run: |
          source .venv/bin/activate
          coverage run -m pytest --ignore tests/test_docs.py # -n auto
          # coverage combine
          # mv .coverage .coverage.${{ matrix.python-version }}
        if: matrix.python-version != '3.9'

      - name: Test with Pytest on Python ${{ matrix.python-version }}
        env:
          UMLS_API_KEY: ${{ secrets.UMLS_API_KEY }}
        run: |
          source .venv/bin/activate
          coverage run -m pytest # -n auto
          # coverage combine
          # mv .coverage .coverage.${{ matrix.python-version }}
        if: matrix.python-version == '3.9'

      - name: Upload coverage data
        uses: actions/upload-artifact@v4
        with:
          name: coverage-data-${{ matrix.python-version }}
          path: .coverage.*
          if-no-files-found: ignore

  coverage:
    name: Coverage
    needs: pytest
    uses: aphp/foldedtensor/.github/workflows/coverage.yml@main
    with:
      base-branch: master
      coverage-data-pattern: coverage-data-*
      coverage-report: coverage.txt
      coverage-badge: coverage.svg
      coverage-branch: coverage

  documentation:
    name: Documentation
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2

    - uses: actions/setup-python@v4
      with:
        python-version: "3.9"
        # cache: 'pip'

    - run: echo WEEK=$(date +%V) >>$GITHUB_ENV
      shell: bash

    - uses: hynek/setup-cached-uv@v1
      with:
        cache-suffix: -docs-${{ matrix.python-version }}-${{ env.WEEK }}

    - name: Install dependencies
      run: |
        uv venv
        uv pip install '.[dev]'

    - name: Build documentation
      run: |
        source .venv/bin/activate
        mkdocs build --clean

  simple-installation:
    name: Simple installation
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        python-version: ["3.7", "3.8", "3.9"]
    steps:
      - uses: actions/checkout@v2

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - run: echo WEEK=$(date +%V) >>$GITHUB_ENV
        shell: bash

      - uses: hynek/setup-cached-uv@v1
        with:
          cache-suffix: -simple-install-${{ matrix.python-version }}-${{ env.WEEK }}

      - name: Install library
        run: |
          uv venv
          uv pip install .


Testing and distribution

name: Testing and distribution
on:
  push:
  pull_request:
  workflow_dispatch:

jobs:
  pre-commit:
    name: Lint and format with pre-commit
    runs-on: ubuntu-latest
    steps:
      - id: checkout
        name: Checkout
        uses: actions/checkout@v4

      - id: setup-python
        name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - id: pre-commit
        name: Pre-commit
        uses: pre-commit/action@v3.0.1

  test-docs:
    name: Test Sphinx build
    runs-on: ubuntu-latest
    steps:
      - id: checkout
        name: Checkout
        uses: actions/checkout@v4

      - id: setup-python
        name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
          cache: 'pipenv'

      - id: install-pipenv
        name: Install pipenv
        run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
        shell: bash

      - id: install-python-dependencies
        name: Install Python dependencies
        run: pipenv install --dev --python=`which python`
        shell: bash

      - id: tally
        name: Tally sources
        run: make tally-sources

      - id: build
        name: Build
        run: make test-docs

      - id: save
        name: Save artifact
        uses: actions/upload-artifact@v4
        with:
          name: docs-${{ github.run_number }}
          path: ./docs
          if-no-files-found: error

  test-python:
    strategy:
      matrix:
        python: ["3.7", "3.8", "3.9", "3.10", "3.12",]
    name: Test Python code
    runs-on: ubuntu-latest
    steps:
      - id: checkout
        name: Checkout
        uses: actions/checkout@v4

      - id: setup-python
        name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
          cache: 'pipenv'

      - id: install-pipenv
        name: Install pipenv
        run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
        shell: bash

      - id: install-python-dependencies
        name: Install Python dependencies
        run: pipenv install --dev --python=`which python`
        shell: bash

      - id: run
        name: Run tests
        run: make test

      - id: coverage
        name: Coverage report
        run: make coverage

  test-build:
    name: Build Python package
    runs-on: ubuntu-latest
    needs: [test-python]
    steps:
      - id: checkout
        name: Checkout
        uses: actions/checkout@v4

      - id: setup-python
        name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
          cache: 'pipenv'

      - id: install-pipenv
        name: Install pipenv
        run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
        shell: bash

      - id: install-python-dependencies
        name: Install Python dependencies
        run: pipenv install --dev --python=`which python`
        shell: bash

      - id: build
        name: Build release
        run: make build-release

      - id: check
        name: Check release
        run: make check-release

      - id: save
        name: Save artifact
        uses: actions/upload-artifact@v4
        with:
          name: test-release-${{ github.run_number }}
          path: ./dist
          if-no-files-found: error

  test-release:
    name: Test PyPI release
    runs-on: ubuntu-latest
    needs: [test-build]
    if: startsWith(github.ref, 'refs/tags') == 0
    steps:
      - id: setup-python
        name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - id: fetch
        name: Fetch artifact
        uses: actions/download-artifact@v4
        with:
          name: test-release-${{ github.run_number }}
          path: ./dist

      - id: publish
        name: Publish release
        uses: pypa/gh-action-pypi-publish@release/v1
        env:
          PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
        if: env.PYPI_API_TOKEN != null
        with:
          user: __token__
          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
          repository-url: https://test.pypi.org/legacy/
          verbose: true
          verify_metadata: false

  tag-release:
    name: Tagged PyPI release
    runs-on: ubuntu-latest
    needs: [test-build]
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
    steps:
      - id: setup-python
        name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - id: fetch
        name: Fetch artifact
        uses: actions/download-artifact@v4
        with:
          name: test-release-${{ github.run_number }}
          path: ./dist

      - id: publish
        name: Publish release
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}
          verbose: true
          verify_metadata: false


test

name: test

on:
  push:
    branches: ["master"]
  pull_request:
    branches: ["master", "release/*"]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']

    steps:
    - uses: actions/checkout@v2

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies
      run: |
        pip install -r requirements.txt

    - name: Black fmt
      run: |
        black . --check

    - name: Unittest
      run: |
        python -m unittest discover -s tests

    - name: Coverage
      run: |
        coverage run -m unittest discover
        coverage report

  test-speed:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']

    steps:
      - uses: actions/checkout@v2

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          pip install -r requirements.txt
          pip install -r requirements-speed.txt

      - name: Unittest
        run: |
          python -m unittest discover -s tests

  deploy:
    needs: test
    runs-on: ubuntu-latest
    environment:
      name: deploy
    if: github.ref == 'refs/heads/master'

    steps:
    - uses: actions/checkout@v2

    - name: Set up Python 3.11
      uses: actions/setup-python@v2
      with:
        python-version: 3.11

    - name: Install dependencies
      run: |
        pip install -r requirements.txt

    - name: Build package and docs
      run: |
        python -m build
        twine check dist/*
        mkdocs build

    - name: Publish
      run: |
        git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
        git config --local user.name "GitHub Action"
        twine upload dist/*
        mkdocs gh-deploy --force
      env:
        TWINE_USERNAME: __token__
        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}


Publish Docker image

name: Publish Docker image

on:
  workflow_dispatch:
    inputs:
      version:
        required: true
jobs:
  push_to_registry:
    name: Push Docker image to Docker Hub
    runs-on: ubuntu-latest
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3

      - name: Log in to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: gleif/keri

      - name: Build and push Docker image
        uses: docker/build-push-action@v3
        with:
          context: .
          file: images/keripy.dockerfile
          push: true
          tags: |
            gleif/keri:${{ github.event.inputs.version }}
            gleif/keri:latest
          labels: ${{ github.event.inputs.version }}


GitHub Actions for keripy

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: GitHub Actions for keripy
on:
  push:
    branches:
      - 'main'
  pull_request:
  workflow_dispatch:

jobs:
  test:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ macos-13, ubuntu-latest ]

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.12.2
        uses: actions/setup-python@v2
        with:
          python-version: 3.12.2
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install flake8 pytest hio
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
      - name: Lint changes
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --ignore=E7,F841,E301,E302,E303 --max-complexity=10 --max-line-length=127 --statistics
      - name: Run core KERI tests
        run: |
          pytest tests/ --ignore tests/demo/ --ignore test/scripts
      - name: Run KERI demo tests
        run: |
          pytest tests/demo/

  coverage:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.12.2
        uses: actions/setup-python@v2
        with:
          python-version: 3.12.2
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pytest pytest-cov hio
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
      - name: Run core KERI tests
        run: |
          pytest --cov=./ --cov-report=xml
      - name: Upload
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  scripts:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.12.2
        uses: actions/setup-python@v2
        with:
          python-version: 3.12.2
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pytest pytest-cov hio pytest-shell
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
      - name: Run KERI kli tests
        run: |
          ./scripts/demo/test_scripts.sh

  interop-setup:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.ref == 'refs/heads/interop'
    outputs:
      CACHE_KEY_INTEROP: ${{ steps.cache.outputs.CACHE_KEY_INTEROP }}
      GITHUB_REPOSITORY_NAME: ${{ steps.cache.outputs.GITHUB_REPOSITORY_NAME }}
    steps:
      - name: Git checkout
        uses: actions/checkout@v3
      - name: Set outputs
        id: cache
        run: |
          echo "::set-output name=CACHE_KEY_INTEROP::${{ hashFiles('.github/workflows/interop/Dockerfile') }}"
          echo "::set-output name=GITHUB_REPOSITORY_NAME::$(echo ${GITHUB_REPOSITORY,,})"

  build-interop-image:
    needs: [ interop-setup, test ]
    runs-on: ubuntu-latest
    env:
      DOCKER_BUILDKIT: 1
      CACHE_KEY_INTEROP: ${{ needs.interop-setup.outputs.CACHE_KEY_INTEROP }}
      GITHUB_REPOSITORY_NAME: ${{ needs.interop-setup.outputs.GITHUB_REPOSITORY_NAME }}
    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
    steps:
      - name: Git checkout
        uses: actions/checkout@v3
      - name: interop
        run: |
          echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
          docker build -f .github/workflows/interop/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/keripy-interop:${{ env.CACHE_KEY_INTEROP }} .
          docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/keripy-interop:${{ env.CACHE_KEY_INTEROP }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/keripy-interop:latest
          docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/keripy-interop:latest
          mkdir -p ${GITHUB_WORKSPACE}/cache
          touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_INTEROP }}


Backport merged pull request

name: Backport merged pull request
on:
  pull_request_target:
    types: [closed]
  issue_comment:
    types: [created]
permissions:
  contents: write # so it can comment
  pull-requests: write # so it can create pull requests
jobs:
  backport:
    name: Backport pull request
    runs-on: ubuntu-latest

    # Only run when pull request is merged
    # or when a comment containing `/backport` is created by someone other than the
    # https://github.com/backport-action bot user (user id: 97796249). Note that if you use your
    # own PAT as `github_token`, that you should replace this id with yours.
    if: >
      (
        github.event_name == 'pull_request_target' &&
        github.event.pull_request.merged
      ) || (
        github.event_name == 'issue_comment' &&
        github.event.issue.pull_request &&
        github.event.comment.user.id != 97796249 &&
        contains(github.event.comment.body, '/backport')
      )
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Create backport pull requests
        uses: korthout/backport-action@v2


Build MicroK8s Installers

name: Build MicroK8s Installers

on:
  push:
    branches:
      - "**install**"
  pull_request:
    branches:
      - "**install**"

jobs:
  windows:
    runs-on: windows-latest
    defaults:
      run:
        working-directory: ${{ github.workspace }}/installer/windows
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python 3.8
        uses: actions/setup-python@v4.6.1
        with:
          python-version: 3.8
      - name: Install Python requirements
        run: python -m pip install -r ../requirements.txt
      - name: Build exe
        working-directory: ${{ github.workspace }}/installer
        run: pyinstaller.exe ./microk8s.spec
      - name: Move exe to installer build directory
        working-directory: ${{ github.workspace }}/installer
        run: move microk8s.exe ./windows/microk8s.exe
      - name: Download EnVar plugin for NSIS
        uses: carlosperate/download-file-action@v2.0.1
        with:
          file-url: https://github.com/GsNSIS/EnVar/releases/download/v0.3.1/EnVar-Plugin.zip
          file-name: envar_plugin.zip
          location: ${{ github.workspace }}
      - name: Extract EnVar plugin
        run: 7z x -o"C:/Program Files (x86)/NSIS" "${{ github.workspace }}/envar_plugin.zip"
      - name: Download Multipass installer
        uses: carlosperate/download-file-action@v2.0.1
        with:
          file-url: https://github.com/canonical/multipass/releases/download/v1.12.2/multipass-1.12.2+win-win64.exe
          file-name: multipass.exe
          location: ${{ github.workspace }}/installer/windows
      - name: Download kubectl
        uses: carlosperate/download-file-action@v2.0.1
        with:
          file-url: https://storage.googleapis.com/kubernetes-release/release/v1.28.3/bin/windows/amd64/kubectl.exe
          file-name: kubectl.exe
          location: ${{ github.workspace }}/installer/windows
      - name: Create installer
        run: makensis.exe ${{ github.workspace }}/installer/windows/microk8s.nsi
      - name: Upload installer
        uses: actions/upload-artifact@v3
        with:
          name: Windows installer
          path: ${{ github.workspace }}/installer/windows/microk8s-installer.exe


Build and test MicroK8s snap

name: Build and test MicroK8s snap

on:
  pull_request:
    branches:
      - master

jobs:
  build:
    name: Create snap package
    runs-on: ubuntu-20.04

    steps:
      - name: Checking out repo
        uses: actions/checkout@v4
      - name: Install lxd
        run: |
          sudo lxd init --auto
          sudo usermod --append --groups lxd $USER
          sg lxd -c 'lxc version'
      - name: Install snapcraft
        run: |
          sudo snap install snapcraft --classic
      - name: Install snapd from candidate
        run: |
          # TODO(neoaggelos): revert this after latest/beta is working again
          sudo snap refresh snapd --channel=latest/stable
      - name: Build snap
        run: |
          sg lxd -c 'snapcraft --use-lxd'
          sudo mv microk8s*.snap microk8s.snap
      - name: Uploading snap
        uses: actions/upload-artifact@v3
        with:
          name: microk8s.snap
          path: microk8s.snap

  test-upgrade:
    name: Upgrade path test
    runs-on: ubuntu-20.04
    needs: build

    steps:
      - name: Checking out repo
        uses: actions/checkout@v4
      - name: Install test dependencies
        run: |
          set -x
          sudo apt-get install python3-setuptools
          sudo pip3 install --upgrade pip
          sudo pip3 install -U pytest sh
          sudo apt-get -y install open-iscsi
          sudo systemctl enable iscsid
      - name: Fetch snap
        uses: actions/download-artifact@v3.0.2
        with:
          name: microk8s.snap
          path: build
      - name: Running upgrade path test
        run: |
          sudo -E UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade-path.py

  test-addons-core:
    name: Test core addons
    runs-on: ubuntu-20.04
    needs: build

    steps:
      - name: Checking out repo
        uses: actions/checkout@v4
      - name: Install test dependencies
        run: |
          set -x
          sudo apt-get install python3-setuptools
          sudo pip3 install --upgrade pip
          sudo pip3 install -U pytest sh
          sudo apt-get -y install open-iscsi
          sudo systemctl enable iscsid
      - name: Fetch snap
        uses: actions/download-artifact@v3.0.2
        with:
          name: microk8s.snap
          path: build
      - name: Running addons tests
        run: |
          set -x
          sudo snap install build/microk8s.snap --classic --dangerous
          ./tests/smoke-test.sh
          export UNDER_TIME_PRESSURE="True"
          export SKIP_PROMETHEUS="False"
          sudo -E bash -c "cd /var/snap/microk8s/common/addons/core/tests; pytest -s -ra test-addons.py"

  test-addons-community:
    name: Test community addons
    runs-on: ubuntu-20.04
    needs: build

    steps:
      - name: Checking out repo
        uses: actions/checkout@v4
      - name: Install test dependencies
        run: |
          set -x
          sudo apt-get install python3-setuptools
          sudo pip3 install --upgrade pip
          sudo pip3 install -U pytest sh
          sudo apt-get -y install open-iscsi
          sudo systemctl enable iscsid
      - name: Fetch snap
        uses: actions/download-artifact@v3.0.2
        with:
          name: microk8s.snap
          path: build
      # - name: Setup tmate session
      #   uses: mxschmitt/action-tmate@v3
      - name: Running addons tests
        run: |
          set -x
          sudo snap install build/microk8s.snap --classic --dangerous
          sudo microk8s enable community
          export UNDER_TIME_PRESSURE="True"
          sudo -E bash -c "cd /var/snap/microk8s/common/addons/community/; pytest -s -ra ./tests/"

  test-addons-core-upgrade:
    name: Test core addons upgrade
    runs-on: ubuntu-20.04
    needs: build

    steps:
      - name: Checking out repo
        uses: actions/checkout@v4
      # - name: Setup tmate session
      #   uses: mxschmitt/action-tmate@v3
      - name: Install test dependencies
        run: |
          set -x
          sudo apt-get install python3-setuptools
          sudo pip3 install --upgrade pip
          sudo pip3 install -U pytest sh
          sudo apt-get -y install open-iscsi
          sudo systemctl enable iscsid
      - name: Fetch snap
        uses: actions/download-artifact@v3.0.2
        with:
          name: microk8s.snap
          path: build
      - name: Running upgrade tests
        run: |
          set -x
          export UNDER_TIME_PRESSURE="True"
          sudo -E bash -c "UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade.py"

  test-cluster-agent:
    name: Cluster agent health check
    runs-on: ubuntu-20.04
    needs: build

    steps:
      - name: Checking out repo
        uses: actions/checkout@v4
      - name: Install test dependencies
        run: |
          set -x
          sudo apt-get install python3-setuptools
          sudo pip3 install --upgrade pip
          sudo pip3 install -U pytest sh requests
      - name: Fetch snap
        uses: actions/download-artifact@v3.0.2
        with:
          name: microk8s.snap
          path: build
      - name: Running cluster agent health check
        run: |
          set -x
          sudo snap install build/microk8s.snap --classic --dangerous
          sudo -E bash -c "pytest -s ./tests/test-cluster-agent.py"

  test-airgap:
    name: Test airgap installation
    runs-on: ubuntu-20.04
    needs: build

    steps:
      - name: Checking out repo
        uses: actions/checkout@v4
      - name: Fetch snap
        uses: actions/download-artifact@v3.0.2
        with:
          name: microk8s.snap
          path: build
      - name: Initialize LXD
        run: |
          sudo lxd init --auto
          sudo lxc network set lxdbr0 ipv6.address=none
          sudo usermod --append --groups lxd $USER
          sg lxd -c 'lxc version'
      - name: Run airgap tests
        run: |
          sudo -E bash -x -c "./tests/libs/airgap.sh --distro ubuntu:20.04 --channel $PWD/build/microk8s.snap"

  test-spread:
    name: Test microk8s on multi distros
    runs-on: ubuntu-20.04
    needs: build
    strategy:
      fail-fast: false
      matrix:
        distro: ["images:centos/7", "images:debian/12"]
    steps:
      - name: Checking out repo
        uses: actions/checkout@v4
      - name: Fetch snap
        uses: actions/download-artifact@v3.0.2
        with:
          name: microk8s.snap
          path: build
      - name: Initialize LXD
        run: |
          sudo apt install apparmor apparmor-utils -y
          sudo lxd init --auto
          sudo lxc network set lxdbr0 ipv6.address=none
          sudo usermod --append --groups lxd $USER
          sg lxd -c 'lxc version'
      - name: Run spread tests
        run: |
          sudo -E bash -x -c "./tests/libs/spread.sh --distro ${{ matrix.distro }} --channel $PWD/build/microk8s.snap"

  security-scan:
    name: Security scan
    runs-on: ubuntu-20.04
    needs: build
    steps:
      - name: Checking out repo
        uses: actions/checkout@v4
      - name: Fetch snap
        uses: actions/download-artifact@v3.0.2
        with:
          name: microk8s.snap
          path: build
      - name: Setup Trivy vulnerability scanner
        run: |
          mkdir -p sarifs
          VER=$(curl --silent -qI https://github.com/aquasecurity/trivy/releases/latest | awk -F '/' '/^location/ {print  substr($NF, 1, length($NF)-1)}');
          wget https://github.com/aquasecurity/trivy/releases/download/${VER}/trivy_${VER#v}_Linux-64bit.tar.gz
          tar -zxvf ./trivy_${VER#v}_Linux-64bit.tar.gz
      - name: Run Trivy vulnerability scanner in repo mode
        uses: aquasecurity/trivy-action@master
        with:
          scan-type: "fs"
          ignore-unfixed: true
          format: "sarif"
          output: "trivy-microk8s-repo-scan--results.sarif"
          severity: "CRITICAL"
      - name: Gather Trivy repo scan results
        run: |
          cp trivy-microk8s-repo-scan--results.sarif ./sarifs/
      - name: Run Trivy vulnerability scanner on images
        run: |
          for i in $(cat ./build-scripts/images.txt) ; do
            name=$(echo  $i | awk -F ':|/' '{print $(NF-1)}')
            ./trivy image $i --format sarif > sarifs/$name.sarif
          done
      - name: Run Trivy vulnerability scanner on the snap
        run: |
          cp build/microk8s.snap .
          unsquashfs microk8s.snap
          ./trivy rootfs ./squashfs-root/ --format sarif > sarifs/snap.sarif
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "sarifs"


Lint Code

name: Lint Code

on:
  - pull_request

jobs:
  check-formatting:
    name: Check Formatting
    runs-on: ubuntu-latest

    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install tox --fix-missing
          sudo snap install node --classic
          sudo npm install --save-dev --save-exact -g prettier
      - name: Check Python formatting
        run: |
          tox -e lint
      - name: Check YAML formatting
        run: |
          set -eux
          prettier --check $(find . -name "*.yaml" -o -name "*.yml" | \
            grep -v "./microk8s-resources/actions/ingress.yaml" | \
            grep -v "./microk8s-resources/actions/metallb.yaml" | \
            grep -v invalid.yaml | \
            grep -v calico)


Unit Tests

name: Unit Tests

on:
  - pull_request

jobs:
  check-unit-tests:
    name: Check Unit Tests
    runs-on: ubuntu-latest

    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install tox --fix-missing
          sudo pip3 install -U pytest==7.1.3 sh==1.14.3
      - name: Check Units
        run: |
          tox -e scripts
          tox -e wrappers
          tox -e cluster
      - name: Verify branches
        run: |
          pytest -s ./tests/verify-branches.py


cla-check

name: cla-check

on:
  pull_request:
    branches: [master, default]

jobs:
  cla-check:
    runs-on: ubuntu-latest
    steps:
      - name: Check if CLA signed
        uses: canonical/has-signed-canonical-cla@v1


Update list of images

name: Update list of images

on:
  workflow_dispatch:
  schedule:
    - cron: "0 10 * * *"

jobs:
  update:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          # Latest branches
          - { branch: master, channel: latest/edge }
          # Stable branches
          - { branch: "1.30", channel: "1.30" }
          - { branch: "1.29", channel: "1.29" }
          - { branch: "1.28", channel: "1.28" }
          - { branch: "1.27", channel: "1.27" }
          # Stable strict branches
          - { branch: 1.30-strict, channel: 1.30-strict }
          - { branch: 1.29-strict, channel: 1.29-strict }
          - { branch: 1.28-strict, channel: 1.28-strict }
          - { branch: 1.27-strict, channel: 1.27-strict }
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ matrix.branch }}
      - name: Update image list
        run: |
          ./build-scripts/update-images.sh ${{ matrix.channel }} build-scripts/images.txt
      - name: Create pull request
        uses: peter-evans/create-pull-request@v5
        with:
          commit-message: update list of images used by ${{ matrix.channel }}
          title: "[${{ matrix.channel }}] Update MicroK8s images"
          body: update list of images used by ${{ matrix.channel }}
          reviewers: neoaggelos,ktsakalozos
          branch: auto-update-images/${{ matrix.branch }}
          delete-branch: true
          base: ${{ matrix.branch }}


Lock inactive closed issues

name: Lock inactive closed issues
# Lock closed issues that have not received any further activity for two weeks.
# This does not close open issues, only humans may do that. It is easier to
# respond to new issues with fresh examples rather than continuing discussions
# on old issues.

on:
  schedule:
    - cron: '0 0 * * *'
permissions:
  issues: write
  pull-requests: write
concurrency:
  group: lock
jobs:
  lock:
    runs-on: ubuntu-latest
    steps:
      - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5.0.1
        with:
          issue-inactive-days: 14
          pr-inactive-days: 14
          discussion-inactive-days: 14


Publish

name: Publish
on:
  push:
    tags:
      - '*'
jobs:
  build:
    runs-on: ubuntu-latest
    outputs:
      hash: ${{ steps.hash.outputs.hash }}
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: '3.x'
          cache: pip
          cache-dependency-path: requirements*/*.txt
      - run: pip install -r requirements/build.txt
      # Use the commit date instead of the current date during the build.
      - run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV
      - run: python -m build
      # Generate hashes used for provenance.
      - name: generate hash
        id: hash
        run: cd dist && echo "hash=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT
      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        with:
          path: ./dist
  provenance:
    needs: [build]
    permissions:
      actions: read
      id-token: write
      contents: write
    # Can't pin with hash due to how this workflow works.
    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0
    with:
      base64-subjects: ${{ needs.build.outputs.hash }}
  create-release:
    # Upload the sdist, wheels, and provenance to a GitHub release. They remain
    # available as build artifacts for a while as well.
    needs: [provenance]
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
      - name: create release
        run: >
          gh release create --draft --repo ${{ github.repository }}
          ${{ github.ref_name }}
          *.intoto.jsonl/* artifact/*
        env:
          GH_TOKEN: ${{ github.token }}
  publish-pypi:
    needs: [provenance]
    # Wait for approval before attempting to upload to PyPI. This allows reviewing the
    # files in the draft release.
    environment:
      name: publish
      url: https://pypi.org/project/Flask-SQLAlchemy/${{ github.ref_name }}
    runs-on: ubuntu-latest
    permissions:
      id-token: write
    steps:
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
      - uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
        with:
          repository-url: https://test.pypi.org/legacy/
          packages-dir: artifact/
      - uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
        with:
          packages-dir: artifact/


Tests

name: Tests
on:
  push:
    branches:
      - main
      - '*.x'
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - '*.rst'
  pull_request:
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - '*.rst'
jobs:
  tests:
    name: ${{ matrix.name || matrix.python }}
    runs-on: ${{ matrix.os || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - {python: '3.12'}
          - {python: '3.11'}
          - {python: '3.10'}
          - {python: '3.9'}
          - {python: '3.8'}
          - {name: Minimum Versions, python: '3.11', tox: py-min}
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: ${{ matrix.python }}
          allow-prereleases: true
          cache: pip
          cache-dependency-path: requirements*/*.txt
      - run: pip install tox
      - run: tox run -e ${{ matrix.tox || format('py{0}', matrix.python) }}
  typing:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: '3.x'
          cache: pip
          cache-dependency-path: requirements*/*.txt
      - name: cache mypy
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: ./.mypy_cache
          key: mypy|${{ hashFiles('pyproject.toml') }}
      - run: pip install tox
      - run: tox run -e typing


pre-commit

name: pre-commit

on:
  push:
    branches: [main]
  pull_request:

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
          architecture: x64
          packages: |
            ufmt==2.5.1
            black==24.2.0
            usort==1.0.8
      - name: Checkout Torchrec
        uses: actions/checkout@v2
      - name: Run pre-commit
        uses: pre-commit/action@v2.0.3


Unit Test CI

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Unit Test CI

on:
  # TODO: re-enable when GPU unit tests are working
  # push:
  #   paths-ignore:
  #     - "docs/*"
  #     - "third_party/*"
  #     - .gitignore
  #     - "*.md"
  # pull_request:
  #   paths-ignore:
  #     - "docs/*"
  #     - "third_party/*"
  #     - .gitignore
  #     - "*.md"
  workflow_dispatch:

jobs:
  # build on cpu hosts and upload to GHA
  build_on_cpu:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
         - os: linux.2xlarge
          # ideally we run on 3.9 and 3.10 as well, however we are limited in resources.
           python-version: 3.8
           python-tag: "py38"
           cuda-tag: "cu11"
    steps:
    # Checkout the repository to the GitHub Actions runner
    - name: Check ldd --version
      run: ldd --version
    - name: Checkout
      uses: actions/checkout@v2
    - name: Update pip
      run: |
        sudo yum update -y
        sudo yum -y install git python3-pip
        sudo pip3 install --upgrade pip
    - name: Setup conda
      run: |
        wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
        bash ~/miniconda.sh -b -p $HOME/miniconda -u
    - name: setup Path
      run: |
        echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
        echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
    - name: create conda env
      run: |
        conda create --name build_binary python=${{ matrix.python-version }}
        conda info
    - name: check python version no Conda
      run: |
        python --version
    - name: check python version
      run: |
        conda run -n build_binary python --version
    - name: Install C/C++ compilers
      run: |
        sudo yum install -y gcc gcc-c++
    - name: Install PyTorch and CUDA
      shell: bash
      run: |
        conda install -n build_binary -y pytorch pytorch-cuda=11.8 -c pytorch-nightly -c nvidia
    - name: Install Dependencies
      shell: bash
      run: |
        conda run -n build_binary python -m pip install -r requirements.txt
    - name: Test Installation of dependencies
      run: |
        conda run -n build_binary python -c "import torch.distributed"
        echo "torch.distributed succeeded"
        conda run -n build_binary python -c "import skbuild"
        echo "skbuild succeeded"
        conda run -n build_binary python -c "import numpy"
        echo "numpy succeeded"
    # for the conda run with quotes, we have to use "\" and double quotes
    # here is the issue: https://github.com/conda/conda/issues/10972
    - name: Build TorchRec Binary
      run: |
        conda run -n build_binary \
          python setup.py bdist_wheel \
          --python-tag=${{ matrix.python-tag }}
    - name: Upload wheel as GHA artifact
      uses: actions/upload-artifact@v2
      with:
        name: torchrec_${{ matrix.python-version }}_${{ matrix.cuda-tag }}.whl
        path: dist/*.whl

  # download from GHA, test on gpu
  test_on_gpu:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [linux.4xlarge.nvidia.gpu]
        python-version: [3.8]
        cuda-tag: ["cu11"]
    needs: build_on_cpu
    # the glibc version should match the version of the one we used to build the binary
    # for this case, it's 2.26
    steps:
    - name: Check ldd --version
    # Run unit tests
      run: ldd --version
    - name: check cpu info
      shell: bash
      run: |
        cat /proc/cpuinfo
    - name: check distribution info
      shell: bash
      run: |
        cat /proc/version
    - name: Display EC2 information
      shell: bash
      run: |
        set -euo pipefail
        function get_ec2_metadata() {
          # Pulled from instance metadata endpoint for EC2
          # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
          category=$1
          curl -fsSL "http://169.254.169.254/latest/meta-data/${category}"
        }
        echo "ami-id: $(get_ec2_metadata ami-id)"
        echo "instance-id: $(get_ec2_metadata instance-id)"
        echo "instance-type: $(get_ec2_metadata instance-type)"
    - name: check gpu info
      shell: bash
      run: |
        sudo yum install lshw -y
        sudo lshw -C display
    # Checkout the repository to the GitHub Actions runner
    - name: Checkout
      uses: actions/checkout@v2
    - name: Update pip
      run: |
        sudo yum update -y
        sudo yum -y install git python3-pip
        sudo pip3 install --upgrade pip
    - name: Setup conda
      run: |
        wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
        bash ~/miniconda.sh -b -p $HOME/miniconda
    - name: setup Path
      run: |
        echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
        echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
    - name: create conda env
      run: |
        conda create --name build_binary python=${{ matrix.python-version }}
        conda info
    - name: check python version no Conda
      run: |
        python --version
    - name: check python version
      run: |
        conda run -n build_binary python --version
    - name: Install C/C++ compilers
      run: |
        sudo yum install -y gcc gcc-c++
    - name: Install PyTorch and CUDA
      shell: bash
      run: |
        conda install -n build_binary -y pytorch pytorch-cuda=11.8 -c pytorch-nightly -c nvidia
    - name: Install fbgemm
      shell: bash
      run: |
        conda run -n build_binary \
          pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cu118
    # download wheel from GHA
    - name: Download wheel
      uses: actions/download-artifact@v2
      with:
        name: torchrec_${{ matrix.python-version }}_${{ matrix.cuda-tag }}.whl
    - name: Display structure of downloaded files
      run: ls -R
    - name: Install TorchRec GPU
      run: |
        rm -r dist || true
        conda run -n build_binary python -m pip install dist/*.whl
    - name: Test torchrec installation
      shell: bash
      run: |
        conda run -n build_binary \
          python -c "import torchrec"
    - name: Test with pytest
      run: |
        conda run -n build_binary \
          python -m pip install pytest
        conda run -n build_binary \
          python -m pytest torchrec -v -s -W ignore::pytest.PytestCollectionWarning --continue-on-collection-errors


Generate documentation

# This workflow builds the torchrec docs and deploys them to gh-pages.
name: Generate documentation
on:
  push:
    branches:
      - main
  workflow_dispatch:
jobs:
  build_docs_job:
    runs-on: ${{ matrix.os }}
    permissions:
      # Grant write permission here so that the doc can be pushed to gh-pages branch
      contents: write
    strategy:
      matrix:
        include:
         - os: linux.2xlarge
           python-version: 3.8
    steps:
    - name: Check ldd --version
      run: ldd --version
    - name: Checkout
      uses: actions/checkout@v2
    # Update references
    - name: Update pip
      run: |
        sudo yum update -y
        sudo yum -y install git python3-pip
        sudo pip3 install --upgrade pip
    - name: Setup conda
      run: |
        wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
        bash ~/miniconda.sh -b -p $HOME/miniconda
    - name: setup Path
      run: |
        echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
        echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
    - name: create conda env
      run: |
        conda create --name build_binary python=${{ matrix.python-version }}
        conda info
    - name: check python version no Conda
      run: |
        python --version
    - name: check python version
      run: |
        conda run -n build_binary python --version
    - name: Install gcc
      shell: bash
      run: |
        sudo yum group install -y "Development Tools"
    - name: setup Path
      run: |
        echo /usr/local/bin >> $GITHUB_PATH
    - name: Install PyTorch
      shell: bash
      run: |
        conda install -n build_binary --yes pytorch cpuonly -c pytorch-nightly
    - name: Install fbgemm
      run: |
        conda run -n build_binary pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu
    - name: Install torchmetrics
      run: |
        conda run -n build_binary pip install torchmetrics==1.0.3
    - name: Install TorchRec
      run: |
        conda run -n build_binary pip install torchrec --index-url https://download.pytorch.org/whl/nightly/cpu
    - name: Test fbgemm_gpu and torchrec installation
      shell: bash
      run: |
        conda run -n build_binary \
          python -c "import fbgemm_gpu"
        conda run -n build_binary \
          python -c "import torchrec"
    - name: Build the docset
      run: |
        conda run -n build_binary python -m pip install -r docs/requirements.txt
        cd ./docs
        conda run -n build_binary make html
        cd ..
    - name: Get output time
      run: echo "The time was ${{ steps.build.outputs.time }}"
    - name: Deploy
      uses: JamesIves/github-pages-deploy-action@releases/v3
      with:
          ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          BRANCH: gh-pages # The branch the action should deploy to.
          FOLDER: docs/build/html # The folder the action should deploy.


Push Binary Release

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Push Binary Release

on:
  workflow_call:
    secrets:
      PYPI_TOKEN:
        required: false
  workflow_dispatch:

jobs:

  # build on cpu hosts and upload to GHA
  build_on_cpu:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
         - os: linux.2xlarge
           python-version: 3.8
           python-tag: "py38"
           cuda-tag: "cu121"
         - os: linux.2xlarge
           python-version: 3.9
           python-tag: "py39"
           cuda-tag: "cu121"
         - os: linux.2xlarge
           python-version: '3.10'
           python-tag: "py310"
           cuda-tag: "cu121"
         - os: linux.2xlarge
           python-version: '3.11'
           python-tag: "py311"
           cuda-tag: "cu121"
         - os: linux.2xlarge
           python-version: '3.12'
           python-tag: "py312"
           cuda-tag: "cu121"
    steps:
    # Checkout the repository to the GitHub Actions runner
    - name: Check ldd --version
      run: ldd --version
    - name: Checkout
      uses: actions/checkout@v2
    - name: Update pip
      run: |
        sudo yum update -y
        sudo yum -y install git python3-pip
        sudo pip3 install --upgrade pip
    - name: Setup conda
      run: |
        wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
        bash ~/miniconda.sh -b -p $HOME/miniconda -u
    - name: setup Path
      run: |
        echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
        echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
    - name: create conda env
      run: |
        conda create --name build_binary python=${{ matrix.python-version }}
        conda info
    - name: check python version no Conda
      run: |
        python --version
    - name: check python version
      run: |
        conda run -n build_binary python --version
    - name: Install C/C++ compilers
      run: |
        sudo yum install -y gcc gcc-c++
    - name: Install PyTorch and CUDA
      shell: bash
      run: |
        conda run -n build_binary pip install torch --index-url https://download.pytorch.org/whl/test/cu121
    - name: Install fbgemm
      shell: bash
      run: |
        conda run -n build_binary pip install numpy
        conda run -n build_binary pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/test/cu121
    - name: Install Dependencies
      shell: bash
      run: |
        conda run -n build_binary python -m pip install -r requirements.txt
    - name: Test Installation of dependencies
      run: |
        conda run -n build_binary python -c "import torch.distributed"
        echo "torch.distributed succeeded"
        conda run -n build_binary python -c "import skbuild"
        echo "skbuild succeeded"
        conda run -n build_binary python -c "import numpy"
        echo "numpy succeeded"
    # for the conda run with quotes, we have to use "\" and double quotes
    # here is the issue: https://github.com/conda/conda/issues/10972
    - name: Build TorchRec
      env:
        OFFICIAL_RELEASE: 1
      run: |
        rm -r dist || true
        conda run -n build_binary \
          python setup.py bdist_wheel \
          --python-tag=${{ matrix.python-tag }}
    - name: Upload wheel as GHA artifact
      uses: actions/upload-artifact@v2
      with:
        name: torchrec_${{ matrix.python-version }}_${{ matrix.cuda-tag }}.whl
        path: dist/torchrec-*.whl

  # download from GHA, sanity check on gpu and push to pypi
  sanity_check_on_gpu_and_push:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [linux.4xlarge.nvidia.gpu]
        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
        cuda-tag: ["cu121"]
    needs: build_on_cpu
    # the glibc version should match the version of the one we used to build the binary
    # for this case, it's 2.26
    steps:
    - name: Check ldd --version
      run: ldd --version
    - name: check cpu info
      shell: bash
      run: |
        cat /proc/cpuinfo
    - name: check distribution info
      shell: bash
      run: |
        cat /proc/version
    - name: Display EC2 information
      shell: bash
      run: |
        set -euo pipefail
        function get_ec2_metadata() {
          # Pulled from instance metadata endpoint for EC2
          # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
          category=$1
          curl -fsSL "http://169.254.169.254/latest/meta-data/${category}"
        }
        echo "ami-id: $(get_ec2_metadata ami-id)"
        echo "instance-id: $(get_ec2_metadata instance-id)"
        echo "instance-type: $(get_ec2_metadata instance-type)"
    - name: check gpu info
      shell: bash
      run: |
        sudo yum install lshw -y
        sudo lshw -C display
    # Checkout the repository to the GitHub Actions runner
    - name: Checkout
      uses: actions/checkout@v2
    - name: Update pip
      run: |
        sudo yum update -y
        sudo yum -y install git python3-pip
        sudo pip3 install --upgrade pip
    - name: Setup conda
      run: |
        wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
        bash ~/miniconda.sh -b -p $HOME/miniconda
    - name: setup Path
      run: |
        echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
        echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
    - name: create conda env
      run: |
        conda create --name build_binary python=${{ matrix.python-version }}
        conda info
    - name: check python version no Conda
      run: |
        python --version
    - name: check python version
      run: |
        conda run -n build_binary python --version
    - name: Install C/C++ compilers
      run: |
        sudo yum install -y gcc gcc-c++
    - name: Install PyTorch and CUDA
      shell: bash
      run: |
        conda run -n build_binary pip install torch --index-url https://download.pytorch.org/whl/test/cu121
    # download wheel from GHA
    - name: Install fbgemm
      shell: bash
      run: |
        conda run -n build_binary pip install numpy
        conda run -n build_binary pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/test/cu121
    - name: Install torchmetrics
      shell: bash
      run: |
        conda run -n build_binary pip install torchmetrics==1.0.3
    - name: Download wheel
      uses: actions/download-artifact@v2
      with:
        name: torchrec_${{ matrix.python-version }}_${{ matrix.cuda-tag }}.whl
    - name: Display structure of downloaded files
      run: ls -R
    - name: Install TorchRec
      run: |
        rm -r dist || true
        conda run -n build_binary python -m pip install *.whl
    - name: Test fbgemm_gpu and torchrec installation
      shell: bash
      run: |
        conda run -n build_binary \
          python -c "import fbgemm_gpu"
        conda run -n build_binary \
          python -c "import torchrec"
    # Push to Pypi
    - name: Push TorchRec Binary to PYPI
      env:
          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
      run: |
        conda run -n build_binary python -m pip install twine
        conda run -n build_binary \
          python -m twine upload \
            --username __token__ \
            --password "$PYPI_TOKEN" \
            --skip-existing \
            torchrec-*.whl


CPU Unit Test CI

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: CPU Unit Test CI

on:
  push:
    paths-ignore:
      - "docs/*"
      - "third_party/*"
      - .gitignore
      - "*.md"
  pull_request:
    paths-ignore:
      - "docs/*"
      - "third_party/*"
      - .gitignore
      - "*.md"

jobs:
  build_test:
    strategy:
        fail-fast: false
        matrix:
          include:
            - os: linux.2xlarge
              python-version: 3.8
              python-tag: "py38"
            - os: linux.2xlarge
              python-version: 3.9
              python-tag: "py39"
            - os: linux.2xlarge
              python-version: '3.10'
              python-tag: "py310"
            - os: linux.2xlarge
              python-version: '3.11'
              python-tag: "py311"
    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
    with:
      runner: ${{ matrix.os }}
      timeout: 15
      script: |
        ldd --version
        conda create -y --name build_binary python=${{ matrix.python-version }}
        conda info
        python --version
        conda run -n build_binary python --version
        conda install -n build_binary \
          --yes \
          pytorch cpuonly -c pytorch-nightly
        conda run -n build_binary \
          python -c "import torch.distributed"
        conda run -n build_binary \
          pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu
        conda run -n build_binary \
          python -c "import fbgemm_gpu"
        echo "fbgemm_gpu succeeded"
        conda run -n build_binary \
          pip install -r requirements.txt
        conda run -n build_binary \
          python setup.py bdist_wheel \
          --python-tag=${{ matrix.python-tag }}
        conda run -n build_binary \
          python -c "import torchrec"
        echo "torch.distributed succeeded"
        conda run -n build_binary \
          python -c "import numpy"
        echo "numpy succeeded"
        conda install -n build_binary -y pytest
        conda run -n build_binary \
          python -m pytest torchrec -v -s -W ignore::pytest.PytestCollectionWarning --continue-on-collection-errors \
          --ignore-glob=**/test_utils/
        echo "Starting C++ Tests"
        conda install -n build_binary -y gxx_linux-64
        conda run -n build_binary \
          x86_64-conda-linux-gnu-g++ --version
        conda install -n build_binary -c anaconda redis -y
        conda run -n build_binary redis-server --daemonize yes
        mkdir cpp-build
        cd cpp-build
        conda run -n build_binary cmake \
            -DBUILD_TEST=ON \
            -DBUILD_REDIS_IO=ON \
            -DCMAKE_PREFIX_PATH=/opt/conda/envs/build_binary/lib/python${{ matrix.python-version }}/site-packages/torch/share/cmake ..
        conda run -n build_binary make -j
        conda run -n build_binary ctest -v .


Build Dynamic Embedding Wheels

name: Build Dynamic Embedding Wheels

on:
  workflow_dispatch:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
  release:
    types:
      - published

jobs:
  build_wheels:
    name: Wheels on ${{ matrix.os }}/${{ matrix.pyver }}/cu${{ matrix.cuver }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest ]
        pyver: [ cp38, cp39, cp310 ]
        cuver: [ "11.8" ]

    steps:
      -
        name: Check disk space
        run: df . -h

      - name: Remove unnecessary files
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      -
        name: Check disk space
        run: df . -h

      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: pypa/cibuildwheel@v2.8.0
        with:
          package-dir: contrib/dynamic_embedding
        env:
          CIBW_BEFORE_BUILD: "env CUDA_VERSION=${{ matrix.cuver }} contrib/dynamic_embedding/tools/before_linux_build.sh"
          CIBW_BUILD: "${{ matrix.pyver }}-manylinux_x86_64"
          CIBW_REPAIR_WHEEL_COMMAND: "env CUDA_VERSION=${{ matrix.cuver }} contrib/dynamic_embedding/tools/repair_wheel.sh {wheel} {dest_dir}"

      - name: Verify clean directory
        run: git diff --exit-code
        shell: bash

      - name: Upload wheels
        uses: actions/upload-artifact@v3
        with:
          path: wheelhouse/*.whl


Build Linux Wheels

name: Build Linux Wheels

on:
  pull_request:
  push:
    branches:
      - nightly
      - main
      - release/*
    tags:
      # Release candidate tags look like: v1.11.0-rc1
      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
  workflow_dispatch:

permissions:
  id-token: write
  contents: read

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: wheel
      os: linux
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      with-rocm: false
  filter-matrix:
    needs: generate-matrix
    runs-on: linux.2xlarge
    outputs:
      matrix: ${{ steps.filter.outputs.matrix }}
    steps:
    - uses: actions/setup-python@v4
    - name: Checkout torchrec repository
      uses: actions/checkout@v3
      with:
        repository: pytorch/torchrec
    - name: Filter Generated Built Matrix
      id: filter
      env:
        MAT: ${{ needs.generate-matrix.outputs.matrix }}
      run: |
        set -ex
        pwd
        ls
        MATRIX_BLOB="$(python .github/scripts/filter.py)"
        echo "${MATRIX_BLOB}"
        echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"
  build:
    needs: filter-matrix
    name: pytorch/torchrec
    uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
    with:
      repository: pytorch/torchrec
      ref: ""
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      build-matrix: ${{ needs.filter-matrix.outputs.matrix }}
      pre-script: ""
      post-script: .github/scripts/install_fbgemm.sh
      package-name: torchrec
      smoke-test-script: ""
      trigger-event: ${{ github.event_name }}


validate-nightly-binaries

# Scheduled validation of the nightly binaries
name: validate-nightly-binaries

on:
  schedule:
    # At 5:30 pm UTC (7:30 am PDT)
    - cron: "30 17 * * *"
  # Have the ability to trigger this job manually through the API
  workflow_dispatch:
  push:
    branches:
      - main
    paths:
      - .github/workflows/validate-nightly-binaries.yml
      - .github/workflows/validate-binaries.yml
      - .github/scripts/validate-binaries.sh
  pull_request:
    paths:
      - .github/workflows/validate-nightly-binaries.yml
      - .github/workflows/validate-binaries.yml
      - .github/scripts/validate-binaries.sh
jobs:
  nightly:
    uses: ./.github/workflows/validate-binaries.yml
    with:
      channel: nightly


Validate binaries

name: Validate binaries

on:
  workflow_call:
    inputs:
      channel:
        description: "Channel to use (nightly, release)"
        required: false
        type: string
        default: release
      ref:
        description: 'Reference to checkout, defaults to empty'
        default: ""
        required: false
        type: string
  workflow_dispatch:
    inputs:
      channel:
        description: "Channel to use (nightly, release, test)"
        required: true
        type: choice
        options:
          - release
          - nightly
          - test
      ref:
        description: 'Reference to checkout, defaults to empty'
        default: ""
        required: false
        type: string

jobs:
  validate-binaries:
    uses: pytorch/test-infra/.github/workflows/validate-domain-library.yml@main
    with:
      package_type: "wheel"
      os: "linux"
      channel: ${{ inputs.channel }}
      repository: "pytorch/torchrec"
      smoke_test: "source ./.github/scripts/validate_binaries.sh"
      with_cuda: enable


Pyre Check

name: Pyre Check

on:
  push:
    branches: [main]
  pull_request:

jobs:
  pyre-check:
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - uses: conda-incubator/setup-miniconda@v2
        with:
          python-version: 3.8
      - name: Checkout Torchrec
        uses: actions/checkout@v2
      - name: Install dependencies
        run: >
          conda install --yes pytorch cpuonly -c pytorch-nightly &&
          pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/nightly/cpu &&
          pip install -r requirements.txt &&
          pip install pyre-check-nightly==$(cat .pyre_configuration | grep version | awk '{print $2}' | sed 's/\"//g')
      - name: Pyre check
        run: pyre check


Deployment

# This code is part of Qiskit.
#
# (C) Copyright IBM 2021.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

name: Deployment

on:
  push:
    tags:
      - '*'

jobs:
  Deploy:
    runs-on: ubuntu-latest
    environment: release
    permissions:
      id-token: write
    strategy:
      matrix:
        python-version: [3.9]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install deps
        run: pip install -U pip setuptools build
      - name: Build sdist
        run: python3 -m build
      - uses: actions/upload-artifact@v4
        with:
          path: ./dist/*
      - name: Deploy to Pypi
        uses: pypa/gh-action-pypi-publish@release/v1


Docs Upload

---
name: Docs Upload
on:
  push:
    tags:
      - "*"
  workflow_dispatch:

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.9.12'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -U virtualenv setuptools wheel tox
        sudo apt-get install -y graphviz pandoc
    - name: Build docs
      run: tox -e docs
    - name: Upload artifact
      uses: actions/upload-artifact@v4
      with:
        name: html_docs
        path: docs/_build/html


CI

# This code is part of Qiskit.
#
# (C) Copyright IBM 2021.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

name: CI
on:
  [ push, pull_request ]
# save resources: cancel redundant workflow runs on the same branch when new commits are pushed
concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true
jobs:
  code-quality:
    if: github.repository_owner == 'Qiskit'
    name: Run code quality checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -c constraints.txt -r requirements-dev.txt -e .
      - name: Run black
        run: make style
      - name: Run lint
        run: make lint
      - name: Run mypy
        run: make mypy
  documentation:
    if: github.repository_owner == 'Qiskit'
    name: Build documentation
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -U tox
          pip install nbqa docutils
          sudo apt install -y graphviz pandoc
          pip install -e .

          wget https://github.com/errata-ai/vale/releases/download/v2.23.0/vale_2.23.0_Linux_64-bit.tar.gz
          mkdir $HOME/bin && tar -xf vale_2.23.0_Linux_64-bit.tar.gz -C $HOME/bin
          echo "$HOME/bin" >> $GITHUB_PATH
      - name: Lint documentation
        run: |
          make docs-test
      - name: Build documentation
        run: tox -edocs
      - name: Upload documentation
        uses: actions/upload-artifact@v4
        with:
          name: html_docs
          path: docs/_build/html
  unit-tests:
    if: github.repository_owner == 'Qiskit'
    # only kick-off test cases when basic code quality checks succeed
    needs: [ "code-quality" , "documentation" ]
    name: Run unit tests - python${{ matrix.python-version }}-${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: [3.8, 3.9, '3.10', '3.11', '3.12']
        os: [ "macos-latest", "ubuntu-latest", "windows-latest" ]
        exclude:
          - os: "macos-latest"
            python-version: 3.8
          - os: "macos-latest"
            python-version: 3.9
    env:
      LOG_LEVEL: DEBUG
      STREAM_LOG: True
      QISKIT_IN_PARALLEL: True
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -c constraints.txt -r requirements-dev.txt -e .
      - name: Run unit tests
        run: make unit-test-coverage
      - name: Report coverage to coveralls.io
        uses: coverallsapp/github-action@v2
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          flag-name: unit-tests_python${{ matrix.python-version }}-${{ matrix.os }}
          parallel: true
          file: coverage.lcov
  integration-tests:
    if: github.event_name == 'push' && github.repository_owner == 'Qiskit'
    # only kick-off resource intensive integration tests if unit tests and all basic checks succeeded
    needs: [ "unit-tests"  ]
    name: Run integration tests - ${{ matrix.environment }}
    runs-on: ${{ matrix.os }}
    strategy:
      # avoid cancellation of in-progress jobs if any matrix job fails
      fail-fast: false
      matrix:
        python-version: [ 3.9 ]
        os: [ "ubuntu-latest" ]
        environment: [ "ibm-quantum-production", "ibm-cloud-production" ]
    environment: ${{ matrix.environment }}
    env:
      QISKIT_IBM_TOKEN: ${{ secrets.QISKIT_IBM_TOKEN }}
      QISKIT_IBM_URL: ${{ secrets.QISKIT_IBM_URL }}
      QISKIT_IBM_INSTANCE: ${{ secrets.QISKIT_IBM_INSTANCE }}
      LOG_LEVEL: DEBUG
      STREAM_LOG: True
      QISKIT_IN_PARALLEL: True
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -c constraints.txt -r requirements-dev.txt -e .
      - name: Run integration tests
        run: make integration-test
  tests-finished:
    if: github.repository_owner == 'Qiskit'
    name: Submit code coverage metrics
    needs: [ unit-tests ]
    runs-on: ubuntu-latest
    steps:
      - name: Notify coveralls.io that all parallel tests have finished
        uses: coverallsapp/github-action@v2
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          parallel-finished: true


Integration Tests

# This code is part of Qiskit.
#
# (C) Copyright IBM 2021.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

name: Integration Tests
on:
  schedule:
    - cron: '0 4 * * *'
  # allow everyone with write access to the repository to trigger the workflow run manually
  workflow_dispatch:
jobs:
  integration-tests:
    if: github.repository_owner == 'Qiskit'
    name: Run integration tests - ${{ matrix.environment }}
    runs-on: ${{ matrix.os }}
    strategy:
      # avoid cancellation of in-progress jobs if any matrix job fails
      fail-fast: false
      matrix:
        python-version: [ 3.9 ]
        os: [ "ubuntu-latest" ]
        environment: [ "ibm-quantum-production", "ibm-quantum-staging", "ibm-cloud-production", "ibm-cloud-staging" ]
    environment: ${{ matrix.environment }}
    env:
      QISKIT_IBM_TOKEN: ${{ secrets.QISKIT_IBM_TOKEN }}
      QISKIT_IBM_URL: ${{ secrets.QISKIT_IBM_URL }}
      QISKIT_IBM_INSTANCE: ${{ secrets.QISKIT_IBM_INSTANCE }}
      LOG_LEVEL: DEBUG
      STREAM_LOG: True
      QISKIT_IN_PARALLEL: True
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -c constraints.txt -r requirements-dev.txt -e .
      - name: Run integration tests
        run: make integration-test


E2E Tests

## This code is part of Qiskit.
#
# (C) Copyright IBM 2022.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

name: E2E Tests
on:
  schedule:
    - cron: '0 8 * * *'
  # allow everyone with write access to the repository to trigger the workflow run manually
  workflow_dispatch:
jobs:
  e2e-tests:
    if: github.repository_owner == 'Qiskit'
    name: Run e2e tests - ${{ matrix.environment }}
    runs-on: ${{ matrix.os }}
    strategy:
      # avoid cancellation of in-progress jobs if any matrix job fails
      fail-fast: false
      matrix:
        python-version: [ 3.9 ]
        os: [ "ubuntu-latest" ]
        environment: [ "ibm-quantum-production", "ibm-quantum-staging", "ibm-cloud-production", "ibm-cloud-staging" ]
    environment: ${{ matrix.environment }}
    env:
      QISKIT_IBM_TOKEN: ${{ secrets.QISKIT_IBM_TOKEN }}
      QISKIT_IBM_URL: ${{ secrets.QISKIT_IBM_URL }}
      QISKIT_IBM_INSTANCE: ${{ secrets.QISKIT_IBM_INSTANCE }}
      LOG_LEVEL: DEBUG
      STREAM_LOG: True
      QISKIT_IN_PARALLEL: True
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -c constraints.txt -r requirements-dev.txt -e .
      - name: Run e2e tests
        run: make e2e-test


Unit Tests

# This code is part of Qiskit.
#
# (C) Copyright IBM 2022.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

name: Unit Tests
on:
  schedule:
    - cron: '0 5 * * *'
  workflow_dispatch:
jobs:
  unit-tests-latest-qiskit-terra:
    if: github.repository_owner == 'Qiskit'
    name: Run unit tests with latest code of Qiskit
    runs-on: "ubuntu-latest"
    env:
      LOG_LEVEL: DEBUG
      STREAM_LOG: True
      QISKIT_IN_PARALLEL: True
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          # Installing the complete environment should happen in one `pip install` step,
          # or pip will generally allow broken combinations of packages to be installed.
          pip install -c constraints.txt -r requirements-dev.txt -e . git+https://github.com/Qiskit/qiskit.git
      - name: Run tests
        # running unit tests against latest (non-released) code of qiskit-terra gives a basic level
        # of confidence that the integration between qiskit-ibm-runtime and qiskit-terra works
        run: make unit-test


Q-CTRL Tests

# This code is part of Qiskit.
#
# (C) Copyright IBM 2023.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

name: Q-CTRL Tests
on:
  schedule:
    - cron: '0 1 * * *'
  push:
      tags:
        - "*"
  workflow_dispatch:
jobs:
  integration-tests:
    name: Run integration tests - ${{ matrix.environment }}
    runs-on: ${{ matrix.os }}
    strategy:
      # avoid cancellation of in-progress jobs if any matrix job fails
      fail-fast: false
      matrix:
        python-version: [ 3.9 ]
        os: [ "ubuntu-latest" ]
        environment: [ "ibm-cloud-staging" ]
    environment: ${{ matrix.environment }}
    env:
      QISKIT_IBM_TOKEN: ${{ secrets.QISKIT_IBM_TOKEN_QCTRL }}
      QISKIT_IBM_URL: ${{ secrets.QISKIT_IBM_URL }}
      QISKIT_IBM_INSTANCE: ${{ secrets.QISKIT_IBM_INSTANCE_QCTRL }}
      CHANNEL_STRATEGY: q-ctrl
      LOG_LEVEL: DEBUG
      STREAM_LOG: True
      QISKIT_IN_PARALLEL: True
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -c constraints.txt -r requirements-dev.txt -e .
      - name: Run q-ctrl tests
        run: python -m unittest test/qctrl/test_qctrl.py


Integration Tests (Qiskit Main)

# This code is part of Qiskit.
#
# (C) Copyright IBM 2024.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

name: Integration Tests (Qiskit Main)
on:
  schedule:
    # run integration tests against Qiskit main once a week
    - cron: '0 0 * * 0'
  workflow_dispatch:
jobs:
  integration-tests:
    if: github.repository_owner == 'Qiskit'
    name: Run integration tests - ${{ matrix.environment }}
    runs-on: ${{ matrix.os }}
    strategy:
      # avoid cancellation of in-progress jobs if any matrix job fails
      fail-fast: false
      matrix:
        python-version: [ 3.9 ]
        os: [ "ubuntu-latest" ]
        environment: [ "ibm-quantum-production", "ibm-quantum-staging", "ibm-cloud-production", "ibm-cloud-staging" ]
    environment: ${{ matrix.environment }}
    env:
      QISKIT_IBM_TOKEN: ${{ secrets.QISKIT_IBM_TOKEN }}
      QISKIT_IBM_URL: ${{ secrets.QISKIT_IBM_URL }}
      QISKIT_IBM_INSTANCE: ${{ secrets.QISKIT_IBM_INSTANCE }}
      LOG_LEVEL: DEBUG
      STREAM_LOG: True
      QISKIT_IN_PARALLEL: True
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -c constraints.txt -r requirements-dev.txt -e . git+https://github.com/Qiskit/qiskit.git
      - name: Run integration tests
        run: make integration-test