sanity checks on opened pull requests

name: sanity checks on opened pull requests

on:
  pull_request:
    branches:
      - main
      - release/**
      - feature/**
    paths:
      - 'aim/cli/**'
      - 'aim/ext/**'
      - 'aim/sdk/**'
      - 'aim/web/api/**'
      - 'aim/storage/**'
      - 'aim/utils/**'
      - '.github/workflows/pull-request.yml'
      - 'tests/**'
      - 'performance_tests/**'
      - 'setup.py'
    types:
      - opened
      - synchronize
      - ready_for_review
      - reopened
      - edited
jobs:
  validate-naming-convention:
    name: Pull Request's title matches naming convention
    runs-on: ubuntu-latest
    steps:
      - uses: deepakputhraya/action-pr-title@master
        with:
          regex: '^\[(?:feat|fix|doc|refactor|deprecation)\]\s[A-Z].*(?<!\.)$'
          github_token: ${{ github.token }}
  run-checks:
    if: github.event.pull_request.draft == false && github.event.action != 'edited'
    runs-on: ubuntu-latest
    name: Run Unit-tests & Code-style checks
    steps:
      - name: checkout
        uses: actions/checkout@v2
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: setup python
        uses: actions/setup-python@v2
        with:
          python-version: '3.7'
          architecture: x64

      - name: install deps
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r requirements.dev.txt
          python -m pip install -r tests/requirements.txt

      - name: code style checks
        run: |
          flake8 aim

      - name: unit-tests
        run: |
          pytest --cov-report term --cov-branch --cov-fail-under=50 --cov=aim/web --cov=aim/storage --cov=aim/sdk tests

  storage-performance-checks:
    needs: run-checks
    concurrency: perf-tests
    runs-on: [self-hosted, performance-tests]
    name: Performance tests
    steps:
      - name: checkout
        uses: actions/checkout@v2
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: install deps
        run: |
          python3 -m venv venv
          . venv/bin/activate
          python -m pip install --upgrade pip
          python -m pip install -r requirements.dev.txt
          python -m pip install -r performance_tests/requirements.txt

      - name: perf tests
        run: |
          . venv/bin/activate
          export AWS_ACCESS_KEY_ID=${{ secrets.AIM_LOGS_AWS_ACCESS_KEY_ID }}
          export AWS_SECRET_ACCESS_KEY=${{ secrets.AIM_LOGS_AWS_SECRET_ACCESS_KEY }}
          pytest performance_tests || pytest --lf performance_tests


aim packaging pipeline

name: aim packaging pipeline

on:
  workflow_call:
    secrets:
      PYPI_PASSWORD:
        required: true
      DOCKER_HUB_USERNAME:
        required: true
      DOCKER_HUB_PASSWORD:
        required: true
    inputs:
      branch:
        required: true
        type: string
  workflow_dispatch:

jobs:
  aim-ui-dist:
    if: ${{ github.event_name == 'schedule' || github.actor == 'mihran113' || github.actor == 'alberttorosyan' || github.actor == 'roubkar' }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout sources
        uses: actions/checkout@v2
        with:
          ref: ${{ inputs.branch || github.ref }}

      - name: Use node v16
        uses: actions/setup-node@v2
        with:
          node-version: '16'

      - name: Build UI
        working-directory: ./aim/web/ui
        run: |
          npm i --legacy-peer-deps
          CI=false npm run build

      - name: Setup python
        uses: actions/setup-python@v2
        with:
          python-version: 3.9
          architecture: x64

      - name: Install dev dependencies
        run: |
          python -m pip install -r requirements.dev.txt

      - name: Build sdist
        working-directory: ./aim/web/ui
        run: |
          python setup.py sdist

      - name: Publish wheel
        env:
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        working-directory: ./aim/web/ui
        run: |
          python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" dist/*

  linux-dist:
    if: ${{ github.event_name == 'schedule' || github.actor == 'mihran113' || github.actor == 'alberttorosyan' || github.actor == 'roubkar' }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        manylinux-version: ['manylinux2014_x86_64', 'manylinux_2_24_x86_64', 'manylinux_2_28_x86_64']
    name: Linux wheels build (using Docker) ${{ matrix.manylinux-version }}
    needs: aim-ui-dist
    steps:
      - name: Install Docker & pull images
        run: |
          apt update && apt install -y docker.io
          sudo systemctl enable --now docker
          docker pull quay.io/pypa/${{ matrix.manylinux-version }}

      - name: Checkout sources
        uses: actions/checkout@v2
        with:
          ref: ${{ inputs.branch || github.ref }}

      - name: Capture aim version
        id: capture-version
        run: |
          AIM_VERSION=$(< ./aim/VERSION)
          echo "::set-output name=AIM_VERSION::$AIM_VERSION"

      - name: Setup python
        uses: actions/setup-python@v2
        with:
          python-version: 3.9
          architecture: x64

      - name: Install dev dependencies
        run: |
          python -m pip install -r requirements.dev.txt
          python -m pip install -r requirements.txt

      - name: Build bdist wheels for 'cp37-cp37m'
        uses: nick-fields/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 30
          retry_wait_seconds: 60
          command: docker run  -e PYTHON_VERSION='cp37-cp37m' --mount type=bind,source=$PWD,target=/opt/aim quay.io/pypa/${{ matrix.manylinux-version }} bash /opt/aim/docker/build-wheels.sh

      - name: Build bdist wheels for 'cp38-cp38'
        uses: nick-fields/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 30
          retry_wait_seconds: 60
          command: docker run  -e PYTHON_VERSION='cp38-cp38' --mount type=bind,source=$PWD,target=/opt/aim quay.io/pypa/${{ matrix.manylinux-version }} bash /opt/aim/docker/build-wheels.sh

      - name: Build bdist wheels for 'cp39-cp39'
        uses: nick-fields/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 30
          retry_wait_seconds: 60
          command: docker run  -e PYTHON_VERSION='cp39-cp39' --mount type=bind,source=$PWD,target=/opt/aim quay.io/pypa/${{ matrix.manylinux-version }} bash /opt/aim/docker/build-wheels.sh

      - name: Build bdist wheels for 'cp310-cp310'
        uses: nick-fields/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 30
          retry_wait_seconds: 60
          command: docker run  -e PYTHON_VERSION='cp310-cp310' --mount type=bind,source=$PWD,target=/opt/aim quay.io/pypa/${{ matrix.manylinux-version }} bash /opt/aim/docker/build-wheels.sh

      - name: Build bdist wheels for 'cp311-cp311'
        uses: nick-fields/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 30
          retry_wait_seconds: 60
          command: docker run  -e PYTHON_VERSION='cp311-cp311' --mount type=bind,source=$PWD,target=/opt/aim quay.io/pypa/${{ matrix.manylinux-version }} bash /opt/aim/docker/build-wheels.sh

      - name: Build bdist wheels for 'cp312-cp312'
        if: matrix.manylinux-version == 'manylinux_2_28_x86_64'
        uses: nick-fields/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 30
          retry_wait_seconds: 60
          command: docker run  -e PYTHON_VERSION='cp312-cp312' --mount type=bind,source=$PWD,target=/opt/aim quay.io/pypa/${{ matrix.manylinux-version }} bash /opt/aim/docker/build-wheels.sh

      - name: Publish wheels
        env:
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" multilinux_dist/*

      - name: Docker login
        env:
          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
        run: docker login -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD

      - name: Build & Push aim image
        if: matrix.manylinux-version == 'manylinux_2_24_x86_64'
        working-directory: ./docker
        env:
          AIM_VERSION: ${{ steps.capture-version.outputs.AIM_VERSION }}
          UPDATE_TAG: ${{ inputs.branch || 'latest' }}
        run: . create-docker-image.sh

  macos-dist:
    if: ${{ github.event_name == 'schedule' || github.actor == 'mihran113' || github.actor == 'alberttorosyan' || github.actor == 'roubkar' }}
    runs-on: m1
    strategy:
      fail-fast: false
      matrix:
        python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ]
        arch: ['arm64', 'x86_64']
        exclude:
          - arch: 'arm64'
            python-version: 3.7

    name: Python ${{ matrix.python-version }} for ${{ matrix.arch }} build
    env:
      MACOSX_DEPLOYMENT_TARGET: 10.14
      AIM_DEP_DIR: /tmp/aim_run/${{ github.run_number }}/${{ matrix.arch }}
      PYTHON: /opt/conda/${{ matrix.arch }}/envs/py${{ matrix.python-version }}/bin/python

    needs: aim-ui-dist
    steps:
      - name: Checkout sources
        uses: actions/checkout@v2
        with:
          ref: ${{ inputs.branch || github.ref }}

      - name: Build bdist wheel
        run: |
          arch -${{matrix.arch}} $PYTHON -m build

      - name: Publish source dist
        if: matrix.python-version == '3.10' && matrix.arch == 'arm64'
        env:
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          $PYTHON -m twine upload -u __token__ -p "${PYPI_PASSWORD}" dist/*.tar.gz

      - name: Publish wheel
        env:
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          $PYTHON -m twine upload -u __token__ -p "${PYPI_PASSWORD}" dist/*.whl


Dependabot auto-approve

name: Dependabot auto-approve
on: pull_request

permissions:
  pull-requests: write

jobs:
  dependabot:
    runs-on: ubuntu-latest
    if: ${{ github.actor == 'dependabot[bot]' }}
    steps:
      - name: Dependabot metadata
        id: metadata
        uses: dependabot/fetch-metadata@v1.1.1
        with:
          github-token: "${{ secrets.GITHUB_TOKEN }}"
      - name: Approve a PR
        run: gh pr review --approve "$PR_URL"
        env:
          PR_URL: ${{github.event.pull_request.html_url}}
          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}


Dependabot auto-merge

name: Dependabot auto-merge
on: pull_request

permissions:
  pull-requests: write
  contents: write

jobs:
  dependabot:
    runs-on: ubuntu-latest
    if: ${{ github.actor == 'dependabot[bot]' }}
    steps:
      - name: Dependabot metadata
        id: metadata
        uses: dependabot/fetch-metadata@v1.1.1
        with:
          github-token: "${{ secrets.GITHUB_TOKEN }}"
      - name: Enable auto-merge for Dependabot PRs
        run: gh pr merge --auto --merge "$PR_URL"
        env:
          PR_URL: ${{github.event.pull_request.html_url}}
          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}


Lint

name: Lint

on:
  push:
    branches: [master]
  pull_request:
    branches: [master]

jobs:
  linter-black:
      name: Check code formatting with Black
      runs-on: ubuntu-latest
      steps:
        - name: Checkout
          uses: actions/checkout@v2
        - name: Set up Python 3.8
          uses: actions/setup-python@v2
          with:
            python-version: 3.8
        - name: Install Black
          run: pip install black[jupyter]
        - name: Run Black
          run: black --check .

  imports-check-isort:
    name: Check valid import formatting with isort
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Install isort
        run: pip install isort==5.6.4
      - name: Run isort
        run: isort --check-only --diff .

  linter-flake8:
    name: Check valid formatting with flake8
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependencies
        run: pip install flake8
      - name: Run checks
        run: flake8

  pre-commit-hooks:
    name: Check that pre-commit hooks pass
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Install dependencies
        run: pip install pre-commit

      - name: Run checks
        run: pre-commit run --all-files


PyPI Release

name: PyPI Release

# https://help.github.com/en/actions/reference/events-that-trigger-workflows
on:
  # Trigger the workflow on push or pull request,
  # but only for the master branch
  push:
    branches:
      - master
  release:
    types:
      - created

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install poetry
        shell: bash
        run: |
          curl -sSL https://install.python-poetry.org | python3 -
          python -m pip install poetry-dynamic-versioning[plugin]

      - name: Set poetry path variable
        run: echo "/Users/runner/.local/bin" >> $GITHUB_PATH

      - name: Build
        run: |
          poetry build

      - name: Publish distribution 📦 to PyPI
        if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
        run: |
          poetry publish --username "${{ secrets.PYPI_USER }}" --password "${{ secrets.PYPI_PASSWORD }}"


Test

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Test

on:
  push:
    branches: [master]
  pull_request:
    branches: [master]

jobs:
  pytest:
    name: Run pytest
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, macOS-latest] # add windows-2019 when poetry allows installation with `-f` flag
        python-version: ["3.8", "3.9", "3.10"]
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v2

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - name: Setup macOS
        if: runner.os == 'macOS'
        run: |
          brew install libomp  # https://github.com/pytorch/pytorch/issues/20030

      - name: Get full Python version
        id: full-python-version
        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")

      - name: Install poetry
        run: |
          curl -sSL https://install.python-poetry.org | python3 -

      - name: Set poetry path variable
        run: echo "/Users/runner/.local/bin" >> $GITHUB_PATH

      - name: Configure poetry
        run: poetry config virtualenvs.in-project true

      - name: Set up cache
        uses: actions/cache@v2
        id: cache
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Ensure cache is healthy
        if: steps.cache.outputs.cache-hit == 'true'
        run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv

      - name: Upgrade pip
        run: poetry run python -m pip install pip -U

      - name: Install dependencies
        run: poetry install -E "github-actions graph mqf2"

      # - name: Install pytorch geometric dependencies
      #   shell: bash
      #   run: poetry run pip install pyg_lib torch-scatter torch-sparse torch-cluster torch-spline-conv torch-geometric -f https://data.pyg.org/whl/torch-2.0.0+cpu.html

      - name: Run pytest
        run: poetry run pytest tests

      - name: Statistics
        run: |
          pip install coverage
          coverage report
          coverage xml

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v1
        if: always()
        continue-on-error: true
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: coverage.xml
          flags: cpu,pytest
          name: CPU-coverage
          fail_ci_if_error: false

  docs:
    name: Docs build
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - name: Check out Git repository
        uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: 3.8

      - name: Cache pip
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('docs/requirements.txt') }}
          restore-keys: ${{ runner.os }}-pip-

      - name: Install dependencies
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get install -y pandoc
          pip install -r docs/requirements.txt

      - name: Build sphinx documentation
        working-directory: docs/
        run: make html --debug --jobs 2 SPHINXOPTS="-W"

      - name: Upload built docs
        uses: actions/upload-artifact@v2
        with:
          name: docs-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}
          path: docs/build/html/
        # Use always() to always run this step to publish test results when there are test failures
        #if: success()


CI

name: CI

on:
  push:
  pull_request:
  schedule:
    - cron: "0 0 * * *"

jobs:
  test:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install libolm
        if: runner.os == 'macos'
        run: |
          brew install libolm
      - name: Install ffmpeg on Ubuntu
        if: runner.os != 'Windows'
        run: |
          if [ $RUNNER_OS = Linux ]; then
              sudo apt update || true
              sudo apt install ffmpeg -y
          fi
      - name: Install ffmpeg on windows
        if: runner.os == 'Windows'
        run: |
              choco feature enable -n=allowGlobalConfirmation
              choco install ffmpeg
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -U tox tox-gh-actions
      - name: Run tests
        env:
          PLATFORM: ${{ runner.os }}
        run: tox -vv
        timeout-minutes: 30
      - name: Upload coverage report
        uses: codecov/codecov-action@v1
        with:
          file: ./cov.xml

  check:
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        component: [docker-full, docker-min, lint]
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.8"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -U tox
      - name: Run checks
        run: tox -vv -e ${{ matrix.component }}
        timeout-minutes: 30


Upload Docker Dev Container

name: Upload Docker Dev Container

on:
  push:
    branches:
      - 'main'
      - 'master'

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout Repos
      uses: actions/checkout@v3
    - name: Setup QEMU
      uses: docker/setup-qemu-action@v2
      with:
        platforms: 'arm64,arm'
    - name: Set up Docker Buildx
      id: builder
      uses: docker/setup-buildx-action@v2
    - name: Docker meta
      id: docker_meta
      uses: docker/metadata-action@v3
      with:
        images: |
          ${{ secrets.DOCKERHUB_REPO }}
          ghcr.io/${{ github.repository_owner }}/opsdroid
        tags: |
          type=raw,value=dev
    - name: Login to Dockerhub
      uses: docker/login-action@v2
      with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
    - name: Login to GitHub Container Registry
      uses: docker/login-action@v2
      with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
    - name: Build and push
      uses: docker/build-push-action@v3
      with:
        push: true
        tags: ${{ steps.docker_meta.outputs.tags }}
        platforms: linux/amd64,linux/arm64,linux/arm
        labels: ${{ steps.docker_meta.outputs.labels }}


Upload Docker Container

name: Upload Docker Container

on:
  release:
    types: [released]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout Repo
      uses: actions/checkout@v3
    - name: Setup QEMU
      uses: docker/setup-qemu-action@v2
      with:
        platforms: 'arm64,arm'
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v2
    - name: Docker meta
      id: docker_meta
      uses: docker/metadata-action@v3
      with:
        images: |
          ${{ secrets.DOCKERHUB_REPO }}
          ghcr.io/${{ github.repository_owner }}/opsdroid
        tags: |
          type=semver,pattern={{raw}}
    - name: Login to Dockerhub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}
    - name: Login to GitHub Container Registry
      uses: docker/login-action@v2
      with:
        registry: ghcr.io
        username: ${{ github.repository_owner }}
        password: ${{ secrets.GITHUB_TOKEN }}
    - name: Build and push
      uses: docker/build-push-action@v3
      with:
        push: true
        tags: ${{ steps.docker_meta.outputs.tags }}
        platforms: linux/amd64,linux/arm64,linux/arm
        labels: ${{ steps.docker_meta.outputs.labels }}


Upload Python Package

name: Upload Python Package

on:
  release:
    types: [released]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install setuptools wheel twine
    - name: Build and publish python package
      env:
        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
      run: |
        python setup.py sdist
        twine upload dist/*


Pull Request Labeler

name: "Pull Request Labeler"
on:
- pull_request_target

jobs:
  triage:
    permissions:
      contents: read
      pull-requests: write
    runs-on: ubuntu-latest
    steps:
    - uses: actions/labeler@v4


Release Drafter

name: Release Drafter

on:
  push:
    branches:
      - master

jobs:
  update_release_draft:
    runs-on: ubuntu-latest
    steps:
      - uses: release-drafter/release-drafter@v5
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


CodeQL

name: "CodeQL"

on:
  push:
    branches: [ 'master' ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ 'master' ]
  schedule:
    - cron: '36 22 * * 0'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v3

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.

        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
        queries: +security-and-quality

    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    # ℹ Command-line programs to run using the OS shell.
    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

    #   If the Autobuild fails above, remove it and uncomment the following three lines.
    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

    # - run: |
    #   echo "Run, Build Application using script"
    #   ./location_of_script_within_repo/buildscript.sh

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2
      with:
        category: "/language:${{matrix.language}}"


Publish

# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Publish

on:
  release:
    types: [created]
  workflow_dispatch: {}

jobs:
  package-source:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: "3.8"
      - name: Prepare C files to include
        run: |
          python -m pip install --upgrade pip build
          python -m pip install -r requirements-cython.txt
          # Make sure we install to have all c files to be shiped with bundle
          python -m pip install -vv -U .  # We set -vv to see compiler exceptions/warnings
      - name: Build source package
        run: python -m build --sdist
      - name: Upload source package
        uses: actions/upload-artifact@v2
        with:
          name: dist
          path: dist/

  package-wheel:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        arch: [auto]
        include:
          - os: ubuntu-latest
            arch: aarch64
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: "3.8"
      - name: Set up QEMU
        if: ${{ matrix.arch == 'aarch64' }}
        uses: docker/setup-qemu-action@v1
      - name: Build wheels
        env:
          CIBW_ARCHS_LINUX: ${{matrix.arch}}
          CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* cp312-*
          CIBW_SKIP: '*-musllinux*'
          CIBW_BEFORE_BUILD_LINUX: pip install -r requirements-cython.txt && yum install -y zlib-devel
          # On windows and mac we should have z library preinstalled
          CIBW_BEFORE_BUILD: pip install -r requirements-cython.txt
          CIBW_BUILD_VERBOSITY: 2
        run: |
          python -m pip install --upgrade pip
          pip install cibuildwheel
          cibuildwheel --output-dir dist
        shell: bash
      - name: Upload wheels
        uses: actions/upload-artifact@v2
        with:
          name: dist
          path: dist/

  test-wheels-windows:
    needs: [package-source, package-wheel]
    runs-on: windows-latest

    strategy:
      matrix:
        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        include:
          - python: "3.8"
            aiokafka_whl: dist/aiokafka-*-cp38-cp38-win_amd64.whl
          - python: "3.9"
            aiokafka_whl: dist/aiokafka-*-cp39-cp39-win_amd64.whl
          - python: "3.10"
            aiokafka_whl: dist/aiokafka-*-cp310-cp310-win_amd64.whl
          - python: "3.11"
            aiokafka_whl: dist/aiokafka-*-cp311-cp311-win_amd64.whl
          - python: "3.12"
            aiokafka_whl: dist/aiokafka-*-cp312-cp312-win_amd64.whl

    steps:
      - uses: actions/checkout@v2
      - name: Download distributions
        uses: actions/download-artifact@v2
        with:
          name: dist
          path: dist/
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python }}

      - name: Install python dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r requirements-win-test.txt
          python -m pip install ${{ matrix.aiokafka_whl }}
        shell: bash

      - name: Run Unit Tests
        run: |
          # Remove source code to be sure we use wheel code
          rm -rf aiokafka
          make ci-test-unit
        shell: bash

  test-wheels-mac:
    needs: [package-source, package-wheel]
    runs-on: macos-latest

    strategy:
      matrix:
        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        include:
          - python: "3.8"
            aiokafka_whl: dist/aiokafka-*-cp38-cp38-macosx_10_9_x86_64.whl
          - python: "3.9"
            aiokafka_whl: dist/aiokafka-*-cp39-cp39-macosx_10_9_x86_64.whl
          - python: "3.10"
            aiokafka_whl: dist/aiokafka-*-cp310-cp310-macosx_10_9_x86_64.whl
          - python: "3.11"
            aiokafka_whl: dist/aiokafka-*-cp311-cp311-macosx_10_9_x86_64.whl
          - python: "3.12"
            aiokafka_whl: dist/aiokafka-*-cp312-cp312-macosx_10_9_x86_64.whl

    steps:
      - uses: actions/checkout@v2
      - name: Download distributions
        uses: actions/download-artifact@v2
        with:
          name: dist
          path: dist/
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python }}

      - name: Install python dependencies
        run: |
          pip install --upgrade pip setuptools wheel
          pip install -r requirements-ci.txt
          pip install ${{ matrix.aiokafka_whl }}

      - name: Run Unit Tests
        run: |
          # Remove source code to be sure we use wheel code
          rm -rf aiokafka
          make ci-test-unit

  test-wheels-linux:
    needs: [package-source, package-wheel]
    runs-on: ubuntu-latest

    strategy:
      matrix:
        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        include:
          - python: "3.8"
            aiokafka_whl: dist/aiokafka-*-cp38-cp38-manylinux*_x86_64.whl
          - python: "3.9"
            aiokafka_whl: dist/aiokafka-*-cp39-cp39-manylinux*_x86_64.whl
          - python: "3.10"
            aiokafka_whl: dist/aiokafka-*-cp310-cp310-manylinux*_x86_64.whl
          - python: "3.11"
            aiokafka_whl: dist/aiokafka-*-cp311-cp311-manylinux*_x86_64.whl
          - python: "3.12"
            aiokafka_whl: dist/aiokafka-*-cp312-cp312-manylinux*_x86_64.whl

    steps:
      - uses: actions/checkout@v2
      - name: Download distributions
        uses: actions/download-artifact@v2
        with:
          name: dist
          path: dist/
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python }}

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libkrb5-dev
      - name: Install python dependencies
        run: |
          pip install --upgrade pip setuptools wheel
          pip install -r requirements-ci.txt
          pip install ${{ matrix.aiokafka_whl }}

      - name: Run Unit Tests
        run: |
          # Remove source code to be sure we use wheel code
          rm -rf aiokafka
          make ci-test-unit

  test-wheels-aarch64-linux:
    needs: [package-source, package-wheel]
    runs-on: ubuntu-latest
    env:
      py: /opt/python/${{ matrix.pyver }}/bin/python
      img: quay.io/pypa/manylinux2014_aarch64

    strategy:
      matrix:
        include:
          - pyver: cp38-cp38
            aiokafka_whl: dist/aiokafka-*-cp38-cp38-manylinux*_aarch64.whl
          - pyver: cp39-cp39
            aiokafka_whl: dist/aiokafka-*-cp39-cp39-manylinux*_aarch64.whl
          - pyver: cp310-cp310
            aiokafka_whl: dist/aiokafka-*-cp310-cp310-manylinux*_aarch64.whl
          - pyver: cp311-cp311
            aiokafka_whl: dist/aiokafka-*-cp311-cp311-manylinux*_aarch64.whl
          - pyver: cp312-cp312
            aiokafka_whl: dist/aiokafka-*-cp312-cp312-manylinux*_aarch64.whl

    steps:
      - uses: actions/checkout@v2
      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@v1
      - name: Download distributions
        uses: actions/download-artifact@v2
        with:
          name: dist
          path: dist/
      - name: Test Wheel
        run: |
              docker run --rm -v ${{ github.workspace }}:/ws:rw --workdir=/ws \
              ${{ env.img }} \
              bash -exc '${{ env.py }} -m venv .env && \
              source .env/bin/activate && \
              yum install -y epel-release && \
              yum-config-manager --enable epel && \
              yum install -y krb5-devel && \
              pip install --upgrade pip setuptools wheel && \
              pip install -r requirements-ci.txt && \
              pip install ${{ matrix.aiokafka_whl }} && \
              rm -rf aiokafka && \
              make ci-test-unit && \
              deactivate'

  deploy:

    runs-on: ubuntu-latest
    needs: [test-wheels-linux, test-wheels-aarch64-linux, test-wheels-mac, test-wheels-windows]

    steps:
      - uses: actions/checkout@v2
      - name: Download distributions
        uses: actions/download-artifact@v2
        with:
          name: dist
          path: dist/
      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@master
        with:
          user: ${{ secrets.PYPI_USERNAME }}
          password: ${{ secrets.PYPI_PASSWORD }}


Tests

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Tests

on:
  push:
    branches: [ master, '**' ]
    tags:
      - "v0.[0-9]+.[0-9]+"
      - "v0.[0-9]+.[0-9]+.dev*"
  pull_request:
    branches: [ master ]

jobs:
  test-sanity:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: 3.9

    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y libkrb5-dev

    - name: Get pip cache dir
      id: pip-cache
      run: |
        python -m pip install -U "pip>=20.1"
        echo "::set-output name=dir::$(pip cache dir)"

    - name: Cache packages
      uses: actions/cache@v1
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-py-3.9-${{ hashFiles('requirements-ci.txt') }}-${{ hashFiles('setup.py') }}
        # If miss on key takes any other cache with different hashes, will download correct ones on next step anyway
        restore-keys: |
          ${{ runner.os }}-py-3.9-

    - name: Install python dependencies
      run: |
        pip install --upgrade pip setuptools wheel
        pip install -r requirements-ci.txt
        pip install -vv -Ue .  # We set -vv to see compiler exceptions/warnings

    - name: Run Unit Tests
      run: |
        make ci-test-unit

    - name: Lint code
      run: |
        make lint

    - name: Check readme for package
      run: |
        make check-readme

    - name: Build doc
      run: |
        pip install -r requirements-docs.txt
        make -C docs html

  test-windows:
    needs: test-sanity
    runs-on: windows-latest

    strategy:
      matrix:
        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]

    steps:
    - uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python }}

    - name: Get pip cache dir
      id: pip-cache
      run: |
        python -m pip install -U "pip>=20.1"
        echo "::set-output name=dir::$(pip cache dir)"

    - name: Cache packages
      uses: actions/cache@v1
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-py-${{ matrix.python }}-${{ hashFiles('requirements-win-test.txt') }}-${{ hashFiles('setup.py') }}
        # If miss on key takes any other cache with different hashes, will download correct ones on next step anyway
        restore-keys: |
         ${{ runner.os }}-py-${{ matrix.python }}-

    - name: Install python dependencies
      run: |
        pip install --upgrade pip setuptools wheel
        pip install -r requirements-win-test.txt
        pip install -vv -Ue .  # We set -vv to see compiler exceptions/warnings

    - name: Run Unit Tests
      run: |
        make ci-test-unit
        mv coverage.xml coverage-ext.xml
      env:
        PYTHONASYNCIODEBUG: "1"

    - name: Run Unit Tests without extensions
      run: |
        make ci-test-unit
        mv coverage.xml coverage-py.xml
      env:
        AIOKAFKA_NO_EXTENSIONS: "1"

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v1
      with:
        file: ./coverage-ext.xml
        flags: unit,cext
        name: test-windows-${{ matrix.python }}-ext
      if: ${{ always() }}

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v1
      with:
        file: ./coverage-py.xml
        flags: unit,purepy
        name: test-windows-${{ matrix.python }}-py
      if: ${{ always() }}

  test-mac:
    needs: test-sanity
    runs-on: macos-latest

    strategy:
      matrix:
        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]

    steps:
    - uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python }}

    - name: Get pip cache dir
      id: pip-cache
      run: |
        python -m pip install -U "pip>=20.1"
        echo "::set-output name=dir::$(pip cache dir)"

    - name: Cache packages
      uses: actions/cache@v1
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-py-${{ matrix.python }}-${{ hashFiles('requirements-ci.txt') }}-${{ hashFiles('setup.py') }}
        # If miss on key takes any other cache with different hashes, will download correct ones on next step anyway
        restore-keys: |
         ${{ runner.os }}-py-${{ matrix.python }}-

    - name: Install python dependencies
      run: |
        pip install --upgrade pip setuptools wheel
        pip install -r requirements-ci.txt
        pip install -vv -Ue .  # We set -vv to see compiler exceptions/warnings

    - name: Run All Tests
      run: |
        make ci-test-unit
        mv coverage.xml coverage-ext.xml
      env:
        PYTHONASYNCIODEBUG: "1"

    - name: Run All Tests without extensions
      run: |
        make ci-test-unit
        mv coverage.xml coverage-py.xml
      env:
        AIOKAFKA_NO_EXTENSIONS: "1"

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v1
      with:
        file: ./coverage-ext.xml
        flags: unit,cext
        name: test-mac-${{ matrix.python }}-ext
      if: ${{ always() }}

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v1
      with:
        file: ./coverage-py.xml
        flags: unit,purepy
        name: test-mac-${{ matrix.python }}-py
      if: ${{ always() }}

  test-linux:
    needs: test-sanity
    runs-on: ubuntu-latest

    strategy:
      matrix:
        include:
          - python: "3.12"
            kafka: "2.8.1"
            scala: "2.13"

          # Older python versions against latest broker
          - python: "3.8"
            kafka: "2.8.1"
            scala: "2.13"
          - python: "3.9"
            kafka: "2.8.1"
            scala: "2.13"
          - python: "3.10"
            kafka: "2.8.1"
            scala: "2.13"
          - python: "3.11"
            kafka: "2.8.1"
            scala: "2.13"

          # Older brokers against latest python version
          - python: "3.12"
            kafka: "0.9.0.1"
            scala: "2.11"
          - python: "3.12"
            kafka: "0.10.2.1"
            scala: "2.11"
          - python: "3.12"
            kafka: "0.11.0.3"
            scala: "2.12"
          - python: "3.12"
            kafka: "1.1.1"
            scala: "2.12"
          - python: "3.12"
            kafka: "2.1.1"
            scala: "2.12"
          - python: "3.12"
            kafka: "2.2.2"
            scala: "2.12"
          - python: "3.12"
            kafka: "2.3.1"
            scala: "2.12"
          - python: "3.12"
            kafka: "2.4.1"
            scala: "2.12"
          - python: "3.12"
            kafka: "2.5.1"
            scala: "2.12"
          - python: "3.12"
            kafka: "2.6.3"
            scala: "2.12"
          - python: "3.12"
            kafka: "2.7.2"
            scala: "2.13"
      fail-fast: false

    steps:
    - uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python }}

    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y libkrb5-dev krb5-user

    - name: Get pip cache dir
      id: pip-cache
      run: |
        python -m pip install -U "pip>=20.1"
        echo "::set-output name=dir::$(pip cache dir)"

    - name: Cache packages
      uses: actions/cache@v1
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: ${{ runner.os }}-py-${{ matrix.python }}-${{ hashFiles('requirements-ci.txt') }}-${{ hashFiles('setup.py') }}
        # If miss on key takes any other cache with different hashes, will download correct ones on next step anyway
        restore-keys: |
         ${{ runner.os }}-py-${{ matrix.python }}-

    - name: Install python dependencies
      run: |
        pip install --upgrade pip setuptools wheel
        pip install -r requirements-ci.txt
        pip install -vv -Ue .  # We set -vv to see compiler exceptions/warnings

    - name: Run All Tests
      run: |
        make ci-test-all
        mv coverage.xml coverage-ext.xml
      env:
        SCALA_VERSION: ${{ matrix.scala }}
        KAFKA_VERSION: ${{ matrix.kafka }}

    - name: Run All Tests without extensions
      run: |
        make ci-test-all
        mv coverage.xml coverage-py.xml
      env:
        AIOKAFKA_NO_EXTENSIONS: "1"
        SCALA_VERSION: ${{ matrix.scala }}
        KAFKA_VERSION: ${{ matrix.kafka }}

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v1
      with:
        file: ./coverage-ext.xml
        flags: integration,cext
        name: test-linux-${{ matrix.python }}-${{ matrix.kafka }}-${{ matrix.scala }}-ext
      if: ${{ always() }}

    - name: Upload coverage without cext to Codecov
      uses: codecov/codecov-action@v1
      with:
        file: ./coverage-py.xml
        flags: integration,purepy
        name: test-linux-${{ matrix.python }}-${{ matrix.kafka }}-${{ matrix.scala }}-py
      if: ${{ always() }}


python tests+artifacts+release

name: python tests+artifacts+release

on:
  pull_request:
  push:
    branches:
    - "*"
    tags:
    - "v*"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  FORCE_COLOR: 1

jobs:

  package:
    name: Build & inspect our package.
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: hynek/build-and-inspect-python-package@v2

  test:
    needs: [package]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.10' ]
        os: [windows-latest, ubuntu-latest] #, macos-latest]
        include:
        - os: windows-latest
          python_version: 'msys2'

    name: ${{ matrix.os }} - Python ${{ matrix.python_version }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Setup python
        uses: actions/setup-python@v5
        if: matrix.python_version != 'msys2'
        with:
          python-version: ${{ matrix.python_version }}
          architecture: x64
      - name: Setup MSYS2
        uses: msys2/setup-msys2@v2
        if: matrix.python_version == 'msys2'
        with:
          msystem: MINGW64
          install: git mingw-w64-x86_64-python mingw-w64-x86_64-python-setuptools
          update: true
      - name: Setup GnuPG
        # At present, the Windows VMs only come with the copy of GnuPG that's bundled
        # with Git for Windows. If we want to use this version _and_ be able to set
        # arbitrary GnuPG home directories, then the test would need to figure out when
        # to convert Windows-style paths into Unix-style paths with cygpath, which is
        # unreasonable.
        #
        # Instead, we'll install a version of GnuPG that can handle Windows-style paths.
        # However, due to <https://dev.gnupg.org/T5593>, installation fails if the PATH
        # environment variable is too long. Consequently, we need to shorten PATH to
        # something minimal before we can install GnuPG. For further details, see
        # <https://github.com/actions/virtual-environments/issues/2876>.
        #
        # Additionally, we'll explicitly set `gpg.program` to ensure Git for Windows
        # doesn't invoke the bundled GnuPG, otherwise we'll run into
        # <https://dev.gnupg.org/T5504>. See also: <https://dev.gnupg.org/T3020>.
        run: |
          $env:PATH = "C:\Program Files\Git\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\ProgramData\Chocolatey\bin"
          [Environment]::SetEnvironmentVariable("Path", $env:PATH, "Machine")
          choco install gnupg -y --no-progress
          echo "C:\Program Files (x86)\gnupg\bin" >> $env:GITHUB_PATH
          git config --system gpg.program "C:\Program Files (x86)\gnupg\bin\gpg.exe"
        if: runner.os == 'Windows'
      - run: pip install -U 'setuptools>=61'
      - uses: actions/download-artifact@v4
        with:
          name: Packages
          path: dist
      - shell: bash
        run: pip install "$(echo -n dist/*whl)[toml,test]"
      - run: |
          $(hg debuginstall --template "{pythonexe}") -m pip install hg-git --user
        if: matrix.os == 'ubuntu-latest'
      # this hopefully helps with os caches, hg init sometimes gets 20s timeouts
      - run: hg version
      - run: pytest
        timeout-minutes: 15

  dist_upload:

    runs-on: ubuntu-latest
    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
    permissions:
      id-token: write
    needs: [test]
    steps:
    - uses: actions/download-artifact@v4
      with:
        name: Packages
        path: dist
    - name: Publish package to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1

  test-pypi-upload:
    runs-on: ubuntu-latest
    needs: [test]
    permissions:
      id-token: write
    steps:
    - uses: actions/download-artifact@v4
      with:
        name: Packages
        path: dist
    - name: Publish package to PyPI
      continue-on-error: true
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        repository-url: https://test.pypi.org/legacy/


Discord Release Notification

name: Discord Release Notification

on:
  release:
    types: [published]

jobs:
  notify-discord:
    runs-on: ubuntu-latest
    steps:
    - name: Send Notification to Discord
      env:
        DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
      # We truncate the description at the models section (starting with ### Models)
      # to keep the message short.
      # We have also have to format the release description for it to be valid JSON.
      # This is done by piping the description to jq.
      run: |
          DESCRIPTION=$(echo '${{ github.event.release.body }}' | awk '/### Models/{exit}1' | jq -aRs .)
          curl -H "Content-Type: application/json" \
               -X POST \
               -d @- \
               "${DISCORD_WEBHOOK}" << EOF
          {
            "username": "Lightly",
            "avatar_url": "https://avatars.githubusercontent.com/u/50146475",
            "content": "Lightly ${{ github.event.release.tag_name }} has been released!",
            "embeds": [
              {
                "title": "${{ github.event.release.name }}",
                "url": "${{ github.event.release.html_url }}",
                "color": 5814783,
                "description": $DESCRIPTION
              }
            ]
          }
          EOF


Build and release lightly

name: Build and release lightly

on:
  workflow_dispatch:
# TODO(Philipp, 03/23): Enable me after proper testing.
#  release:
#    types: [published]

jobs:
  build:
    name: Build and release
    runs-on: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Checkout latest release tag
      id: checkout_latest_release_tag
      run: |
        LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)
        git checkout $LATEST_TAG
        echo "tag_name=$LATEST_TAG" >> $GITHUB_OUTPUT;
    - name: Set up Python 3.10
      uses: actions/setup-python@v4
      with:
        python-version: "3.10"
    - name: Build and release
      id: build_and_release
      run: |
        pip3 install wheel
        pip3 install twine
        make dist
        twine upload -u ${{ secrets.PYPI_USER_NAME }} -p ${{ secrets.PYPI_PASSWORD }} dist/*
    - name: Convert success/failure strings to emojis
      id: emoji_status
      run: |
        function set-emoji-output {
          if [ "$2" == "success" ];
          then echo "$1=:github-check-mark:" >> $GITHUB_OUTPUT;
          else echo "$1=:github-changes-requested:" >> $GITHUB_OUTPUT;
          fi
        }
        set-emoji-output status ${{ steps.build_and_release.outcome }}
    - name: Slack notification
      if: always()
      uses: rtCamp/action-slack-notify@v2
      env:
        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_RELEASES }}
        SLACK_ICON_EMOJI: ":python:"
        SLACK_USERNAME: Release of Lightly PIP Package ${{steps.checkout_latest_release_tag.outputs.tag_name}}
        SLACK_COLOR: ${{ steps.build_and_release.outcome }}
        SLACK_FOOTER: ""
        SLACK_MESSAGE: |
          Release status: ${{steps.emoji_status.outputs.status}}
          https://pypi.org/project/lightly/${{steps.checkout_latest_release_tag.outputs.tag_name}}


Unit Tests

name: Unit Tests

on:
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:

jobs:
  detect-code-changes:
    name: Detect Code Changes
    runs-on: ubuntu-latest
    outputs:
      run-tests: ${{ steps.filter.outputs.run-tests }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          list-files: shell
          filters: |
            run-tests:
              - '!docs/**'
              - '!examples/**'
              - '!benchmarks/**'

  test:
    name: Test
    needs: detect-code-changes
    if: needs.detect-code-changes.outputs.run-tests == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python: ["3.7", "3.10"]

    steps:
      - name: Checkout Code
        uses: actions/checkout@v3
      - name: Hack to get setup-python to work on nektos/act
        run: |
          if [ ! -f "/etc/lsb-release" ] ; then
            echo "DISTRIB_RELEASE=18.04" > /etc/lsb-release
          fi
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
      - uses: actions/cache@v2
        with:
          path: ${{ env.pythonLocation }}
          key: general-env-${{ runner.os }}-python-${{ matrix.python }}-${{ hashFiles('requirements/**') }}
      - name: Install Dependencies and lightly
        run: pip install -e '.[all]'
      - name: Clean Previous Coverage Data
        run: rm -f .coverage
      - name: Run Pytest (Python 3.7)
        if: matrix.python == '3.7'
        run: |
          export LIGHTLY_SERVER_LOCATION="localhost:-1"
          python -m pytest -s -v --runslow --ignore=./lightly/openapi_generated/
      - name: Run Pytest with Coverage (Python 3.10)
        if: matrix.python == '3.10'
        run: |
          export LIGHTLY_SERVER_LOCATION="localhost:-1"
          pip install pytest-cov==5.0.0
          python -m pytest -s -v --runslow --cov=./lightly --cov-report=xml --ignore=./lightly/openapi_generated/
      - name: Upload coverage to Codecov
        if: matrix.python == '3.10'
        uses: codecov/codecov-action@v4
        with:
          fail_ci_if_error: false
          files: ./coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}


Code Format Check

name: Code Format Check

on:
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:

jobs:
  test:
    name: Check
    runs-on: ubuntu-latest
    steps:
    - name: Checkout Code
      uses: actions/checkout@v3
    - name: Hack to get setup-python to work on nektos/act
      run: |
        if [ ! -f "/etc/lsb-release" ] ; then
          echo "DISTRIB_RELEASE=18.04" > /etc/lsb-release
        fi
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.7"
    - uses: actions/cache@v2
      with:
        path: ${{ env.pythonLocation }}
        key: cache_v2_${{ env.pythonLocation }}-${{ hashFiles('requirements/**') }}
    - name: Install Dependencies and lightly
      run: pip install -e '.[all]'
    - name: Run Format Check
      run: |
        make format-check
    - name: Run Type Check
      run: |
        make type-check


Minimal Dependency Tests

name: Minimal Dependency Tests

on:
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:

jobs:
  detect-code-changes:
    name: Detect Code Changes
    runs-on: ubuntu-latest
    outputs:
      run-tests: ${{ steps.filter.outputs.run-tests }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          list-files: shell
          filters: |
            run-tests:
              - '!docs/**'
              - '!examples/**'
              - '!benchmarks/**'

  test:
    name: Test
    needs: detect-code-changes
    if: needs.detect-code-changes.outputs.run-tests == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python: ["3.7"]

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
      - name: Cache Python Dependencies
        uses: actions/cache@v2
        with:
          path: ${{ env.pythonLocation }}
          key: minimal-env-${{ runner.os }}-python-${{ matrix.python }}-${{ hashFiles('requirements/minimal_requirements.txt') }}
      - name: Install Minimal Dependencies
        run: pip install -r requirements/minimal_requirements.txt
      - name: Install Package Without Dependencies
        run: pip install --no-deps .
      - name: Run Tests
        run: |
          export LIGHTLY_SERVER_LOCATION="localhost:-1"
          python -m pytest -s -v --runslow


check setup.py

name: check setup.py

on:
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:

jobs:
  detect-code-changes:
    name: Detect Code Changes
    runs-on: ubuntu-latest
    outputs:
      run-tests: ${{ steps.filter.outputs.run-tests }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          list-files: shell
          filters: |
            run-tests:
              - '!docs/**'

  test:
    name: Test setup.py
    needs: detect-code-changes
    if: needs.detect-code-changes.outputs.run-tests == 'true'
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Code
        uses: actions/checkout@v3
      - name: Hack to get setup-python to work on nektos/act
        run: |
          if [ ! -f "/etc/lsb-release" ] ; then
            echo "DISTRIB_RELEASE=18.04" > /etc/lsb-release
          fi
      - name: Set up Python 3.7
        uses: actions/setup-python@v4
        with:
          python-version: 3.7
      - uses: actions/cache@v2
        with:
          path: ${{ env.pythonLocation }}
          key: cache_v2_${{ env.pythonLocation }}-${{ hashFiles('requirements/**') }}
      - name: Install Dependencies and lightly
        run: pip install .
      - name: basic tests of CLI
        run: |
          LIGHTLY_SERVER_LOCATION="localhost:-1"
          lightly-crop --help
          lightly-train --help
          lightly-embed --help
          lightly-magic --help
          lightly-download --help
          lightly-version
      - name: test of CLI on a real dataset
        run: |
          LIGHTLY_SERVER_LOCATION="localhost:-1"
          git clone https://github.com/alexeygrigorev/clothing-dataset-small clothing_dataset_small
          INPUT_DIR_1="clothing_dataset_small/test/dress"
          lightly-train input_dir=$INPUT_DIR_1 trainer.max_epochs=1 loader.num_workers=6
          lightly-embed input_dir=$INPUT_DIR_1


run unmocked tests

name: run unmocked tests
on: [workflow_dispatch]

# Why is this only triggered manually with workflow_dispatch?
# - We have many unittests anyway
# - We ran into quota problems with too many tests (3000mins for the pip package)
# - The tests are unreliable / not deterministic, as they depend on the API from staging
# - The tests take 10mins, which can slow down development

jobs:
  test:
    name: Run unmocked tests
    runs-on: ubuntu-latest
    steps:
    - name: Checkout Code
      uses: actions/checkout@v3
    - name: Set up Python 3.7
      uses: actions/setup-python@v4
      with:
        python-version: 3.7
    - uses: actions/cache@v2
      with:
        path: ${{ env.pythonLocation }}
        key: cache_v2_${{ env.pythonLocation }}-${{ hashFiles('requirements/**') }}
    - name: Install Dependencies and lightly
      run: pip install .
    - name: run unmocked tests
      run: |
        export LIGHTLY_SERVER_LOCATION=${{ secrets.LIGHTLY_SERVER_LOCATION }}
        bash tests/UNMOCKED_end2end_tests/run_all_unmocked_tests.sh ${{ secrets.DUMMY_USER_TOKEN_STAGING }}


Weekly Dependency Test

# Install lightly from scratch and run tests.
# Python environment for regular unit tests is cached. In this workflow,
# we always pick up the latest dependencies and therefore check if a new
# external release breaks lightly.
name: Weekly Dependency Test

on:
  workflow_dispatch:
  schedule:
    - cron: '0 22 * * THU' # each Thursday night, at 22:00 UTC

jobs:
  test_fresh_install:
    name: Test fresh install
    runs-on: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v3
    - name: Set up Python 3.10
      uses: actions/setup-python@v4
      with:
        python-version: "3.10"
    - name: Install Lightly from scratch
      run: |
        pip3 install ".[dev]"
    - name: Run tests
      id: run_tests
      run: |
        pytest -n auto
    - name: Slack notification
      if: always()
      uses: rtCamp/action-slack-notify@v2
      env:
        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_NIGHTLY }}
        SLACK_ICON_EMOJI: ":github:"
        SLACK_USERNAME: Lightly PIP Package ${{ github.workflow }}
        SLACK_COLOR: ${{ steps.run_tests.outcome }}
        SLACK_FOOTER: ""


Docker Build and Publish

name: Docker Build and Publish

permissions:
  packages: write

on:
  workflow_dispatch:
    inputs:
      tags:
        description: 'Manual supplied image tag like 1.16.0'
        required: true
        type: string
  push:
    tags:
      - "*.*.*"
jobs:
  build-and-publish:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            ghcr.io/${{ github.repository }}/mapproxy
          tags: |
            type=semver,pattern={{version}}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to ghcr.io
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push base image
        uses: docker/build-push-action@v5
        if: ${{ inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ inputs.tags }}
          target: base
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ inputs.tags }}
          platforms: linux/amd64,linux/arm64

      - name: Build and push base image
        uses: docker/build-push-action@v5
        if: ${{ !inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
          target: base
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
          platforms: linux/amd64,linux/arm64

      - name: Build and push base alpine image
        uses: docker/build-push-action@v5
        if: ${{ inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile-alpine
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ inputs.tags }}
          target: base
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ inputs.tags }}-alpine
          platforms: linux/amd64,linux/arm64

      - name: Build and push base alpine image
        uses: docker/build-push-action@v5
        if: ${{ !inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile-alpine
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
          target: base
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}-alpine
          platforms: linux/amd64,linux/arm64

      - name: Build and push development image
        uses: docker/build-push-action@v5
        if: ${{ inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ inputs.tags }}
          target: development
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ inputs.tags }}-dev
          platforms: linux/amd64,linux/arm64

      - name: Build and push development image
        uses: docker/build-push-action@v5
        if: ${{ !inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
          target: development
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}-dev
          platforms: linux/amd64,linux/arm64

      - name: Build and push nginx image
        uses: docker/build-push-action@v5
        if: ${{ inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ inputs.tags }}
          target: nginx
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ inputs.tags }}-nginx
          platforms: linux/amd64,linux/arm64

      - name: Build and push nginx image
        uses: docker/build-push-action@v5
        if: ${{ !inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
          target: nginx
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}-nginx
          platforms: linux/amd64,linux/arm64

      - name: Build and push alpine based nginx image
        uses: docker/build-push-action@v5
        if: ${{ inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile-alpine
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ inputs.tags }}
          target: nginx
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ inputs.tags }}-alpine-nginx
          platforms: linux/amd64,linux/arm64

      - name: Build and push alpine based nginx image
        uses: docker/build-push-action@v5
        if: ${{ !inputs.tags }}
        with:
          context: docker/
          file: ./docker/Dockerfile-alpine
          push: true
          build-args: |
            MAPPROXY_VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
          target: nginx
          tags: |
            ghcr.io/${{ github.repository }}/mapproxy:${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}-alpine-nginx
          platforms: linux/amd64,linux/arm64


name: Build and deploy documentation on github pages

on:
  push:
    tags:
      - "*.*.*"

permissions:
  contents: write

jobs:
  build:
    runs-on: ubuntu-20.04

    steps:
    - name: Checkout sources
      uses: actions/checkout@v4
      with:
        ref: upgrade-docs

    - name: Install dependencies ⏬
      run: pip install sphinx sphinx-book-theme sphinx-copybutton

    - name: Run documentation build
      run: sphinx-build doc/ docs -D html_context.current_version=${{ github.ref_name }}

    - name: Deploy docs to folder `latest` to GitHub Pages
      uses: JamesIves/github-pages-deploy-action@v4.5.0
      with:
        folder: docs
        target-folder: docs/latest

    - name: Deploy docs to a folder named after the new tag to GitHub Pages
      uses: JamesIves/github-pages-deploy-action@v4.5.0
      with:
        folder: docs
        target-folder: docs/${{ github.ref_name }}

    - name: Update versions config
      # read existing versions from gh-pages branch, add new version, sort, dedupe and write results to config/versions.json
      run: |
        mkdir config
        echo -e "${{ github.ref_name }}\n$(git show gh-pages:config/versions.json | jq -r '.[]')" \
        | sort -V | uniq | jq -R -s -c 'split("\n")[:-1]' > config/versions.json

    - name: Deploy config folder to GitHub Pages
      uses: JamesIves/github-pages-deploy-action@v4.5.0
      with:
        folder: config
        target-folder: docs/config


Running mapproxy lint

name: Running mapproxy lint

on:
  workflow_dispatch:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

jobs:
  test:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout sources
        uses: actions/checkout@v4

      - name: Use python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: 3.12

      - name: flake8 Lint
        uses: py-actions/flake8@v2


Running mapproxy tests

name: Running mapproxy tests

on:
  workflow_dispatch:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

jobs:
  test:
    runs-on: ubuntu-20.04
    services:
      redis-server:
        image: redis
        ports:
          - 6379:6379
      couchdb:
        image: couchdb:2
        ports:
          - 5984:5984
      riak:
        image: basho/riak-kv
        ports:
          - 8087:8087
          - 8098:8098
      azure-blob:
        image: mcr.microsoft.com/azure-storage/azurite
        ports:
          - 10000:10000

    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]

    env:
      MAPPROXY_TEST_COUCHDB: 'http://localhost:5984'
      MAPPROXY_TEST_REDIS: '127.0.0.1:6379'
      MAPPROXY_TEST_RIAK_HTTP: 'http://localhost:8098'
      MAPPROXY_TEST_RIAK_PBC: 'pbc://localhost:8087'
      MAPPROXY_TEST_AZURE_BLOB: 'http://localhost:10000'
      # do not load /etc/boto.cfg with Python 3 incompatible plugin
      # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
      BOTO_CONFIG: '/doesnotexist'

    steps:
    - name: Install packages
      run: |
        sudo apt update
        sudo apt install proj-bin libgeos-dev libgdal-dev libxslt1-dev libxml2-dev build-essential python-dev libjpeg-dev zlib1g-dev libfreetype6-dev protobuf-compiler libprotoc-dev -y

    - name: Checkout sources
      uses: actions/checkout@v4

    - name: Use python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache python deps 💾
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.OS }}-python-${{ hashFiles('**/requirements-tests.txt') }}
        restore-keys: |
          ${{ runner.OS }}-python-
          ${{ runner.OS }}-

    - name: Install dependencies ⏬
      run: |
        pip install .
        pip install -r requirements-tests.txt
        pip freeze

    - name: Run tests 🏗
      run: |
        export LD_PRELOAD=/lib/x86_64-linux-gnu/libstdc++.so.6:$LD_PRELOAD
        pytest mapproxy


Issue Labeler

name: "Issue Labeler"
on:
  issues:
    types: [opened]
  pull_request_target:
    types: [opened]

# '*: write' permissions for https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#add-labels-to-an-issue
permissions:
  contents: read
  issues: write
  pull-requests: write

jobs:
  triage:
    runs-on: ubuntu-latest
    steps:
    - name: Add agent-python label
      uses: actions-ecosystem/action-add-labels@v1
      with:
        labels: agent-python
    - id: is_elastic_member
      uses: elastic/apm-pipeline-library/.github/actions/is-member-elastic-org@current
      with:
        username: ${{ github.actor }}
        token: ${{ secrets.APM_TECH_USER_TOKEN }}
    - name: Add community and triage labels
      if: contains(steps.is_elastic_member.outputs.result, 'false') && github.actor != 'dependabot[bot]' && github.actor != 'apmmachine'
      uses: actions-ecosystem/action-add-labels@v1
      with:
        labels: |
          community
          triage


pre-commit

name: pre-commit

on:
  pull_request:
  push:
    branches: [main]

permissions:
  contents: read

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
    - uses: pre-commit/action@v3.0.1


test

name: test # The name must be the same as in test-docs.yml

on:
  workflow_call:
    inputs:
      full-matrix:
        description: "Run the full matrix"
        required: true
        type: boolean
      ref:
        description: "The git ref of elastic/apm-agent-python to run test workflow from."
        required: false
        type: string
  pull_request:
    paths-ignore:
      - "**/*.md"
      - "**/*.asciidoc"
  push:
    branches:
      - main
    paths-ignore:
      - "**/*.md"
      - "**/*.asciidoc"
  schedule:
    - cron: "0 2 * * *"
  workflow_dispatch:
    inputs:
      full-matrix:
        description: "Run the full matrix"
        required: true
        type: boolean

permissions:
  contents: read

jobs:
  build-distribution:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/build-distribution

  create-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.generate.outputs.matrix }}
      data: ${{ steps.split.outputs.data }}
      chunks: ${{ steps.split.outputs.chunks }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.ref }}
      - id: generate
        uses: elastic/apm-pipeline-library/.github/actions/version-framework@current
        with:
          # Use .ci/.matrix_python_full.yml if it's a scheduled workflow, otherwise use .ci/.matrix_python.yml
          versionsFile: .ci/.matrix_python${{ (github.event_name == 'schedule' || github.event_name == 'push' || inputs.full-matrix) && '_full' || '' }}.yml
          # Use .ci/.matrix_framework_full.yml if it's a scheduled workflow, otherwise use .ci/.matrix_framework.yml
          frameworksFile: .ci/.matrix_framework${{ (github.event_name == 'schedule' || github.event_name == 'push' || inputs.full-matrix) && '_full' || '' }}.yml
          excludedFile: .ci/.matrix_exclude.yml
      - name: Split matrix
        shell: python
        id: split
        run: |
          import os
          import json

          def split(lst, n):
            return [lst[i::n] for i in range(n)]

          matrix = json.loads(os.environ['GENERATED_MATRIX'])

          # Using the number 4 because the full matrix has roughly 400+ items
          # Hence, it is split into chunks of size ~100
          # We are doing this because the matrix in GH actions has a max limit of 256
          chunks = split(matrix['include'], 4)
          chunks_json = json.dumps(chunks)

          with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
            print(f'chunks={chunks_json}', file=f)

        env:
          GENERATED_MATRIX: ${{ steps.generate.outputs.matrix }}

  chunks-0:
    needs: create-matrix
    uses: ./.github/workflows/run-matrix.yml
    with:
      include: ${{ toJSON(fromJSON(needs.create-matrix.outputs.chunks)[0]) }}

  chunks-1:
    needs: create-matrix
    uses: ./.github/workflows/run-matrix.yml
    with:
      include: ${{ toJSON(fromJSON(needs.create-matrix.outputs.chunks)[1]) }}

  chunks-2:
    needs: create-matrix
    uses: ./.github/workflows/run-matrix.yml
    with:
      include: ${{ toJSON(fromJSON(needs.create-matrix.outputs.chunks)[2]) }}

  chunks-3:
    needs: create-matrix
    uses: ./.github/workflows/run-matrix.yml
    with:
      include: ${{ toJSON(fromJSON(needs.create-matrix.outputs.chunks)[3]) }}

  windows:
    name: "windows (version: ${{ matrix.version }}, framework: ${{ matrix.framework }}, asyncio: ${{ matrix.asyncio }})"
    runs-on: windows-2019
    strategy:
      fail-fast: false
      matrix:
        include:
          #  - version: "3.6"
          #    framework: "none"
          #    asyncio: "true"
          #  - version: "3.7"
          #    framework: none
          #    asyncio: true
          - version: "3.8"
            framework: none
            asyncio: true
          - version: "3.9" # waiting for greenlet to have binary wheels for 3.9
            framework: none
            asyncio: true
    env:
      VERSION: ${{ matrix.version }}
      FRAMEWORK: ${{ matrix.framework }}
      ASYNCIO: ${{ matrix.asyncio }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.ref }}
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.version }}
          cache: pip
          cache-dependency-path: "tests/requirements/reqs-${{ matrix.framework }}.txt"
      - name: Install tools
        run: .\scripts\install-tools.bat
      - name: Run tests
        run: .\scripts\run-tests.bat
      - if: success() || failure()
        name: Upload JUnit Test Results
        uses: actions/upload-artifact@v3
        with:
          name: test-results
          path: "**/*-python-agent-junit.xml"
      - if: success() || failure()
        name: Upload Coverage Reports
        uses: actions/upload-artifact@v3
        with:
          name: coverage-reports
          path: "**/.coverage*"
  # This job is here to have a single status check that can be set as required.
  # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idneeds
  # If a run contains a series of jobs that need each other, a failure applies to all jobs in the dependency chain from the point of failure onwards.
  all:
    if: always()
    runs-on: ubuntu-latest
    needs:
      - chunks-0
      - chunks-1
      - chunks-2
      - chunks-3
      - windows
    steps:
      - id: check
        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
        with:
          needs: ${{ toJSON(needs) }}
      - run: ${{ steps.check.outputs.isSuccess }}
      - if: failure() && (github.event_name == 'schedule' || github.event_name == 'push')
        uses: elastic/oblt-actions/slack/notify-result@v1
        with:
          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
          status: ${{ steps.check.outputs.status }}
          channel-id: "#apm-agent-python"

  coverage:
    name: Combine & check coverage.
    needs: all
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.ref }}

      - uses: actions/setup-python@v5
        with:
          # Use latest Python, so it understands all syntax.
          python-version: 3.11

      - run: python -Im pip install --upgrade coverage[toml]

      - uses: actions/download-artifact@v3
        with:
          name: coverage-reports

      - name: Combine coverage & fail if it's <84%.
        run: |
          python -Im coverage combine
          python -Im coverage html --skip-covered --skip-empty

          # Report and write to summary.
          python -Im coverage report | sed 's/^/    /' >> $GITHUB_STEP_SUMMARY

          # Report again and fail if under 84%.
          python -Im coverage report --fail-under=84

      - name: Upload HTML report
        uses: actions/upload-artifact@v4
        with:
          name: html-coverage-report
          path: htmlcov
      - uses: geekyeggo/delete-artifact@24928e75e6e6590170563b8ddae9fac674508aa1
        with:
          name: coverage-reports


release

name: release

on:
  push:
    tags:
      - "v*.*.*"
    branches:
      - main

permissions:
  contents: read

jobs:
  test:
    uses: ./.github/workflows/test-release.yml
    with:
      full-matrix: true
      enabled: ${{ startsWith(github.ref, 'refs/tags') }}

  packages:
    permissions:
      attestations: write
      id-token: write
      contents: write
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/packages
      - name: generate build provenance
        uses: actions/attest-build-provenance@534b352d658f90498fd148d231fdbf88f3886a3a  # v1.3.1
        with:
          subject-path: "${{ github.workspace }}/dist/*"

  publish-pypi:
    needs:
      - test
      - packages
    runs-on: ubuntu-latest
    environment: release
    permissions:
      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: packages
          path: dist
      - name: Upload pypi.org
        if: startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0
        with:
          repository-url: https://upload.pypi.org/legacy/
      - name: Upload test.pypi.org
        if: ${{ ! startsWith(github.ref, 'refs/tags') }}
        uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0
        with:
          repository-url: https://test.pypi.org/legacy/

  build-distribution:
    permissions:
      attestations: write
      id-token: write
      contents: write
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/build-distribution
      - name: generate build provenance
        uses: actions/attest-build-provenance@534b352d658f90498fd148d231fdbf88f3886a3a  # v1.3.1
        with:
          subject-path: "${{ github.workspace }}/build/dist/elastic-apm-python-lambda-layer.zip"

  publish-lambda-layers:
    needs:
      - build-distribution
    runs-on: ubuntu-latest
    env:
      # TODO: use keyless
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with:
          name: build-distribution
          path: ./build
      - name: Publish lambda layers to AWS
        if: startsWith(github.ref, 'refs/tags')
        run: |
          # Convert v1.2.3 to ver-1-2-3
          VERSION=${GITHUB_REF_NAME/v/ver-}
          VERSION=${VERSION//./-}

          ELASTIC_LAYER_NAME="elastic-apm-python-${VERSION}" .ci/publish-aws.sh
      - uses: actions/upload-artifact@v4
        if: startsWith(github.ref, 'refs/tags')
        with:
          name: arn-file
          path: ".arn-file.md"
          if-no-files-found: error

  publish-docker:
    needs:
      - build-distribution
    runs-on: ubuntu-latest
    permissions:
      attestations: write
      id-token: write
      contents: write
    strategy:
      fail-fast: false
      matrix:
        dockerfile: [ 'Dockerfile', 'Dockerfile.wolfi' ]
    env:
      DOCKER_IMAGE_NAME: docker.elastic.co/observability/apm-agent-python
    steps:
      - uses: actions/checkout@v4

      - name: Log in to the Elastic Container registry
        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
        with:
          registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }}
          username: ${{ secrets.ELASTIC_DOCKER_USERNAME }}
          password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }}

      - uses: actions/download-artifact@v3
        with:
          name: build-distribution
          path: ./build

      - name: Extract metadata (tags, labels)
        id: docker-meta
        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81  # v5.5.1
        with:
          images: ${{ env.DOCKER_IMAGE_NAME }}
          tags: |
            type=raw,value=latest,prefix=test-,enable={{is_default_branch}}
            type=semver,pattern={{version}}
          flavor: |
            suffix=${{ contains(matrix.dockerfile, 'wolfi') && '-wolfi' || '' }}

      - name: Build and push image
        id: push
        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25  # v5.4.0
        with:
          context: .
          push: true
          file: ${{ matrix.dockerfile }}
          tags: ${{ steps.docker-meta.outputs.tags }}
          labels: ${{ steps.docker-meta.outputs.labels }}
          build-args: |
            AGENT_DIR=./build/dist/package/python

      - name: generate build provenance (containers)
        uses: actions/attest-build-provenance@534b352d658f90498fd148d231fdbf88f3886a3a  # v1.3.1
        with:
          subject-name: "${{ env.DOCKER_IMAGE_NAME }}"
          subject-digest: ${{ steps.push.outputs.digest }}
          push-to-registry: true

  github-draft:
    permissions:
      contents: write
    needs:
      - publish-lambda-layers
    if: startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: arn-file
      - name: Create GitHub Draft Release
        run: >-
          gh release create "${GITHUB_REF_NAME}"
          --title="${GITHUB_REF_NAME}"
          --generate-notes
          --notes-file=".arn-file.md"
          --draft
        env:
          GH_TOKEN: ${{ github.token }}

  notify:
    runs-on: ubuntu-latest
    if: always()
    needs:
      - publish-lambda-layers
      - publish-pypi
      - publish-docker
      - github-draft
    steps:
      - id: check
        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
        with:
          needs: ${{ toJSON(needs) }}
      - if: startsWith(github.ref, 'refs/tags')
        uses: elastic/oblt-actions/slack/notify-result@v1.7.0
        with:
          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
          channel-id: "#apm-agent-python"
          status: ${{ steps.check.outputs.status }}


packages

name: packages

on:
  workflow_call: ~
  pull_request:
    paths-ignore:
      - '**/*.md'
      - '**/*.asciidoc'
  push:
    branches:
      - main
    paths-ignore:
      - '**/*.md'
      - '**/*.asciidoc'

permissions:
  contents: read

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/packages


run-matrix

name: run-matrix

on:
  workflow_call:
    inputs:
      include:
        required: true
        description: Matrix include JSON string
        type: string

permissions:
  contents: read

jobs:
  docker:
    name: "docker (version: ${{ matrix.version }}, framework: ${{ matrix.framework }})"
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 10
      matrix:
        include: ${{ fromJSON(inputs.include) }}
    steps:
      - uses: actions/checkout@v4
      - name: Run tests
        run: ./tests/scripts/docker/run_tests.sh ${{ matrix.version }} ${{ matrix.framework }}
        env:
          LOCALSTACK_VOLUME_DIR: localstack_data
      - if: success() || failure()
        name: Upload JUnit Test Results
        uses: actions/upload-artifact@v3
        with:
          name: test-results
          path: "**/*-python-agent-junit.xml"
      - if: success() || failure()
        name: Upload Coverage Reports
        uses: actions/upload-artifact@v3
        with:
          name: coverage-reports
          path: "**/.coverage*"


test

---
# This workflow sets the test / all status check to success in case it's a docs only PR and test.yml is not triggered
# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
name: test # The name must be the same as in test.yml

on:
  pull_request:
    paths-ignore: # This expression needs to match the paths ignored on test.yml.
      - '**'
      - '!**/*.md'
      - '!**/*.asciidoc'

permissions:
  contents: read

jobs:
  all:
    runs-on: ubuntu-latest
    steps:
      - run: 'echo "No build required"'
      # As long as this workflow uses the same name than .github/workflows/test.yml and
      # the test-reporter requires artifacts with the test results to exist.
      # let's produce a dummy test so the test-report don't fail in this particular
      # case
      - name: Generate junit placeholder - to integrate with test-reporter
        run: |-
          cat > docs-python-agent-junit.xml << ENDOFFILE
          <?xml version="1.0" encoding="UTF-8"?>
          <testsuites name="apm-agent-python" tests="1" failures="0" errors="0" time="1">
            <testsuite name="apm-agent-python-docs" errors="0" failures="0" skipped="1" time="1" tests="1">
              <testcase classname="docs" name="Skipped test" time="0">
                <skipped/>
              </testcase>
            </testsuite>
          </testsuites>
          ENDOFFILE
      - if: success() || failure()
        name: Upload JUnit Test Results
        uses: actions/upload-artifact@v3
        with:
          name: test-results
          path: "docs-python-agent-junit.xml"


updatecli

name: updatecli

on:
  workflow_dispatch: ~
  schedule:
    - cron: '0 6 * * *'

permissions:
  contents: read

jobs:
  bump:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: elastic/oblt-actions/updatecli/run@v1
        with:
          command: "apply --config .ci/updatecli/updatecli.d --values .ci/updatecli/values.yml"
        env:
          GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}

      - if: failure()
        uses: elastic/oblt-actions/slack/send@v1
        with:
          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
          channel-id: "#apm-agent-python"
          message: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, @robots-ci please look what's going on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"


microbenchmark

name: microbenchmark

on:
  workflow_dispatch:
  push:
    branches:
      - main
    paths-ignore:
      - '**.md'
      - '**.asciidoc'

# limit the access of the generated GITHUB_TOKEN
permissions:
  contents: read

jobs:
  microbenchmark:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Run microbenchmark
        uses: elastic/oblt-actions/buildkite/run@v1.5.0
        with:
          pipeline: "apm-agent-microbenchmark"
          token: ${{ secrets.BUILDKITE_TOKEN }}
          wait-for: false
          env-vars: |
            script=.ci/bench.sh
            repo=apm-agent-python
            sha=${{ github.sha }}
            BRANCH_NAME=${{ github.ref_name }}


matrix-command

name: matrix-command

on:
  pull_request_review:
    types:
      - submitted

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

permissions:
  contents: read

jobs:
  command-validation:
    if: startsWith(github.event.review.body, '/test matrix')
    runs-on: ubuntu-latest
    timeout-minutes: 5
    permissions:
      pull-requests: write
    steps:
      - name: Is comment allowed?
        uses: actions/github-script@v7
        with:
          script: |
            const actorPermission = (await github.rest.repos.getCollaboratorPermissionLevel({
              ...context.repo,
              username: context.actor
            })).data.permission
            const isPermitted = ['write', 'admin'].includes(actorPermission)
            if (!isPermitted) {
              const errorMessage = 'Only users with write permission to the repository can run GitHub commands'
              await github.rest.issues.createComment({
                ...context.repo,
                issue_number: context.issue.number,
                body: errorMessage,
              })
              core.setFailed(errorMessage)
              return
            }

  test:
    needs:
      - command-validation
    uses: ./.github/workflows/test.yml
    with:
      full-matrix: true
      ref: ${{ github.event.pull_request.head.sha }}


test-release

name: test-release

on:
  workflow_call:
    inputs:
      full-matrix:
        description: "Run the full matrix"
        required: true
        type: boolean
      ref:
        description: "The git ref of elastic/apm-agent-python to run test workflow from."
        required: false
        type: string
      enabled:
        description: "Whether to run the workfow"
        required: true
        type: boolean
  workflow_dispatch:
    inputs:
      full-matrix:
        description: "Run the full matrix"
        required: true
        type: boolean
      enabled:
        description: "Whether to run the workfow"
        required: true
        type: boolean

jobs:
  test:
    if: ${{ inputs.enabled }}
    uses: ./.github/workflows/test.yml
    with:
      full-matrix: ${{ inputs.full-matrix }}

  run-if-disabled:
    if: ${{ ! inputs.enabled }}
    runs-on: ubuntu-latest
    steps:
      - run: echo "do something to help with the reusable workflows with needs"


test-reporter

---
## Workflow to process the JUnit test results and add a report to the checks.
name: test-reporter
on:
  workflow_run:
    workflows:
      - test
    types:
      - completed

permissions:
  contents: read
  actions: read
  checks: write

jobs:
  report:
    runs-on: ubuntu-latest
    steps:
      - uses: elastic/apm-pipeline-library/.github/actions/test-report@current
        with:
          artifact: test-results
          name: JUnit Tests
          path: "**/*-python-agent-junit.xml"
          reporter: java-junit


CI

name: CI

on:
  push:
    branches: [master]
  pull_request:
    branches: [master]

env:
  DEFAULT_PYTHON: "3.10"

permissions:
  contents: read

jobs:
  ruff:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: chartboost/ruff-action@v1

  black:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - uses: psf/black@24.4.2

  pylint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install -r requirements-dev.txt
      - name: Run Pylint
        run: |
          pylint office365

  pytest:
    runs-on: ubuntu-latest
    needs:
      - ruff
      - black
      - pylint
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install -r requirements-dev.txt
      - name: Test with pytest
        env:
          office365_python_sdk_securevars: ${{ secrets.OFFICE365_PYTHON_SDK_SECUREVARS }}
        run: |
          echo "${{env.office365_python_sdk_securevars}}"
          pytest


Gnocchi CI Image

name: Gnocchi CI Image

on:
  workflow_dispatch:
  push:
    paths:
      - 'images/Dockerfile.ci'
      - 'images/entrypoint.sh.ci'

jobs:
  publish:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - uses: docker/build-push-action@v2
        with:
          context: ./images
          file: ./images/Dockerfile.ci
          push: true
          tags: ghcr.io/gnocchixyz/ci:latest