Build wheels

# A single CI script with github workflow
name: Build wheels

on:
  push:
  release:
    types:
      - published

jobs:
  build:
    strategy:
      matrix:
        os: [ubuntu-latest, macos-13, windows-latest]
        python-version: [3.9, '3.10', '3.11']
    uses: qiboteam/workflows/.github/workflows/deploy-pip-poetry.yml@main
    with:
      os: ${{ matrix.os }}
      python-version: ${{ matrix.python-version }}
      publish: ${{ github.event_name == 'release' && github.event.action == 'published' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.9' }}
      poetry-extras: "--with docs,tests,analysis --all-extras"
    secrets: inherit


docs

name: docs

on:
  workflow_dispatch:
  push:
    branches: [main]
    tags:
      - "*"

jobs:
  evaluate-label:
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.label_step.outputs.version}}
    steps:
      - name: checks for the label
        id: label_step
        run: |
          if [[ "${{ github.ref  }}" == "refs/heads/main" ]]; then
                echo "version=latest" >> $GITHUB_OUTPUT
          fi
          if [[ "${{ github.ref_type }}" == "branch" ]] && [[ "${{ github.ref  }}" != "refs/heads/main" ]]; then
                exit 1
          fi
          if [[ "${{ github.ref_type }}" == "tag" ]]; then
                echo "version=stable" >> $GITHUB_OUTPUT
          fi

  deploy-docs:
    needs: [evaluate-label]
    uses: qiboteam/workflows/.github/workflows/deploy-ghpages-latest-stable.yml@main
    with:
      python-version: '3.10'
      package-manager: "poetry"
      dependency-path: "**/poetry.lock"
      trigger-label: "${{needs.evaluate-label.outputs.label}}"
      project: qibolab
      poetry-extras: --with docs --all-extras


tests

# A single CI script with github workflow
name: tests

on:
  workflow_dispatch:
  push:
  pull_request:
    types: opened

jobs:
  build:
    strategy:
      matrix:
        os: [ubuntu-latest, macos-13, windows-latest]
        python-version: [3.9, '3.10', '3.11']
    uses: qiboteam/workflows/.github/workflows/rules-poetry.yml@main
    with:
      os: ${{ matrix.os }}
      python-version: ${{ matrix.python-version }}
      doctests: ${{ matrix.os == 'ubuntu-latest'}}
      poetry-extras: "--with docs,tests,analysis --all-extras"
    secrets: inherit


Rust API

# Test Qibolab Rust API
name: Rust API

on:
  push:
  workflow_dispatch:

jobs:
  tests:
    strategy:
      matrix:
        os: [ubuntu-latest, macos-13]
    runs-on: ${{ matrix.os }}

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
      - name: Prepare virtual environment
        run: |
          pip install .
      - name: Install Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
      - name: Check and lint
        working-directory: crate
        run: |
          cargo check
          cargo clippy --all-targets
      - name: Build & run example
        working-directory: crate
        run: |
          cargo run --example example


Tests with qpu

# CI workflow that runs on qpu
name: Tests with qpu

on:
  pull_request:
    types: [labeled]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref }}
  cancel-in-progress: true

jobs:
  prepare:
    if: contains(join(github.event.pull_request.labels.*.name), 'run-on')
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Read platforms from labels
        id: set-matrix
        env:
          LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }}
        run: |
          echo $LABELS
          platforms="["
          combined=""
          shopt -s lastpipe
          jq -c '.[]' <<< "$LABELS" | while read label; do
              platform=(${label//-/ })
              if [ ${platform[0]} == "\"run" ] && [ ${platform[1]} == "on" ]; then
                platforms+="${combined:+,}\"${platform[2]}"
                combined=${platforms}
              fi
          done
          platforms+="]"
          echo ${platforms}
          echo matrix="${platforms}" >> $GITHUB_OUTPUT

  tests:
    needs: prepare
    strategy:
      matrix:
        platform: ${{ fromJSON(needs.prepare.outputs.matrix) }}
      fail-fast: false
    runs-on: self-hosted
    steps:
    - name: Cleanup workspace manually
      run: |
        rm -rf _work/*
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.10"
    - name: Install and configure poetry
      uses: snok/install-poetry@v1
      with:
        virtualenvs-create: false
    - name: Install qibolab main
      run: |
        python -m venv testenv
        source testenv/bin/activate
        poetry install --no-interaction --all-extras --with tests
    - name: Execute on ${{ matrix.platform }}
      run: |
        source testenv/bin/activate
        export platform=${{ matrix.platform }}
        git clone https://github.com/qiboteam/qibolab_platforms_qrc
        queues=`cat qibolab_platforms_qrc/queues.json`
        export QIBOLAB_PLATFORMS=./qibolab_platforms_qrc
        partition="$(jq -r -n --argjson data "$queues" '$data.'$platform)"
        srun -p $partition selfhosted
        mkdir coverage
        mv coverage.xml coverage/
        mv htmlcov coverage/
    - name: Upload coverage report as artifact
      uses: actions/upload-artifact@v3
      with:
        name: coverage-from-self-hosted
        path: coverage/
    - name: Notify the Pull Request
      uses: thollander/actions-comment-pull-request@v2
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        message: |
          Run on QPU `${{ matrix.platform }}` completed! :atom:

          > *You can download the coverage report as an artifact, from the workflow summary page:*
          > ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}


CI | Code Quality

name: CI | Code Quality

on:
  push:
    branches-ignore:
      - "main-ci"
      - "release"

  pull_request:
    branches:
      - main

  workflow_dispatch:

jobs:
  lint:
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.8, 3.9, 3.10.12, 3.11.0]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Dependencies
        run: |
          python -m pip install --upgrade pip
          pip install .[test]

      - name: Pylint Source
        run: pylint --ignore-paths='build/*' --ignore='_version.py' $(find . -type f -name '*.py') \


name: CD | Publish Python 🐍 distributions 📦 to PyPI

on:
  release:
    types: [published]

  workflow_dispatch:

jobs:
  build-n-publish:
    name: Build and publish Python 🐍 distributions 📦 to PyPI
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.11.0
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.0

      - name: Install pypa/build
        run: >-
          python -m
          pip install
          build
          --user

      - name: Build a binary wheel and a source tarball
        run: >-
          python -m
          build
          --sdist
          --wheel
          --outdir dist/
          .

      - name: Publish distribution 📦 to Test PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
          repository_url: https://test.pypi.org/legacy/

      - name: Publish distribution 📦 to PyPI
        # if: startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}

  notify-runpod-workers:
    name: Notify workers
    needs: build-n-publish
    strategy:
      matrix:
        repo:
          [
            "runpod-workers/worker-faster_whisper",
            "runpod-workers/worker-stable_diffusion_v1",
            "runpod-workers/worker-kandinsky",
            "runpod-workers/worker-stable_diffusion_v2",
            "runpod-workers/worker-template",
            "runpod-workers/worker-whisper",
            "runpod-workers/worker-esrgan",
            "runpod-workers/worker-github_runner",
            "runpod-workers/worker-a1111",
            "runpod-workers/worker-dreambooth",
            "runpod-workers/worker-bark",
            "runpod-workers/worker-gpt",
            "runpod-workers/worker-iseven",
            "runpod-workers/worker-controlnet",
            "runpod-workers/worker-blip",
            "runpod-workers/worker-deforum",
            runpod-workers/mock-worker,
          ]

    runs-on: ubuntu-latest

    steps:
      - name: Wait for propegation
        run: sleep 300s
        shell: bash

      - name: Repository Dispatch
        uses: peter-evans/repository-dispatch@v3
        with:
          token: ${{ secrets.RUNPOD_WORKERS_PAT }}
          repository: ${{ matrix.repo }}
          event-type: python-package-release


name: CD | Test Publish Python 🐍 distributions 📦 to TestPyPI

on:
  push:
    branches:
      - "staging"

  workflow_dispatch:

jobs:
  build-n-publish:
    name: Build and publish Python 🐍 distributions 📦 to TestPyPI
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.11.0
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.0

      - name: Install pypa/build
        run: >-
          python -m
          pip install
          build
          --user

      - name: Build a binary wheel and a source tarball
        run: >-
          python -m
          build
          --sdist
          --wheel
          --outdir dist/
          .

      - name: Publish distribution 📦 to Test PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
          repository_url: https://test.pypi.org/legacy/


CI | CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CI | CodeQL"

on:
  push:
    branches: ["main"]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: ["main"]
  schedule:
    - cron: "15 1 * * 1"

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: ["python"]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Use only 'java' to analyze code written in Java, Kotlin or both
        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: ${{ matrix.language }}
          # If you wish to specify custom queries, you can do so here or in a config file.
          # By default, queries listed here will override any specified in a config file.
          # Prefix the list here with "+" to use these queries and those in the config file.

          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
          # queries: security-extended,security-and-quality

      # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
      # If this step fails, then you should remove it and run the build manually (see below)
      - name: Autobuild
        uses: github/codeql-action/autobuild@v3

      # ℹ Command-line programs to run using the OS shell.
      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

      #   If the Autobuild fails above, remove it and uncomment the following three lines.
      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

      # - run: |
      #   echo "Run, Build Application using script"
      #   ./location_of_script_within_repo/buildscript.sh

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3
        with:
          category: "/language:${{matrix.language}}"


CI | End-to-End RunPod Python Tests

# Performs a full test of the package within production environment.

name: CI | End-to-End RunPod Python Tests

on:
  push:
    branches:
      - main

  pull_request:
    branches:
      - main

  workflow_dispatch:

jobs:
  e2e-build:
    name: Build and push mock-worker Docker image
    runs-on: ubuntu-latest
    outputs:
      docker_tag: ${{ steps.output_docker_tag.outputs.docker_tag }}

    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Clone and patch mock-worker
        run: |
          git clone https://github.com/runpod-workers/mock-worker
          GIT_SHA=${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
          echo "git+https://github.com/runpod/runpod-python.git@$GIT_SHA" > mock-worker/builder/requirements.txt

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Define Docker Tag
        id: docker_tag
        run: |
          DOCKER_TAG=${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
          echo "DOCKER_TAG=$(echo $DOCKER_TAG | cut -c 1-7)" >> $GITHUB_ENV

      - name: Set Docker Tag as Output
        id: output_docker_tag
        run: echo "::set-output name=docker_tag::${{ env.DOCKER_TAG }}"

      - name: Build and push Docker image
        env:
          DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO }}
          DOCKERHUB_IMG: ${{ vars.DOCKERHUB_IMG }}
        run: |
          docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_TOKEN }}
          docker build -t $DOCKERHUB_REPO/$DOCKERHUB_IMG:${{ env.DOCKER_TAG }} -f mock-worker/Dockerfile mock-worker
          docker push $DOCKERHUB_REPO/$DOCKERHUB_IMG:${{ env.DOCKER_TAG }}

  test:
    name: Run End-to-End Tests
    runs-on: ubuntu-latest
    needs: [e2e-build]

    steps:
      - uses: actions/checkout@v4

      - name: Run Tests
        id: run-tests
        uses: runpod/runpod-test-runner@v2.1.0
        with:
          image-tag: ${{ vars.DOCKERHUB_REPO }}/${{ vars.DOCKERHUB_IMG }}:${{ needs.e2e-build.outputs.docker_tag }}
          runpod-api-key: ${{ secrets.RUNPOD_API_KEY }}
          request-timeout: 1200

      - name: Verify Tests
        env:
          TOTAL_TESTS: ${{ steps.run-tests.outputs.total-tests }}
          SUCCESSFUL_TESTS: ${{ steps.run-tests.outputs.succeeded }}
        run: |
          echo "Total tests: $TOTAL_TESTS"
          echo "Successful tests: $SUCCESSFUL_TESTS"
          if [ "$TOTAL_TESTS" != "$SUCCESSFUL_TESTS" ]; then
              exit 1
          fi


vhs

name: vhs
on:
  #   push:
  #     branches-ignore:
  #       - main

  workflow_dispatch:

jobs:
  vhs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Install runpod
        run: |
          pip install .

      # runpod --help
      - uses: charmbracelet/vhs-action@v2
        with:
          path: ".github/tape_collection/cli_help.tape"

      # runpod config
      - uses: charmbracelet/vhs-action@v2
        with:
          path: ".github/tape_collection/cli_config.tape"

      # runpod ssh
      - uses: charmbracelet/vhs-action@v2
        with:
          path: ".github/tape_collection/cli_ssh.tape"

      # Add gifs to commit
      - uses: stefanzweifel/git-auto-commit-action@v5
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          commit_message: Update generated VHS GIF
          commit_user_name: vhs-action 📼
          commit_user_email: actions@github.com
          commit_author: vhs-action 📼 <actions@github.com>
          file_pattern: "*.gif"


CI | Unit Tests

name: CI | Unit Tests

on:
  push:
    branches-ignore:
      - main-ci
      - release

  pull_request:
    branches:
      - main

  workflow_dispatch:

jobs:
  run_tests:
    strategy:
      matrix:
        python-version: [3.8, 3.9, 3.10.12, 3.11.0]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Dependencies
        run: |
          python -m pip install --upgrade pip
          pip install .[test]

      - name: Run Tests
        run: pytest --cov-config=.coveragerc --timeout=120 --timeout_method=thread --cov=runpod --cov-report=xml --cov-report=term-missing --cov-fail-under=100 -W error -p no:cacheprovider -p no:unraisableexception


code-check

name: code-check

on:
  push:
    branches: ["main"]
    tags: ["*"]
  pull_request:
    branches:
      - "*"
      - "!gh-pages"

jobs:
  python-format-black:
    name: Python lint [black]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Install black
        run: |
          pip install black>=24

      - name: Lint
        run: black --check .

  python-lint-ruff:
    name: Python lint [ruff]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Install ruff
        run: |
          pip install ruff

      - name: Lint
        run: ruff .


doc

name: doc

on:
  push:
    branches: ["main"]

# Prevent doc action on `main` to conflict with each others.
concurrency:
  group: doc-${{ github.ref }}
  cancel-in-progress: true

jobs:
  doc:
    runs-on: "ubuntu-latest"
    timeout-minutes: 30

    defaults:
      run:
        shell: bash -l {0}

    steps:
      - name: Checkout the code
        uses: actions/checkout@v4

      - name: Setup mamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: env.yml
          environment-name: my_env
          cache-environment: true
          cache-downloads: true

      - name: Install library
        run: python -m pip install --no-deps .

      - name: Configure git
        run: |
          git config --global user.name "${GITHUB_ACTOR}"
          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"

      - name: Deploy the doc
        run: |
          echo "Get the gh-pages branch"
          git fetch origin gh-pages

          echo "Build and deploy the doc on main"
          mike deploy --push main


release

name: release

on:
  workflow_dispatch:
    inputs:
      release-version:
        description: "A valid Semver version string"
        required: true

permissions:
  contents: write
  pull-requests: write

jobs:
  release:
    # Do not release if not triggered from the default branch
    if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch)

    runs-on: ubuntu-latest
    timeout-minutes: 30

    defaults:
      run:
        shell: bash -l {0}

    steps:
      - name: Checkout the code
        uses: actions/checkout@v4

      - name: Setup mamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: env.yml
          environment-name: my_env
          cache-environment: true
          cache-downloads: true
          create-args: >-
            pip
            semver
            python-build
            setuptools_scm

      - name: Check the version is valid semver
        run: |
          RELEASE_VERSION="${{ inputs.release-version }}"

          {
            pysemver check $RELEASE_VERSION
          } || {
            echo "The version '$RELEASE_VERSION' is not a valid Semver version string."
            echo "Please use a valid semver version string. More details at https://semver.org/"
            echo "The release process is aborted."
            exit 1
          }

      - name: Check the version is higher than the latest one
        run: |
          # Retrieve the git tags first
          git fetch --prune --unshallow --tags &> /dev/null

          RELEASE_VERSION="${{ inputs.release-version }}"
          LATEST_VERSION=$(git describe --abbrev=0 --tags)

          IS_HIGHER_VERSION=$(pysemver compare $RELEASE_VERSION $LATEST_VERSION)

          if [ "$IS_HIGHER_VERSION" != "1" ]; then
            echo "The version '$RELEASE_VERSION' is not higher than the latest version '$LATEST_VERSION'."
            echo "The release process is aborted."
            exit 1
          fi

      - name: Build Changelog
        id: github_release
        uses: mikepenz/release-changelog-builder-action@v4
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          toTag: "main"

      - name: Configure git
        run: |
          git config --global user.name "${GITHUB_ACTOR}"
          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"

      - name: Create and push git tag
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Tag the release
          git tag -a "${{ inputs.release-version }}" -m "Release version ${{ inputs.release-version }}"

          # Checkout the git tag
          git checkout "${{ inputs.release-version }}"

          # Push the modified changelogs
          git push origin main

          # Push the tags
          git push origin "${{ inputs.release-version }}"

      - name: Install library
        run: python -m pip install --no-deps .

      - name: Build the wheel and sdist
        run: python -m build --no-isolation

      - name: Publish package to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}
          packages-dir: dist/

      - name: Create GitHub Release
        uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844
        with:
          tag_name: ${{ inputs.release-version }}
          body: ${{steps.github_release.outputs.changelog}}

      - name: Deploy the doc
        run: |
          echo "Get the gh-pages branch"
          git fetch origin gh-pages

          echo "Build and deploy the doc on ${{ inputs.release-version }}"
          mike deploy --push stable
          mike deploy --push ${{ inputs.release-version }}


test

name: test

on:
  push:
    branches: ["main"]
    tags: ["*"]
  pull_request:
    branches:
      - "*"
      - "!gh-pages"
  schedule:
    - cron: "0 4 * * MON"

jobs:
  test:
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11"]
        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
        rdkit-version: ["2023.09", "2024.03"]

    runs-on: ${{ matrix.os }}
    timeout-minutes: 30

    defaults:
      run:
        shell: bash -l {0}

    name: |
      os=${{ matrix.os }}
      - python=${{ matrix.python-version }}
      - rdkit=${{ matrix.rdkit-version }}

    steps:
      - name: Checkout the code
        uses: actions/checkout@v4

      - name: Setup mamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: env.yml
          environment-name: my_env
          cache-environment: true
          cache-downloads: true
          create-args: >-
            python=${{ matrix.python-version }}
            rdkit=${{ matrix.rdkit-version }}

      - name: Install library
        run: python -m pip install --no-deps -e . # `-e` required for correct `coverage` run.

      - name: Run tests
        run: pytest

      - name: Codecov Upload
        uses: codecov/codecov-action@v4
        with:
          files: ./coverage.xml
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: false
          verbose: false
          env_vars: ${{ matrix.os }},${{ matrix.python-version }},${{ matrix.rdkit-version }}

      - name: Test building the doc
        run: mkdocs build


On Push

name: On Push

on: push

jobs:
  #depending on pytest workflow, see https://stackoverflow.com/a/71489231/364388
  pytest:
    uses: ./.github/workflows/pytest.yml

  slack:
    needs:
      - pytest
    if: always() # also execute when pytest fails
    runs-on: ubuntu-latest
    steps:
      - name: Determine if we need to notify
        uses: Jimdo/should-i-notify-action@main
        id: should_notify
        with:
          needs_context: ${{ toJson(needs) }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
      - name: Slack workflow notification
        if: steps.should_notify.outputs.should_send_message == 'yes'
        uses: Gamesight/slack-workflow-status@master
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          slack_webhook_url: ${{ secrets.SLACK_ROBOTICS_CI_WEBHOOK }}
          channel: "robotik-ci"
          name: "RoSys"


Upload Python Package

name: Upload Python Package

on:
  workflow_dispatch:
  push:
    tags:
      - v**

jobs:
  #depending on pytest workflow, see https://stackoverflow.com/a/71489231/364388
  pytest:
    uses: ./.github/workflows/pytest.yml

  pypi:
    runs-on: ubuntu-latest
    needs: pytest
    steps:
      - uses: actions/checkout@v2
      - name: set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.11"
      - name: set up Poetry
        uses: abatilo/actions-poetry@v2.0.0
        with:
          poetry-version: "1.3.1"
      - name: Update version.py
        run: |
          echo "__version__ = '${GITHUB_REF#refs/tags/v}'" > rosys/version.py
      - name: get version
        id: get_version
        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
      - name: set version
        run: poetry version ${{ steps.get_version.outputs.VERSION }}
      - name: publish
        env:
          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
        run: poetry publish --build
      - name: Create GitHub release entry
        uses: softprops/action-gh-release@v1
        id: create_release
        with:
          draft: false
          prerelease: false
          name: ${{ steps.get_version.outputs.VERSION }}
          tag_name: ${{ steps.get_version.outputs.VERSION }}
        env:
          GITHUB_TOKEN: ${{ github.token }}
      - name: verify
        shell: bash
        run: for i in {1..100}; do python -m pip install 'rosys==${{ steps.get_version.outputs.VERSION }}' && break || sleep 2; done

  docker:
    needs: pypi
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Prepare
        id: prep
        run: |
          DOCKER_IMAGE=${{ secrets.DOCKER_USERNAME }}/${GITHUB_REPOSITORY#*/}
          VERSION=latest
          SHORTREF=${GITHUB_SHA::8}

          # If this is git tag, use the tag name as a docker tag
          if [[ $GITHUB_REF == refs/tags/* ]]; then
            VERSION=${GITHUB_REF#refs/tags/v}
          fi
          TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:${SHORTREF}"

          # If the VERSION looks like a version number, assume that
          # this is the most recent version of the image and also
          # tag it 'latest'.
          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
            TAGS="$TAGS,${DOCKER_IMAGE}:latest"
          fi

          # Set output parameters.
          echo ::set-output name=tags::${TAGS}
          echo ::set-output name=docker_image::${DOCKER_IMAGE}
      - name: Set up QEMU
        uses: docker/setup-qemu-action@master
        with:
          platforms: all
      - name: Login to DockerHub
        if: github.event_name != 'pull_request'
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@master
        with:
          install: true
      - name: Build
        uses: docker/build-push-action@v2
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          build-args: INSTALL_DEV=true
          push: true
          tags: ${{ steps.prep.outputs.tags }}

      # Uploading the README.md is not a core feature of docker/build-push-action yet
      - name: Update README
        uses: christian-korneck/update-container-description-action@v1
        env:
          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
          DOCKER_PASS: ${{ secrets.DOCKER_PASSWORD }}
        with:
          destination_container_repo: zauberzeug/rosys
          provider: dockerhub
          short_description: "A robot system based on web technologies with focus on autonomous mobile robots."

  docs:
    runs-on: ubuntu-latest
    needs: pypi
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.11"
      - name: Install dependencies
        run: |
          python3 -m pip install -r mkdocs_requirements.txt
          python3 -m pip install -e .
      - name: Build docs
        run: mkdocs build -v
      - name: Deploy gh-pages
        uses: JamesIves/github-pages-deploy-action@v4.2.5
        with:
          branch: gh-pages
          folder: site


Run Tests

name: Run Tests

on: workflow_call

jobs:
  pytest:
    strategy:
      matrix:
        python: ["3.10", "3.11"]
      fail-fast: false
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v2
      - name: set up Python
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python }}
      - name: set up Poetry
        uses: abatilo/actions-poetry@v2.0.0
        with:
          poetry-version: "1.3.1"
      - name: install dependencies
        run: |
          poetry config virtualenvs.create false --local
          poetry install
          sudo apt-get update && sudo apt-get install -y v4l-utils
      - name: test
        run: pytest
      - name: examples
        run: cd tests && ./test_examples.py


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
    push:
        branches: ["master"]
    pull_request:
        # The branches below must be a subset of the branches above
        branches: ["master"]
    schedule:
        - cron: "28 14 * * 1"

jobs:
    analyze:
        name: Analyze
        runs-on: ubuntu-latest
        permissions:
            actions: read
            contents: read
            security-events: write

        strategy:
            fail-fast: false
            matrix:
                language: ["python"]
                # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
                # Use only 'java' to analyze code written in Java, Kotlin or both
                # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
                # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

        steps:
            - name: Checkout repository
              uses: actions/checkout@v4

            # Initializes the CodeQL tools for scanning.
            - name: Initialize CodeQL
              uses: github/codeql-action/init@v3
              with:
                  languages: ${{ matrix.language }}
                  # If you wish to specify custom queries, you can do so here or in a config file.
                  # By default, queries listed here will override any specified in a config file.
                  # Prefix the list here with "+" to use these queries and those in the config file.

                  # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
                  # queries: security-extended,security-and-quality

            # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
            # If this step fails, then you should remove it and run the build manually (see below)
            - name: Autobuild
              uses: github/codeql-action/autobuild@v3

            # ℹ Command-line programs to run using the OS shell.
            # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

            #   If the Autobuild fails above, remove it and uncomment the following three lines.
            #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

            # - run: |
            #   echo "Run, Build Application using script"
            #   ./location_of_script_within_repo/buildscript.sh

            - name: Perform CodeQL Analysis
              uses: github/codeql-action/analyze@v3
              with:
                  category: "/language:${{matrix.language}}"


Lint PR

name: "Lint PR"

on:
    pull_request_target:
        types:
            - opened
            - edited
            - synchronize

jobs:
    main:
        runs-on: ubuntu-latest
        steps:
            # Please look up the latest version from
            # https://github.com/amannn/action-semantic-pull-request/releases
            - uses: amannn/action-semantic-pull-request@v5.5.2
              env:
                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


PyPI Release

# This is a basic workflow to help you get started with Actions

name: PyPI Release

# Controls when the workflow will run
on:
    release:

    # Allows you to run this workflow manually from the Actions tab
    workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
    deploy:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v4
            - name: Set up Python
              uses: actions/setup-python@v5
              with:
                  python-version: "3.x"
            - name: Install dependencies
              run: |
                  python -m pip install --upgrade pip
                  pip install build
            - name: Build package
              run: python -m build
            - name: Publish package to PyPi Test
              uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0
              with:
                  user: __token__
                  password: ${{ secrets.TEST_PYPI_API_TOKEN }}
                  repository_url: https://test.pypi.org/legacy/
            - name: Publish package to PyPi Live
              uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0
              with:
                  user: __token__
                  password: ${{ secrets.PYPI_API_TOKEN }}


Test Builds

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Test Builds

on:
    pull_request_target:
        branches:
            - master

jobs:
    build:
        runs-on: ubuntu-latest
        strategy:
            fail-fast: false
            matrix:
                python-version: ["3.10"]

        steps:
            - uses: actions/checkout@v4
              with:
                  ref: ${{github.event.pull_request.head.ref}}
                  repository: ${{github.event.pull_request.head.repo.full_name}}
            - name: Set up Python ${{ matrix.python-version }}
              uses: actions/setup-python@v5
              with:
                  python-version: ${{ matrix.python-version }}
            - name: Install dependencies
              run: |
                  python -m pip install --upgrade pip
                  python -m pip install flake8 pytest
                  if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - name: Lint with flake8
              run: |
                  # stop the build if there are Python syntax errors or undefined names
                  flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
                  # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
                  flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
            - name: Test with pytest
              env:
                  KIA_CA_CDNNINJA_USERNAME: ${{ secrets.KIA_CA_CDNNINJA_USERNAME }}
                  KIA_CA_CDNNINJA_PASSWORD: ${{ secrets.KIA_CA_CDNNINJA_PASSWORD }}
                  KIA_CA_CDNNINJA_PIN: ${{ secrets.KIA_CA_CDNNINJA_PIN }}
                  KIA_EU_FUATAKGUN_USERNAME: ${{ secrets.KIA_EU_FUATAKGUN_USERNAME }}
                  KIA_EU_FUATAKGUN_PASSWORD: ${{ secrets.KIA_EU_FUATAKGUN_PASSWORD }}
              run: |
                  pytest


Release

name: Release
# Controls when the workflow will run
on:
    # Triggers the workflow on push or pull request events but only for the master branch
    push:
        branches: [master]
    workflow_dispatch:
jobs:
    build:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v4
            - name: Gets semantic release info
              id: semantic_release_info
              uses: jossef/action-semantic-release-info@v2
              env:
                  GITHUB_TOKEN: ${{ github.token }}
            - name: Update Version and Commit
              if: ${{steps.semantic_release_info.outputs.version != ''}}
              run: |
                  echo "Version: ${{steps.semantic_release_info.outputs.version}}"
                  sed -i "s/version=\".*\",/version=\"${{steps.semantic_release_info.outputs.version}}\",/g" setup.py
                  git config --local user.email "action@github.com"
                  git config --local user.name "GitHub Action"
                  git add -A
                  git commit -m "chore: bumping version to ${{steps.semantic_release_info.outputs.version}}"
                  git tag ${{ steps.semantic_release_info.outputs.git_tag }}
            - name: Push changes
              if: ${{steps.semantic_release_info.outputs.version != ''}}
              uses: ad-m/github-push-action@v0.8.0
              with:
                  github_token: ${{ github.token }}
                  tags: true
            - name: Create GitHub Release
              if: ${{steps.semantic_release_info.outputs.version != ''}}
              uses: actions/create-release@v1
              env:
                  GITHUB_TOKEN: ${{ github.token }}
              with:
                  tag_name: ${{ steps.semantic_release_info.outputs.git_tag }}
                  release_name: ${{ steps.semantic_release_info.outputs.git_tag }}
                  body: ${{ steps.semantic_release_info.outputs.notes }}
                  draft: false
                  prerelease: false
            - name: Install dependencies
              if: ${{steps.semantic_release_info.outputs.version != ''}}
              run: |
                  python -m pip install --upgrade pip
                  pip install build
            - name: Build package
              if: ${{steps.semantic_release_info.outputs.version != ''}}
              run: python -m build
            - name: Publish package to PyPi Test
              if: ${{steps.semantic_release_info.outputs.version != ''}}
              uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0
              with:
                  user: __token__
                  password: ${{ secrets.TEST_PYPI_API_TOKEN }}
                  repository_url: https://test.pypi.org/legacy/
            - name: Publish package to PyPi Live
              if: ${{steps.semantic_release_info.outputs.version != ''}}
              uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0
              with:
                  user: __token__
                  password: ${{ secrets.PYPI_API_TOKEN }}


Test and Publish

name: Test and Publish
on:
  push:
    branches:
      - main
      - series/*
  release:
    types: [published]
  pull_request:

concurrency:
  group: test-${{github.ref}}
  cancel-in-progress: true

defaults:
  run:
    shell: bash -l {0}

jobs:
  test-conda:
    name: Test with Conda Python ${{matrix.python}} on ${{matrix.platform}}
    runs-on: ${{matrix.platform}}-latest
    strategy:
      fail-fast: false
      matrix:
        platform:
        - macos
        - windows
        - ubuntu
        python:
        - "3.8"
        - "3.9"
        - "3.10"
        - "3.11"
        - "3.12"

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Create Conda environment file
      run: |
        ct_opts="-p $PYVER -e test"
        ct_args="pyproject.toml"
        plat=$(uname -m)
        if [ "$plat" != arm64 ]; then
          ct_opts="$ct_opts -e dev -e mkl"
          ct_args="$ct_args mkl-devel.yml"
        fi
        pipx run ./utils/conda-tool.py --env -o environment.yml $ct_opts $ct_args
      env:
        PYVER: ${{matrix.python}}

    - name: 👢 Set up Conda environment
      uses: mamba-org/setup-micromamba@v1
      id: setup
      with:
        environment-file: environment.yml
        environment-name: csr
        init-shell: bash

    - name: 🔎 Inspect environment
      run: |
        numba -s

    - name: 🛠 Compile MKL helper
      if: matrix.platform != 'macos'
      run: |
        invoke build-mkl

    - name: Run tests
      run: |
        python -m pytest --cov=csr -v

    - name: Run tests without JIT
      run: |
        python -m pytest --cov=csr --cov-append --log-file=test-nojit.log --hypothesis-profile=nojit
      env:
        NUMBA_DISABLE_JIT: 1

    - name: Aggregate coverage data
      run: coverage xml

    - name: Save test results
      uses: lenskit/lkbuild/actions/save-test-results@main
      with:
        artifact-name: test-conda-${{matrix.platform}}-py${{matrix.python}}

  test-vanilla:
    name: Test w/ Vanilla Python ${{matrix.python}} on ${{matrix.platform}}
    runs-on: ${{matrix.platform}}-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        python:
        - "3.8"
        - "3.9"
        - "3.10"
        - "3.11"
        platform:
        - windows
        - ubuntu
    steps:
      - name: Check out source
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Set up Python and environment
        uses: lenskit/lkbuild/actions/setup-vanilla-env@main
        with:
          python-version: ${{matrix.python}}

      - name: 🔎 Inspect environment
        run: |
          numba -s

      - name: Run tests
        run: |
          python -m pytest --cov=csr --cov-report=xml --log-file=test.log

      - name: Save test results
        uses: lenskit/lkbuild/actions/save-test-results@main
        with:
          artifact-name: test-vanilla-${{matrix.platform}}-py${{matrix.python}}

  test-mindeps:
    name: Test w/ Oldest Deps
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Check out source
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Set up Python and environment
        uses: lenskit/lkbuild/actions/setup-vanilla-env@main
        with:
          python-version: "3.8"
          include-dev-dependencies: false
          extras: test
          constraints-file: min-constraints.txt

      - name: 🔎 Inspect environment
        run: |
          numba -s

      - name: Run tests
        run: |
          python -m pytest --cov=csr --cov-report=xml --log-file=test.log

      - name: Run tests without JIT
        run: |
          python -m pytest --cov=csr --cov-append --log-file=test-nojit.log --hypothesis-profile=nojit
        env:
          NUMBA_DISABLE_JIT: 1

      - name: Save test results
        uses: lenskit/lkbuild/actions/save-test-results@main
        with:
          artifact-name: test-mindeps

  process-tests:
    name: Process Test Results
    runs-on: ubuntu-latest
    needs: [test-conda, test-vanilla, test-mindeps]

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Process test reports
      uses: lenskit/lkbuild/actions/report-test-results@main

  package:
    name: Package and Publish
    runs-on: ubuntu-latest
    needs: [process-tests]
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Fetch Git tags
      run: git fetch --tags

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.10"

    - name: Install Python deps
      run: pip install -U flit

    - name: Build distribution
      run: flit build

    - name: Save distribution archive
      uses: actions/upload-artifact@v3
      with:
        name: pypi-pkgs
        path: dist

    - name: List dist dir
      run: ls -R dist

    - name: Publish PyPI packages
      if: github.event_name == 'release'
      run: |
        flit publish
      shell: bash
      env:
        TWINE_NON_INTERACTIVE: y
        FLIT_USERNAME: __token__
        FLIT_PASSWORD: ${{ secrets.PYPI_TOKEN }}


Upload Python Package

name: Upload Python Package

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    branches:
      - dev
  push:
    branches:
      - dev
    tags:
      - 'v*.*.*'

env:
  # Change this to invalidate existing cache.
  CACHE_PREFIX: v0
  PYTHON_PATH: ./

permissions:
  contents: write

jobs:
  checks:
    name: Python ${{ matrix.python }} - ${{ matrix.task.name }}
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        python: ['3.10']
#        task:
#          - name: Test
#            run: |
#              pytest -v --color=yes tests/

        include:
#          - python: '3.10'
#            task:
#              name: Lint
#              run: flake8 .

#          - python: '3.10'
#            task:
#              name: Type check
#              run: mypy .

          - python: '3.10'
            task:
              name: Build
              run: |
                python setup.py check
                python setup.py bdist_wheel sdist

#          - python: '3.10'
#            task:
#              name: Style
#              run: black --check .

#          - python: '3.10'
#            task:
#              name: Docs
#              run: cd docs && make html

    steps:
      - uses: actions/checkout@v3

      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '18.x'

      - name: Setup Python
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python }}

      - name: Install prerequisites
        run: |
          pip install --upgrade pip setuptools wheel virtualenv

      - name: Set build variables
        shell: bash
        run: |
          # Get the exact Python version to use in the cache key.
          echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
          echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
          echo "NODE_VERSION=$(node --version)" >> $GITHUB_ENV
          # Use week number in cache key so we can refresh the cache weekly.
          echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV

      - uses: actions/cache@v3
        id: virtualenv-cache
        with:
          path: |
            .venv
            src/web/node_modules
          key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt') }}-${{ env.NODE_VERSION }}-${{ hashFiles('src/web/package-lock.json') }}
          restore-keys: |
            ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ env.NODE_VERSION }}

      - name: Setup virtual environment (no cache hit)
        if: steps.virtualenv-cache.outputs.cache-hit != 'true'
        run: |
          test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
          . .venv/bin/activate
          pip install -e .[dev]
          cd src/web
          npm install

      - name: Install editable (cache hit)
        if: steps.virtualenv-cache.outputs.cache-hit == 'true'
        run: |
          . .venv/bin/activate
          pip install --no-deps -e .[dev]

      - name: Show environment info
        run: |
          . .venv/bin/activate
          which python
          python --version
          pip freeze
          which node
          node --version

      - name: Build Web
        run: |
          cd src/web
          npm run build
        env:
          CI: false

      - name: ${{ matrix.task.name }}
        run: |
          . .venv/bin/activate
          ${{ matrix.task.run }}

      - name: Upload package distribution files
        if: matrix.task.name == 'Build'
        uses: actions/upload-artifact@v2
        with:
          name: package
          path: dist

      - name: Clean up
        if: always()
        run: |
          . .venv/bin/activate
          pip uninstall -y my-package

  release:
    name: Release
    runs-on: ubuntu-latest
    needs: [checks]
    if: startsWith(github.ref, 'refs/tags/')
    steps:
      - uses: actions/checkout@v1

      - name: Setup Python
        uses: actions/setup-python@v3
        with:
          python-version: '3.10'

      - name: Install requirements
        run: |
          pip install --upgrade pip setuptools wheel
          pip install -r requirements.txt

      - name: Prepare environment
        run: |
          echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
          echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV

      - name: Download package distribution files
        uses: actions/download-artifact@v2
        with:
          name: package
          path: dist

      - name: Generate release notes
        run: |
          python scripts/release_notes.py > ${{ github.workspace }}-RELEASE_NOTES.md

      - name: Publish GitHub release
        uses: softprops/action-gh-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          body_path: ${{ github.workspace }}-RELEASE_NOTES.md
          prerelease: ${{ contains(env.TAG, 'rc') }}
          files: |
            dist/*


Build images and test for multiple python versions


GH Pages Redirect

name: GH Pages Redirect

on:
  workflow_dispatch:

permissions:
  contents: read
  pages: write
  id-token: write

concurrency:
  group: "pages"
  cancel-in-progress: false

jobs:
  deploy:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Setup Pages
        uses: actions/configure-pages@v3

      - name: Upload artifact
        uses: actions/upload-pages-artifact@v1
        with:
          path: "./docs/gh_pages"

      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2


Lint and test

name: Lint and test

on:
  workflow_dispatch:
  push:
  pull_request:

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout ⬇
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          submodules: recursive

      - name: Setup python 🐍
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install dependencies ☕
        run: |
          source setup.sh ""

      - name: Lint 🔍
        run: |
          source setup.sh ""
          ./tests/run_linting

  lint_docs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout ⬇
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          submodules: recursive

      - name: Setup python 🐍
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install dependencies ☕
        run: |
          source setup.sh ""

      - name: Lint 🔍
        run: |
          source setup.sh ""
          ./tests/run_docs lint

  test:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout ⬇
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          submodules: recursive

      - name: Setup python 🐍
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install dependencies ☕
        run: |
          source setup.sh ""

      - name: Test 🎢
        run: |
          source setup.sh ""
          ./tests/run_tests

  pypi:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout ⬇
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          submodules: recursive

      - name: Setup python 🐍
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install dependencies ☕
        run: |
          pip install -U pip
          pip install twine

      - name: Check bundling 📦
        run: python setup.py sdist

      - name: Check setup 🚦
        run: twine check "dist/$( python setup.py --fullname ).tar.gz"

  coverage:
    runs-on: ubuntu-latest
    env:
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
    steps:
      - name: Checkout ⬇
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          submodules: recursive

      - name: Setup python 🐍
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install dependencies ☕
        run: |
          source setup.sh ""

      - name: Run coverage test 🎢
        run: |
          source setup.sh ""
          ./tests/run_coverage
          ls -al

      - name: Upload report 🔝
        uses: codecov/codecov-action@v4
        with:
          flags: unittests
          files: coverage_test_util.xml,coverage_test_columnar_util.xml
          verbose: true


Publish on PyPI

name: Publish on PyPI

on:
  workflow_dispatch:

jobs:
  publish:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout ⬇
        uses: actions/checkout@master
        with:
          persist-credentials: false
          submodules: recursive

      - name: Setup python 🐍
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Install dependencies ☕
        run: |
          pip install -U pip
          pip install twine

      - name: Bundle 📦
        run: python setup.py sdist

      - name: Publish package 🐍
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_TOKEN }}
          skip-existing: true


Clear cache

name: Clear cache

on:
    schedule:
      - cron: '0 0 1 * *'
    workflow_dispatch:

permissions:
  actions: write

jobs:
  clear-cache:
    runs-on: ubuntu-latest
    steps:
      - name: Clear cache
        uses: actions/github-script@v7
        with:
          script: |
            console.log("About to clear")
            const caches = await github.rest.actions.getActionsCacheList({
              owner: context.repo.owner,
              repo: context.repo.repo,
            })
            for (const cache of caches.data.actions_caches) {
              console.log(cache)
              github.rest.actions.deleteActionsCacheById({
                owner: context.repo.owner,
                repo: context.repo.repo,
                cache_id: cache.id,
              })
            }
            console.log("Clear completed")


Docs WorkFlow - Develop Tag 📚

name: Docs WorkFlow - Develop Tag 📚

on:
  push:
    branches:
      - develop
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.ref}}
  cancel-in-progress: true

permissions:
  contents: write
  pages: write
  pull-requests: write

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - name: 🔄 Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: 📦 Install mkdocs-material
        run: pip install "mkdocs-material"
      - name: 📦 Install mkdocstrings[python]
        run: pip install "mkdocstrings[python]"
      - name: 📦 Install mkdocs-material[imaging]
        run: pip install "mkdocs-material[imaging]"
      - name: 📦 Install mike
        run: pip install "mike"
      - name: 📦 Install mkdocs-git-revision-date-localized-plugin
        run: pip install "mkdocs-git-revision-date-localized-plugin"
      - name: 📦 Install JupyterLab
        run: pip install jupyterlab
      - name: 📦 Install mkdocs-jupyter
        run: pip install mkdocs-jupyter
      - name: 📦 Install mkdocs-git-committers-plugin-2
        run: pip install mkdocs-git-committers-plugin-2
      - name: ⚙ Configure git for github-actions
        run: |
          git config --global user.name "github-actions[bot]"
          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
      - name: 🚀 Deploy MkDoc-Material with mike
        run: |
          MKDOCS_GIT_COMMITTERS_APIKEY=${{ secrets.GITHUB_TOKEN }} mike deploy --push develop


name: Supervision Release Documentation Workflow 📚
on:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.ref}}
  cancel-in-progress: true

permissions:
  contents: write
  pages: write
  pull-requests: write

jobs:
  doc-build-deploy:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10"]
    steps:
      - name: 🛎 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.head_ref }}

      - name: 🐍 Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: 📦 Install mkdocs-material
        run: pip install "mkdocs-material"
      - name: 📦 Install mkdocstrings[python]
        run: pip install "mkdocstrings[python]"
      - name: 📦 Install mkdocs-material[imaging]
        run: pip install "mkdocs-material[imaging]"
      - name: 📦 Install mike
        run: pip install "mike"
      - name: 📦 Install mkdocs-git-revision-date-localized-plugin
        run: pip install "mkdocs-git-revision-date-localized-plugin"
      - name: 📦 Install JupyterLab
        run: pip install jupyterlab
      - name: 📦 Install mkdocs-jupyter
        run: pip install mkdocs-jupyter
      - name: 📦 Install mkdocs-git-committers-plugin-2
        run: pip install mkdocs-git-committers-plugin-2
      - name: ⚙ Configure git for github-actions 👷
        run: |
          git config --global user.name "github-actions[bot]"
          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
      - name: 🚀 Deploy MkDoc-Material 📚
        run: |
            latest_tag=$(git describe --tags `git rev-list --tags --max-count=1`)
            MKDOCS_GIT_COMMITTERS_APIKEY=${{ secrets.GITHUB_TOKEN }} mike deploy --push --update-aliases $latest_tag latest


Supervision Test Releases to PyPi

name: Supervision Test Releases to PyPi
on:
  push:
    tags:
      - '[0-9]+.[0-9]+[0-9]+.[0-9]+a[0-9]'
      - '[0-9]+.[0-9]+[0-9]+.[0-9]+b[0-9]'
      - '[0-9]+.[0-9]+[0-9]+.[0-9]+rc[0-9]'

  workflow_dispatch:

jobs:
  build-n-publish:
    name: Build and publish to PyPI
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10"]
    steps:
      - name: 🛎 Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
      - name: 🐍 Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: 🏗 Build source and wheel distributions
        run: |
          python -m pip install --upgrade build twine
          python -m build
          twine check --strict dist/*
      - name: 🚀 Publish distribution to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: ${{ secrets.PYPI_USERNAME }}
          password: ${{ secrets.PYPI_PASSWORD }}
      - name: 🚀 Publish to Test-PyPi
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: https://test.pypi.org/legacy/
          user: ${{ secrets.PYPI_TEST_USERNAME }}
          password: ${{ secrets.PYPI_TEST_PASSWORD }}


Supervision Releases to PyPi

name: Supervision Releases to PyPi
on:
  push:
    tags:
      - '[0-9]+.[0-9]+[0-9]+.[0-9]'

  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10"]
    steps:
      - name: 🛎 Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
      - name: 🐍 Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name:  🏗 Build source and wheel distributions
        run: |
          python -m pip install --upgrade build twine
          python -m build
          twine check --strict dist/*
      - name: 🚀 Publish to PyPi
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: ${{ secrets.PYPI_USERNAME }}
          password: ${{ secrets.PYPI_PASSWORD }}
      - name: 🚀 Publish to Test-PyPi
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: https://test.pypi.org/legacy/
          user: ${{ secrets.PYPI_TEST_USERNAME }}
          password: ${{ secrets.PYPI_TEST_PASSWORD }}


🧪 Docs Test WorkFlow 📚

name: 🧪 Docs Test WorkFlow 📚

on:
  pull_request:
    branches: [main, develop]

jobs:
  docs-build-test:
    runs-on: ubuntu-latest
    steps:
      - name: 🔄 Checkout code
        uses: actions/checkout@v4
      - name: 🐍 Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: 📦 Install mkdocs-material
        run: pip install "mkdocs-material[all]"
      - name: 📦 Install mkdocstrings[python]
        run: pip install "mkdocstrings[python]"
      - name: 📦 Install mkdocs-material[imaging]
        run: pip install "mkdocs-material[imaging]"
      - name: 📦 Install mike
        run: pip install "mike"
      - name: 📦 Install mkdocs-git-revision-date-localized-plugin
        run: pip install "mkdocs-git-revision-date-localized-plugin"
      - name: 📦 Install JupyterLab
        run: pip install jupyterlab
      - name: 📦 Install mkdocs-jupyter
        run: pip install mkdocs-jupyter
      - name: 📦 Install mkdocs-git-committers-plugin-2
        run: pip install mkdocs-git-committers-plugin-2
      - name: 🧪 Test documentation build
        run: mkdocs build --verbose


Python 3.8 - Min Dep Test WorkFlow

name: Python 3.8 - Min Dep Test WorkFlow

on:
  pull_request:
    branches: [main, develop]

jobs:
  build-min-dep-test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8"]
    steps:
      - name: 🛎 Checkout
        uses: actions/checkout@v4
      - name: 🐍 Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        # id based on python version
        id: python-setup
        with:
          python-version: ${{ matrix.python-version }}
          check-latest: true

      - name: 📦 Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install \
              attrs==23.1.0 \
              certifi==2023.7.22 \
              charset-normalizer==2.0.12 \
              cycler==0.12.1 \
              exceptiongroup==1.1.3 \
              fonttools==4.43.1 \
              idna==3.4 \
              iniconfig==2.0.0 \
              kiwisolver==1.4.5 \
              matplotlib==3.5.0 \
              numpy==1.21.2 \
              opencv-python==4.5.5.64 \
              Pillow==10.1.0 \
              packaging==23.2 \
              pluggy==1.3.0 \
              pyparsing==3.1.1 \
              pytest==7.2.0 \
              python-dateutil==2.8.2 \
              PyYAML==5.3 \
              requests==2.26.0 \
              scipy==1.10.0 \
              setuptools-scm==8.0.4 \
              six==1.16.0 \
              tomli==2.0.1 \
              tqdm==4.62.3 \
              typing_extensions==4.8.0 \
              urllib3==1.26.18 \
              defusedxml==0.7.1

      - name: 🧪 Test
        run: "python -m pytest ./test"


Test WorkFlow

name: Test WorkFlow

on:
  pull_request:
    branches: [main, develop]

jobs:
  build-dev-test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
    steps:
      - name: 🛎 Checkout
        uses: actions/checkout@v4
      - name: 🐍 Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          check-latest: true

      - name: 📦 Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install .
          pip install pytest

      - name: 🧪 Test
        run: "python -m pytest ./test"


Welcome WorkFlow

name: Welcome WorkFlow

on:
  issues:
    types: [opened]
  pull_request_target:
    types: [opened]

jobs:
  build:
    name: 👋 Welcome
    runs-on: ubuntu-latest
    steps:
      - uses: actions/first-interaction@v1.3.0
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          issue-message: "Hello there, thank you for opening an Issue ! 🙏🏻 The team was notified and they will get back to you asap."
          pr-message:  "Hello there, thank you for opening an PR ! 🙏🏻 The team was notified and they will get back to you asap."


benchmarks

name: benchmarks

on:
  push:
    branches:
      - "main" # or "master"
  pull_request:
  # `workflow_dispatch` allows CodSpeed to trigger backtest
  # performance analysis in order to generate initial data.
  workflow_dispatch:

jobs:
  benchmarks:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v3
        with:
          python-version: "3.9"

      - name: Install dependencies
        run: pip install .[test]

      - name: Run benchmarks
        uses: CodSpeedHQ/action@v2
        with:
          token: ${{ secrets.CODSPEED_TOKEN }}
          run: pytest . --codspeed


Check for .rej files

# .rej files occur when cruft update could not merge two files.
# They need to be handled, but are easy to miss if there's no CI
name: Check for .rej files

on:
  pull_request:
    types: [opened, synchronize]

jobs:
  check-for-rej-files:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: Check for .rej files
        run: |
          files=`find . -type f -name "*.rej"`
          count=`echo $files | grep -o "\.rej" | wc -l`
          if [[ $count != 0 ]]; then
            echo "Found .rej files in the repository."
            echo $files |
            exit 1
          else
            echo "No .rej files found in the repository."
          fi


automerge-dependabot-prs

# GitHub action to automerge dependabot PRs. Only merges if tests pass due to branch protections in the repo.

name: automerge-dependabot-prs

on: pull_request

permissions:
  contents: write
  pull-requests: write

jobs:
  dependabot-automerge:
    runs-on: ubuntu-latest
    if: ${{ github.actor == 'dependabot[bot]'}}

    steps:
    # Checkout action is required for token to persist
    - name: Enable auto-merge for Dependabot PRs
      run: gh pr merge --auto --merge "$PR_URL" # Use Github CLI to merge automatically the PR
      env:
        PR_URL: ${{github.event.pull_request.html_url}}
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

    - name: Auto approve dependabot PRs
      if: github.actor == 'dependabot[bot]'
      uses: hmarr/auto-approve-action@v2
      with:
        github-token: ${{ secrets.GITHUB_TOKEN }}


Documentation

# creates the documentation on pushes it to the gh-pages branch
name: Documentation

on:
  push:
    branches: [main]
  pull_request:

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
          token: ${{ secrets.PAT }}

      - name: Install dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[docs,tutorials,text]"

      - name: Build and Commit
        uses: sphinx-notes/pages@2.1
        with:
          documentation_path: docs
          sphinx_version: "5.3.0"

      - name: Push changes
        if: ${{ github.event_name == 'push' }}
        uses: ad-m/github-push-action@v0.6.0
        with:
          github_token: ${{ secrets.PAT }}
          branch: gh-pages


Create paper pdf

name: Create paper pdf

on:
  push:
    paths:
      - "paper/*.md"

jobs:
  paper:
    runs-on: ubuntu-latest
    name: Paper Draft
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Build draft PDF
        uses: openjournals/openjournals-draft-action@master
        with:
          journal: joss
          # This should be the path to the paper within your repo.
          paper-path: paper/paper.md
      - name: Upload
        uses: actions/upload-artifact@v1
        with:
          name: paper
          # This is the output path where Pandoc will write the compiled
          # PDF. Note, this should be the same directory as the input
          # paper.md
          path: paper/paper.pdf


Pytest

# Workflow which does the following
# 1) run pytest
# 2) updates the version according to semantic versioning
# 3) creates a release
# 4) uploads the package to pypi
name: Pytest

on:
  pull_request:
  push:
    branches:
      - main

env:
  cache-version: 0.0.9 # Change this number if you want to manually invalidate all caches

jobs:
  test:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    env:
      python-version: ${{ matrix.python-version }}

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Run tests
        uses: ./.github/actions/test
        with:
          python-version: ${{ env.python-version }}

  jupyter-tutorials:
    runs-on: ${{ matrix.os }}
    env:
      python-version: "3.9"

    strategy:
      matrix:
        os: [ubuntu-latest]

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Run tests
        uses: ./.github/actions/test_tutorials
        with:
          python-version: ${{ env.python-version }}

  release:
    runs-on: ubuntu-latest
    needs: test
    if: ${{ github.ref == 'refs/heads/main' }}
    steps:
      # Checkout action is required for token to persist
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.RELEASE_BOT }}

      - name: Python Semantic Release
        uses: relekang/python-semantic-release@v7.34.4
        with:
          github_token: ${{ secrets.RELEASE_BOT }}
          # Remember to copy the tool.semantic_release section from pyproject.toml
          # as well
          # To enable pypi,
          # 1) Set upload_to_pypi to true in pyproject.toml and
          # 2) Set the pypi_token in the repo
          # 3) Uncomment the two lines below
          repository_username: __token__
          repository_password: ${{ secrets.PYPI_TOKEN }}


run-pre-commit

# GitHub action to check if pre-commit has been run. Runs from .pre-commit-config.yaml, where the pre-commit actions are.

name: run-pre-commit

on:
  pull_request:
  push:
    branches: [main]

jobs:
  pre-commit:
    permissions:
      pull-requests: write
    if: ${{ github.actor != 'dependabot[bot]' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          token: ${{ secrets.PAT }}

      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Install pre-commit
        run: pip install pre-commit

      - name: Run pre-commit
        id: pre_commit
        continue-on-error: true
        run: |
            if pre-commit run --color always --all-files; then
              echo "Succeeded check passed"
            else
              echo "Failed, exiting"
              exit 1
            fi


Close stale issues and PRs

name: "Close stale issues and PRs"
on:
  schedule:
    - cron: "0 13 * * 2-5"
  workflow_dispatch:

jobs:
  stale:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/stale@v6
        with:
          days-before-issue-stale: 14
          days-before-issue-close: 7
          stale-issue-message: "This issue is stale because it has been open for 14 days with no activity. Feel free to either 1) remove the stale label or 2) comment. If nothing happens, this will be closed in 7 days."
          close-issue-message: "This issue was closed automatically. Feel free to re-open it if it's important."
          days-before-pr-stale: 2
          stale-pr-message: "This PR is stale because it has been open 1+ days with no activity. Feel free to either 1) remove the stale label or 2) comment. If nothing happens, this will be closed in 7 days."
          days-before-pr-close: 7
          close-pr-message: "This PR was closed automatically. Feel free to re-open it if you still want to work on it."
          close-pr-label: "closed-by-stalebot"
          operations-per-run: 20
          exempt-pr-labels: "dependencies,bot"


# We do not include static_type_checks as a pre-commit hook because pre-commit hooks
# are installed in their own virtual environment, so static_type_checks cannot
# use stubs from imports
name: static_type_checks

on:
  pull_request:
  push:
    branches: [main]

jobs:
  static_type_checks:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - name: Checkout (GitHub)
        uses: actions/checkout@v4

      - name: Create github hosts file
        run: | # If this file is not created, the dev container fails because of non-existent mount
          mkdir -p ~/.config/gh
          touch ~/.config/gh/hosts.yml

      - name: Pre-build dev container image
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            inv types


Run tests

name: Run tests

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

env:
  WEBSOCKETS_TESTS_TIMEOUT_FACTOR: 10

jobs:
  coverage:
    name: Run test coverage checks
    runs-on: ubuntu-latest
    steps:
    - name: Check out repository
      uses: actions/checkout@v4
    - name: Install Python 3.x
      uses: actions/setup-python@v5
      with:
        python-version: "3.x"
    - name: Install tox
      run: pip install tox
    - name: Run tests with coverage
      run: tox -e coverage
    - name: Run tests with per-module coverage
      run: tox -e maxi_cov

  quality:
    name: Run code quality checks
    runs-on: ubuntu-latest
    steps:
    - name: Check out repository
      uses: actions/checkout@v4
    - name: Install Python 3.x
      uses: actions/setup-python@v5
      with:
        python-version: "3.x"
    - name: Install tox
      run: pip install tox
    - name: Check code formatting
      run: tox -e black
    - name: Check code style
      run: tox -e ruff
    - name: Check types statically
      run: tox -e mypy

  matrix:
    name: Run tests on Python ${{ matrix.python }}
    needs:
      - coverage
      - quality
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
          - "pypy-3.8"
          - "pypy-3.9"
        is_main:
          - ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
        exclude:
          - python: "pypy-3.8"
            is_main: false
          - python: "pypy-3.9"
            is_main: false
    steps:
    - name: Check out repository
      uses: actions/checkout@v4
    - name: Install Python ${{ matrix.python }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python }}
    - name: Install tox
      run: pip install tox
    - name: Run tests
      run: tox -e py


Make release

name: Make release

on:
  push:
    tags:
      - '*'
  workflow_dispatch:

jobs:
  sdist:
    name: Build source distribution and architecture-independent wheel
    runs-on: ubuntu-latest
    steps:
    - name: Check out repository
      uses: actions/checkout@v4
    - name: Install Python 3.x
      uses: actions/setup-python@v5
      with:
        python-version: 3.x
    - name: Build sdist
      run: python setup.py sdist
    - name: Save sdist
      uses: actions/upload-artifact@v4
      with:
        path: dist/*.tar.gz
    - name: Install wheel
      run: pip install wheel
    - name: Build wheel
      env:
        BUILD_EXTENSION: no
      run: python setup.py bdist_wheel
    - name: Save wheel
      uses: actions/upload-artifact@v4
      with:
        path: dist/*.whl

  wheels:
    name: Build architecture-specific wheels on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - ubuntu-latest
          - windows-latest
          - macOS-latest
    steps:
    - name: Check out repository
      uses: actions/checkout@v4
    - name: Install Python 3.x
      uses: actions/setup-python@v5
      with:
        python-version: 3.x
    - name: Set up QEMU
      if: runner.os == 'Linux'
      uses: docker/setup-qemu-action@v3
      with:
        platforms: all
    - name: Build wheels
      uses: pypa/cibuildwheel@v2.16.2
      env:
        BUILD_EXTENSION: yes
    - name: Save wheels
      uses: actions/upload-artifact@v4
      with:
        path: wheelhouse/*.whl

  upload:
    name: Upload
    needs:
      - sdist
      - wheels
    runs-on: ubuntu-latest
    # Don't release when running the workflow manually from GitHub's UI.
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
    permissions:
      id-token: write
    steps:
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: artifact
          path: dist
      - name: Upload to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
      - name: Create GitHub release
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh release -R python-websockets/websockets create ${{ github.ref_name }} --notes "See https://websockets.readthedocs.io/en/stable/project/changelog.html for details."


Multi-software test

name: Multi-software test

on:
  pull_request:
  push:
    branches:
    - master
    - develop

jobs:
  test:
    name: Tests | python ${{ matrix.python-version }}, ${{ matrix.os }}, ${{ matrix.pandas }}, ${{ matrix.numpy }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-22.04 ]
        python-version: [ 3.8 ]
        pandas: ["pandas>1.1"]
        numpy: ["numpy"]
        include:
          - os: ubuntu-22.04
            python-version: 3.7.16
            pandas: "pandas>1.1"
            numpy: "numpy"
          - os: ubuntu-22.04
            python-version: 3.8
            pandas: "pandas>1.1"
            numpy: "numpy"
          - os: ubuntu-22.04
            python-version: 3.9
            pandas: "pandas>1.1"
            numpy: "numpy"
          - os: ubuntu-22.04
            python-version: "3.10"
            pandas: "pandas>1.1"
            numpy: "numpy"
          - os: ubuntu-22.04
            python-version: "3.11"
            pandas: "pandas>1.1"
            numpy: "numpy"
          - os: ubuntu-22.04
            python-version: "3.12"
            pandas: "pandas>1.1"
            numpy: "numpy"

    steps:
    - uses: actions/checkout@v4

    - name: Setup python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        architecture: x64

    - uses: actions/cache@v4
      if: startsWith(runner.os, 'Linux')
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-${{ matrix.pandas }}-pip-
    - uses: actions/cache@v4
      if: startsWith(runner.os, 'macOS')
      with:
        path: ~/Library/Caches/pip
        key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-${{ matrix.pandas }}-pip-
    - uses: actions/cache@v4
      if: startsWith(runner.os, 'Windows')
      with:
        path: ~\AppData\Local\pip\Cache
        key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-${{ matrix.pandas }}-pip-
    - run: |
        pip install --upgrade pip setuptools wheel
        pip install -r requirements.txt "${{ matrix.pandas }}" "${{ matrix.numpy }}"
        pip install -r requirements-test.txt
    - run: make install

    - run: make test

  coverage:
    name: Coverage | python ${{ matrix.python-version }}, ${{ matrix.os }}, ${{ matrix.pandas }}, ${{ matrix.numpy }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-22.04 ]
        python-version: [ 3.8 ]
        pandas: [ "pandas>1.1"]
        numpy: ["numpy"]

    steps:
    - uses: actions/checkout@v4

    - name: Setup python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        architecture: x64

    - uses: actions/cache@v4
      if: startsWith(runner.os, 'Linux')
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-${{ matrix.pandas }}-pip-
    - uses: actions/cache@v4
      if: startsWith(runner.os, 'macOS')
      with:
        path: ~/Library/Caches/pip
        key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-${{ matrix.pandas }}-pip-
    - uses: actions/cache@v4
      if: startsWith(runner.os, 'Windows')
      with:
        path: ~\AppData\Local\pip\Cache
        key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-${{ matrix.pandas }}-pip-
    - run: |
        pip install --upgrade pip setuptools wheel
        pip install -r requirements.txt "${{ matrix.pandas }}" "${{ matrix.numpy }}"
        pip install -r requirements-test.txt
    - run: make install

    - run: make test_cov

    - uses: actions/cache@v4
      if: startsWith(runner.os, 'Windows')
      with:
        path: ~\AppData\Local\pip\Cache
        key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-${{ matrix.pandas }}-pip-
    - run: |
        pip install --upgrade pip setuptools wheel
        pip install -r requirements.txt "${{ matrix.pandas }}" "${{ matrix.numpy }}"
        pip install -r requirements-test.txt
    - run: make install
    - run: make test_cov
    - run: codecov -F py${{ matrix.python-version }}-${{ matrix.os }}-${{ matrix.pandas }}-${{ matrix.numpy }}

  test_spark:
    runs-on: ${{ matrix.os }}
    continue-on-error: True
    strategy:
      matrix:
        os: [ ubuntu-22.04 ]
        python-version: [3.7, 3.8]
        pandas: ["pandas>1.1"]
        spark: ["3.0.1"]
        hadoop: [ 2.7 ]
        numpy: ["numpy"]
        java_home: [ /usr/lib/jvm/java-8-openjdk-amd64 ]

    name: Tests Spark | python ${{ matrix.python-version }}, ${{ matrix.os }}, spark${{ matrix.spark }}, ${{ matrix.pandas }}, ${{ matrix.numpy }}
    env:
      JAVA_HOME: ${{ matrix.java_home }}
      SPARK_VERSION: ${{ matrix.spark }}
      HADOOP_VERSION: ${{ matrix.hadoop }}
      SPARK_DIRECTORY: ${{ github.workspace }}/../
      SPARK_HOME: ${{ github.workspace }}/../spark/
    steps:
      - uses: actions/checkout@v4
      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64
      - uses: actions/cache@v4
        if: startsWith(runner.os, 'Linux')
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.pandas }}-pip-\
      - uses: actions/cache@v4
        if: startsWith(runner.os, 'macOS')
        with:
          path: ~/Library/Caches/pip
          key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.pandas }}-pip-
      - uses: actions/cache@v4
        if: startsWith(runner.os, 'Windows')
        with:
          path: ~\AppData\Local\pip\Cache
          key: ${{ runner.os }}-${{ matrix.pandas }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.pandas }}-pip-
      - run: |
          pip install --upgrade pip setuptools wheel
          pip install pytest-spark>=0.6.0 pyarrow==1.0.1 pyspark=="${{ matrix.spark }}"
          pip install -r requirements.txt
          pip install -r requirements-test.txt
          pip install "${{ matrix.pandas }}" "${{ matrix.numpy }}"
      - if: ${{ matrix.spark != '3.0.1' }}
        run: echo "ARROW_PRE_0_15_IPC_FORMAT=1" >> $GITHUB_ENV
      - run: echo "SPARK_LOCAL_IP=127.0.0.1" >> $GITHUB_ENV
      - run: make install
      - run: make install-spark-ci
      - run: pip install -r requirements-spark.txt # Make sure the proper version of pandas is install after everything
      - run: make test_spark


Release

name: Release

on:
  release:
    types:
    - released

jobs:
  package:
    name: Build and Package
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4

    - name: Version
      id: version
      run: echo "value=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT

    - name: Setup Python 3.10
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"

    - uses: actions/cache@v4
      name: Cache pip dependencies
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -r requirements.txt
        python -m pip install -r requirements-dev.txt
        python -m pip install -r requirements-test.txt

    - name: Install
      run: make install

    - name:  Build package
      run: make package version=${{ steps.version.outputs.value }}

    - name: Upload Pipeline Artifacts
      uses: actions/upload-artifact@v4
      with:
        retention-days: 1
        name: built-artifacts
        path: dist/

  upload-assets:
    name: Upload Assets to Release
    runs-on: ubuntu-22.04

    needs: package

    steps:
    - uses: actions/download-artifact@v4
      with:
        name: built-artifacts
        path: dist/

    - uses: AButler/upload-release-assets@v3.0
      with:
        files: "dist/*"
        repo-token: ${{ secrets.ACCESS_TOKEN }}
        release-tag: ${{ needs.package.outputs.version }}

  publish-pypi:
    name: Publish packages to PyPi
    runs-on: ubuntu-22.04

    needs: package

    steps:
    - uses: actions/download-artifact@v4
      with:
        name: built-artifacts
        path: dist/

    - uses: pypa/gh-action-pypi-publish@v1.9.0
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}


Label issues

name: Label issues

on:
  issues:
    types:
      - reopened
      - opened

jobs:
  label_issues:
    runs-on: ubuntu-22.04

    permissions:
      issues: write

    steps:
    - name: Label issues
      uses: andymckay/labeler@master
      with:
        add-labels: "needs-triage"
        repo-token: ${{ secrets.ACCESS_TOKEN }}
        ignore-if-labeled: true


Pull Request

name: Pull Request

on:
  push:
    branches:
    - renovate/**
  pull_request:

jobs:
  commitlint:
    name: Lint commit message
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - uses: wagoid/commitlint-github-action@v6

  lint:
    if: github.actor != 'renovate[bot]'
    name: Lint source code
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4
      with:
        ref: ${{ github.head_ref }}
        fetch-depth: 0
        persist-credentials: false

    - name: Configurating Git
      run: |
        git config user.email "azory@ydata.ai"
        git config user.name "Azory YData Bot"
        git config core.autocrlf false

    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"

    - uses: actions/cache@v4
      name: Cache pip dependencies
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-

    - name: Install pip dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -r requirements.txt
        python -m pip install -r requirements-dev.txt
        python -m pip install -r requirements-test.txt

    - name: Install the package
      run: make install

    - name: Validate code formatting
      run: make lint

    - name: Commit code formatting changes
      if: always()
      id: commit
      run: |
        if [[ `git status --porcelain --untracked-files=no` ]]; then
          echo "CHANGES=true" >> $GITHUB_OUTPUT
          git add -u
          git commit -m "fix(linting): code formatting" -a
        else
          echo "CHANGES=false" >> $GITHUB_OUTPUT
        fi

    - name: Push changes
      uses: ad-m/github-push-action@v0.8.0
      if: always() && steps.commit.outputs.CHANGES == 'true'
      with:
        branch: ${{ github.head_ref }}
        github_token: ${{ secrets.ACCESS_TOKEN }}

  validate-docs:
    name: Validate Docs
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4

    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"

    - name: Cache pip dependencies
      id: cache
      uses: actions/cache@v4
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -r requirements.txt
        python -m pip install -r requirements-dev.txt
        python -m pip install -r requirements-test.txt
        python -m pip install -r requirements-docs.txt

    - name: Install the package
      run: make install

    - name: Build docs
      run: mkdocs build


Merge into develop branch

name: Merge into develop branch

on:
  push:
    branches:
    - develop

jobs:
  prepare:
    name: Calculate Version and Build Number
    runs-on: ubuntu-22.04

    outputs:
      version: ${{ steps.version.outputs.value }}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Find Latest Tag
      id: latest_tag
      uses: oprypin/find-latest-tag@v1.1.2
      with:
        repository: ${{ github.repository }}
        regex: '^v\d+\.\d+\.\d+'
        releases-only: true

    - name: Extract semantic version
      id: semantic
      run: echo "value=$(echo ${{ steps.latest_tag.outputs.tag }} | sed -r 's|([0-9]+.[0-9]+.[0-9]+).*|\1|g')" >> $GITHUB_OUTPUT

    - name: Count number or releases for tag
      id: build_number
      run: echo "value=$(($(git tag | grep -c ${{ steps.semantic.outputs.value }}.pre) + 1))" >> $GITHUB_OUTPUT

    - name: Compute version
      id: version
      run: echo "value=${{ steps.semantic.outputs.value }}.pre${{ steps.build_number.outputs.value }}" >> $GITHUB_OUTPUT

  prerelease-tag:
    name: Create Prerelease Tag
    runs-on: ubuntu-22.04

    needs:
    - prepare

    steps:
    - uses: ydataai/create-tag@v1.0.0
      with:
        tag: ${{ needs.prepare.outputs.version }}
        message: ${{  github.event.head_commit.message }}
        github_token: ${{ secrets.ACCESS_TOKEN }}


Merge into master branch

name: Merge into master branch

on:
  push:
    branches:
    - master

jobs:
  prepare:
    name: Calculate Version and Build Number
    runs-on: ubuntu-22.04

    outputs:
      build_number: ${{ steps.build_number.outputs.value }}
      new_release: ${{ steps.semantic.outputs.new_release_published }}
      release: v${{ steps.semantic.outputs.new_release_version }}
      release_notes: ${{ steps.semantic.outputs.new_release_notes }}
      version: v${{ steps.version.outputs.value }}

    steps:
    - uses: actions/checkout@v4
      with:
        token: ${{ secrets.ACCESS_TOKEN }}

    - uses: cycjimmy/semantic-release-action@v3
      id: semantic
      env:
        GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
      with:
        branch: master
        dry_run: true

    - name: Count number or releases for tag
      id: build_number
      run: echo "value=$(($(git tag | grep -c ${{ steps.semantic.outputs.new_release_version }}.rc) + 1))" >> $GITHUB_OUTPUT

    - name: Compute version
      id: version
      run: echo "value=${{ steps.semantic.outputs.new_release_version }}.rc${{ steps.build_number.outputs.value }}" >> $GITHUB_OUTPUT

  prerelease-tag:
    if: needs.prepare.outputs.new_release == 'true'

    name: Create Prerelease Tag
    runs-on: ubuntu-22.04

    needs:
    - prepare

    steps:
    - uses: ydataai/create-tag@v1.0.0
      env:
        GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
      with:
        tag: ${{ needs.prepare.outputs.version }}
        message: ${{  github.event.head_commit.message }}

  draft-release:
    if: needs.prepare.outputs.new_release == 'true'

    name: Draft Release
    runs-on: ubuntu-22.04

    needs:
    - prepare

    steps:
    - name: Delete Previous drafts
      uses: hugo19941994/delete-draft-releases@v1.0.1
      env:
        GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}

    - name: Create Draft Release
      uses: actions/create-release@v1
      env:
        GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
      with:
        tag_name: ${{ needs.prepare.outputs.release }}
        release_name: ${{ needs.prepare.outputs.release }}
        body: ${{ needs.prepare.outputs.release_notes }}
        draft: true
        commitish: master


Release Deprecated PP

name: Release Deprecated PP

on:
  workflow_dispatch:
    inputs:
      version:
        description: 'Version to release'
        required: true
        default: '3.6.4'

jobs:
  package:
    name: Build and Package
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4

    - name: Setup Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: "3.8"

    - uses: actions/cache@v4
      name: Cache pip dependencies
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -r requirements.txt
        python -m pip install -r requirements-dev.txt
        python -m pip install -r requirements-test.txt

    - name: Install
      run: make install

    - name: Lint
      run: make lint

    - name:  Build package
      run: make package version=${{ github.event.inputs.version }}

    - name: Upload Pipeline Artifacts
      uses: actions/upload-artifact@v4
      with:
        retention-days: 1
        name: built-artifacts
        path: dist/

  publish-pypi:
    name: Publish packages to PyPi
    runs-on: ubuntu-22.04

    needs: package

    steps:
    - uses: actions/download-artifact@v4
      with:
        name: built-artifacts
        path: dist/

    - uses: pypa/gh-action-pypi-publish@v1.9.0
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}
        repository_url: https://pypi.org/project/pandas-profiling