mypy

name: "mypy"
on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main

concurrency:
  group: "${{ github.workflow }}-${{ github.ref }}"
  cancel-in-progress: true

defaults:
  run:
    shell: bash -l {0}

jobs:
  mypy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Get current date
        id: date
        run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}"

      - name: "Setup Micromamba"
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: environment.yml
          environment-name: openfe_env
          cache-environment: true
          cache-downloads: true
          cache-environment-key: environment-${{ steps.date.outputs.date }}
          cache-downloads-key: downloads-${{ steps.date.outputs.date }}
          create-args: >-
            python=3
            rdkit=2023.09.5
          init-shell: bash

      - name: "Install steps"
        run: |
          python -m pip install --no-deps git+https://github.com/OpenFreeEnergy/gufe@main
          python -m pip install mypy
          python -m pip install types-pkg_resources
          python -m pip install --no-deps -e .

      - name: "Environment Information"
        run: |
          micromamba info
          micromamba list

      - name: "Lint with mypy"
        run: mypy


Package Install Tests

name: "Package Install Tests"
on:
  schedule:
    # At 07:00 UTC on Monday and Thursday.
    - cron: "0 7 * * *"

defaults:
  run:
    shell: bash -l {0}

jobs:
  package-tests:
    runs-on: ubuntu-latest
    name: "package install tests"
    steps:
      - uses: actions/checkout@v4

      - name: Get current date
        id: date
        run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}"

      - name: "Setup Micromamba"
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: environment.yml
          environment-name: openfe_env
          cache-environment: true
          cache-downloads: true
          cache-environment-key: environment-${{ steps.date.outputs.date }}
          cache-downloads-key: downloads-${{ steps.date.outputs.date }}
          create-args: >-
            python=3.10
          init-shell: bash

      - name: "install extra deps"
        run: pip install pipx wheel twine readme-renderer

      - name: "build sdist"
        run: pipx run build --sdist --outdir dist

      - name: "check package build"
        run: |
          dist=$(ls -t1 dist/openfe-*tar.gz | head -n1)
          test -n "${dist}" || { echo "no distribution found"; exit 1; }
          twine check $dist

      - name: "install from source dist"
        working-directory: ./dist
        run: python -m pip install openfe-*tar.gz

      - name: "run tests"
        working-directory: ./dist
        env:
          OFE_SLOW_TESTS: "true"
        run: |
          pytest -n auto -v --pyargs openfe.tests
          pytest -n auto -v --pyargs openfecli.tests


CD

name: CD

on:
  release:
    types:
    - published

jobs:
  release:
    name: Release
    runs-on: ubuntu-latest
    if: "startsWith(github.ref, 'refs/tags/')"
    outputs:
      bytewax-version: ${{ steps.get-info.outputs.bytewax-version }}
      version-image-tag: ${{ steps.get-info.outputs.version-image-tag }}
      latest-image-tag: ${{ steps.get-info.outputs.latest-image-tag }}
    steps:
      - name: Get Version and Tags
        id: get-info
        run: |
          BYTEWAX_VERSION=${GITHUB_REF:11}
          echo "::set-output name=bytewax-version::${BYTEWAX_VERSION}"
          REPOSITORY=bytewax/bytewax
          IMAGE_TAG="${REPOSITORY}:${GITHUB_REF:11}"
          echo "::set-output name=version-image-tag::${IMAGE_TAG}"
          echo "::set-output name=latest-image-tag::${REPOSITORY}:latest"
      - name: Report Info
        run: |
          [ -n "${{steps.get-info.outputs.bytewax-version}}" ]
          echo "::notice title=Bytewax Version::${{ steps.get-info.outputs.bytewax-version }}"
          [ -n "${{steps.get-info.outputs.version-image-tag}}" ]
          echo "::notice title=Base Tag::${{ steps.get-info.outputs.version-image-tag }}"
          [ -n "${{steps.get-info.outputs.latest-image-tag}}" ]
          echo "::notice title=Latest Tag::${{ steps.get-info.outputs.latest-image-tag }}"
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.ECR_ACCESS_KEY }}
          aws-secret-access-key: ${{ secrets.ECR_SECRET_KEY }}
          aws-region: us-west-2
      - name: Download files from S3 with AWS CLI
        run: |
          aws s3 sync s3://${{ secrets.WHEELS_S3_BUCKET }}/${{ github.sha }} .
          tree
      - name: Check the Version and Quantity of Wheel files
        env:
          NEEDED_VERSION: ${{steps.get-info.outputs.bytewax-version}}
          WHEELS: 30
        run: |
          cat << EOF > ./check_versions.sh
          #!/bin/bash
          echo "Looking for version \$NEEDED_VERSION in \$WHEELS wheel files..."
          wheels_found=0
          for file in *.whl
          do
            echo "Processing file: \$file"
            file_version=\$(echo \$file | sed 's/bytewax-//' | sed 's/-.*//')
            echo "Version in file: \$file_version"
            if [ "\$NEEDED_VERSION" = "\$file_version" ]; then
              echo "OK"
              wheels_found=\$((wheels_found+1))
            else
              echo "Versions mismatch. Canceling..."
              exit 1
            fi
          done
          echo "\$wheels_found wheel files with the right version found."
          if [ "\$wheels_found" = "\$WHEELS" ]; then
            echo "Verification passed."
          else
            echo "Wrong number of wheel files."
            echo "You should check if the last CI Workflow ends successfully."
            echo "https://github.com/bytewax/bytewax/actions/workflows/CI.yml"
            exit 1
          fi
          EOF
          chmod +x ./check_versions.sh
          ./check_versions.sh
      - uses: actions/setup-python@v2
        with:
          python-version: 3.9
      - name: Publish to PyPI
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
        run: |
          pip install --upgrade twine
          twine upload --skip-existing *.whl

  check-pypi:
    name: Check PyPI
    runs-on: ubuntu-latest
    if: "startsWith(github.ref, 'refs/tags/')"
    needs: [ release ]
    steps:
    - name: Install ansible
      run: |
        pip install ansible==3.4.0
        ansible -m ping localhost
    - name: Create task file
      run: |
        cat << EOF > ./pip_task.yaml
        ---
        - name: Try install bytewax using pip
          hosts: localhost
          tasks:
          - name: Loop
            shell: "pip install bytewax=={{ bytewax_version }} -v"
            register: pip_result
            retries: 60
            until: pip_result.rc == 0
            delay: 10
        EOF
    - name: Upgrade pip version
      run: |
        pip --version
        pip install --upgrade pip
    - name: Check PyPI index before building docker images
      run: |
        ansible-playbook ./pip_task.yaml -e bytewax_version=${{ needs.release.outputs.bytewax-version }} -vvv

  publish-docker-hub:
    name: Publish to Docker Hub
    if: "startsWith(github.ref, 'refs/tags/')"
    needs: [ release, check-pypi ]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11']
    steps:
    - uses: actions/checkout@v2
    - name: Set up QEMU
      uses: docker/setup-qemu-action@v2
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v1
    - name: Login to Docker Hub
      uses: docker/login-action@v1
      with:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_PASSWORD }}
    - name: Build and push to Docker Hub
      id: docker_build
      uses: docker/build-push-action@v4
      with:
        push: true
        platforms: linux/amd64, linux/arm64
        tags: |
          ${{ needs.release.outputs.version-image-tag }}-python${{ matrix.python-version }}
          ${{ needs.release.outputs.latest-image-tag }}-python${{ matrix.python-version }}
        build-args: |
          BYTEWAX_VERSION=${{ needs.release.outputs.bytewax-version }}
          PYTHON_VERSION=${{ matrix.python-version }}
        file: Dockerfile.release
    - name: Tag latest
      if: ${{ matrix.python-version == '3.9' }}
      run: |
        docker buildx imagetools create -t ${{ needs.release.outputs.latest-image-tag }} ${{ needs.release.outputs.version-image-tag }}-python${{ matrix.python-version }}


CI

name: CI

on:
  push:
    branches:
      - main
    paths-ignore:
      - ".github/ISSUE_TEMPLATE/**"
  pull_request: {}
  merge_group:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  repo-checks:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: 3.12
      - name: Install uv
        run: pipx install uv
      - uses: extractions/setup-just@v2
      # Need this because we're running `cargo clippy` and `fmt`
      # directly.
      - uses: dsherret/rust-toolchain-file@v1
      - name: Install cbfmt
        run: |
            mkdir /tmp/cbfmt && cd $_
            curl -fsSL -o cbfmt.tar.gz "https://github.com/lukas-reineke/cbfmt/releases/download/v0.2.0/cbfmt_linux-x86_64_v0.2.0.tar.gz"
            tar --strip-components 1 -xvf cbfmt.tar.gz
            mv cbfmt /usr/local/bin/
      - name: Prep venv
        shell: bash
        run: |
          uv venv -p 3.12 venv
          . venv/bin/activate
          uv pip sync --strict requirements/dev.txt
      - name: Check pre-commit hooks
        shell: bash
        run: |
          . venv/bin/activate
          just lint-pc
      - name: Run lints
        shell: bash
        run: |
          . venv/bin/activate
          just lint
      - name: Run Rust tests
        shell: bash
        run: |
          . venv/bin/activate
          just test-rs
      - name: Run documentation tests
        shell: bash
        run: |
          . venv/bin/activate
          just test-doc

  linux:
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
        target: ['x86_64', 'aarch64', 'armv7']
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: pipx install uv
      - uses: extractions/setup-just@v2
      - name: Prep build venv
        shell: bash
        run: |
          uv venv -p ${{ matrix.python-version }} venv/
          . venv/bin/activate
          uv pip sync --strict requirements/build-py${{ matrix.python-version }}.txt
      - name: Build wheel
        uses: PyO3/maturin-action@v1
        env:
          RUSTFLAGS: -C force-frame-pointers=yes
        with:
          target: ${{ matrix.target }}
          args: --release --out dist/ -i ${{ matrix.python-version }}
          sccache: true
      - name: Run Python tests
        # Only run tests for the host architecture, as aarch64 and
        # armv7 are cross-compiled on Ubuntu runners.
        if: ${{ matrix.target == 'x86_64' }}
        shell: bash
        env:
          TEST_KAFKA_BROKER: ${{ secrets.TEST_KAFKA_BROKER }}
          CLUSTER_API_KEY: ${{ secrets.CLUSTER_API_KEY }}
          CLUSTER_API_SECRET: ${{ secrets.CLUSTER_API_SECRET }}
        run: |
          . venv/bin/activate
          WHEEL_FILE=$(ls ./dist/*.whl)
          uv pip install -v --no-deps $WHEEL_FILE
          just test-py
      - name: Upload wheels
        uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: dist

  macos:
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
        target: ['x86_64', 'aarch64']
    runs-on: macos-latest
    env:
      MACOSX_DEPLOYMENT_TARGET: 10.9
    steps:
      - uses: actions/checkout@v3
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: pipx install uv
      - uses: extractions/setup-just@v2
      - name: Prep build venv
        shell: bash
        run: |
          uv venv -p ${{ matrix.python-version }} venv/
          . venv/bin/activate
          uv pip sync --strict requirements/build-py${{ matrix.python-version }}.txt
      - name: Build wheel
        uses: PyO3/maturin-action@v1
        env:
          RUSTFLAGS: -C force-frame-pointers=yes
        with:
          target: ${{ matrix.target }}
          args: --release --out dist/ -i ${{ matrix.python-version }}
      - name: Run Python tests
        # Only run tests for the host architecture, as x86_64 is
        # cross-compiled on macOS runners.
        if: ${{ matrix.target == 'aarch64' }}
        shell: bash
        env:
          TEST_KAFKA_BROKER: ${{ secrets.TEST_KAFKA_BROKER }}
          CLUSTER_API_KEY: ${{ secrets.CLUSTER_API_KEY }}
          CLUSTER_API_SECRET: ${{ secrets.CLUSTER_API_SECRET }}
        run: |
          . venv/bin/activate
          WHEEL_FILE=$(ls ./dist/*.whl)
          uv pip install -v --no-deps $WHEEL_FILE
          just test-py
      - name: Upload wheels
        uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: dist

  windows:
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
    runs-on: windows-latest
    steps:
      - uses: ilammy/msvc-dev-cmd@v1
      - shell: cmd
        run: |
          set PATH=%PATH%;%wix%bin;"%WindowsSdkVerBinPath%"\x86
      - uses: actions/checkout@v3
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: pipx install uv
      - uses: extractions/setup-just@v2
      - name: Prep build venv
        run: |
          uv venv -p ${{ matrix.python-version }} venv/
          venv/Scripts/activate
          uv pip sync --strict requirements/build-py${{ matrix.python-version }}.txt
      - name: Build wheel
        uses: PyO3/maturin-action@v1
        env:
          RUSTFLAGS: -C force-frame-pointers=yes
        with:
          target: ${{ matrix.target }}
          args: --release --out dist/ -i ${{ matrix.python-version }}
          sccache: true
      - name: Run Python tests
        run: |
          venv/Scripts/activate
          $WHEEL_FILE=@((gci -path dist *.whl|Resolve-path -relative) -replace "","")[0]
          uv pip install -v --no-deps $WHEEL_FILE
          just test-py
      - name: Upload wheels
        uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: dist

  upload:
    name: Store wheels in S3
    runs-on: ubuntu-20.04
    if: "github.ref == 'refs/heads/main'"
    needs: [ linux, macos, windows ]
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: wheels
          path: wheels
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.ECR_ACCESS_KEY }}
          aws-secret-access-key: ${{ secrets.ECR_SECRET_KEY }}
          aws-region: us-west-2
      - name: Upload files to S3 with AWS CLI
        run: |
          aws s3 sync ./wheels s3://${{ secrets.WHEELS_S3_BUCKET }}/${{ github.sha }} --delete
          aws s3 ls s3://${{ secrets.WHEELS_S3_BUCKET }}/${{ github.sha }}


benchmarks

name: benchmarks

on:
  push:
    branches:
      - "main"
  pull_request:
  # `workflow_dispatch` allows CodSpeed to trigger backtest
  # performance analysis in order to generate initial data.
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  benchmarks:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install uv
        run: python3 -m pip install uv
      - uses: mkroening/rust-toolchain-toml@main
      - uses: extractions/setup-just@v2
      - name: Prep build venv
        shell: bash
        run: |
          uv venv -p 3.12 venv/
          . venv/bin/activate
          uv pip sync --strict requirements/build-py3.12.txt
      - name: Build wheel
        shell: bash
        run: |
          . venv/bin/activate
          just build
      - name: Run benchmarks
        uses: CodSpeedHQ/action@v2
        with:
          token: ${{ secrets.CODSPEED_TOKEN }}
          run: |
            . venv/bin/activate
            WHEEL_FILE=$(ls ./dist/*.whl)
            uv pip install -v --no-deps $WHEEL_FILE
            just test-benchmark


issue-automation

name: issue-automation

on:
  issues:
    types: [opened]

jobs:
  automate-issues-labels:
    runs-on: ubuntu-latest
    steps:
      - name: initial labeling
        uses: andymckay/labeler@1.0.4
        with:
          add-labels: "needs triage"
          ignore-if-assigned: true


Clear all caches

name: Clear all caches
on:
  workflow_dispatch:
  schedule:
    - cron: '0 0 * * 0' # Every week

permissions:
    actions: write

jobs:
  clear:
    name: Clear all cache
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.0
        with:
          fetch-depth: 1
      - name: Clear cache
        run: gh cache delete --all
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Full CI

# https://docs.github.com/en/actions/using-workflows/reusing-workflows#calling-a-reusable-workflow
# https://docs.github.com/fr/actions/using-workflows/workflow-syntax-for-github-actions#exemple--inclusion-de-chemins-dacc%C3%A8s

name: Full CI
on:
  push:
    branches:
      - main
  pull_request:
  workflow_dispatch:
    inputs:
      run-integration-tests:
        description: "If integration test should be run"
        required: true
        type: boolean
        default: false
      use-cache:
        description: "If cache should be used"
        required: true
        type: boolean
        default: true
      is-dispatch:
        description: "Just to identify manual dispatch"
        required: true
        type: boolean
        default: true
  workflow_call:
    inputs:
      run-integration-tests:
        description: "If integration test should be run"
        required: true
        type: boolean
        default: false
      use-cache:
        description: "If cache should be used"
        required: true
        type: boolean
        default: false
# Concurrency : auto-cancel "old" jobs ie when pushing again
# https://docs.github.com/fr/actions/using-jobs/using-concurrency
concurrency:
  group: ${{ github.workflow }}-${{ inputs.run-integration-tests }}-${{ inputs.is-dispatch }}-${{ github.ref || github.run_id }}
  cancel-in-progress: true
env:
  GSK_DISABLE_ANALYTICS: true
  GSK_DISABLE_SENTRY: true
defaults:
  run:
    shell: bash
jobs:
  build-python:
    name: "Python ${{ matrix.python-version }}${{ matrix.pandas_v1 && ' (Pandas V1)' || ''}}${{ matrix.pydantic_v1 && ' (Pydantic V1)' || ''}} ${{ matrix.langchain_minimal && ' (Minimal langchain)' || ''}} on ${{ matrix.os }}"
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false # Do not stop when any job fails
      matrix:
        python-version: ["3.9", "3.10", "3.11"]
        os: [ubuntu-latest]
        pydantic_v1: [false]
        pandas_v1: [false]
        langchain_minimal: [false]
        # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
        include:
          # - python-version: "3.10" # Deactivating windows-2019, since it's trying to use python 3.7 to install PDM. Maybe try to reactivate later ?
          #   os: windows-2019
          #   pydantic_v1: false
          #   pandas_v1: false
          #   langchain_minimal: false
          - python-version: "3.10"
            os: windows-2022
            pydantic_v1: false
            pandas_v1: false
            langchain_minimal: false
          - python-version: "3.10"
            os: macos-12
            pydantic_v1: false
            pandas_v1: false
            langchain_minimal: false
          - python-version: "3.10"
            os: ubuntu-latest
            pydantic_v1: true
            pandas_v1: false
            langchain_minimal: false
          - python-version: "3.10"
            os: ubuntu-latest
            pydantic_v1: false
            pandas_v1: true
            langchain_minimal: false
          - python-version: "3.10"
            os: ubuntu-latest
            pydantic_v1: false
            pandas_v1: false
            langchain_minimal: true
    continue-on-error: false # https://ncorti.com/blog/howto-github-actions-build-matrix
    steps:
      - name: Check disk space
        run: |
          df -h
      - name: Free disk space
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          sudo swapoff -a
          sudo rm -f /swapfile
          sudo rm -rf /usr/local/lib/android
          sudo apt update && sudo apt remove -y \
          google-cloud-cli microsoft-edge-stable dotnet-sdk-* llvm-* google-chrome-stable temurin-*
          sudo apt autoremove -y
          sudo apt autoclean -y

      - name: Check new disk space
        run: |
          df -h

      - name: Checkout code
        uses: actions/checkout@v4.1.0
        with:
          fetch-depth: 0

      - name: Setup PDM
        uses: pdm-project/setup-pdm@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: false

      - name: Cache Giskard test resources
        uses: actions/cache@v4
        if: ${{ github.event_name == 'pull_request' || inputs.use-cache }}
        with:
          path: ~/.giskard
          key: ${{ matrix.os }}-${{ matrix.python-version }}-python-test-resources-${{ hashFiles('tests/fixtures/**/*py')}}
          restore-keys: ${{ matrix.os }}-${{ matrix.python-version }}-python-giskard-test-resources

      - name: Check again new disk space
        run: |
          df -h

      - name: Install dependencies
        run: |
          pdm install -G :all
          pdm list

      - name: Check yet again new disk space
        run: |
          df -h

      - name: Re-install lightgbm from sources for MacOS
        if: ${{ matrix.os == 'macos-12' }}
        run: |
          pdm run pip uninstall lightgbm -y
          pdm run pip install --no-binary lightgbm lightgbm --config-settings=cmake.define.USE_OPENMP=OFF
      - name: Install pydantic v1
        if: ${{ matrix.pydantic_v1 }}
        run: |
          pdm run pip uninstall pydantic pydantic_core -y
          pdm run pip install "pydantic>=1,<2"

      - name: Check Pydantic installed version
        run: |
          pdm run pip freeze | grep '^pydantic'
          pdm run pip freeze | grep -q '^pydantic==${{ matrix.pydantic_v1 && '1' || '2' }}\.'

      - name: Install langchain minimal version
        if: ${{ matrix.langchain_minimal }}
        run: |
          pdm run pip uninstall langchain -y
          pdm run pip install "langchain==0.0.275"

      - name: Check langchain installed version
        if: ${{ matrix.langchain_minimal }}
        run: |
          pdm run pip freeze | grep '^langchain'
          pdm run pip freeze | grep -q '^langchain==0.0.275'

      - name: Install pandas v1
        if: ${{ matrix.pandas_v1 }}
        run: |
          pdm run pip uninstall pandas -y
          pdm run pip install "pandas<2"

      - name: Check Pandas installed version
        run: |
          pdm run pip freeze | grep '^pandas'
          pdm run pip freeze | grep -q '^pandas==${{ matrix.pandas_v1 && '1' || '2' }}\.'

      - name: Lint code
        run: pdm run lint

      - name: Test code (concurrency)
        run: pdm test-worker

      - name: Test code
        run: pdm test-fast
        env:
          PYTEST_XDIST_AUTO_NUM_WORKERS: 2

      - name: SonarCloud Scan
        if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' && !matrix.langchain_minimal && !matrix.pandas_v1 && !matrix.pydantic_v1 && (github.event.ref == 'refs/heads/main' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }}
        uses: SonarSource/sonarcloud-github-action@v2.2.0
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

      - name: Build
        run: pdm build

      - name: "Python client: archive built artifacts"
        # Upload needs to be done only once by workflo run, so we need to select one only in the matrix
        if: ${{ github.event_name == 'push' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' && !matrix.langchain_minimal && !matrix.pandas_v1 && !matrix.pydantic_v1 }}
        uses: actions/upload-artifact@v4
        with:
          path: dist/*whl

      - name: Run integration tests for python
        if: ${{ inputs.run-integration-tests }}
        env:
          PYTEST_XDIST_AUTO_NUM_WORKERS: 2
        run: pdm test-slow tests/

      - name: "Memory csv"
        if: ${{ always() && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' && !matrix.langchain_minimal && !matrix.pandas_v1 && !matrix.pydantic_v1 }}
        uses: actions/upload-artifact@v4
        with:
          path: memory*.csv
          name: memory-usage
          retention-days: 7

  install-poetry:
    name: "Check if wheel can be installed with using Poetry"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.0
      - name: Setup PDM
        uses: pdm-project/setup-pdm@v4
        with:
          python-version: "3.10"
          cache: false
      - name: Build wheel
        run: pdm build
      - name: Install Poetry
        uses: snok/install-poetry@v1
      - name: Create new project, install wheel and import (Poetry)
        run: |
          poetry new ./install-run
          cd ./install-run
          sed -i 's/^\(python *= *\).*$/\1">=3.10,<3.12"/' pyproject.toml
          poetry add "$(ls ../dist/*.whl)"
          poetry run python -c "import giskard"
  install-pip:
    name: "Check if wheel can be installed with pip"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.0
      - name: Setup PDM
        uses: pdm-project/setup-pdm@v4
        with:
          python-version: "3.10"
          cache: false
      - name: Build wheel
        run: pdm build
      - name: Create new project, install wheel and import (Pip)
        run: |
          python -m venv .venv-test-pip
          source .venv-test-pip/bin/activate
          python -m pip install "$(ls ./dist/*.whl)"
          python -c "import giskard"
  install-pdm:
    name: "Check if wheel can be installed with PDM"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.0
      - name: Setup PDM
        uses: pdm-project/setup-pdm@v4
        with:
          python-version: "3.10"
          version: "2.10.4" # Fix to repair the CI, use latest version when fixed on pdm
          cache: false
      - name: Build wheel
        run: pdm build
      - name: Create new project, install wheel and import (PDM)
        run: |
          mkdir ./install-run
          cd ./install-run
          pdm init --python 3.10 -n .
          sed -i 's/^\(requires-python *= *\).*$/\1">=3.10,<3.12"/' pyproject.toml
          pdm add "$(ls ../dist/*.whl)"
          pdm run python -c "import giskard"
  check-doc:
    name: "Build and check doc"
    runs-on: ubuntu-latest
    steps:
      - name: Check disk space
        run: |
          df -h

      - name: Free disk space
        run: |
          sudo swapoff -a
          sudo rm -f /swapfile
          sudo rm -rf /usr/local/lib/android
          docker rmi $(docker image ls -aq)
          sudo apt update && sudo apt remove -y \
          google-cloud-cli microsoft-edge-stable dotnet-sdk-* llvm-* google-chrome-stable temurin-*
          sudo apt autoremove -y
          sudo apt autoclean -y

      - name: Check new disk space
        run: |
          df -h

      - name: Checkout code
        uses: actions/checkout@v4.1.0
        with:
          fetch-depth: 0

      - name: Setup PDM
        uses: pdm-project/setup-pdm@v4
        with:
          python-version: "3.10"
          cache: false

      - name: Set up Pandoc (needed for doc)
        uses: r-lib/actions/setup-pandoc@v2
        with:
          pandoc-version: "3.1.7" # https://github.com/jgm/pandoc/releases

      - name: Install dependencies
        run: pdm install -G :all

      - name: Check new disk space
        run: |
          df -h

      - name: Build doc
        run: pdm  doc

      - name: Check doc
        run: pdm check-doc


Release and publish a new version

name: Release and publish a new version

on:
  workflow_dispatch:
    inputs:
        version:
          description: 'Tag to be created, in the form X.Y.Z'
          required: true
          type: string

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Check if organization member
        id: is_organization_member
        uses: JamesSingleton/is-organization-member@1.0.1
        with:
          organization: Giskard-AI
          username: ${{ github.actor }}
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Interrupt job
        if: ${{ steps.is_organization_member.outputs.result == 'false' }}
        shell: bash
        run: |
          echo "Job failed due to user not being a member of Giskard-AI organization and the 'safe for build' label not being set on the PR"
          exit 1

      - name: Write release version env vars (with/without v)
        run: |
          VERSION_NAME="v${{ inputs.version }}"
          VERSION_NUMBER="${VERSION_NAME:1}"
          echo "VERSION_NUMBER=${VERSION_NUMBER}" >> $GITHUB_ENV
          echo "VERSION_NAME=${VERSION_NAME}" >> $GITHUB_ENV

      - name: Checkout code
        uses: actions/checkout@v4.1.0
        with:
          fetch-tags: true
          ref: main
          token: ${{ secrets.RELEASE_PAT_TOKEN }} # Needed to trigger other actions

      - name: Edit pyproject.toml
        run:  sed -i 's/^\(version *= *\).*$/\1"${{ env.VERSION_NUMBER }}"/' pyproject.toml

      - name: Remove dark theme logo from README
        run:  sed -i 's/.*#gh-dark-mode-only.*//' README.md

      - name: Setup PDM
        uses: pdm-project/setup-pdm@v4
        with:
          python-version: '3.10'
          cache: false

      - name: "@slack Release process started"
        id: slack
        uses: slackapi/slack-github-action@v1.26.0
        with:
          channel-id: ${{ vars.SLACK_CHANNEL_ID }}
          slack-message: |-
            Release *${{ env.VERSION_NAME }}* is on the way :rocket:
            <${{ github.server_url }}/${{ github.actor }}|@${{ github.actor }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|commit> <!channel>
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

      - name: Configure git
        run:  |
          git config --global user.name 'BotReleaser'
          git config --global user.email 'bot.releaser@users.noreply.github.com'

      - name: Adding file
        run:  |
          git add pyproject.toml
          git fetch --quiet --tags
          git commit -m "${{ env.VERSION_NAME }}" --allow-empty
          git tag ${{ env.VERSION_NAME }}

      - name: Push to main and tags
        run: |
          git push origin main
          git push origin ${{ env.VERSION_NAME }}

      # build .tar.gz sdist tarball
      - name: Build source distribution tarball
        run: pdm build

      - name: Create Github Release
        id: github-release
        uses: softprops/action-gh-release@v2
        with:
          tag_name: ${{ env.VERSION_NAME }}
          fail_on_unmatched_files: true
          generate_release_notes: true
          files: |
            dist/giskard-*.tar.gz
            dist/giskard-*.whl

      - name: Push to Pipy
        run: pdm publish --no-build --username "${{ secrets.PIPY_USERNAME }}" --password "${{ secrets.PIPY_PASSWORD }}"

      - name: Set job success env var
        run: |
          echo "JOB_SUCCESS=true" >> $GITHUB_ENV

      - name: "@slack Share release process completion"
        # cancellable always() https://docs.github.com/en/actions/learn-github-actions/expressions#status-check-functions
        if: ${{ !cancelled() }}
        uses: slackapi/slack-github-action@v1.26.0
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          ON_SUCCESS: |-
            *${{ env.VERSION_NAME }}* has been published to PyPI ! :python: :tada:
            <${{ steps.github-release.outputs.url }}|Release notes> | <https://pypi.org/project/giskard/${{ env.VERSION_NUMBER }}|PyPI> <!channel>
          ON_FAILURE: |-
            Could not publish *${{ env.VERSION_NAME }}* to PyPI :x:
            <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|logs> <!channel>
        with:
          channel-id: ${{ vars.SLACK_CHANNEL_ID }}
          slack-message: ${{ env.JOB_SUCCESS == 'true' && env.ON_SUCCESS || env.ON_FAILURE }}
          # reploy_broadcast == also send to channel
          payload: |
            {
              "thread_ts": "${{ steps.slack.outputs.thread_ts }}",
              "reply_broadcast": true
            }


Lock dependencies

# https://docs.github.com/en/actions/using-workflows/reusing-workflows#calling-a-reusable-workflow
# https://docs.github.com/fr/actions/using-workflows/workflow-syntax-for-github-actions#exemple--inclusion-de-chemins-dacc%C3%A8s

name: Lock dependencies
on:
  pull_request:
    types: [opened, reopened, synchronize, labeled, unlabeled]
  workflow_dispatch:
# Concurrency : auto-cancel "old" jobs ie when pushing again
# https://docs.github.com/fr/actions/using-jobs/using-concurrency
concurrency:
  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
  cancel-in-progress: true
defaults:
  run:
    shell: bash
permissions:
  pull-requests: write
  contents: write

jobs:
  lock-deps:
    if: ${{ !github.event.pull_request || contains( github.event.pull_request.labels.*.name, 'Lockfile') }}
    name: "Lock dependencies using PDM"
    runs-on: ubuntu-latest
    continue-on-error: false # https://ncorti.com/blog/howto-github-actions-build-matrix
    steps:
      - name: Check if organization member
        id: is_organization_member
        uses: JamesSingleton/is-organization-member@1.0.1
        with:
          organization: Giskard-AI
          username: ${{ github.actor }}
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Interrupt job
        if: ${{ steps.is_organization_member.outputs.result == 'false' }}
        shell: bash
        run: |
          echo "Job failed due to user not being a member of Giskard-AI organization and the 'safe for build' label not being set on the PR"
          exit 1

      - name: Extract branch name
        shell: bash
        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
        id: extract_branch

      - name: Checkout code
        uses: actions/checkout@v4.1.0
        with:
          token: ${{ secrets.RELEASE_PAT_TOKEN }} # Needed to trigger other actions
          ref: ${{ steps.extract_branch.outputs.branch }}

      - name: Setup PDM
        uses: pdm-project/setup-pdm@v4
        with:
          python-version: "3.10"
          cache: false

      - name: Install dependencies
        run: rm -rf pdm.lock && pdm lock -G :all

      - name: Configure git
        run:  |
          git config --global user.name 'BotLocker'
          git config --global user.email 'bot.locker@users.noreply.github.com'

      - name: Adding file
        run:  |
          git add pdm.lock
          git commit -m "Regenerating pdm.lock" --allow-empty

      - name: Remove label
        if: ${{ github.event.pull_request}}
        run: |
          gh pr edit --remove-label Lockfile
        env:
          GH_TOKEN: ${{ github.token }}

      - name: Push to target branch
        run: |
          git push origin ${{ steps.extract_branch.outputs.branch }}


Pre-commit checks

name: Pre-commit checks
on:
  push:
    branches:
      - main
  pull_request:
  workflow_dispatch:

env:
  GSK_DISABLE_ANALYTICS: true
  SENTRY_ENABLED: false
  GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
defaults:
  run:
    shell: bash
jobs:
  pre-commit:
    name: Pre-commit checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
      - uses: pre-commit/action@v3.0.1
        env:
          SKIP: ggshield


Nightly test

name: Nightly test
on:
  schedule:
    - cron: '0 1 * * *' # Always run at 1:00 a.m.
  workflow_dispatch:

jobs:
  call-full-ci:
    uses: ./.github/workflows/build-python.yml
    with:
      run-integration-tests: true
      use-cache: false


Pegasus CI

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions

name: Pegasus CI

on:
  pull_request:
    branches:
      - master

jobs:
  build:

    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        sudo apt -q update
        sudo apt install -y libfftw3-dev default-jdk git
        python -m pip install --upgrade pip
        python -m pip install flake8 pytest setuptools wheel cython
        python -m pip install zarr
        python -m pip install git+https://github.com/lilab-bcb/pegasusio@master
        python -m pip install -e .[all]
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
    - name: Fetch test data
      run: |
        sudo apt install -y git
        git clone https://github.com/lilab-bcb/pegasus-test-data.git ./tests/data
    - name: Pipeline test
      run: |
        bash tests/run_pipeline.sh
        pytest tests/test_pipeline.py
    - name: One sample input test
      run: |
        bash tests/run_one_sample.sh
    - name: iNMF test
      run: |
        bash tests/run_inmf.sh
        pytest tests/test_inmf.py


CI to Docker hub [Dev-only latest tag]

name: CI to Docker hub [Dev-only latest tag]

on:
  workflow_dispatch:

jobs:
  docker-build-webapp:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v3
        with:
          images: chaosgenius/chaosgenius-webapp
          tags: latest

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1

      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Build and push
        uses: docker/build-push-action@v2
        with:
          context: frontend
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          file: frontend/Dockerfile.prod

  docker-build-server:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v3
        with:
          images: chaosgenius/chaosgenius-server
          tags: latest

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1

      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Build and push
        uses: docker/build-push-action@v2
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}


Greetings

name: Greetings

on: [pull_request, issues]

jobs:
  greeting:
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    steps:
    - uses: actions/first-interaction@v1
      continue-on-error: true
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        issue-message: 'Hello there, thanks for opening your first issue. We welcome you to the community!'
        pr-message: 'Hello there, thanks for opening your first Pull Request. Someone will review it soon.'


Python application

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Python application

on:
  push:
    branches:
    - main
    - develop
    - release/*
  pull_request:
    branches:
    - main
    - develop
    - release/*
jobs:
  build:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python 3.8
      uses: actions/setup-python@v2
      with:
        python-version: 3.8
    - name: Install dependencies
      run: |
        pip install --upgrade pip
        pip install wheel flake8 pytest
        if [ -f requirements/dev.txt ]; then pip install -r requirements/dev.txt; fi
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 chaos_genius --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings.
        flake8 chaos_genius --count --exit-zero --statistics
    - name: Test with pytest
      run: |
        pytest
    - name: Check diverged migrations
      run: |
        ./scripts/check_diverged_migrations.sh


release

name: release

on:
  release:
    types: [published]

jobs:
  release:
    runs-on: ubuntu-latest

    environment:
      name: pypi
      url: https://pypi.org/p/scvi-tools

    permissions:
      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing

    steps:
      - uses: actions/checkout@v4
        with:
          filter: blob:none
          fetch-depth: 0

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: "pip"

      - name: Install dependencies
        run: pip install build

      - name: Build package
        run: python -m build

      - uses: pypa/gh-action-pypi-publish@release/v1


build

name: build

on:
  push:
    branches: [main, "[0-9]+.[0-9]+.x"]
  pull_request:
    branches: [main, "[0-9]+.[0-9]+.x"]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  package:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: "pip"
          cache-dependency-path: "**/pyproject.toml"

      - name: Install build dependencies
        run: python -m pip install --upgrade pip wheel twine build

      - name: Build package
        run: python -m build

      - name: Check package
        run: twine check --strict dist/*.whl


test (cuda)

name: test (cuda)

on:
  pull_request:
    branches: [main, "[0-9]+.[0-9]+.x"]
    types: [labeled, synchronize, opened]
  schedule:
    - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    # if PR has label "cuda tests" or "all tests" or if scheduled or manually triggered
    if: >-
      (
        contains(github.event.pull_request.labels.*.name, 'cuda tests') ||
        contains(github.event.pull_request.labels.*.name, 'all tests') ||
        contains(github.event_name, 'schedule') ||
        contains(github.event_name, 'workflow_dispatch')
      )

    runs-on: [self-hosted, Linux, X64, CUDA]

    defaults:
      run:
        shell: bash -e {0} # -e to fail on error

    strategy:
      fail-fast: false
      matrix:
        python: ["3.11"]
        cuda: ["11", "12"]

    container:
      image: scverse/scvi-tools:py${{ matrix.python }}-cu${{ matrix.cuda }}-base
      options: --user root --gpus all

    name: integration

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python }}

    steps:
      - uses: actions/checkout@v4

      - run: |
          python -m pip install --upgrade pip wheel uv
          python -m uv pip install --system "scvi-tools[tests] @ ."

      - name: Run pytest
        env:
          MPLBACKEND: agg
          PLATFORM: ${{ matrix.os }}
          DISPLAY: :42
          COLUMNS: 120
        run: |
          coverage run -m pytest -v --color=yes --accelerator cuda --devices auto
          coverage report

      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


test

name: test

on:
  push:
    branches: [main, "[0-9]+.[0-9]+.x"]
  pull_request:
  schedule:
    - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    runs-on: ${{ matrix.os }}

    defaults:
      run:
        shell: bash -e {0} # -e to fail on error

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python: ["3.9", "3.10", "3.11"]

    name: integration

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "**/pyproject.toml"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip wheel uv
          python -m uv pip install --system "scvi-tools[tests] @ ."

      - name: Run pytest
        env:
          MPLBACKEND: agg
          PLATFORM: ${{ matrix.os }}
          DISPLAY: :42
          COLUMNS: 120
        run: |
          coverage run -m pytest -v --color=yes
          coverage report

      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


test (macos)

name: test (macos)

on:
  pull_request:
    branches: [main, "[0-9]+.[0-9]+.x"]
    types: [labeled, synchronize, opened]
  schedule:
    - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    # if PR has label "macos tests" or "all tests" or if scheduled or manually triggered or on push
    if: >-
      (
        contains(github.event.pull_request.labels.*.name, 'macos tests') ||
        contains(github.event.pull_request.labels.*.name, 'all tests') ||
        contains(github.event_name, 'schedule') ||
        contains(github.event_name, 'workflow_dispatch') ||
        contains(github.event_name, 'push')
      )

    runs-on: ${{ matrix.os }}

    defaults:
      run:
        shell: bash -e {0} # -e to fail on error

    strategy:
      fail-fast: false
      matrix:
        os: [macos-14]
        python: ["3.9", "3.10", "3.11"]

    name: integration

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "**/pyproject.toml"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip wheel uv
          python -m uv pip install --system "scvi-tools[tests] @ ."

      - name: Run pytest
        env:
          MPLBACKEND: agg
          PLATFORM: ${{ matrix.os }}
          DISPLAY: :42
          COLUMNS: 120
        run: |
          coverage run -m pytest -v --color=yes
          coverage report

      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


test (windows)

name: test (windows)

on:
  pull_request:
    branches: [main, "[0-9]+.[0-9]+.x"]
    types: [labeled, synchronize, opened]
  schedule:
    - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    # if PR has label "windows tests" or "all tests" or if scheduled or manually triggered or on push
    if: >-
      (
        contains(github.event.pull_request.labels.*.name, 'windows tests') ||
        contains(github.event.pull_request.labels.*.name, 'all tests') ||
        contains(github.event_name, 'schedule') ||
        contains(github.event_name, 'workflow_dispatch') ||
        contains(github.event_name, 'push')
      )

    runs-on: ${{ matrix.os }}

    defaults:
      run:
        shell: bash -e {0} # -e to fail on error

    strategy:
      fail-fast: false
      matrix:
        os: [windows-latest]
        python: ["3.9", "3.10", "3.11"]

    name: integration

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "**/pyproject.toml"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip wheel uv
          python -m uv pip install --system "scvi-tools[tests] @ ."

      - name: Run pytest
        env:
          MPLBACKEND: agg
          PLATFORM: ${{ matrix.os }}
          DISPLAY: :42
          COLUMNS: 120
        run: |
          coverage run -m pytest -v --color=yes
          coverage report

      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


test (private)

name: test (private)

on:
  pull_request:
    branches: [main, "[0-9]+.[0-9]+.x"]
    types: [labeled, synchronize, opened]
  schedule:
    - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    # if PR has label "private tests" or "all tests" or if scheduled or manually triggered or on push
    if: >-
      (
        contains(github.event.pull_request.labels.*.name, 'private tests') ||
        contains(github.event.pull_request.labels.*.name, 'all tests') ||
        contains(github.event_name, 'schedule') ||
        contains(github.event_name, 'workflow_dispatch') ||
        contains(github.event_name, 'push')
      )

    runs-on: ${{ matrix.os }}

    defaults:
      run:
        shell: bash -e {0} # -e to fail on error

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python: ["3.11"]

    permissions:
      id-token: write

    name: unit

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python }}

    steps:
      - uses: actions/checkout@v4

      - uses: aws-actions/configure-aws-credentials@v4
        with:
          audience: sts.amazonaws.com
          role-to-assume: arn:aws:iam::${{ secrets.AWS_IAM }}:role/scvi-tools_private_test
          aws-region: us-west-1
          role-duration-seconds: 3600 # generous upper limit for the duration of the test

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "**/pyproject.toml"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip wheel uv
          python -m uv pip install --system "scvi-tools[tests] @ ."

      - name: Run pytest
        env:
          MPLBACKEND: agg
          PLATFORM: ${{ matrix.os }}
          DISPLAY: :42
          HF_API_TOKEN: ${{ secrets.HF_API_TOKEN }}
          COLUMNS: 120
        run: |
          coverage run -m pytest -v --color=yes --private
          coverage report

      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


test (resolution)

name: test (resolution)

on:
  pull_request:
    branches: [main, "[0-9]+.[0-9]+.x"]
    types: [labeled, synchronize, opened]
  schedule:
    - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    # if PR has label "resolution tests" or "all tests" or if scheduled or manually triggered
    if: >-
      (
        contains(github.event.pull_request.labels.*.name, 'resolution tests') ||
        contains(github.event.pull_request.labels.*.name, 'all tests') ||
        contains(github.event_name, 'schedule') ||
        contains(github.event_name, 'workflow_dispatch')
      )

    runs-on: ${{ matrix.os }}

    defaults:
      run:
        shell: bash -e {0} # -e to fail on error

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python: ["3.9", "3.10", "3.11", "3.12"]
        install-flags:
          [
            "--prerelease if-necessary-or-explicit",
            "--resolution lowest-direct",
            "--resolution lowest",
          ]

    name: integration

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "**/pyproject.toml"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip wheel uv
          python -m uv pip install ${{ matrix.install-flags }} --system "scvi-tools[tests] @ ."

      - name: Run pytest
        env:
          MPLBACKEND: agg
          PLATFORM: ${{ matrix.os }}
          DISPLAY: :42
          COLUMNS: 120
        run: |
          coverage run -m pytest -v --color=yes
          coverage report

      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


test-docs

name: test-docs
run-name: Documentation tests

on:
  pull_request:
  push:
    branches: [main]

permissions:
  contents: read

jobs:
  doc-test:
    name: Sphinx-${{ matrix.builder }}
    runs-on: ubuntu-latest
    continue-on-error: ${{ matrix.experimental || false }}
    strategy:
      fail-fast: false
      matrix:
        builder: [ html ]
        include:
          # Run default html builder with warnings as error
          - builder: html
            args: -W
    steps:
      - uses: actions/checkout@v4
      - name: Run sphinx builder ${{ matrix.builder }}
        run: |
          pip install hatch
          hatch run docs:${{ matrix.builder }} ${{ matrix.args }}


pre-commit

name: pre-commit

on:
  pull_request:
  push:
    branches: [main]

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    env:
      SKIP: no-commit-to-branch
    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with:
        python-version: |
          3.9
          3.x
    - uses: pre-commit/action@v3.0.0


release

name: release

on:
  release:
    types: [published]
  workflow_dispatch:
    inputs:
      ref:
        description: 'Tag to release'
        required: true

jobs:
  release:
    runs-on: ubuntu-latest

    # Needed for trusted publishing
    permissions:
      id-token: write

    steps:

      # Check out the repo
      - uses: actions/checkout@v4
        if: ${{ github.event_name == 'release' }}
      - uses: actions/checkout@v4
        if: ${{ github.event_name == 'workflow_dispatch' }}
        with:
          ref: ${{ github.event.inputs.ref }}

      # Setup python
      - uses: actions/setup-python@v5
        with:
          python-version: "3.x"

      # Build it
      - name: Build a binary wheel and a source tarball
        run: |
            python -m pip install hatch
            hatch run docs:man
            hatch build

      # Publish it
      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1


Differential ShellCheck

name: Differential ShellCheck
on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

permissions:
  contents: read

jobs:
  lint:
    runs-on: ubuntu-latest

    permissions:
      security-events: write

    steps:
      - name: Repository checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - id: ShellCheck
        name: Differential ShellCheck
        uses: redhat-plumbers-in-action/differential-shellcheck@v4
        with:
          exclude-path: |
            tests/**
            examples/**/test.sh
            tmt/templates/**
          token: ${{ secrets.GITHUB_TOKEN }}

      - if: ${{ always() }}
        name: Upload artifact with ShellCheck defects in SARIF format
        uses: actions/upload-artifact@v3
        with:
          name: Differential ShellCheck SARIF
          path: ${{ steps.ShellCheck.outputs.sarif }}


Lock threads

name: 'Lock threads'
# Lock closed issues that have not received any further activity for
# two weeks. This does not close open issues, only humans may do that.
# We find that it is easier to respond to new issues with fresh examples
# rather than continuing discussions on old issues.
on:
  schedule:
    - cron: '0 0 * * *'
permissions:
  issues: write
  pull-requests: write
concurrency:
  group: lock
jobs:
  lock:
    runs-on: ubuntu-latest
    steps:
      - uses: dessant/lock-threads@7de207be1d3ce97a9abe6ff1306222982d1ca9f9
        with:
          issue-inactive-days: 14
          pr-inactive-days: 14


Publish

name: Publish
on:
  push:
    tags:
      - '*'
jobs:
  build:
    runs-on: ubuntu-latest
    outputs:
      hash: ${{ steps.hash.outputs.hash }}
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
        with:
          python-version: '3.x'
          cache: pip
          cache-dependency-path: requirements*/*.txt
      - run: pip install -r requirements/build.txt
      # Use the commit date instead of the current date during the build.
      - run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV
      - run: python -m build
      # Generate hashes used for provenance.
      - name: generate hash
        id: hash
        run: cd dist && echo "hash=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT
      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
        with:
          name: dist
          path: ./dist
  provenance:
    needs: [build]
    permissions:
      actions: read
      id-token: write
      contents: write
    # Can't pin with hash due to how this workflow works.
    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.9.0
    with:
      base64-subjects: ${{ needs.build.outputs.hash }}
  create-release:
    # Upload the sdist, wheels, and provenance to a GitHub release. They remain
    # available as build artifacts for a while as well.
    needs: [provenance]
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
      - name: create release
        run: >
          gh release create --draft --repo ${{ github.repository }}
          ${{ github.ref_name }}
          *.intoto.jsonl/* dist/*
        env:
          GH_TOKEN: ${{ github.token }}
  publish-pypi:
    needs: [provenance]
    # Wait for approval before attempting to upload to PyPI. This allows reviewing the
    # files in the draft release.
    environment: publish
    runs-on: ubuntu-latest
    permissions:
      id-token: write
    steps:
      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
      - uses: pypa/gh-action-pypi-publish@f946db0f765b9ae754e44bfd5ae5b8b91cfb37ef
        with:
          repository-url: https://test.pypi.org/legacy/
      - uses: pypa/gh-action-pypi-publish@f946db0f765b9ae754e44bfd5ae5b8b91cfb37ef


Tests

name: Tests
on:
  push:
    branches:
      - main
      - '*.x'
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - '*.rst'
  pull_request:
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - '*.rst'
jobs:
  tests:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - {name: Linux, python: '3.12', os: ubuntu-latest, tox: py312}
          - {name: Windows, python: '3.12', os: windows-latest, tox: py312}
          - {name: Mac, python: '3.12', os: macos-latest, tox: py312}
          - {name: '3.11', python: '3.11', os: ubuntu-latest, tox: py311}
          - {name: '3.10', python: '3.10', os: ubuntu-latest, tox: py310}
          - {name: '3.9', python: '3.9', os: ubuntu-latest, tox: py39}
          - {name: '3.8', python: '3.8', os: ubuntu-latest, tox: py38}
          - {name: 'PyPy', python: 'pypy-3.10', os: ubuntu-latest, tox: pypy310}
          - {name: 'Minimum Versions', python: '3.12', os: ubuntu-latest, tox: py312-min}
          - {name: 'Development Versions', python: '3.8', os: ubuntu-latest, tox: py38-dev}
          - {name: Typing, python: '3.12', os: ubuntu-latest, tox: typing}
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
        with:
          python-version: ${{ matrix.python }}
          cache: 'pip'
          cache-dependency-path: requirements*/*.txt
      - name: cache mypy
        uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2
        with:
          path: ./.mypy_cache
          key: mypy|${{ matrix.python }}|${{ hashFiles('pyproject.toml') }}
        if: matrix.tox == 'typing'
      - run: pip install tox
      - run: tox run -e ${{ matrix.tox }}


Test all warehouse platforms

name: Test all warehouse platforms
on:
  pull_request:
    branches: ["master"]
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
    inputs:
      dbt-version:
        type: string
        required: false
        description: dbt's version to test with
      elementary-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'elementary' repository
      dbt-data-reliability-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'dbt-data-reliability' repository

  workflow_call:
    inputs:
      dbt-version:
        type: string
        required: false
      elementary-ref:
        type: string
        required: false
      dbt-data-reliability-ref:
        type: string
        required: false

jobs:
  test:
    strategy:
      fail-fast: false
      matrix:
        dbt-version:
          ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) ||
          ! contains(github.event_name, 'pull_request') && fromJSON('["1.3.0", "latest_pre"]') ||
          fromJSON('["latest_pre"]') }}
        warehouse-type:
          [
            postgres,
            snowflake,
            bigquery,
            redshift,
            databricks,
            databricks_catalog,
            athena,
            trino,
          ]
        include:
          # If we're not running on a specific dbt version, then always add postgres on 1.3.0
          - dbt-version: "${{ inputs.dbt-version || '1.3.0' }}"
            warehouse-type: postgres
        exclude:
          - dbt-version: "1.3.0"
            warehouse-type: athena
          - dbt-version: "1.3.0"
            warehouse-type: trino
    uses: ./.github/workflows/test-warehouse.yml
    with:
      warehouse-type: ${{ matrix.warehouse-type }}
      dbt-version: ${{ matrix.dbt-version }}
      elementary-ref: ${{ inputs.elementary-ref }}
      dbt-data-reliability-ref: ${{ inputs.dbt-data-reliability-ref }}
    secrets: inherit

  notify_failures:
    name: Notify Slack
    secrets: inherit
    needs: [test]
    if: |
      always() &&
      ! contains(needs.test.result, 'success') &&
      ! contains(needs.test.result, 'cancelled') &&
      contains(github.event_name, 'schedule') &&
      ! cancelled()
    uses: elementary-data/elementary/.github/workflows/notify_slack.yml@master
    with:
      result: "failure"
      run_id: ${{ github.run_id }}
      workflow_name: ${{ github.workflow }}


Test warehouse platform

name: Test warehouse platform
on:
  workflow_dispatch:
    inputs:
      warehouse-type:
        type: choice
        required: true
        description: Type of warehouse platform
        options:
          - postgres
          - snowflake
          - bigquery
          - redshift
          - databricks
          - databricks_catalog
          - spark
          - athena
          - trino
      elementary-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'elementary' repository
      dbt-data-reliability-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'dbt-data-reliability' repository
      dbt-version:
        type: string
        required: false
        default: "latest_official"
        description: dbt's version to test with

  workflow_call:
    inputs:
      warehouse-type:
        type: string
        required: true
      elementary-ref:
        type: string
        required: false
      dbt-data-reliability-ref:
        type: string
        required: false
      dbt-version:
        type: string
        default: "latest_official"
        required: false

env:
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
  TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests

jobs:
  test:
    runs-on: ubuntu-20.04
    concurrency:
      # This is what eventually defines the schema name in the data platform.
      group: tests_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}_${{ github.head_ref || github.ref_name }}
      cancel-in-progress: true
    steps:
      - name: Checkout Elementary
        uses: actions/checkout@v3
        with:
          repository: elementary-data/elementary
          path: elementary
          ref: ${{ inputs.elementary-ref }}

      - name: Checkout dbt package
        uses: actions/checkout@v3
        with:
          path: dbt-data-reliability
          ref: ${{ inputs.dbt-data-reliability-ref }}

      - name: Start Postgres
        if: inputs.warehouse-type == 'postgres'
        working-directory: ${{ env.TESTS_DIR }}
        run: docker-compose up -d postgres

      - name: Start Trino
        if: inputs.warehouse-type == 'trino'
        working-directory: ${{ env.TESTS_DIR }}
        run: docker-compose -f docker-compose-trino.yml up -d

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"
          cache: "pip"

      - name: Install Spark requirements
        if: inputs.warehouse-type == 'spark'
        run: sudo apt-get install python-dev libsasl2-dev gcc

      - name: Install compatible databricks connector (not limited in older dbt-databricks versions)
        if: startsWith(inputs.warehouse-type, 'databricks') && inputs.dbt-version < '1.7.0'
        run: pip install databricks-sql-connector==2.9.3

      - name: Install dbt
        run:
          pip install${{ (inputs.dbt-version == 'latest_pre' && ' --pre') || '' }}
          "dbt-core${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }}"
          "dbt-${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || (inputs.warehouse-type == 'spark' && 'spark[PyHive]') || (inputs.warehouse-type == 'athena' && 'athena-community') || inputs.warehouse-type }}${{ (!startsWith(inputs.dbt-version, 'latest') && format('<={0}', inputs.dbt-version)) || '' }}"

      - name: Install Elementary
        run: pip install "./elementary[${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || inputs.warehouse-type }}]"

      - name: Install dependencies
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          dbt deps --project-dir dbt_project
          pip install -r requirements.txt

      - name: Write dbt profiles
        env:
          PROFILES_YML: ${{ secrets.CI_PROFILES_YML }}
        run: |
          mkdir -p ~/.dbt
          DBT_VERSION=$(pip show dbt-core | grep -i version | awk '{print $2}' | sed 's/\.//g')
          UNDERSCORED_REF_NAME=$(echo "${{ inputs.warehouse-type }}_dbt_${DBT_VERSION}_${BRANCH_NAME}" | awk '{print tolower($0)}' | head -c 40 | sed "s/-/_/g")
          echo "$PROFILES_YML" | base64 -d | sed "s/<SCHEMA_NAME>/dbt_pkg_$UNDERSCORED_REF_NAME/g" > ~/.dbt/profiles.yml

      - name: Check DWH connection
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          dbt debug -t "${{ inputs.warehouse-type }}"

      - name: Test
        working-directory: "${{ env.TESTS_DIR }}/tests"
        run: py.test -n8 -vvv --target "${{ inputs.warehouse-type }}" --junit-xml=test-results.xml --html=detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}.html --self-contained-html

      - name: Upload test results
        if: always()
        uses: pmeier/pytest-results-action@main
        with:
          path: ${{ env.TESTS_DIR }}/tests/test-results.xml
          summary: true
          display-options: fEX
          fail-on-empty: true

      - name: Upload HTML report
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}
          path: ${{ env.TESTS_DIR }}/tests/detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}.html


Bump version

name: Bump version
on:
  workflow_dispatch:
    inputs:
      dbt-package-version:
        type: string
        required: true
        description: New elementary package version

  workflow_call:
    inputs:
      dbt-package-version:
        type: string
        required: true

jobs:
  validate-version:
    runs-on: ubuntu-latest
    outputs:
      validated-dbt-package-version: ${{ steps.validate-dbt-package-input.outputs.dbt-package-validation }}
    steps:
      - name: validate dbt package version
        id: validate-dbt-package-input
        run: echo "dbt-package-validation=$(echo ${{ inputs.dbt-package-version }} | sed -n '/^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/p')" >> $GITHUB_OUTPUT
      - name: echo versions
        run: |
          echo "dbt package version: ${{ steps.validate-dbt-package-input.outputs.dbt-package-validation }}"
      - name: fail on invalid input
        if: ${{ steps.validate-dbt-package-input.outputs.dbt-package-validation == '' }}
        uses: actions/github-script@v6
        with:
          script: |
            core.setFailed("Invalid version input - ${{ inputs.dbt-package-version }}")

  bump-version:
    needs: validate-version
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
      - name: Create release branch
        run: git checkout -b release/${{ inputs.dbt-package-version }}
      - name: Initial config
        run: |
          git config user.name "GitHub Actions"
          git config user.email noreply@github.com
      - name: Bump package version
        run: |
          sed -i 's/version: "[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*"$/version: "${{ inputs.dbt-package-version }}"/' ./dbt_project.yml
      - name: Bump readme package version
        run: |
          sed -i 's/version: [0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/version: ${{ inputs.dbt-package-version }}/' ./README.md
      - name: Commit changes
        run: git commit -am "release ${{ inputs.dbt-package-version }}"
      - name: Push code
        run: git push origin release/${{ inputs.dbt-package-version }}

  create-pr:
    needs: bump-version
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: create pull request
        uses: repo-sync/pull-request@v2
        with:
          source_branch: "release/${{ inputs.dbt-package-version }}"
          destination_branch: "master"
          pr_title: "release/${{ inputs.dbt-package-version }}"
          pr_body: "Open automatically using bump version workflow"
          github_token: ${{ secrets.GITHUB_TOKEN }}


Remind docs and tests

name: Remind docs and tests
on:
  pull_request:
    branches: ["master"]
jobs:
  run:
    runs-on: ubuntu-latest
    steps:
      - uses: wow-actions/auto-comment@v1
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          pullRequestOpened: |
            👋 @{{ author }}
            Thank you for raising your pull request.
            Please make sure to add tests and document all user-facing changes.
            You can do this by editing the `docs` files in the [`elementary`](https://github.com/elementary-data/elementary) repository.


Run pre-commit hooks

name: Run pre-commit hooks
on:
  workflow_dispatch:
  pull_request:

jobs:
  code-quality:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Elementary
        uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4.3.0
        with:
          python-version: "3.8"

      - name: Install dev requirements
        run: pip install -r dev-requirements.txt

      - name: Run pre-commit hooks
        run: pre-commit run --all-files --show-diff-on-failure


name: Test all warehouse platforms on dbt pre-releases
on:
  workflow_dispatch:

jobs:
  test:
    uses: ./.github/workflows/test-all-warehouses.yml
    secrets: inherit
    with:
      dbt-version: latest_pre


karrio-build

name: karrio-build

on:
  push:
    branches: [ "main" ]

jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.changes.outputs.version }}
    steps:
    - uses: actions/checkout@v3

    - uses: dorny/paths-filter@v2
      id: changes
      with:
        filters: |
          version:
            - 'apps/api/karrio/server/VERSION'

  server-build:

    needs: changes
    if: ${{ needs.changes.outputs.version == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
      id-token: write

    steps:
      - uses: actions/checkout@v3

      - id: get_tag
        run: |
          cat ./apps/api/karrio/server/VERSION
          echo "tag=$(cat ./apps/api/karrio/server/VERSION)" >> "$GITHUB_ENV"

      - name: Build karrio server image
        run: |
          echo 'Building karrio server:${{ env.tag }}...'
          ./bin/build-server-image-from-source ${{ env.tag }}

      - name: Push karrio server image
        run: |
          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}

          echo 'Pushing karrio server:${{ env.tag }}...'
          docker push karrio/server:${{ env.tag }}
          docker tag karrio/server:${{ env.tag }} karrio/server:latest-rc
          docker push karrio/server:latest-rc

  dashboard-build:

    needs: changes
    if: ${{ needs.changes.outputs.version == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
      id-token: write

    steps:
      - uses: actions/checkout@v3

      - id: get_tag
        run: |
          cat ./apps/api/karrio/server/VERSION
          echo "tag=$(cat ./apps/api/karrio/server/VERSION)" >> "$GITHUB_ENV"

      - name: Build karrio dashboard image
        run: |
          echo 'Building karrio dashboard:${{ env.tag }}...'
          ./bin/build-dashboard-image ${{ env.tag }}

      - name: Push karrio dashboard image
        run: |
          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}

          echo 'Pushing karrio dashboard:${{ env.tag }}...'
          docker push karrio/dashboard:${{ env.tag }}
          docker tag karrio/dashboard:${{ env.tag }} karrio/dashboard:latest-rc
          docker push karrio/dashboard:latest-rc


karrio-tests

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: karrio-tests

on: [push]

jobs:
  sdk-tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.12"]

    steps:
      - uses: actions/checkout@v2
      - name: Run Tests
        run: |
          ./bin/setup-sdk-env &&
          ./bin/run-sdk-typecheck &&
          ./bin/run-sdk-tests

  server-tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.12"]

    steps:
      - uses: actions/checkout@v2
      - name: Run Tests
        run: |
          ./bin/setup-server-env &&
          ./bin/migrate &&
          ./bin/run-server-tests

  dashbaord-ci:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
      - name: Install
        run: |
          npm install
      - name: Test Build
        run: |
          npm run build -w apps/dashboard


Release

---
name: Release

on:
  workflow_dispatch:

jobs:
  release:
    runs-on: ubuntu-latest
    environment: deployment
    name: Build, publish, and release
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          ssh-key: ${{ secrets.DEPLOY_KEY }}

      - name: Setup Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Cache pip repository
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}

      - name: Prepare python environment
        run: |
          pip install -r requirements.txt
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project true

      - name: Cache poetry virtual environment
        uses: actions/cache@v3
        with:
          path: .venv
          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}

      - name: Configure git
        run: |
          git config user.name "${{ github.actor }}"
          git config user.email "${{ github.actor }}@users.noreply.github.com"

      - name: Determine version and create changelog
        id: bumper
        uses: tomerfi/version-bumper-action@1.2.2

      - name: Set new project version
        uses: sandstromviktor/toml-editor@2.0.0
        with:
          file: pyproject.toml
          key: tool.poetry.version
          value: ${{ steps.bumper.outputs.new_version }}

      - name: Commit, tag, and push
        # yamllint disable rule:line-length
        # editorconfig-checker-disable
        run: |
          git add pyproject.toml
          git commit -m "build: bump version to ${{ steps.bumper.outputs.new_version }} [skip ci]"
          git push
          git tag ${{ steps.bumper.outputs.new_version }} -m "${{ steps.bumper.outputs.new_version }}"
          git push origin ${{ steps.bumper.outputs.new_version }}
        # yamllint enable rule:line-length
        # editorconfig-checker-enable

      - name: Verify documentation site build
        run: |
          make install-docs-only-deps
          make docs-build

      - name: Publish build to PyPi
        run: PYPI_TOKEN="${{ secrets.PYPI_TOKEN }}" make publish

      - name: Set development project version
        uses: sandstromviktor/toml-editor@2.0.0
        with:
          file: pyproject.toml
          key: tool.poetry.version
          value: ${{ steps.bumper.outputs.next_dev_iteration }}

      - name: Commit and push
        # yamllint disable rule:line-length
        run: |
          git add pyproject.toml
          git commit -m "build: bump version to ${{ steps.bumper.outputs.next_dev_iteration }} [skip ci]"
          git push
        # yamllint enable rule:line-length

      - name: Create a release
        id: gh_release
        uses: actions/github-script@v6
        with:
          github-token: ${{ secrets.RELEASE_PAT }}
          script: |
            const repo_name = context.payload.repository.full_name
            const response = await github.request('POST /repos/' + repo_name + '/releases', {
              tag_name: '${{ steps.bumper.outputs.new_version }}',
              generate_release_notes: true
            })
            core.setOutput('html_url', response.data.html_url)


Stage

---
name: Stage

on:
  workflow_dispatch:
  push:
    branches:
      - dev
    paths-ignore:
      - ".github"
      - ".vscode"

jobs:
  stage:
    runs-on: ubuntu-latest
    environment: staging
    name: Stage project
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Setup timezone
        uses: zcong1993/setup-timezone@v2.0.0
        with:
          timezone: Asia/Jerusalem

      - name: Setup Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Cache pip repository
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}

      - name: Prepare python environment
        run: |
          pip install -r requirements.txt
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project true

      - name: Cache poetry virtual environment
        uses: actions/cache@v3
        with:
          path: .venv
          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}

      - name: Install, test with coverage report, and build
        run: |
          make install-all-deps
          make test-with-coverage-reports
          make build

      - name: Push to CodeCov
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: coverage.xml
          fail_ci_if_error: true


Pages Deploy

---
name: Pages Deploy

on:
  release:
    types: [published]

jobs:
  deploy-pages:
    runs-on: ubuntu-latest
    environment: github-pages
    name: Build documentation site and deploy to GH-Pages
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3
        with:
          ref: ${{ github.ref }}

      - name: Setup Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Cache pip repository
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}

      - name: Prepare python environment
        run: |
          pip install -r requirements.txt
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project true

      - name: Cache poetry virtual environment
        uses: actions/cache@v3
        with:
          path: .venv
          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}

      - name: Build documentation site
        run: |
          make install-docs-only-deps
          make docs-build

      - name: Deploy to GH-Pages
        uses: peaceiris/actions-gh-pages@v3.9.3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./site
          cname: aioswitcher.tomfi.info
          commit_message: 'docs: deployed documentation site '


Pull request build

---
name: Pull request build

on:
  pull_request:

env:
  MAIN_PY_VER: "3.10"

jobs:
  lint:
    runs-on: ubuntu-latest
    name: Lint source files
    permissions:
      pull-requests: read
    steps:
      - name: Source checkout
        uses: actions/checkout@v3

      - name: Set up Python ${{ env.MAIN_PY_VER }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.MAIN_PY_VER }}

      - name: Cache pip repository
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ env.MAIN_PY_VER }}

      - name: Prepare python environment
        run: |
          pip install -r requirements.txt
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project true

      - name: Cache poetry virtual environment
        uses: actions/cache@v3
        with:
          path: .venv
          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}-${{ env.MAIN_PY_VER }}

      - name: Install deno runtime
        uses: denoland/setup-deno@v1.1.2

      - name: Lint the project
        run: |
          make install-all-deps
          make lint

  test:
    runs-on: ubuntu-latest
    needs: [lint]
    strategy:
      matrix:
        python: ["3.9", "3.10"]
    name: Test project with Python ${{ matrix.python }}
    permissions:
      checks: write
      pull-requests: write
    steps:
      - name: Source checkout
        uses: actions/checkout@v3

      - name: Setup timezone
        uses: zcong1993/setup-timezone@v2.0.0
        with:
          timezone: Asia/Jerusalem

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}

      - name: Cache pip repository
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ matrix.python }}

      - name: Prepare python environment
        run: |
          pip install -r requirements.txt
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project true

      - name: Cache poetry virtual environment
        uses: actions/cache@v3
        with:
          path: .venv
          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}-${{ matrix.python }}

      - name: Install project build dependencies
        run: make install-all-deps

      - name: Test the project
        run: >
          if [ ${{ matrix.python }} == ${{ env.MAIN_PY_VER }} ];
          then make test-with-coverage-reports;
          else make test; fi

      - name: Report test summary
        uses: EnricoMi/publish-unit-test-result-action@v2
        if: ${{ matrix.python == env.MAIN_PY_VER && always() }}
        with:
          test_changes_limit: 0
          junit_files: ./junit.xml
          report_individual_runs: true

      - name: Push to CodeCov
        uses: codecov/codecov-action@v3
        if: ${{ matrix.python == env.MAIN_PY_VER }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml

  docs:
    runs-on: ubuntu-latest
    needs: [lint]
    name: Verify documentation site
    permissions:
      pull-requests: read
    steps:
      - name: Source checkout
        uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.MAIN_PY_VER }}

      - name: Cache pip repository
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ env.MAIN_PY_VER }}

      - name: Prepare python environment
        run: |
          pip install -r requirements.txt
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project true

      - name: Cache poetry virtual environment
        uses: actions/cache@v3
        with:
          path: .venv
          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}-${{ env.MAIN_PY_VER }}

      - name: Build documentation site
        run: |
          make install-all-deps
          make docs-build


Upload Python Package

# This workflow will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.

name: Upload Python Package

on:
  release:
    types: [published]

jobs:
  deploy:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4.1.1
      with:
        submodules: true
    - name: Set up Python
      uses: actions/setup-python@v5.0.0
      with:
        python-version: '3.x'
    - name: Install SSH key
      uses: shimataro/ssh-key-action@v2
      with:
        key: ${{ secrets.WORKFLOW_SSH_KEY }}
        known_hosts: ${{ secrets.KNOWN_HOSTS }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install build
    - name: Build package
      run: python -m build
    - name: Publish package
      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}
  push_to_store:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4.1.1

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1

      - name: Login to ostorlab store
        uses: docker/login-action@v1
        with:
          registry: ostorlab.store
          username: ${{ secrets.store_user }}
          password: ${{ secrets.store_password }}
      - name: truncate current tag
        run: echo "TAG=$(echo $GITHUB_REF | cut -c12-17)" >> $GITHUB_ENV
      - name: Build image and push to ostorlab Container Registry
        uses: docker/build-push-action@v2
        with:
          # relative path to the place where source code with Dockerfile is located
          context: .
          tags: |
            ostorlab.store/ostorlab/ostorlab:${{env.TAG}}
            ostorlab.store/ostorlab/ostorlab:latest
          push: true


Pytest-Windows

# This workflow use pytest:
#  - Install Python dependencies.
#  - Run pytest for each of the supported Python versions ["3.8", "3.9", "3.10"].
# Running pytest with -m "no docker" to disable test that require a docker installation.

name: Pytest.

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ "*" ]

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true
jobs:
  build:

    runs-on:  ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.10", "3.11"]
        os: [windows-latest]

    steps:
    - uses: actions/checkout@v4.1.1
      with:
        submodules: true
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5.0.0
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies.
      run: |
        python -m pip install --upgrade pip
        pip install .[testing]
        pip install .[scanner]
        pip install .[agent]
        python -m pip install -e .
    - name: Running tests with pytest.
      run: |
        pytest -m "not docker and not nats"


Pytest-Linux

# This workflow use pytest:
#  - Install Python dependencies.
#  - Run pytest for each of the supported Python versions ["3.8", "3.9", "3.10"].
# Running pytest with -m "no docker" to disable test that require a docker installation.

name: Pytest.

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ "*" ]

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true
jobs:
  build:

    runs-on:  ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.10", "3.11"]
        os: [ubuntu-latest]

    steps:
    - uses: actions/checkout@v4.1.1
      with:
        submodules: true
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5.0.0
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies.
      run: |
        python -m pip install --upgrade pip
        pip install .[testing]
        pip install .[scanner]
        pip install .[agent]
        python -m pip install -e .
    - name: Running tests with pytest.
      run: |
        set -o pipefail
        pytest -m "not docker and not nats"


Lint And Format Checker.

name: Lint And Format Checker.

on:
  pull_request:
    branches: [ "*" ]
  push:
    branches: [ "main" ]
jobs:
  ruff:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [ "3.9", "3.10" ]
    steps:
      - uses: actions/checkout@v4.1.1
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5.0.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Checking code formatting.
        uses: chartboost/ruff-action@v1
        with:
          args: format --check
      - name: Running linter.
        uses: chartboost/ruff-action@v1


Static Types Checker.

name: Static Types Checker.

on:
  pull_request:
    branches: [ "*" ]
  push:
    branches: [ "main" ]
jobs:
  mypy:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [ "3.9", "3.10" ]
    steps:
      - uses: actions/checkout@v4.1.1
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5.0.0
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
          cache-dependency-path: |
            **/typing_requirements.txt
      - name: Install dependencies.
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r typing_requirements.txt
          python -m pip install -e .
      - name: Running static types checker.
        run: |
          mypy src/ostorlab/agent/schema
          mypy src/ostorlab/agent/kb
          mypy src/ostorlab/agent/message
          mypy src/ostorlab/utils
          mypy src/ostorlab/apis/runners
          mypy src/ostorlab/agent/mixins/agent_report_vulnerability_mixin.py
          mypy src/ostorlab/assets


on:
  release:
    types: [published]

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        os: ['ubuntu-latest', 'macos-latest']

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies.
      run: |
        python -m pip install --upgrade pip
        pip install .
        pip install .[scanner]
        pip install .[agent]
        pip install pyinstaller

    - run: |
        pyinstaller --name oxo_${{ matrix.os }} --onefile --add-data="./*:." --hidden-import alembic --hidden-import docker --hidden-import click src/ostorlab/main.py
    - name: Release
      uses: softprops/action-gh-release@v2
      with:
        files: dist/oxo_${{ matrix.os }}


on:
  release:
    types: [published]

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        os: ['windows-latest']

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies.
      run: |
        python -m pip install --upgrade pip
        pip install .
        pip install .[scanner]
        pip install .[agent]
        pip install pyinstaller

    - run: |
        pyinstaller --name oxo_${{ matrix.os }} --onefile --add-data="./*:." --hidden-import alembic --hidden-import docker --hidden-import click src/ostorlab/main.py
    - name: Release
      uses: softprops/action-gh-release@v2
      with:
        files: dist/oxo_${{ matrix.os }}.exe