detection-testing

# name: detection-testing
# on:
#   push:
#   pull_request:
#     types: [opened, reopened]
#   schedule:
#     - cron: "44 4 * * *"
# jobs:

#   validate-tag-if-present:
#     runs-on: ubuntu-latest

#     steps:
#       - name: TAGGED, Validate that the tag is in the correct format

#         run: |
#           echo "The GITHUB_REF: $GITHUB_REF"
#           #First check to see if the release is a tag
#           if [[ $GITHUB_REF =~ refs/tags/* ]]; then
#             #Yes, this is a tag, so we need to test to make sure that the tag
#             #is in the correct format (like v1.10.20)
#             if [[ $GITHUB_REF =~ refs/tags/v[0-9]+.[0-9]+.[0-9]+ ]]; then
#               echo "PASS: Tagged release with good format"
#               exit 0
#             else
#               echo "FAIL: Tagged release with bad format"
#               exit 1
#             fi
#           else
#           echo "PASS: Not a tagged release"
#           exit 0
#           fi

#   quit-for-dependabot:
#     runs-on: ubuntu-latest
#     if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]'
#     steps:
#       - name: "Placeholder"
#         run: |
#           echo "yes it ran"

#   docker-detection-testing-setup:
#     runs-on: ubuntu-latest
#     if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job
#     needs: [validate-tag-if-present, quit-for-dependabot]
#     steps:
#         - name: Get branch and PR required for detection testing main.py
#           id: vars
#           run: |
#             echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"

#         - name: Checkout Repo
#           uses: actions/checkout@v2
#           #with:
#           #  ref: develop

#         - uses: actions/setup-python@v2
#           with:
#             python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
#             architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
#             cache: 'pip'

#         - name: Install Python Dependencies
#           run: |
#             python -m venv .venv
#             source .venv/bin/activate
#             python -m pip install wheel
#             python -m pip install -r requirements.txt

#         - name: Run the CI
#           run: |
#             source .venv/bin/activate
#             cd bin/docker_detection_tester
#             echo "github.event.issue.pull_request    : [${{ github.event.issue.pull_request }}]"
#             echo "github.event.pull_request.number   : [${{ github.event.pull_request.number }}]"
#             echo "steps.vars.outputs.branch          : [${{ steps.vars.outputs.branch }}]"
#             echo "github.event.pull_request.head.ref : [${{ github.event.pull_request.head.ref }}]"
#             echo "github.event_name                  : [${{ github.event_name }}]"

#             if [[ ${{ github.event_name }} == schedule ]]; then
#               # Note that scheduled actions ONLY run on the default branch, so it won't run on all other branches!
#               echo "Running a nightly test on all detections OR a commit was made directly to develop"
#               python detection_testing_execution.py run --branch develop --mode all --mock --config_file test_config_github_actions.json
#             elif [[ ! -z "${{  github.event.pull_request.head.ref }}" &&  ! -z "${{ github.event.pull_request.number }}" ]]; then
#               echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]"
#               python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }}  --mode changes --mock --config_file test_config_github_actions.json
#             else
#               echo "Push from branch [${{ steps.vars.outputs.branch }}]"
#               python detection_testing_execution.py run --branch  ${{ steps.vars.outputs.branch }}  --mode changes --mock --config_file test_config_github_actions.json
#             fi

#             mv *-test-run.json replicate_test.json
#         - name: Upload Test Results Files
#           uses: actions/upload-artifact@v2
#           with:
#             name: testing-results-config
#             path: |
#               bin/docker_detection_tester/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz
#               bin/docker_detection_tester/prior_config/config_tests_0.json
#               bin/docker_detection_tester/prior_config/config_tests_1.json
#               bin/docker_detection_tester/prior_config/config_tests_2.json
#               bin/docker_detection_tester/prior_config/config_tests_3.json
#               bin/docker_detection_tester/prior_config/config_tests_4.json
#               bin/docker_detection_tester/prior_config/config_tests_5.json
#               bin/docker_detection_tester/prior_config/config_tests_6.json
#               bin/docker_detection_tester/prior_config/config_tests_7.json
#               bin/docker_detection_tester/prior_config/config_tests_8.json
#               bin/docker_detection_tester/prior_config/config_tests_9.json

#         - name: Upload File to Enable Replication of the Test at a Different Time or Place
#           uses: actions/upload-artifact@v2
#           with:
#             name: replicate_test
#             path: |
#               bin/docker_detection_tester/replicate_test.json

#   docker-detection-testing-execution:
#     runs-on: ubuntu-latest
#     if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job
#     needs: [docker-detection-testing-setup]
#     strategy:
#       matrix:
#         manifest_filename: ["config_tests_0.json",
#                         "config_tests_1.json",
#                         "config_tests_2.json",
#                         "config_tests_3.json",
#                         "config_tests_4.json",
#                         "config_tests_5.json",
#                         "config_tests_6.json",
#                         "config_tests_7.json",
#                         "config_tests_8.json",
#                         "config_tests_9.json"]
#     steps:
#         - name: Get branch and PR required for detection testing main.py
#           id: vars
#           run: |
#             echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"

#         - name: Checkout Repo
#           uses: actions/checkout@v2
#           #with:
#           #  ref: develop

#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: testing-results-config
#             path: bin/docker_detection_tester/prior_config

#         - uses: actions/setup-python@v2
#           with:
#             python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
#             architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
#             cache: 'pip'

#         - name: Install Python Dependencies
#           run: |
#             python -m venv .venv
#             source .venv/bin/activate
#             python -m pip install wheel
#             python -m pip install -r requirements.txt

#         - name: Run the CI
#           run: |
#             source .venv/bin/activate
#             cd bin/docker_detection_tester

#             python detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}}

#         - name: Upload Test Results Files
#           uses: actions/upload-artifact@v2
#           with:
#             name: ${{ matrix.manifest_filename}}.results
#             path: |
#               bin/docker_detection_tester/test_results/success.csv
#               bin/docker_detection_tester/test_results/error.csv
#               bin/docker_detection_tester/test_results/failure.csv
#               bin/docker_detection_tester/test_results/combined.csv
#               bin/docker_detection_tester/test_results/success.json
#               bin/docker_detection_tester/test_results/error.json
#               bin/docker_detection_tester/test_results/failure.json
#               bin/docker_detection_tester/test_results/combined.json

#               bin/docker_detection_tester/test_results/summary.json

#   docker-detection-testing-execution-merge-results:
#     runs-on: ubuntu-latest
#     if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job
#     needs: [docker-detection-testing-setup, docker-detection-testing-execution]

#     steps:
#         - name: Get branch and PR required for detection testing main.py
#           id: vars
#           run: |
#             echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"

#         - name: Checkout Repo
#           uses: actions/checkout@v2
#           #with:
#           #  ref: develop

#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_0.json.results
#             path: bin/docker_detection_tester/results_0
#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_1.json.results
#             path: bin/docker_detection_tester/results_1
#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_2.json.results
#             path: bin/docker_detection_tester/results_2
#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_3.json.results
#             path: bin/docker_detection_tester/results_3
#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_4.json.results
#             path: bin/docker_detection_tester/results_4
#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_5.json.results
#             path: bin/docker_detection_tester/results_5
#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_6.json.results
#             path: bin/docker_detection_tester/results_6
#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_7.json.results
#             path: bin/docker_detection_tester/results_7
#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_8.json.results
#             path: bin/docker_detection_tester/results_8
#         - name: Download artifacts
#           uses: actions/download-artifact@v2
#           with:
#             name: config_tests_9.json.results
#             path: bin/docker_detection_tester/results_9

#         - uses: actions/setup-python@v2
#           with:
#             python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
#             architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
#             cache: 'pip'

#         - name: Install Python Dependencies
#           run: |
#             python -m venv .venv
#             source .venv/bin/activate
#             python -m pip install wheel
#             python -m pip install -r requirements.txt

#         - name: Merge Detections into single File
#           run: |
#             source .venv/bin/activate
#             cd bin/docker_detection_tester
#             python summarize_json.py --files results_*/summary.json --output_filename summary_test_results.json

#         - name: Upload Summary Test Results JSON
#           uses: actions/upload-artifact@v2
#           if: always()
#           with:
#             name: SummaryTestResults
#             path: |
#               bin/docker_detection_tester/summary_test_results.json

#         - name: Upload Failures Manifest on Failure
#           uses: actions/upload-artifact@v2
#           if: failure()
#           with:
#             name: DetectionFailureManifest
#             path: |
#               bin/docker_detection_tester/detection_failure_manifest.json

#         #Always clean these up, they make the output messy
#         - name: Clean up intermediate Files
#           uses: geekyeggo/delete-artifact@v1
#           if: always()
#           with:
#               name: |
#                 config_tests_0.json.results
#                 config_tests_1.json.results
#                 config_tests_2.json.results
#                 config_tests_3.json.results
#                 config_tests_4.json.results
#                 config_tests_5.json.results
#                 config_tests_6.json.results
#                 config_tests_7.json.results
#                 config_tests_8.json.results
#                 config_tests_9.json.results

#         - name: Log in to S3 for Artifact Uploads
#           if: ${{ github.event_name == 'schedule' }}
#           uses: aws-actions/configure-aws-credentials@v1
#           with:
#             aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
#             aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

#             aws-region: us-west-2

#         - name: Upload S3 Badge and Summary Artifacts for Nightly Scheduled Run
#           if: ${{ github.event_name == 'schedule' }}
#           run: |
#             cd bin/docker_detection_tester
#             python generate_detection_coverage_badge.py --input_summary_file summary_test_results.json --output_badge_file detection_coverage.svg --badge_string "Pass Rate"

#             #Upload artifact (summary test results)
#             aws s3 cp summary_test_results.json s3://security-content/reporting/summary_test_results.json

#             #Since these reside in a public bucket, no need to explicitly mark as public
#             # make the file public since it is not by default
#             #aws s3api put-object-acl --bucket security-content --key reporting/summary_test_results.json --acl public-read

#             #Upload artifact (test results coverage badge)
#             aws s3 cp detection_coverage.svg s3://security-content/reporting/detection_coverage.svg

#             #Since these reside in a public bucket, no need to explicitly mark as public
#             # make the file public since it is not by default
#             #aws s3api put-object-acl --bucket security-content --key reporting/detection_coverage.svg --acl public-read


Container

name: Container

# on:
#   release:
#     types: [published]

on:
  push:
    branches:
      - main
    tags:
      - '**'

env:
  IMAGE_NAME: pycryptobot
  DOCKER_FILE: Dockerfile
  DOCKER_BUILDKIT: 1
  # PLATFORM: linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64  <--  build fails with: docker build . --platform "linux/arm/v7"
  PLATFORM: linux/arm64,linux/amd64

jobs:
  # Build and push image to GitHub Packages.
  # See also https://docs.docker.com/docker-hub/builds/
  push:
    runs-on: ubuntu-latest
    if: github.event_name == 'push'

    steps:
      - uses: actions/checkout@v2

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1

      - name: Cache Docker layers
        uses: actions/cache@v2
        id: cache
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-
      - name: Log into registry
        run: echo "${{ secrets.PACKAGE_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

      - name: Create Tags
        id: tags
        run: |
          IMAGE_ID=ghcr.io/${{ github.repository }}/$IMAGE_NAME
          # Change all uppercase to lowercase
          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
          # Strip git ref prefix from version
          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
          # Use Docker `latest` tag convention
          [ "$VERSION" == "main" ] && VERSION=latest
          echo IMAGE_ID=$IMAGE_ID
          echo VERSION=$VERSION
          echo "::set-output name=IMAGE_ID::$IMAGE_ID"
          echo "::set-output name=VERSION::$VERSION"
      - name: Generate Docker Metadata
        id: meta
        uses: docker/metadata-action@v3
        with:
          images: ${{ steps.tags.outputs.IMAGE_ID }}

      - name: Build and Push Image
        uses: docker/build-push-action@v2
        with:
          build-args: REPO=${{ github.repository }}
          context: .
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
          file: ${{ env.DOCKER_FILE }}
          platforms: ${{ env.PLATFORM }}
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}

      - name: Inspect Image
        run: |
          # Inspect Image
          docker buildx imagetools inspect ${{ steps.tags.outputs.IMAGE_ID }}:${{ steps.tags.outputs.VERSION }}
      # Temp fix stops cache growing too big
      # https://github.com/docker/build-push-action/issues/252
      # https://github.com/moby/buildkit/issues/1896
      - name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: "${{ steps.tags.outputs.IMAGE_ID }}:${{ steps.tags.outputs.VERSION }}"
          format: "sarif"
          output: "trivy-results.sarif"
          ignore-unfixed: true
          exit-code: 0
          scan-type: "image"
          vuln-type: "os,library"
          severity: "CRITICAL,HIGH"

      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v1
        with:
          sarif_file: 'trivy-results.sarif'


Unit Tests

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Unit Tests

on:
  push:
    branches: [ beta ]
  pull_request:
    branches: [ beta ]
  workflow_dispatch:

jobs:
  build:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python 3.11
      uses: actions/setup-python@v2
      with:
        python-version: 3.11
    - name: Find typos with codespell
      uses: codespell-project/actions-codespell@master
      with:
        ignore_words_list: vai,zar,hist
        skip: ".venv,.git,*.json,./tests/unit_tests/responses"
    - uses: actions/cache@v2
      with:
        path: ${{ env.pythonLocation }}
        key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install flake8 pytest mock pytest-mock
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E203,E266,E501,W503,F403,F401,C901 --max-line-length=160 --show-source --statistics
        # exit-zero treats all errors as warnings.
        flake8 . --count --exit-zero --max-complexity=10 --statistics
    - name: Test with pytest
      run: |
        pytest tests/unit_tests


---
name: Build and deploy documentation on merge to master

concurrency:
  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
  cancel-in-progress: true

on:
  pull_request:
    paths:
      - 'docs/**'
      - '.github/worklows/documentation.yml'
      - 'requirements.docs.txt'
      - 'Dockerfile.docs'
      - 'mkdocs.yml'
      - 'CNAME'
  push:
    branches:
      - master
    paths:
      - 'docs/**'
      - '.github/worklows/documentation.yml'
      - 'requirements.docs.txt'
      - 'Dockerfile.docs'
      - 'mkdocs.yml'
      - 'CNAME'

jobs:
  docs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout kapitan recursively
        uses: actions/checkout@v4
      - uses: actions/setup-python@v2
      - run: pip install --upgrade pip && pip install -r requirements.docs.txt
      - run: mkdocs build
      - name: Deploy docs
        if: github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch
        uses: mhausenblas/mkdocs-deploy-gh-pages@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          CONFIG_FILE: ./mkdocs.yml
          REQUIREMENTS: requirements.docs.txt


Housekeeping

name: 'Housekeeping'
on:
  schedule:
    - cron: '0 0 * * *'

jobs:
  stale:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/stale@v8
        with:
          stale-issue-message: |
            This issue is stale because it has been open for 1 year with no activity.
            Remove the stale label or comment if this issue is still relevant for you.
            If not, please close it yourself.
          days-before-issue-stale: 365
          day-before-pr-stale: -1
          days-before-close: -1
          days-before-pr-close: -1


PEX Build and Upload

---
name: PEX Build and Upload

on:
  workflow_run:
    workflows:
      - Upload Python Package
    types:
      - completed

jobs:
  pex_build_publish:
    if: false  # disable as this is not working with multiple python versions
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.x
      - name: Set up Go
        uses: actions/setup-go@v4
        with:
          go-version: 1.17
      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pex
      - name: Build Pex
        run: |-
          mkdir -p dist
          pex .[gojsonnet] -r requirements.txt --python-shebang='#!/usr/bin/env python3' --python=python3.7 --python=python3.8 --python=python3.9 -m kapitan -o dist/kapitan.linux-x86_64.pex
      - name: Add linux-x86_64 pex to assets
        uses: softprops/action-gh-release@v1
        if: startsWith(github.ref, 'refs/tags/')
        with:
          files: dist/kapitan.linux-x86_64.pex


Upload Python Package

---
name: Upload Python Package

concurrency:
  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
  cancel-in-progress: true

on:
  workflow_dispatch:
  release:
    types: [created]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.x
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel twine build
      - name: Build and check
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
        run: |-
          python -m build
          twine check dist/*
      - name: Test package upload
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_TEST_TOKEN }}
        run: |-
          twine upload --repository testpypi dist/*
      - name: Package upload
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
        run: |-
          twine upload dist/*


Test, Build and Publish docker image

---
name: Test, Build and Publish docker image
run-name: Docker Build for ${{ github.actor }} on branch ${{ github.ref_name }}

concurrency:
  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
  cancel-in-progress: true

on:
  push:
    branches:
      - master
      - main
      - test/*
    paths-ignore:
      - 'docs/**'
      - '.github/worklows/documentation.yml'
      - 'requirements.docs.txt'
      - 'mkdocs.yml'
      - 'CNAME'
      - 'Dockerfile.docs'

  release:
    types: [created]

  pull_request:
    paths-ignore:
      - 'docs/**'
      - '.github/worklows/documentation.yml'
      - 'requirements.docs.txt'
      - 'mkdocs.yml'
      - 'CNAME'
      - 'Dockerfile.docs'

jobs:
  lint:
    name: linter
    runs-on: ubuntu-latest
    if: success() || failure()  # Continue running if other jobs fail
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
      - uses: psf/black@main

  test:
    name: python ${{ matrix.python-version }} tests
    runs-on: ubuntu-latest
    if: success() || failure()   # Continue running if other jobs fail
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.10', '3.11']
    steps:
      - name: Checkout kapitan recursively
        uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          cache: 'poetry'
          python-version: ${{ matrix.python-version }}
      - name: Install libraries dependencies
        run: |
          poetry install --no-root
      - name: Install testing dependencies (Helm)
        run: |
          sudo apt-get -qq update
          sudo apt-get install -y gnupg2 git curl
          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
      - name: Run pytest
        uses: pavelzw/pytest-action@v2
        with:
          verbose: true
          emoji: false
          job-summary: true
          custom-pytest: poetry run pytest
          custom-arguments: '-q'
          click-to-expand: true
          report-title: 'Kapitan tests'

  build:
    name: build ${{ matrix.platform }} image
    if: success() || failure()   # Continue running if other jobs fail
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        platform:
          - linux/amd64
          - linux/arm64
    steps:
      - name: Checkout kapitan recursively
        uses: actions/checkout@v4
        with:
          submodules: recursive

      # Setup QEMU and Buildx to build multi-platform image
      # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      # Builds docker image and allow scoped caching
      - name: build Kapitan Image
        uses: docker/build-push-action@v5
        with:
          push: False
          platforms: ${{ matrix.platform }}
          load: True
          file: Dockerfile
          tags: local-test-${{ matrix.platform }}
          cache-from: type=gha,scope=$GITHUB_REF_NAME-${{ matrix.platform }}
          cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-${{ matrix.platform }}

      - name: Test Kapitan for ${{ matrix.platform }}
        run: |
          docker run -t --rm local-test-${{ matrix.platform }} --version

  publish:
    name: publish platform images
    # Only starts if everything else is successful
    needs: [lint, test, build]
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        platform:
          - linux/amd64
          - linux/arm64
    steps:
      - name: Checkout kapitan recursively
        uses: actions/checkout@v4
        with:
          submodules: recursive

      # Setup QEMU and Buildx to build multi-platform image
      # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to DockerHub
        uses: docker/login-action@v3
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
        if: env.DOCKERHUB_USERNAME != null
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
        with:
          # list of Docker images to use as base name for tags
          images: |
            name=${{ vars.DOCKERHUB_REPOSITORY }}/kapitan
          # generate Docker tags based on the following events/attributes
          tags: |
            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}
          flavor: |
            suffix=-${{ matrix.platform }}

      - name: Build and push by digest
        id: push-digest
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
        if: env.DOCKERHUB_USERNAME != null
        uses: docker/build-push-action@v5
        with:
          platforms: ${{ matrix.platform }}
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{steps.meta.output.labels}}
          cache-from: type=gha,scope=$GITHUB_REF_NAME-${{ matrix.platform }}
          cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-${{ matrix.platform }}

  build-multi-architecture:
    name: combine platform images
    needs:
      - publish
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      # Setup QEMU and Buildx to build multi-platform image
      # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to DockerHub
        uses: docker/login-action@v3
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
        if: env.DOCKERHUB_USERNAME != null
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
        with:
          # list of Docker images to use as base name for tags
          images: |
            name=${{ vars.DOCKERHUB_REPOSITORY }}/kapitan
          # generate Docker tags based on the following events/attributes
          tags: |
            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}

      - uses: int128/docker-manifest-create-action@v1
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
        if: env.DOCKERHUB_USERNAME != null
        with:
          tags: ${{ steps.meta.outputs.tags }}
          builder: buildx
          suffixes: |
            -linux-amd64
            -linux-arm64


Codecov upload

name: Codecov upload
on:
  push:
    branches:
      - 'master'
jobs:
  codecov:
    name: Codecov Workflow
    runs-on: ubuntu-20.04

    steps:
      - uses: actions/checkout@v1
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Generate coverage report
        run: |
          python -m pip install --upgrade pip setuptools wheel
          pip install pytest pytest-cov
          pip install -r requirements_testing.txt
          pip install -e .
          pytest --cov=./ --cov-report=xml
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          flags: unittests


explainerdashboard

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: explainerdashboard

on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]
  schedule:
    - cron:  '0 0 * * *'
  workflow_dispatch:

jobs:
  build:

    runs-on: ubuntu-latest
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ['3.9', '3.10', '3.11']

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip setuptools wheel
        pip install flake8 pytest
        pip install --upgrade --upgrade-strategy eager -r requirements_testing.txt
        pip install --upgrade --upgrade-strategy eager -e .
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
    - name: Test with pytest
      run: |
        pytest -k "not selenium"


Publish to PyPI.org

name: Publish to PyPI.org
on:
  release:
    types: [published]
jobs:
  pypi:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - run: python3 -m pip install --upgrade build && python3 -m build
      - name: Publish package
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_TOKEN }}


build

name: build

on:
  push:
    branches:
      - "branch-*"
    tags:
      - v[0-9].[0-9][0-9].[0-9][0-9]
  workflow_dispatch:
    inputs:
      branch:
        required: true
        type: string
      date:
        required: true
        type: string
      sha:
        required: true
        type: string
      build_type:
        type: string
        default: nightly

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  conda-python-build:
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08
    with:
      build_type: ${{ inputs.build_type || 'branch' }}
      branch: ${{ inputs.branch }}
      date: ${{ inputs.date }}
      sha: ${{ inputs.sha }}
  upload-conda:
    needs: [conda-python-build]
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.08
    with:
      build_type: ${{ inputs.build_type || 'branch' }}
      branch: ${{ inputs.branch }}
      date: ${{ inputs.date }}
      sha: ${{ inputs.sha }}
  wheel-build:
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08
    with:
      build_type: ${{ inputs.build_type || 'branch' }}
      branch: ${{ inputs.branch }}
      sha: ${{ inputs.sha }}
      date: ${{ inputs.date }}
      script: ci/build_wheel.sh
  wheel-publish:
    needs: wheel-build
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08
    with:
      build_type: ${{ inputs.build_type || 'branch' }}
      branch: ${{ inputs.branch }}
      sha: ${{ inputs.sha }}
      date: ${{ inputs.date }}
      package-name: ucx_py


pr

name: pr

on:
  push:
    branches:
      - "pull-request/[0-9]+"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  pr-builder:
    needs:
      - checks
      - conda-python-build
      - conda-python-tests
      - wheel-build
      - wheel-tests
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08
  checks:
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08
  conda-python-build:
    needs: checks
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08
    with:
      build_type: pull-request
  conda-python-tests:
    needs: conda-python-build
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08
    with:
      build_type: pull-request
      container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000"
  wheel-build:
    needs: checks
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08
    with:
      build_type: pull-request
      script: ci/build_wheel.sh
  wheel-tests:
    needs: wheel-build
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08
    with:
      build_type: pull-request
      script: ci/test_wheel.sh
      container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000"


test

name: test

on:
  workflow_dispatch:
    inputs:
      branch:
        required: true
        type: string
      date:
        required: true
        type: string
      sha:
        required: true
        type: string

jobs:
  conda-python-tests:
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08
    with:
      build_type: nightly
      branch: ${{ inputs.branch }}
      date: ${{ inputs.date }}
      sha: ${{ inputs.sha }}
      container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000"
  wheel-tests:
    secrets: inherit
    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08
    with:
      build_type: nightly
      branch: ${{ inputs.branch }}
      date: ${{ inputs.date }}
      sha: ${{ inputs.sha }}
      script: ci/test_wheel.sh
      container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000"


Continuous delivery - Linux

name: Continuous delivery - Linux

on:
  release:
    types: [prereleased, released]

env:
  FLIT_ROOT_INSTALL: 1

jobs:
  version-check:
    name: Check versioning
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Check version tag format
        run: |
          TAG_VERSION="${{ github.event.release.tag_name }}"
          if [[ $TAG_VERSION =~ ^v[0-9]+.[0-9]+.[0-9]+$ ]]; then exit 0; else exit 1; fi
      - name: Check if version tag and package version are equal
        run: |
          TAG_VERSION="${{ github.event.release.tag_name }}"
          if [ ${TAG_VERSION:1} == $(cat pynitrokey/VERSION) ]; then exit 0; else exit 1; fi
  build-onefile:
    name: Build onefile
    runs-on: ubuntu-latest
    container: python:3.9-slim
    needs: version-check
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Install required packages
        run: |
          apt update
          apt install -y binutils gcc git libpcsclite-dev libusb-1.0-0 make swig
      - name: Create virtual environment
        run: |
          python -m venv venv
          . venv/bin/activate
          pip install flit
          flit install --symlink
      - name: Patch oscrypto
        run: |
          . venv/bin/activate
          pip uninstall -y oscrypto
          : # This uses an oscrypto version to avoid issue #431
          pip install "oscrypto @ git+https://github.com/wbond/oscrypto.git@1547f535001ba568b239b8797465536759c742a3"
      - name: Build
        run: |
          . venv/bin/activate
          pyinstaller \
            ci-scripts/linux/pyinstaller/pynitrokey-onefile.spec
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: nitropy-onefile
          path: dist/nitropy
  publish-binary:
    name: Publish binary
    runs-on: ubuntu-latest
    container: python:3.9-slim
    needs: build-onefile
    permissions:
      contents: write
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: nitropy-onefile
      - name: Rename binary
        run: |
          mv \
            nitropy \
            nitropy-${{ github.event.release.tag_name }}-x64-linux-binary
      - name: Create archive
        run: |
          tar \
          -czvf \
          nitropy-${{ github.event.release.tag_name }}-x64-linux-binary.tar.gz \
          nitropy-${{ github.event.release.tag_name }}-x64-linux-binary
      - name: Publish release
        uses: softprops/action-gh-release@v2
        with:
          files: nitropy-${{ github.event.release.tag_name }}-x64-linux-binary.tar.gz


Continuous delivery - Pypi

name: Continuous delivery - Pypi

on:
  release:
    types: [released]

env:
  FLIT_ROOT_INSTALL: 1

jobs:
  version-check:
    name: Check versioning
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Check version tag format
        run: |
          TAG_VERSION="${{ github.event.release.tag_name }}"
          if [[ $TAG_VERSION =~ ^v[0-9]+.[0-9]+.[0-9]+$ ]]; then exit 0; else exit 1; fi
      - name: Check if version tag and package version are equal
        run: |
          TAG_VERSION="${{ github.event.release.tag_name }}"
          if [ ${TAG_VERSION:1} == $(cat pynitrokey/VERSION) ]; then exit 0; else exit 1; fi
  build:
    name: Build
    runs-on: ubuntu-latest
    container: python:3.9-slim
    needs: version-check
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Install required packages
        run: |
          apt update
          apt install -y binutils gcc libpcsclite-dev libusb-1.0-0 make swig
      - name: Create virtual environment
        run: |
          python -m venv venv
          . venv/bin/activate
          pip install flit
          flit install --symlink
      - name: Build
        run: |
          . venv/bin/activate
          flit build
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: nitropy-pypi
          path: dist
  publish-binary:
    name: Publish
    runs-on: ubuntu-latest
    container: python:3.9-slim
    needs: build
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: nitropy-pypi
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Install required packages
        run: |
          apt update
          apt install -y binutils gcc libpcsclite-dev libusb-1.0-0 make swig
      - name: Create virtual environment
        run: |
          python -m venv venv
          . venv/bin/activate
          pip install flit
          flit install --symlink
      - name: Prepare pypi configuration file
        run: sed -e "s|\${passw}|${{ secrets.PYPI_TOKEN }}|" ci-scripts/pypi/.pypirc_template > ~/.pypirc
      - name: Publish release
        run: |
          . venv/bin/activate
          flit --repository pypi publish


Continuous delivery - Windows

name: Continuous delivery - Windows

on:
  release:
    types: [prereleased, released]

env:
  FLIT_ROOT_INSTALL: 1

jobs:
  version-check:
    name: Check versioning
    runs-on: windows-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Check version tag format
        run: |
          $VERSION_TAG="${{ github.event.release.tag_name }}"
          if ($VERSION_TAG -match "^v[0-9]+.[0-9]+.[0-9]+$") {exit 0} else {exit 1}
      - name: Check if version tag and package version are equal
        run: |
          $VERSION_TAG="${{ github.event.release.tag_name }}"
          $VERSION_FILE=Get-Content .\pynitrokey\VERSION
          if ($VERSION_TAG.Substring(1) -eq $VERSION_FILE) {exit 0} else {exit 1}
  build-onedir:
    name: Build onedir
    runs-on: windows-latest
    needs: version-check
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Create virtual environment
        run: |
          python -m venv venv
          .\venv\Scripts\Activate.ps1
          .\venv\Scripts\pip install pip
          .\venv\Scripts\pip install flit
          .\venv\Scripts\flit install --symlink
      - name: Create Windows version info file
        run: |
          .\venv\Scripts\Activate.ps1
          create-version-file `
            --outfile .\ci-scripts\windows\pyinstaller\file_version_info.txt `
            --version "$(Get-Content .\pynitrokey\VERSION)" `
            .\ci-scripts\windows\pyinstaller\file_version_info_metadata.yaml
      - name: Build onedir
        run: |
          .\venv\Scripts\Activate.ps1
          pyinstaller ci-scripts/windows/pyinstaller/pynitrokey-onedir.spec
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: pynitrokey-onedir
          path: dist/nitropy
  build-onefile:
    name: Build onefile
    runs-on: windows-latest
    needs: version-check
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Create virtual environment
        run: |
          python -m venv venv
          .\venv\Scripts\Activate.ps1
          .\venv\Scripts\pip install pip
          .\venv\Scripts\pip install flit
          .\venv\Scripts\flit install --symlink
      - name: Create Windows version info file
        run: |
          .\venv\Scripts\Activate.ps1
          create-version-file `
            --outfile .\ci-scripts\windows\pyinstaller\file_version_info.txt `
            --version "$(Get-Content .\pynitrokey\VERSION)" `
            .\ci-scripts\windows\pyinstaller\file_version_info_metadata.yaml
      - name: Build onefile
        run: |
          .\venv\Scripts\Activate.ps1
          pyinstaller ci-scripts/windows/pyinstaller/pynitrokey-onefile.spec
      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: pynitrokey-onefile
          path: dist/nitropy.exe
  build-msi-installer:
    name: Build MSI installer
    runs-on: windows-latest
    needs: build-onedir
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: pynitrokey-onedir
          path: dist/nitropy
      - name: Create sources file
        run: |
          $Env:Path += ";" + "$Env:WIX" + "bin"
          heat `
            dir .\dist\nitropy\ `
            -dr INSTALLFOLDER `
            -ag `
            -cg ApplicationFilesDynamic `
            -ke `
            -srd -sfrag -suid -sreg `
            -nologo `
            -pog:Binaries `
            -pog:Documents `
            -pog:Satellites `
            -pog:Sources `
            -pog:Content `
            -o Sources.wxs
      - name: Build object files
        run: |
          $Env:Path += ";" + "$Env:WIX" + "bin"
          candle .\ci-scripts\windows\wix\Product.wxs -o Product.wixobj
          candle .\Sources.wxs -o .\Sources.wixobj
      - name: Build installer package
        run: |
          $Env:Path += ";" + "$Env:WIX" + "bin"
          light `
            -b .\dist\nitropy\ `
            -sice:ICE80 `
            .\Product.wixobj `
            .\Sources.wixobj `
            -o nitropy.msi
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: nitropy-installer
          path: nitropy.msi
  publish-binary:
    name: Publish binary
    runs-on: windows-latest
    needs: build-onefile
    permissions:
      contents: write
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: pynitrokey-onefile
      - name: Rename binary
        run: |
          mv `
            nitropy.exe `
            nitropy-${{ github.event.release.tag_name }}-x64-windows-binary.exe
      - name: Create archive
        run: |
          7z a -tzip -mx9 `
            nitropy-${{ github.event.release.tag_name }}-x64-windows-binary.zip `
            nitropy-${{ github.event.release.tag_name }}-x64-windows-binary.exe
      - name: Publish release
        uses: softprops/action-gh-release@v2
        with:
          files: nitropy-${{ github.event.release.tag_name }}-x64-windows-binary.zip
  publish-msi-installer:
    name: Publish MSI installer
    runs-on: windows-latest
    needs: build-msi-installer
    permissions:
      contents: write
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: nitropy-installer
      - name: Rename installer
        run: |
          mv `
            nitropy.msi `
            nitropy-${{ github.event.release.tag_name }}-x64-windows-installer.msi
      - name: Publish release
        uses: softprops/action-gh-release@v2
        with:
          files: nitropy-${{ github.event.release.tag_name }}-x64-windows-installer.msi


Continuous integration

name: Continuous integration
on:
  push:
    branches-ignore:
      - "master"
  pull_request:

env:
  FLIT_ROOT_INSTALL: 1

jobs:
  format-code:
    name: Check code format
    runs-on: ubuntu-latest
    container: python:3.9-slim
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Install required packages
        run: |
          apt update
          apt install -y gcc libpcsclite-dev make swig git
      - name: Create virtual environment
        run: make init
      - name: Check code format
        run: |
          . venv/bin/activate
          make check-format
  format-import:
    name: Check imports format
    runs-on: ubuntu-latest
    container: python:3.9-slim
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Install required packages
        run: |
          apt update
          apt install -y gcc libpcsclite-dev make swig git
      - name: Create virtual environment
        run: make init
      - name: Check code import format
        run: |
          . venv/bin/activate
          make check-import-sorting
  lint-style:
    name: Check code style
    runs-on: ubuntu-latest
    container: python:3.9-slim
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Install required packages
        run: |
          apt update
          apt install -y gcc libpcsclite-dev make swig git
      - name: Create virtual environment
        run: make init
      - name: Check code style
        run: |
          . venv/bin/activate
          make check-style
  lint-typing:
    name: Check static typing
    runs-on: ubuntu-latest
    container: python:3.9-slim
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Install required packages
        run: |
          apt update
          apt install -y gcc libpcsclite-dev make swig git
      - name: Create virtual environment
        run: make init
      - name: Check code static typing
        run: |
          . venv/bin/activate
          make check-typing


Add new issues to the backlog

name: Add new issues to the backlog

on:
  issues:
    types:
      - opened

jobs:
  add-to-project:
    name: Add issue to project
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/add-to-project@v0.4.0
        with:
          project-url: https://github.com/orgs/Clinical-Genomics/projects/46
          github-token: ${{ secrets.REPO_AND_PROJECT_PAT }}


Black linter and code formatter

name: Black linter and code formatter
on: [pull_request]
jobs:
  lint:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - uses: psf/black@stable
        with:
          options: "--check --verbose"
          version: "22.3.0"


Conventional PRs

name: Conventional PRs
on: ["pull_request"]

jobs:
  title-format:
    runs-on: ubuntu-22.04
    steps:
      - uses: amannn/action-semantic-pull-request@v5
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


name: create_release_zenodo

on:
  push:
    tags:
      - "**"

jobs:
  build:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Release
        uses: softprops/action-gh-release@v1
        if: startsWith(github.ref, 'refs/tags/')
        with:
          token: ${{ secrets.GITHUB_TOKEN}}


Build, push, and test develop containers


Docker develop containers

name: Docker develop containers
on:
  push:
    branches:
      - "develop"
  pull_request:
    branches-ignore:
      - "master"
    paths:
      - "BALSAMIC/containers/**"
jobs:
  main:
    name: Build and publish
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: true
      matrix:
        container-name: [align_qc, annotate, ascatNgs, cadd, cnvkit, cnvpytor, coverage_qc, delly, gatk, htslib, purecn, somalier, varcall_py3, varcall_py27, vcf2cytosure]
    steps:
      - name: Git checkout
        id: git_checkout
        uses: actions/checkout@v3
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Login to Docker Hub
        id: docker_login
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Build and push
        id: docker_build_push
        uses: docker/build-push-action@v3
        with:
          file: BALSAMIC/containers/${{ matrix.container-name }}/Dockerfile
          push: true
          tags: clinicalgenomics/balsamic:develop-${{ matrix.container-name }}
          build-args: CONTAINER_NAME=${{ matrix.container-name }}
          provenance: false
      - name: Prune containers
        id: docker_prune_containers
        shell: bash
        run: |
          docker system prune -a -f
      - name: Test container
        id: docker_test_container
        shell: bash
        run: |
          docker run \
          -v $(readlink -f container_tests):/container_tests \
          clinicalgenomics/balsamic:develop-${{ matrix.container-name }} \
          bash /container_tests/${{ matrix.container-name }}/${{ matrix.container-name }}.sh


Docker release containers

name: Docker release containers
on:
  pull_request:
    branches:
      - "master"
jobs:
  main:
    name: Build and publish
    if:  startsWith(github.head_ref, 'release_v')
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: true
      matrix:
        container-name: [align_qc, annotate, ascatNgs, cadd, cnvkit, cnvpytor, coverage_qc, delly, gatk, htslib, purecn, somalier, varcall_py3, varcall_py27, vcf2cytosure]
    steps:
      - name: Git checkout
        id: git_checkout
        uses: actions/checkout@v3
      - name: Get branch name
        id: get_branch_name
        shell: bash
        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF##*/}}" >> $GITHUB_OUTPUT
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Login to Docker Hub
        id: docker_login
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Build and push
        id: docker_build_push
        uses: docker/build-push-action@v3
        with:
          file: BALSAMIC/containers/${{ matrix.container-name }}/Dockerfile
          push: true
          tags: clinicalgenomics/balsamic:${{ steps.get_branch_name.outputs.branch }}-${{ matrix.container-name }}
          build-args: CONTAINER_NAME=${{ matrix.container-name }}
          provenance: false
      - name: Prune containers
        id: docker_prune_containers
        shell: bash
        run: |
          docker system prune -a -f
      - name: Test container
        id: docker_test_container
        shell: bash
        run: |
          docker run \
          -v $(readlink -f container_tests):/container_tests \
          clinicalgenomics/balsamic:${{ steps.get_branch_name.outputs.branch }}-${{ matrix.container-name }} \
          bash /container_tests/${{ matrix.container-name }}/${{ matrix.container-name }}.sh


Build and push master and develop container


Check changelog

name: "Check changelog"
on:
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
jobs:
  # Enforces the update of a changelog file on every pull request
  changelog:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v3
      - uses: dangoslen/changelog-enforcer@v3
        with:
          changeLogPath: 'CHANGELOG.rst'
          skipLabels: 'Skip-Changelog'


pytests

name: pytests
on:
  pull_request:
    paths-ignore:
      - "CHANGELOG.rst"
      - "docs/**"
  push:
    branches:
      - master
    paths-ignore:
      - "CHANGELOG.rst"
      - "docs/**"

jobs:
  pytest_coveralls:
    name: run PyTest
    runs-on: ubuntu-22.04
    steps:
      # Checkout BALSAMIC
      - name: Git checkout
        id: git_checkout
        uses: actions/checkout@v3
      # Conda env create
      - name: setup conda
        id: setup_conda
        uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: balsamic
          environment-file: BALSAMIC/conda/balsamic.yaml
      - name: Install the HTML to PDF renderer
        run: sudo apt-get update && sudo apt-get install -y wkhtmltopdf
      # Install BALSAMIC
      - name: Install BALSAMIC
        id: install_balsamic
        shell: bash -l {0}
        run: |
          conda activate balsamic
          pip install --no-cache-dir .
      # Install pytest coveralls dependencies
      - name: Install PyTest and Coveralls
        id: install_pytest
        shell: bash -l {0}
        run: |
          conda activate balsamic
          pip install --no-cache-dir -r requirements-dev.txt
      # Run PyTest
      - name: Run PyTest
        id: pytest
        shell: bash -l {0}
        run: |
          conda activate balsamic
          py.test --cov-report=xml --cov=BALSAMIC -rsxv tests/*
        env:
          SENTIEON_LICENSE: dummy_license
          SENTIEON_INSTALL_DIR: dummy_install_dir
      # Run Codecov
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          flags: unittests
          fail_ci_if_error: true
          verbose: true


Pull Request Docs Check

name: "Pull Request Docs Check"
on:
  pull_request:
    paths:
      - "docs/**"
    branches:
      - master
jobs:
  docs:
    runs-on: ubuntu-22.04
    steps:
      # Checkout BALSAMIC
      - name: Git checkout
        id: git_checkout
        uses: actions/checkout@v3
      - name: Sphinx build action
        id: sphinx_docs
        uses: ammaraskar/sphinx-action@master
        with:
          docs-folder: "docs/"
          pre-build-command: "pip install -r requirements-dev.txt && python setup.py install"
          build-command: "sphinx-build -T -E -b html -d _build/doctrees-readthedocs -D language=en . _build/html"


Check URLs

name: Check URLs
on:
  push:
    branches:
      - master
      - develop
jobs:
  main:
    name: Find broken cache and documentation links
    runs-on: ubuntu-22.04
    steps:
      - name: Git checkout
        id: git_checkout
        uses: actions/checkout@v3
      - name: Link Checker
        uses: lycheeverse/lychee-action@v1.8.0
        with:
          args: --verbose './BALSAMIC/constants/cache.py' './docs/*.rst'
          fail: true


Deploy static content to Pages

# Simple workflow for deploying static content to GitHub Pages generated by Github
# except for added job steps "Copy Static Files" through "Build MkDocs Pages".
name: Deploy static content to Pages

on:
  # Runs on pushes targeting the default branch
  push:
    branches: ["master"]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
  contents: read
  pages: write
  id-token: write

# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
  group: "pages"
  cancel-in-progress: false

jobs:
  # Single deploy job since we're just deploying
  deploy:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Setup Pages
        uses: actions/configure-pages@v3
      - name: Copy Static Files
        run: |
          cp -R docs/website site/
      - name: Replace GITHUB token
        # Use different sed delimiter to avoid clashing with forward slash in URL
        run: |
          find docs/ -type f -exec sed -i "s@GITHUB@${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}\/blob\/master@g" {} \;
      - name: Replace WEBSITE token
        # Use different sed delimiter to avoid clashing with forward slash in URL
        run: |
          WEBSITE_URL="https://${GITHUB_REPOSITORY_OWNER}.github.io/automlbenchmark"
          find docs/ -type f -exec sed -i "s@WEBSITE@${WEBSITE_URL}@g" {} \;
          sed -i "s@WEBSITE@${WEBSITE_URL}@g" mkdocs.yml
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - name: Install MkDocs
        run: |
           python -m pip install mkdocs-material
      - name: Build MkDocs Pages
        run: |
          mkdocs build --site-dir site/docs
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v2
        with:
          # Upload entire repository
          path: './site/'
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2


Run All Frameworks

name: Run All Frameworks

on: pull_request

jobs:

  detect_changes:  # Figure out which frameworks need to be evaluated based on which files are changed.
    name: Detect Changes
    runs-on: ubuntu-latest
    outputs:
      frameworks: ${{ steps.find-required-tests.outputs.frameworks }}
      tasks: ${{ steps.find-required-tests.outputs.tasks }}
      benchmark: ${{ steps.find-required-tests.outputs.benchmark }}
      skip_baseline: ${{ steps.find-required-tests.outputs.skip_baseline }}
      skip_evaluation: ${{ steps.find-required-tests.outputs.skip_evaluation }}
    steps:
      - uses: actions/checkout@v3
      - name: pull base branch
        run: |
          git fetch --unshallow origin $GITHUB_BASE_REF
          git branch --track $GITHUB_BASE_REF origin/$GITHUB_BASE_REF
      - id: find-required-tests
        name: Detect Common Changes  # detect if any changes occurred that should trigger all frameworks.
        run: |
          changed_files=$(git diff --name-only $GITHUB_BASE_REF...HEAD)

          shopt -s globstar
          common_files="amlb/** "\
          "resources/** "\
          "frameworks/shared/** "\
          ".github/workflows/run_all_frameworks.yml "\
          ".github/runbenchmark/action.yml "\
          "runbenchmark.py "\
          "requirements.txt"

          echo Common files: $common_files
          echo Changed files: $changed_files

          is_common=1
          for f in $changed_files
          do
            if echo $common_files | grep -e $f >> /dev/null;
            then
                echo "File $f detected as common file"
                is_common=0
                break
            fi
          done

          # Indicates which jobs should be executed (0) or not (1)
          skip_evaluation=0
          if [[ $is_common -eq 0 ]];
          then
            FRAMEWORKS='["autogluon", "autosklearn", "gama", "h2oautoml", "mlplanweka", "mlr3automl", "naiveautoml", "tpot", "tunedrandomforest"]'
            TASKS='["iris", "kc2", "cholesterol"]'
            BENCHMARK='["test"]'
          else
            TASKS='["APSFailure", "bioresponse", "dresses-sales", "eucalyptus", "internet-advertisements", "kc1", "micro-mass"]'
            BENCHMARK='["validation"]'
            changed_frameworks=$(git diff --name-only HEAD..$GITHUB_BASE_REF | grep -o -i -P 'frameworks/(?!shared).*/' | uniq | sed -e 's/frameworks//' -e 's/\///g')
            if [ ! -z "$changed_frameworks" ];
            then
              json_array=[
              for framework in $changed_frameworks; do json_array=$json_array\"$framework\",; done
              FRAMEWORKS=${json_array::-1}]  #remove trailing comma and add closing bracket
            else
              # No changes to common files or frameworks - must be e.g. docs. No need to run tests.
              skip_evaluation=1
              FRAMEWORKS=[]
            fi
          fi

          echo Building matrix for frameworks: $FRAMEWORKS

          echo "::set-output name=frameworks::$FRAMEWORKS"
          echo "::set-output name=tasks::$TASKS"
          echo "::set-output name=benchmark::$BENCHMARK"
          echo "::set-output name=skip_baseline::$is_common"
          echo "::set-output name=skip_evaluations::$skip_evaluation"

  baseline:
    name: ${{ matrix.framework }}/${{ matrix.task }}
    runs-on: ubuntu-latest
    needs: detect_changes
    if: needs.detect_changes.outputs.skip_baseline == 0
    strategy:
      matrix:
        framework: [constantpredictor, randomforest]
        task: [iris, kc2, cholesterol]
      fail-fast:  false
    steps:
      - uses: actions/checkout@v3
      - name: Setup Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Create venv
        run: python -m venv venv
      - uses: actions/cache@v3
        id: cache
        with:
          path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
          key: pip-v3-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            pip-v3-
      - name: Install Requirements
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          source venv/bin/activate
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip show openml
      - name: Run constantpredictor on openml iris
        run: |
          source venv/bin/activate
          python runbenchmark.py ${{ matrix.framework }} -t ${{ matrix.task }} -f 0 -e

  run_frameworks:
    name: ${{ matrix.framework }}/${{ matrix.task }}
    runs-on: ubuntu-latest
    needs:
     - baseline
     - detect_changes
    if: ${{ success() }} || ${{ cancelled() }}

    strategy:
      matrix:
        python-version: [3.9]
        framework: ${{ fromJson(needs.detect_changes.outputs.frameworks) }}
        task: ${{ fromJson(needs.detect_changes.outputs.tasks) }}
        benchmark: ${{ fromJson(needs.detect_changes.outputs.benchmark) }}
      fail-fast:  true  # not sure about this one, but considering the big workload it might be nicer

    steps:
    - uses: actions/checkout@v3
    - name: Setup Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Create venv
      run: python -m venv venv
    - uses: actions/cache@v3
      id: cache
      with:
        path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
        key: pip-v3-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          pip-v3-
    - name: Install Requirements
      if: steps.cache.outputs.cache-hit != 'true'
      run: |
        source venv/bin/activate
        python -m pip install --upgrade pip
        python -m pip install -r requirements.txt
    - name: Run ${{ matrix.framework }} on ${{ matrix.task }}
      run: |
        source venv/bin/activate
        python runbenchmark.py ${{ matrix.framework }} ${{ matrix.benchmark }} test -f 0 -t ${{ matrix.task }} -e
      env:
        GITHUB_PAT: ${{ secrets.PUBLIC_ACCESS_GITHUB_PAT }}


Light-weight Unit Tests

name: Light-weight Unit Tests

on: pull_request

jobs:
  pytest_not_stress:
    name: pytest
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Create venv
        run: python -m venv venv
      - uses: actions/cache@v3
        id: cache
        with:
          path: /home/runner/work/automlbenchmark/automlbenchmark/venv/lib/python3.9/site-packages
          key: pip-v3-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/requirements-dev.txt') }}
          restore-keys: |
            pip-v3-
      - name: Install Requirements
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          source venv/bin/activate
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install -r requirements-dev.txt
      - name: Run unit tests
        run: |
          source venv/bin/activate
          python -m pytest -vv -m "not stress and not openml_upload"


Run `runbenchmark.py`

name: Run `runbenchmark.py`

on:
  workflow_dispatch:
    inputs:
      python_version:
        default: '3.9'
      os:
        default: 'ubuntu-latest'
      framework:
        default: 'randomforest'
      options:
        default: ''

jobs:
  run_benchmark:
    runs-on: ${{ inputs.os }}
    steps:
      - name: Runs on ${{ inputs.os }}
        run: echo Preparing to benchmark ${{ inputs.framework }} ${{ inputs.options }}
      - name: Checkout ${{ github.ref_name }}
        uses: actions/checkout@v3
        with:
          ref: ${{ github.ref_name }}
      - name: Install Py${{ inputs.python_version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ inputs.python_version }}
      - name: Install benchmark dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r requirements.txt
      - name: Install ${{ inputs.framework }}
        run: python runbenchmark.py ${{ inputs.framework }} -s only
        env:
          GITHUB_PAT: ${{ secrets.PUBLIC_ACCESS_GITHUB_PAT }}
      - name: Benchmark ${{ inputs.framework }}
        run: python runbenchmark.py ${{ inputs.framework }} ${{ inputs.options }}


Version reset to dev

name: Version reset to dev
# reset the version if merging a release branch into main

on:
  push:
    branches:
      - main
      - master
    paths:
      - 'amlb/__version__.py'

  workflow_dispatch:

env:
  BRANCH: ${{ github.event.release.target_commitish }}

jobs:
  deploy:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ env.BRANCH }}
          fetch-depth: 2  # we need the previous commit as we amend the last one
      - name: Reset version to dev
        run: ./.github/scripts/version-set.sh "" push no-add-commit


Version Bump

name: Version Bump

on:
  release:
    types:
      - published

  workflow_dispatch:

env:
  BRANCH: ${{ github.event.release.target_commitish }}
  TAG: ${{ github.event.release.tag_name }}

jobs:
  validation:
    runs-on: ubuntu-latest
    steps:
      - name: Release only from stable-v* branch
        if: ${{ !startsWith(env.BRANCH, 'stable-v') }}
        uses: actions/github-script@v6
        with:
          script: core.setFailed("Releases can only be made from 'stable-v*' branches")
  rollback:
    needs: validation
    if: ${{ failure() }}
    runs-on: ubuntu-latest
    steps:
      - name: Rollback if invalid release
        uses: author/action-rollback@stable
        with:
          tag: ${{ env.TAG }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  versioning:
    needs: validation
    if: ${{ success() }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ env.BRANCH }}
      - name: Bump version from tag
        run: ./.github/scripts/version-set.sh $TAG push
      - name: Update tag
        run: ./.github/scripts/tag-set.sh $TAG push
      - name: Restore dev version
        run: ./.github/scripts/version-set.sh "" push


Linux

name: Linux

on:
  push:
    branches:
      - '**'
    paths:
      - '**'
    tags:
      - 'v*.*.*'
  pull_request:
    branches:
      - 'master'
    paths:
      - '**'

jobs:
  test:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    env:
      COVERAGE_PROCESS_START: "${{ github.workspace }}/.coveragerc"
    defaults:
      run:
        shell: bash
    strategy:
      matrix:
        julia-version: ['1']
        python-version: ['3.12']
        os: [ubuntu-latest]
        test-id: [main]
        include:
          - julia-version: '1.6'
            python-version: '3.8'
            os: ubuntu-latest
            test-id: include
          - julia-version: '1'
            python-version: '3.12'
            os: ubuntu-latest
            test-id: include

    steps:
      - uses: actions/checkout@v4
      - name: "Set up Julia"
        uses: julia-actions/setup-julia@v2
        with:
          version: ${{ matrix.julia-version }}
      - name: "Cache Julia"
        uses: julia-actions/cache@v2
        with:
          cache-name: ${{ matrix.os }}-test-${{ matrix.julia-version }}-${{ matrix.python-version }}
          cache-packages: false
      - name: "Set up Python"
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - name: "Install PySR"
        run: |
            python -m pip install --upgrade pip
            pip install .
            python -c 'import pysr'
      - name: "Assert Julia version"
        if: ${{ matrix.julia-version != '1'}}
        run: python3 -c "from pysr import jl; assert jl.VERSION.major == jl.seval('v\"${{ matrix.julia-version }}\"').major; assert jl.VERSION.minor == jl.seval('v\"${{ matrix.julia-version }}\"').minor"
      - name: "Install test dependencies"
        run: pip install coverage coveralls pytest nbval
      - name: "Set up coverage for subprocesses"
        run: echo 'import coverage; coverage.process_startup()' > "${{ github.workspace }}/sitecustomize.py"
      - name: "Run tests"
        run: coverage run -m pysr test main,cli,startup
      - name: "Install JAX"
        run: pip install jax jaxlib # (optional import)
        if: ${{ matrix.test-id == 'main' }}
      - name: "Run JAX tests"
        run: coverage run --append -m pysr test jax
        if: ${{ matrix.test-id == 'main' }}
      - name: "Install Torch"
        run: pip install torch # (optional import)
        if: ${{ matrix.test-id == 'main' }}
      - name: "Run Torch tests"
        run: coverage run --append -m pysr test torch
        if: ${{ matrix.test-id == 'main' }}
      - name: "Coveralls"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          COVERALLS_FLAG_NAME: test-${{ matrix.julia-version }}-${{ matrix.python-version }}-${{ matrix.test-id }}
          COVERALLS_PARALLEL: true
        run: coveralls --service=github

  dev_install:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: ['ubuntu-latest']
        python-version: ['3.12']
        julia-version: ['1']
        include:
          - os: ubuntu-latest
            python-version: '3.8'
            julia-version: '1.6'
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
      - name: "Install PySR"
        run: |
            python -m pip install --upgrade pip
            pip install .
      - name: "Run development test"
        run: PYSR_TEST_JULIA_VERSION=${{ matrix.julia-version }} PYSR_TEST_PYTHON_VERSION=${{ matrix.python-version }} python -m pysr test dev

  conda_test:
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ['3.12']
        os: ['ubuntu-latest']

    steps:
      - uses: actions/checkout@v4
      - name: "Cache conda"
        uses: actions/cache@v4
        env:
          CACHE_NUMBER: 0
        with:
          path: ~/conda_pkgs_dir
          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }}
      - name: "Set up Conda"
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-variant: Mambaforge
          miniforge-version: latest
          auto-activate-base: true
          python-version: ${{ matrix.python-version }}
          activate-environment: pysr-test
          environment-file: environment.yml
      - name: "Cache Julia"
        uses: julia-actions/cache@v2
        with:
          cache-name: ${{ matrix.os }}-conda-${{ matrix.python-version }}
          cache-packages: false
      - name: "Install PySR"
        run: |
            python3 -m pip install .
            python3 -c 'import pysr'
      - name: "Run tests"
        run: cd /tmp && python -m pysr test main

  coveralls:
    name: Indicate completion to coveralls.io
    needs:
      - test
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    container: python:3-slim
    steps:
      - name: Finished
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
            pip install coveralls
            coveralls --finish

  types:
    name: Check types
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version:
          - '3.12'
          - '3.8'
        os: ['ubuntu-latest']

    steps:
      - uses: actions/checkout@v4
      - name: "Set up Python"
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - name: "Install PySR and all dependencies"
        run: |
            python -m pip install --upgrade pip
            pip install .
            pip install mypy
      - name: "Install additional dependencies"
        run: python -m pip install jax jaxlib torch
        if: ${{ matrix.python-version != '3.8' }}
      - name: "Run mypy"
        run: python -m mypy --install-types --non-interactive pysr
        if: ${{ matrix.python-version != '3.8' }}
      - name: "Run compatible mypy"
        run: python -m mypy --ignore-missing-imports pysr
        if: ${{ matrix.python-version == '3.8' }}


Windows

name: Windows

on:
  push:
    branches:
      - 'master'
    paths:
      - '**'
    tags:
      - 'v*.*.*'
  pull_request:
    branches:
      - 'master'
    paths:
      - '**'

jobs:
  test:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    defaults:
      run:
        shell: bash
    strategy:
      matrix:
        julia-version: ['1']
        python-version: ['3.12']
        os: [windows-latest]

    steps:
      - uses: actions/checkout@v4
      - name: "Set up Julia"
        uses: julia-actions/setup-julia@v2
        with:
          version: ${{ matrix.julia-version }}
      - name: "Cache Julia"
        uses: julia-actions/cache@v2
        with:
          cache-name: ${{ matrix.os }}-test-${{ matrix.julia-version }}-${{ matrix.python-version }}
          cache-packages: false
      - name: "Set up Python"
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - name: "Install PySR"
        run: |
            python -m pip install --upgrade pip
            pip install pytest nbval "numpy<2.0.0"
            pip install .
            python -c 'import pysr'
      - name: "Run tests"
        run: |
            python -m pysr test main,cli,startup
      - name: "Install Torch"
        run: pip install torch # (optional import)
      - name: "Run Torch tests"
        run: python -m pysr test torch


conda-forge

name: conda-forge

# This CI tries the conda-forge version of PySR

on:
  schedule:
    # Run at the 0th minute of the 10th hour (UTC).
    # This means the job will run at 5am EST.
    - cron: "0 10 * * *"
    # This will automatically run on master branch only.
  workflow_dispatch:

jobs:
  conda_test:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.9', '3.10', '3.11', '3.12']
        os: ['ubuntu-latest']
        use-mamba: [true, false]
        include:
          - python-version: 3.9
            os: 'windows-latest'
            use-mamba: true
          - python-version: 3.12
            os: 'windows-latest'
            use-mamba: true
          - python-version: 3.9
            os: 'macos-latest'
            use-mamba: true
          - python-version: 3.12
            os: 'macos-latest'
            use-mamba: true

    steps:
      - name: "Set up Conda"
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-variant: Mambaforge
          miniforge-version: latest
          auto-activate-base: true
          python-version: ${{ matrix.python-version }}
          activate-environment: pysr-test
      - name: "Install pysr with mamba"
        run: conda activate pysr-test && mamba install pysr
        if: ${{ matrix.use-mamba }}
      - name: "Install pysr with conda"
        run: conda activate pysr-test && conda install pysr
        if: ${{ !matrix.use-mamba }}
      - name: "Run tests"
        run: |
          pip install pytest nbval
          python -m pysr test main,startup


Docker

name: Docker

on:
  push:
    branches:
      - 'master'
    paths:
      - '**'
    tags:
      - 'v*.*.*'
  pull_request:
    branches:
      - 'master'
    paths:
      - '**'

jobs:
  test:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    defaults:
      run:
        shell: bash
    strategy:
      matrix:
        os: [ubuntu-latest]
        arch: ['linux/amd64']

    steps:
      - uses: actions/checkout@v4
      - name: Build docker
        run: docker build --platform=${{ matrix.arch }} -t pysr .
      - name: Test docker
        run: docker run --platform=${{ matrix.arch }} --rm pysr /bin/bash -c 'pip install pytest nbval && python3 -m pysr test main,cli,startup'


name: Docker_Large_Nightly

on:
  schedule:
    # Run at the 0th minute of the 10th hour (UTC).
    # This means the job will run at 5am EST.
    - cron: "0 10 * * *"
    # This will automatically run on master branch only.
  workflow_dispatch:

jobs:
  test:
    runs-on: ${{ matrix.os }}
    continue-on-error: ${{ matrix.arch == 'linux/arm64' }}
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix:
        julia-version: ['1.6', '1']
        python-version: ['3.8', '3.12']
        os: [ubuntu-latest]
        arch: ['linux/amd64', 'linux/arm64']

    steps:
      - uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
        with:
          platforms: all
      - name: Build docker
        run: docker build --platform=${{ matrix.arch }} -t pysr --build-arg JLVERSION=${{ matrix.julia-version }} --build-arg PYVERSION=${{ matrix.python-version }} .
      - name: Test docker
        run: docker run --platform=${{ matrix.arch }} --rm pysr /bin/bash -c 'pip install pytest nbval && python3 -m pysr test main,cli,startup'


name: large_nightly

# This CI only runs once per day, but tries
# many different configurations.

on:
  schedule:
    # Run at the 0th minute of the 10th hour (UTC).
    # This means the job will run at 5am EST.
    - cron: "0 10 * * *"
    # This will automatically run on master branch only.
  workflow_dispatch:

jobs:
  test:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    # Windows jobs are flaky:
    continue-on-error: ${{ matrix.os == 'windows-latest' }}
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix:
        julia-version: ['1.6', '1.8', '1.10']
        python-version: ['3.8', '3.10', '3.12']
        os: [ubuntu-latest, macos-latest, windows-latest]

    steps:
      - uses: actions/checkout@v4
      - name: "Set up Julia"
        uses: julia-actions/setup-julia@v2
        with:
          version: ${{ matrix.julia-version }}
      - name: "Set up Python"
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: "Install PySR"
        run: |
            python -m pip install --upgrade pip
            pip install pytest nbval
            pip install .
            python -c 'import pysr'
      - name: "Assert Julia version"
        if: ${{ matrix.julia-version != '1'}}
        run: python3 -c "from pysr import jl; assert jl.VERSION.major == jl.seval('v\"${{ matrix.julia-version }}\"').major; assert jl.VERSION.minor == jl.seval('v\"${{ matrix.julia-version }}\"').minor"
      - name: "Run tests"
        run: python -m pysr test main,cli,startup


macOS

name: macOS

on:
  push:
    branches:
      - 'master'
    paths:
      - '**'
    tags:
      - 'v*.*.*'
  pull_request:
    branches:
      - 'master'
    paths:
      - '**'

jobs:
  test:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    defaults:
      run:
        shell: bash
    strategy:
      matrix:
        julia-version: ['1']
        python-version: ['3.12']
        os: [macos-latest]

    steps:
      - uses: actions/checkout@v4
      - name: "Set up Julia"
        uses: julia-actions/setup-julia@v2
        with:
          version: ${{ matrix.julia-version }}
      - name: "Cache Julia"
        uses: julia-actions/cache@v2
        with:
          cache-name: ${{ matrix.os }}-test-${{ matrix.julia-version }}-${{ matrix.python-version }}
          cache-packages: false
      - name: "Set up Python"
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - name: "Install PySR"
        run: |
            python -m pip install --upgrade pip
            pip install pytest nbval
            pip install .
            python -c 'import pysr'
      - name: "Run tests"
        run: |
            python -m pysr test main,cli,startup
      - name: "Install JAX"
        run: pip install jax jaxlib # (optional import)
      - name: "Run JAX tests"
        run: python -m pysr test jax
      - name: "Install Torch"
        run: pip install torch # (optional import)
      - name: "Run Torch tests"
        run: python -m pysr test torch


CodeQL

name: "CodeQL"

on:
  push:
    branches: [ "master" ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ "master" ]
  schedule:
    - cron: '28 17 * * 1'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.

        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
        # queries: security-extended,security-and-quality

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v3

    # ℹ Command-line programs to run using the OS shell.
    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

    #   If the Autobuild fails above, remove it and uncomment the following three lines.
    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

    # - run: |
    #   echo "Run, Build Application using script"
    #   ./location_of_script_within_repo/buildscript.sh

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3


Deploy Docker

name: Deploy Docker

on:
  schedule:
    - cron: "0 10 * * *"
  push:
    branches:
      - "**"
    tags:
      - "v*.*.*"
  workflow_dispatch:

jobs:
  docker:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        arch: [linux/amd64]
        python-version: [3.12.3]
        julia-version: [1.10.3]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Login to GitHub registry
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          #  List of Docker images to use as base name for tags
          images: |
            mcranmer/pysr
            ghcr.io/${{ github.repository }}
          # generate Docker tags based on the following events/attributes
          tags: |
            type=schedule
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}
            type=sha
            type=raw,value=latest,enable={{is_default_branch}}
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Build and push
        uses: docker/build-push-action@v6
        with:
          context: .
          platforms: ${{ matrix.arch }}
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=registry,ref=mcranmer/pysr:buildcache
          cache-to: type=registry,ref=mcranmer/pysr:buildcache,mode=max
          build-args: |
            PYVERSION=${{ matrix.python-version }}
            JLVERSION=${{ matrix.julia-version }}


docs

name: docs

on:
  push:
    branches:
      - 'master'
    paths:
      - 'pysr/**'
      - '.github/workflows/docs.yml'
      - 'docs/**'
      - 'setup.py'
      - 'README.md'
      - 'mkdocs.yml'
  workflow_dispatch:

jobs:
  test:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash

    steps:
      - uses: actions/checkout@v4
      - name: "Set up Python"
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
          cache: pip
      - name: "Install packages for docs building"
        run: pip install -r docs/requirements.txt
      - name: "Install PySR"
        run: pip install . && python -c 'import pysr'
      - name: "Build API docs"
        run: cd docs && ./gen_docs.sh
      - name: "Deploy documentation"
        run: mkdocs gh-deploy --force


Deploy PyPI

name: Deploy PyPI
on:
  push:
    tags:
      - 'v*.*.*'
  workflow_dispatch:

jobs:
  pypi:
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/p/pysr
    permissions:
      id-token: write
    steps:
      - name: "Checkout"
        uses: actions/checkout@v4
      - name: "Set up Python"
        uses: actions/setup-python@v5
        with:
          python-version: 3.10.8
      - name: "Install building tools"
        run: pip install build
      - name: "Build package"
        run: python -m build
      - name: "Publish distribution 📦 to Test PyPI"
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
          repository-url: https://test.pypi.org/legacy/
          skip-existing: true
          verbose: true
      - name: "Publish distribution 📦 to PyPI"
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}
          verbose: true


PySR backend update

name: PySR backend update
on:
  schedule:
    - cron: '00 00 * * *'
  workflow_dispatch:
jobs:
  update_compat:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.12
          cache: pip

      - name: "Install dependencies"
        run: |
          python -m pip install --upgrade pip
          pip install tomlkit

      - name: "Get SymbolicRegression.jl latest version"
        id: get-latest
        run: |
          cd $(mktemp -d)
          git clone https://github.com/MilesCranmer/SymbolicRegression.jl
          cd SymbolicRegression.jl
          echo "version=$(git describe --tags --match='v*' --abbrev=0 | sed 's/^v//')" >> $GITHUB_OUTPUT

      - name: "Update SymbolicRegression.jl version in PySR"
        run: |
          python .github/workflows/update_backend_version.py ${{ steps.get-latest.outputs.version }}

      - name: "Restore changes if no diff to `pysr/juliapkg.json`"
        run: |
          if git diff --quiet pysr/juliapkg.json; then
            echo "No changes to pysr/juliapkg.json. Restoring changes."
            git restore pyproject.toml
          fi

      - name: "Create PR if necessary"
        uses: peter-evans/create-pull-request@v6
        with:
          title: "Automated update to backend: v${{ steps.get-latest.outputs.version }}"
          body: |
            This PR was automatically generated by the GitHub Action `.github/workflows/update-backend.yml`

            It updates the backend version to v${{ steps.get-latest.outputs.version }}. For a full description of the changes, see the backend changelog: [v${{ steps.get-latest.outputs.version }}](https://github.com/MilesCranmer/SymbolicRegression.jl/releases/tag/v${{ steps.get-latest.outputs.version }}).
          delete-branch: true
          commit-message: "Update backend version to v${{ steps.get-latest.outputs.version }}"
          add-paths: |
            pyproject.toml
            pysr/juliapkg.json


Build from repos

name: Build from repos

on:
    schedule:
        - cron: '0 4 * * *' # 04:00 each day

jobs:
    build:
        name: Build
        runs-on: ubuntu-20.04
        if: github.repository_owner == 'managarm' && github.ref == 'refs/heads/master'
        steps:
            - name: Checkout repository
              uses: actions/checkout@v2
              with:
                path: src
            - name: Install prerequisites
              run: |
                  pip3 install xbstrap
                  xbstrap prereqs cbuildrt xbps
            - name: Download and unpack rootfs
              run: |
                  curl https://repos.managarm.org/buildenv/managarm-buildenv.tar.gz -o managarm-rootfs.tar.gz
                  tar xvf managarm-rootfs.tar.gz
            - name: Prepare build directory
              run: |
                  mkdir build
                  cd build
                  cat << EOF > bootstrap-site.yml
                  define_options:
                    mount-using: 'loopback'
                  pkg_management:
                    format: xbps
                  container:
                    runtime: cbuildrt
                    rootfs:  $GITHUB_WORKSPACE/rootfs
                    uid: 1000
                    gid: 1000
                    src_mount: /var/lib/managarm-buildenv/src
                    build_mount: /var/lib/managarm-buildenv/build
                    allow_containerless: true
                  EOF
                  xbstrap init ../src
            - name: Pull packages
              run: |
                  xbstrap pull-pack --deps-of weston-desktop mlibc mlibc-headers
                  xbstrap install --deps-of weston-desktop
              working-directory: ./build
            - name: Download tool archives
              run: |
                  xbstrap download-tool-archive --build-deps-of managarm-system --build-deps-of managarm-kernel --build-deps-of mlibc
              working-directory: ./build
            - name: Build mlibc from source
              run: |
                  xbstrap install --rebuild mlibc mlibc-headers
              working-directory: ./build
            - name: Build managarm-kernel from source
              run: |
                  xbstrap install --rebuild managarm-kernel
              working-directory: ./build
            - name: Build managarm-system from source
              run: |
                  xbstrap install --rebuild managarm-system
              working-directory: ./build
            - name: Create image
              run: |
                  xbstrap run initialize-empty-image make-image
              working-directory: ./build