Stop Scaleway

name: Stop Scaleway

on:
  workflow_dispatch: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch

  schedule: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
  - cron:  '36 21 * * *'

jobs:
  down:
    runs-on: ubuntu-latest
    steps:
    - name: Use CLI
      uses: jawher/action-scw@v2.24.0
      env:
        SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
        SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
        SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
        SCW_ZONE: ${{ secrets.SCW_ZONE }}
      with:
        args: container container list name=${{ secrets.CONTAINER_NAME }} --output json

    - name: Get CONTAINER_ID env var
      run: echo "CONTAINER_ID=$(cat "${GITHUB_WORKSPACE}/scw.output" | jq -r '.[0].id')"  >> $GITHUB_ENV

    - name: 0 instances
      uses: jawher/action-scw@v2.24.0
      env:
        SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
        SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
        SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
        SCW_ZONE: ${{ secrets.SCW_ZONE }}
      with:
        args: container container update min-scale=0 ${{ env.CONTAINER_ID }}


Start Scaleway

name: Start Scaleway

on:
  workflow_dispatch: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch

  schedule: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
  - cron:  '00 07 * * *'

jobs:
  up:
    runs-on: ubuntu-latest
    steps:
    - name: Use CLI
      uses: jawher/action-scw@v2.24.0
      env:
        SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
        SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
        SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
        SCW_ZONE: ${{ secrets.SCW_ZONE }}
      with:
        args: container container list name=${{ secrets.CONTAINER_NAME }} --output json

    - name: Get CONTAINER_ID env var
      run: echo "CONTAINER_ID=$(cat "${GITHUB_WORKSPACE}/scw.output" | jq -r '.[0].id')"  >> $GITHUB_ENV

    - name: start 1 instances
      uses: jawher/action-scw@v2.24.0
      env:
        SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
        SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
        SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
        SCW_ZONE: ${{ secrets.SCW_ZONE }}
      with:
        args: container container update min-scale=1 ${{ env.CONTAINER_ID }}


Import job Scaleway

name: Import job Scaleway

on:
  workflow_dispatch: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch

jobs:
  start-job-image:
    runs-on: ubuntu-latest
    steps:
    - name: start import job to reapply logic to all elements offset 0
      uses: jawher/action-scw@v2.27.0
      env:
        SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
        SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
        SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
        SCW_ZONE: ${{ secrets.SCW_ZONE }}
      with:
        args: jobs definition start ${{ secrets.SCALEWAY_JOB_IMPORT_ID }} environment-variables.UPDATE=true environment-variables.START_OFFSET=0
    - name: start import job to reapply logic to all elements offset 300000
      uses: jawher/action-scw@v2.27.0
      env:
        SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
        SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
        SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
        SCW_ZONE: ${{ secrets.SCW_ZONE }}
      with:
        args: jobs definition start ${{ secrets.SCALEWAY_JOB_IMPORT_ID }} environment-variables.UPDATE=true environment-variables.START_OFFSET=300000
    - name: start import job to reapply logic to all elements offset 600000
      uses: jawher/action-scw@v2.27.0
      env:
        SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
        SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
        SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
        SCW_ZONE: ${{ secrets.SCW_ZONE }}
      with:
        args: jobs definition start ${{ secrets.SCALEWAY_JOB_IMPORT_ID }} environment-variables.UPDATE=true environment-variables.START_OFFSET=600000
    - name: start import job to reapply logic to all elements offset 900000
      uses: jawher/action-scw@v2.27.0
      env:
        SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
        SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
        SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
        SCW_ZONE: ${{ secrets.SCW_ZONE }}
      with:
        args: jobs definition start ${{ secrets.SCALEWAY_JOB_IMPORT_ID }} environment-variables.UPDATE=true environment-variables.START_OFFSET=900000


Build Gusto

name: Build Gusto

on:
  # Push to main or PR
  push:
    branches:
      - main
  pull_request:
  schedule:
    # * is a special character in YAML so you have to quote this string
    # Scheduled build at 0330 UTC on Monday mornings to detect bitrot.
    - cron:  '30 3 * * 1'

jobs:
  build:
    name: "Build Gusto"
    # The type of runner that the job will run on
    runs-on: self-hosted
    # The docker container to use.
    container:
      image: firedrakeproject/firedrake-vanilla:latest
    steps:
      - uses: actions/checkout@v4
      - name: Cleanup
        if: ${{ always() }}
        run: |
          cd ..
          rm -rf build
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install Gusto
        run: |
          . /home/firedrake/firedrake/bin/activate
          python -m pip install -r requirements.txt
          python -m pip install -e .
          python -m pip install  \
            pytest-cov pytest-timeout pytest-xdist
      - name: Gusto tests
        run: |
          . /home/firedrake/firedrake/bin/activate
          which firedrake-clean
          python -m pytest \
            -n 12 --dist worksteal \
            --durations=100 \
            --cov gusto \
            -v unit-tests integration-tests examples


Check docs build cleanly

name: Check docs build cleanly

on:
  # Run on pushes to master
  push:
    branches:
      - master
  # And all pull requests
  pull_request:

concurrency:
  # Cancels jobs running if new commits are pushed
  group: >
    ${{ github.workflow }}-
    ${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  build_docs:
    name: Run doc build
    # The type of runner that the job will run on
    runs-on: ubuntu-latest
    # The docker container to use.
    container:
      image: firedrakeproject/firedrake-docdeps:latest
      options: --user root
      volumes:
        - ${{ github.workspace }}:/home/firedrake/output
    # Steps represent a sequence of tasks that will be executed as
    # part of the jobs
    steps:
      - uses: actions/checkout@v3
      - name: Install checkedout Gusto
        run: |
          . /home/firedrake/firedrake/bin/activate
          python -m pip install -e .
      - name: Install Read the Docs theme
        run: |
          . /home/firedrake/firedrake/bin/activate
          python -m pip install sphinx_rtd_theme
      - name: Check documentation links
        if: ${{ github.ref == 'refs/heads/master' }}
        run: |
          . /home/firedrake/firedrake/bin/activate
          cd docs
          make linkcheck
      - name: Build docs
        run: |
          . /home/firedrake/firedrake/bin/activate
          cd docs
          make html
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v1
        with:
          name: github-pages
          path: /__w/gusto/gusto/docs/build/html
          retention-days: 1


Run lint

name: Run lint

on:
  # Push to master or PR
  push:
    branches:
      - master
  pull_request:

jobs:
  linter:
    name: "Run linter"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Setup flake8 annotations
        uses: rbialon/flake8-annotations@v1
      - name: Install linter
        run: |
          pip install flake8 pylint
      - name: Lint codebase
        run: |
          make lint GITHUB_ACTIONS_FORMATTING=1
  actionlint:
    name: "Lint Github actions YAML files"
    # There's a way to add error formatting so GH actions adds messages to code,
    # but I can't work out the right number of quotes to get it to work
    # https://github.com/rhysd/actionlint/blob/main/docs/usage.md
    # #example-error-annotation-on-github-actions
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Check workflow files
        uses: docker://rhysd/actionlint:latest
        with:
          args: -color


# Sample workflow for building and deploying a Jekyll site to GitHub Pages
name: Deploy Jekyll with GitHub Pages dependencies preinstalled

on:
  # Runs on pushes targeting the default branch
  push:
    branches: ["main"]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
  contents: read
  pages: write
  id-token: write

# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
  group: "pages"
  cancel-in-progress: false

jobs:
  # Build job
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Setup Pages
        uses: actions/configure-pages@v3
      - name: Build with Jekyll
        uses: actions/jekyll-build-pages@v1
        with:
          source: ./docs
          destination: ./docs/_site
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v1
        with:
          path: ./docs/_site/

  # Deployment job
  deploy:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    needs: build
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2


Tests

name: Tests

on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:
  schedule:
    - cron: '5 1 * * *'  # every day at 01:05

env:
  FORCE_COLOR: "1"

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  tests:
    timeout-minutes: 10
    runs-on: ubuntu-latest
    strategy:
      matrix:
        context:
          - {project_name: testing-123}
          - {project_name: testing-123-with-docs, docs: true}

    name: ${{ matrix.context.project_name }}
    steps:
      - name: Check out the repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup nox
        uses: wntrblm/nox@2024.04.15
        with:
          python-versions: "3.8, 3.9, 3.10, 3.11, 3.12, pypy-3.8, pypy-3.9, pypy-3.10"

      - name: Generate template via cruft
        run: |
          pip install cruft
          cruft create -y --extra-context '${{ toJSON(matrix.context) }}' .

      - name: Show directory structure and file contents
        run: |
          cd ${{ matrix.context.project_name }}
          echo "::group::tree" && tree -a && echo "::endgroup::"
          echo "::group::.cruft.json" && cat .cruft.json && echo "::endgroup::"
          echo "::group::pyproject.toml" && cat pyproject.toml && echo "::endgroup::"
          echo "::group::CONTRIBUTING.rst" && cat CONTRIBUTING.rst && echo "::endgroup::"
          echo "::group::noxfile.py" && cat noxfile.py && echo "::endgroup::"
          echo "::group::tests.yml" && cat .github/workflows/tests.yml && echo "::endgroup::"

      - name: Initialize repository
        run: |
          cd ${{ matrix.context.project_name }}
          git init
          git config user.name 'github-actions[bot]'
          git config user.email 'github-actions[bot]@users.noreply.github.com'
          git add .
          git commit -m "init"

      - name: Run nox
        run: |
          cd ${{ matrix.context.project_name }}
          nox
          nox -s lint build dev ${{ matrix.context.docs && 'docs'}}


Build pghoard

name: Build pghoard

# Default to read-only access to all APIs.
permissions: read-all

on:
  push:
    branches:
      - main
    tags:
      - '**'
  pull_request:

jobs:

  lint:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        # only use one version for the lint step
        python-version: [3.9]

    steps:

      - id: checkout-code
        uses: actions/checkout@v3
        with:
          # Do not persist the token during execution of this job.
          persist-credentials: false

      - id: prepare-python
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}

      - id: dependencies
        run: |
          pip install .
          pip install ".[dev]"

      - id: pylint
        run: make lint

      - id: mypy
        run: make mypy

      - id: validate-style
        run: |
          make fmt
          if [ $(git diff --name-only --diff-filter=ACMR | wc -l ) != 0 ]; then
            echo "Reformatting failed! Please run make fmt on your commits and resubmit!" 1>&2;
            git diff;
            exit 1;
          fi

  test:
    runs-on: ubuntu-22.04
    needs: lint
    strategy:
      max-parallel: 5
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]

    steps:
      - id: checkout-code
        uses: actions/checkout@v3

      - id: prepare-python
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}

      - id: dependencies
        run: |
          # Setup the Postgres repositories
          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main 14" > /etc/apt/sources.list.d/pgdg.list'
          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
          sudo apt-get update
          # Setup build deps
          sudo apt-get install -y libsnappy-dev postgresql-10 postgresql-11 postgresql-12 postgresql-13 postgresql-14 postgresql-15 postgresql-16
          # Setup common python dependencies
          python -m pip install --upgrade pip
          pip install .
          pip install ".[constraints]"
          pip install -e .

      - id: unittest
        run: make coverage

      - id: upload-codecov
        # Third-party action pinned to v3.1.1
        uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70
        with:
          verbose: true


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [main]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [main]
  schedule:
    - cron: "42 20 * * 6"

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        language: ["python"]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        with:
          languages: ${{ matrix.language }}
          # If you wish to specify custom queries, you can do so here or in a config file.
          # By default, queries listed here will override any specified in a config file.
          # Prefix the list here with "+" to use these queries and those in the config file.
          # queries: ./path/to/local/query, your-org/your-repo/queries@main

      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
      # If this step fails, then you should remove it and run the build manually (see below)
      - name: Autobuild
        uses: github/codeql-action/autobuild@v2

      # ℹ Command-line programs to run using the OS shell.
      # 📚 https://git.io/JvXDl

      # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
      #    and modify them (or add more) to build your code if your project
      #    uses a compiled language

      #- run: |
      #   make bootstrap
      #   make release

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2


Sphinx Pages

name: Sphinx Pages

on:
  push:
    branches:
      - main

jobs:
  build:
    name: Sphinx Pages
    runs-on: ubuntu-latest
    steps:
      - name: "Checkout Main Branch"
        uses: actions/checkout@v2
        with:
          persist-credentials: false
          path: main
          ref: main

      - name: "Checkout GitHub Pages Branch"
        uses: actions/checkout@v2
        with:
          persist-credentials: false
          path: gh-pages
          ref: gh-pages

      - name: "Setup Python & Install Spinx"
        uses: actions/setup-python@v3

      - name: "Install Sphinx & Theme"
        run: pip install sphinx==4.5.0 sphinx-rtd-theme==1.0.0

      - name: "Run Sphinx"
        run: sphinx-build -b html main/docs gh-pages -E -d $GITHUB_WORKSPACE/.doctree

      - name: "Commit & Push Changes (If Any)"
        run: |
          cd gh-pages
          git add -A
          git config --global user.email "$(git show --format=%ae -s)"
          git config --global user.name "$(git show --format=%an -s)"
          git commit -m "From $GITHUB_REF $(echo ${GITHUB_SHA} | cut -c 1-8)"
          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
          git push


Sync master from main

name: "Sync master from main"
on:
  push:
    branches:
      - main

jobs:
  sync_master_from_main:
    runs-on: ubuntu-latest
    name: Sync master from main

    steps:
      - name: Checkout main
        uses: actions/checkout@v3
        with:
          ref: main
          fetch-depth: 0

      - name: Push to master
        run: git push origin main:master


Ansible Lint

name: Ansible Lint # feel free to pick your own name

on: [push, pull_request]

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      # Important: This sets up your GITHUB_WORKSPACE environment variable
      - uses: actions/checkout@v3

      - name: Lint Ansible Playbook
        uses: ansible/ansible-lint-action@v6
        # Let's point it to the path
        with:
          path: "ansible/"


Unit tests

---
name: Unit tests

#
# Documentation:
# https://help.github.com/en/articles/workflow-syntax-for-github-actions
#

#############################
# Start the job on all push #
#############################
on: [push, pull_request]

###############
# Set the Job #
###############
jobs:
  build:
    # Name the Job
    name: Unit Test Code Base
    # Set the agent to run on
    runs-on: ubuntu-latest

    ##################
    # Load all steps #
    ##################
    steps:
      ##########################
      # Checkout the code base #
      ##########################
      - name: Checkout Code
        uses: actions/checkout@v3
        with:
          # Full git history is needed to get a proper list of changed files within `super-linter`
          fetch-depth: 0
      - name: Setup helm
        uses: azure/setup-helm@v3
        with:
          version: 'v3.13.2'
        id: install

      ################################
      # Run Linter against code base #
      ################################
      # - name: Lint Code Base
      #   uses: github/super-linter@v4
      #   env:
      #     VALIDATE_ALL_CODEBASE: false
      #     DEFAULT_BRANCH: main
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Run make test
        run: |
          make test
      - name: Run make helmlint
        run: |
          make helmlint


Super linter

---
name: Super linter

on: [push, pull_request]

jobs:
  build:
    # Name the Job
    name: Super linter
    # Set the agent to run on
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Code
        uses: actions/checkout@v3
        with:
          # Full git history is needed to get a proper list of changed files within `super-linter`
          fetch-depth: 0

      ################################
      # Run Linter against code base #
      ################################
      - name: Lint Code Base
        uses: github/super-linter/slim@v5
        env:
          VALIDATE_ALL_CODEBASE: true
          DEFAULT_BRANCH: main
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          # These are the validation we disable atm
          VALIDATE_BASH: false
          VALIDATE_JSCPD: false
          VALIDATE_KUBERNETES_KUBECONFORM: false
          VALIDATE_YAML: false
          VALIDATE_ANSIBLE: false
          VALIDATE_DOCKERFILE_HADOLINT: false
          # VALIDATE_MARKDOWN: false
          # VALIDATE_NATURAL_LANGUAGE: false
          #VALIDATE_TEKTON: false


Docker Image for sbi toolkit

name: Docker Image for sbi toolkit

on:
  release:
    types: [published]

jobs:
  build_devcontainer_base:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push Docker image
        uses: docker/build-push-action@v3
        with:
          context: .
          file: .devcontainer/Dockerfile
          platforms: linux/amd64
          push: true
          tags: ghcr.io/${{ github.repository }}/base-image-codespaces:latest


Continuous Deployment

name: Continuous Deployment

on:
  push:
    branches: [main]
  workflow_dispatch:

defaults:
  run:
    shell: bash

concurrency:
  group:  ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  cd:
    name: CD
    runs-on: ubuntu-latest

    steps:
    - name: Checkout
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: false

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.8'

    - name: Cache dependency
      id: cache-dependencies
      uses: actions/cache@v4
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e .[dev]

    - name: Run the fast and the slow CPU tests with coverage
      run: |
        pytest -v -x -n auto -m "not gpu" --cov=sbi --cov-report=xml tests/

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v4-beta
      with:
        env_vars: OS,PYTHON
        file: ./coverage.xml
        flags: unittests
        name: codecov-sbi-all-cpu
        token: ${{ secrets.CODECOV_TOKEN }}

    - name: Check doc building
      run: |
        jupyter nbconvert --to markdown tutorials/*.ipynb --output-dir docs/tutorial/
        jupyter nbconvert --to markdown examples/*.ipynb --output-dir docs/examples/
        mkdocs build -f docs/mkdocs.yml --site-dir site


Continuous Integration

name: Continuous Integration

on: [pull_request, workflow_dispatch]

defaults:
  run:
    shell: bash

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  ci:
    name: CI
    runs-on: ubuntu-latest
    if: |
      github.event_name == 'push' ||
      (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.8']
        torch-version: ['1.11', '2.2']

    steps:
    - name: Checkout
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: false

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache dependency
      id: cache-dependencies
      uses: actions/cache@v4
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ matrix.torch-version }}$
        restore-keys: |
          ${{ runner.os }}-pip-${{ matrix.python-version }}-
          ${{ runner.os }}-pip-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install torch==${{ matrix.torch-version }} --extra-index-url https://download.pytorch.org/whl/cpu
        pip install -e .[dev]

    - name: Run the fast CPU tests with coverage
      run: |
        pytest -v -x -n auto -m "not slow and not gpu" --cov=sbi --cov-report=xml tests/

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v4
      with:
        env_vars: OS,PYTHON
        file: ./coverage.xml
        flags: unittests
        name: codecov-sbi-fast-cpu
        token: ${{ secrets.CODECOV_TOKEN }}


Linting

name: Linting

on:
  pull_request:
  push:
    branches: [main]
  workflow_dispatch:

defaults:
  run:
    shell: bash

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  pre-commit:
    name: ruff and hooks.
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.8'
      - uses: pre-commit/action@v3.0.1
        with:
          extra_args: --all-files --show-diff-on-failure

  pyright:
    name: Check types
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: false

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'

      - name: Cache dependency
        id: cache-dependencies
        uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: ubuntu-latest-pip-3.8
          restore-keys: |
            ubuntu-latest-pip-

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install torch --extra-index-url https://download.pytorch.org/whl/cpu
          pip install -e .[dev]

      - name: Check types with pyright
        run: |
          pyright sbi


Manual-Test

name: Manual-Test

on:
  workflow_dispatch:
    inputs:
      pytest-marker:
        description: "Combination of markers to restrict the tests, use '' to run all tests."
        type: choice
        options:
          - 'not slow and not gpu'
          - 'not gpu'
          - 'not slow'
          - ''
        default: ''
        required: true

defaults:
  run:
    shell: bash

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  test:
    name: manual-test
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.8']
        torch-version: ['1.11', '2.2']

    steps:
    - name: Checkout
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: false

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache dependency
      id: cache-dependencies
      uses: actions/cache@v4
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ matrix.torch-version }}$
        restore-keys: |
          ${{ runner.os }}-pip-${{ matrix.python-version }}-
          ${{ runner.os }}-pip-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install torch==${{ matrix.torch-version }} --extra-index-url https://download.pytorch.org/whl/cpu
        pip install -e .[dev]

    - name: Run the selected tests without coverage
      run: |
        pytest -v -x -m ${{ inputs.pytest-marker }} tests/


Continuous Integration

# NOTES:
# - The map syntax used for matrix is flagged red but actually works
# - This runs everything in Python 3.12, except the fulltest which is also run in 3.10
# - Only coretest and fulltest environments are cached due to space limit

name: Continuous Integration
on:
  push:
    branches: main
  pull_request:
    branches: main
  workflow_dispatch:

env:
  COVERAGE_OVERALL_THRESH: 70      # threshold for overall coverage check
  COVERAGE_INDIVIDUAL_THRESH: 45   # threshold for individual coverage check

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changelog:
    name: "Changelog"
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'pull_request' }}
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
      - name: Assert Changelog Edit
        run: |
          git fetch origin main:main
          NUM_ADDED=`git diff --numstat main HEAD CHANGELOG.md | cut -d$'\t' -f1`
          echo "Changelog differences: $NUM_ADDED"
          if [ "$NUM_ADDED" -gt "0" ]; then
            echo "SUCCESS"
            exit 0
          else
            echo "ERROR - Nothing has been added to CHANGELOG.md"
            exit 1
          fi

  reminder:
    runs-on: ubuntu-latest
    name: "Reminder"
    continue-on-error: true
    permissions:
      issues: write
    steps:
      - name: check reminders and notify
        uses: agrc/reminder-action@v1

  lint:
    strategy:
      matrix:
        py-version: [ {semantic: '3.12', tox: 'py312'} ]
    name: Lint ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - name: Run linting
        run: |
          pip install tox-uv
          tox -e lint-${{ matrix.py-version.tox }}

  build-docs:
    name: "Build Docs"
    runs-on: ubuntu-latest
    needs: [lint]
    permissions:
      contents: read
      pages: write
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: {python-version: "3.12"}
      - name: Build Docs
        run: |
          pip install tox-uv
          tox -e docs-py312 -- -r

  typecheck:
    needs: [lint]
    strategy:
      matrix:
        py-version: [ {semantic: '3.12', tox: 'py312'} ]
    name: Type Check ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - name: Run type check
        run: |
          pip install tox-uv
          tox -e mypy-${{ matrix.py-version.tox }}

  audit:
    needs: [lint]
    strategy:
      matrix:
        py-version: [ {semantic: '3.12', tox: 'py312'} ]
    name: Audit ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - name: Run pip-audit
        run: |
          pip install tox-uv
          tox -e audit-${{ matrix.py-version.tox }}

  coretest:
    needs: [typecheck, audit]
    strategy:
      matrix:
        py-version: [ {semantic: '3.10', tox: 'py310'} ]
    name: Core Tests ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - uses: actions/cache@v4
        with:
          path: .tox/coretest-${{ matrix.py-version.tox }}
          key: coretest-${{ matrix.py-version.tox }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('tox.ini') }}
      - name: Run core tests
        run: |
          pip install tox-uv
          tox -e coretest-${{ matrix.py-version.tox }}

  fulltest:
    needs: [typecheck, audit]
    strategy:
      matrix:
        py-version: [ {semantic: '3.10', tox: 'py310'}, {semantic: '3.12', tox: 'py312'} ]
    name: Full Tests ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - uses: actions/cache@v4
        with:
          path: .tox/fulltest-${{ matrix.py-version.tox }}
          key: fulltest-${{ matrix.py-version.tox }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('tox.ini') }}
      - name: Run full tests
        run: |
          pip install tox-uv
          tox -e fulltest-${{ matrix.py-version.tox }} -- --cov-report=xml
      - name: "Assert Overall Coverage"
        run: |
          pip install coverage
          coverage report --fail-under=${{ env.COVERAGE_OVERALL_THRESH }}
      - name: "Assert Individual Coverage"
        shell: bash
        run: |
          coverage report |
          grep -E -o '[0-9]+%' |
          tr -d '%' |
          sed '$d' |
          awk '{if ( $1<${{ env.COVERAGE_INDIVIDUAL_THRESH }} ) exit 1 }'


Documentation

name: Documentation

on:
  release:
    types:
      - published
  workflow_dispatch:
    inputs:
      force:
        description: 'Force'
        default: false
        required: false
        type: boolean
      tag:
        description: 'Specify tag (empty builds branch HEAD)'
        required: false
        default: ''

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  build:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pages: write
      id-token: write
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.tag || github.ref }}
      - uses: actions/setup-python@v5
        with: {python-version: "3.10"}
      - name: Install tox
        run: pip install tox-uv
      - name: Build Docs ${{ inputs.force == true && '(Force)' || '' }}
        run: tox -e docs-py310 ${{ inputs.force == true && '-- -f -r' || '-- -r' }}
      - name: Upload docs artifact
        uses: actions/upload-pages-artifact@v1
        with:
          path: 'docs/build'

  deploy:
    needs: build
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v1


Regular Checks

# NOTES:
# - The map syntax used for matrix is flagged red but actually works
# - This runs everything in Python 3.10, 3.11 and 3.12
# - No environments are cached due to space limit

name: Regular Checks
on:
  schedule:
    # Run roughly every 15 days at 02:08 UTC
    - cron: '8 2 1,16 * *'
  workflow_dispatch:

env:
  COVERAGE_OVERALL_THRESH: 70      # threshold for overall coverage check
  COVERAGE_INDIVIDUAL_THRESH: 45   # threshold for individual coverage check

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:

  reminder:
    runs-on: ubuntu-latest
    name: "Reminder"
    continue-on-error: true
    permissions:
      issues: write
    steps:
      - name: check reminders and notify
        uses: agrc/reminder-action@v1

  # Warns about broken links in the docs. Especially useful for
  # those that point to our own github.io page (linkcheck is disabled in the
  # CI pipeline, because the respective pages are yet to be created at that time).
  docs:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: {python-version: "3.10"}
      - name: Build Docs
        run: |
          pip install tox-uv
          tox -e docs-py310 -- -r

  lint:
    strategy:
      fail-fast: false
      matrix:
        py-version: [ {semantic: '3.10', tox: 'py310'},
                      {semantic: '3.11', tox: 'py311'},
                      {semantic: '3.12', tox: 'py312'} ]
    name: Lint ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - name: Run linting
        run: |
          pip install tox-uv
          tox -e lint-${{ matrix.py-version.tox }}

  typecheck:
    needs: [lint]
    strategy:
      fail-fast: false
      matrix:
        py-version: [ {semantic: '3.10', tox: 'py310'},
                      {semantic: '3.11', tox: 'py311'},
                      {semantic: '3.12', tox: 'py312'} ]
    name: Type Check ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - name: Run type check
        run: |
          pip install tox-uv
          tox -e mypy-${{ matrix.py-version.tox }}

  audit:
    needs: [lint]
    strategy:
      fail-fast: false
      matrix:
        py-version: [ {semantic: '3.10', tox: 'py310'},
                      {semantic: '3.11', tox: 'py311'},
                      {semantic: '3.12', tox: 'py312'} ]
    name: Audit ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - name: Run pip-audit
        run: |
          pip install tox-uv
          tox -e audit-${{ matrix.py-version.tox }}

  coretest:
    needs: [typecheck, audit]
    strategy:
      fail-fast: false
      matrix:
        py-version: [ {semantic: '3.10', tox: 'py310'},
                      {semantic: '3.11', tox: 'py311'},
                      {semantic: '3.12', tox: 'py312'} ]
    name: Core Tests ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - uses: actions/cache@v4
        with:
          path: .tox/coretest-${{ matrix.py-version.tox }}
          key: coretest-${{ matrix.py-version.tox }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('tox.ini') }}
      - name: Run core tests
        run: |
          pip install tox-uv
          tox -e coretest-${{ matrix.py-version.tox }}

  fulltest:
    needs: [typecheck, audit]
    strategy:
      fail-fast: false
      matrix:
        py-version: [ {semantic: '3.10', tox: 'py310'},
                      {semantic: '3.11', tox: 'py311'},
                      {semantic: '3.12', tox: 'py312'} ]
    name: Full Tests ${{ matrix.py-version.semantic }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.py-version.semantic }}
      - uses: actions/cache@v4
        with:
          path: .tox/fulltest-${{ matrix.py-version.tox }}
          key: fulltest-${{ matrix.py-version.tox }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('tox.ini') }}
      - name: Run full tests
        run: |
          pip install tox-uv
          tox -e fulltest-${{ matrix.py-version.tox }} -- --cov-report=xml
      - name: "Assert Overall Coverage"
        run: |
          pip install coverage
          coverage report --fail-under=${{ env.COVERAGE_OVERALL_THRESH }}
      - name: "Assert Individual Coverage"
        shell: bash
        run: |
          coverage report |
          grep -E -o '[0-9]+%' |
          tr -d '%' |
          sed '$d' |
          awk '{if ( $1<${{ env.COVERAGE_INDIVIDUAL_THRESH }} ) exit 1 }'


Build & Publish Package

name: Build & Publish Package

on:
  push:
    tags:
      - '*'
  release:
    types:
      - published

concurrency:
  group: ${{ github.event_name }}_${{  github.ref_name }}

jobs:
  # Build and verify wheels
  build:
    name: Build & Verify Package
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Clean
        run: |
          python -c 'import shutil; [shutil.rmtree(p, True) for p in ("build", "dist")]'
          python -c 'import pathlib, shutil; [shutil.rmtree(p, True) for p in pathlib.Path(".").glob("*.egg-info")]'
      - name: Create Wheel and Dist
        run: |
          pip install build
          python -m build --sdist --wheel --outdir dist/ .
          ls -lat dist
      - name: Check Wheel
        shell: bash
        run: |
          pip install check-wheel-contents
          check-wheel-contents dist/*.whl
      - name: Upload Artifact
        uses: actions/upload-artifact@v4
        with:
          name: Dist_${{  github.ref_name }}
          path: dist

  # Upload to Test PyPI on every tag
  release-test-pypi:
    needs: build
    name: Publish PyPI TEST
    environment: release-test-pypi
    if: github.repository_owner == 'emdgroup' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-latest
    steps:
      - name: Download packages built
        uses: actions/download-artifact@v4
        with:
          name: Dist_${{  github.ref_name }}
          path: dist
      - name: Upload package to Test PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: https://test.pypi.org/legacy/
          password: ${{ secrets.TEST_PYPI_TOKEN }}

  # Upload to real PyPI on GitHub Releases.
  release-pypi:
    needs: build
    name: Publish PyPI PROD
    environment: release-pypi
    if: github.repository_owner == 'emdgroup' && github.event.action == 'published'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: '3.12'
      - name: Install test-package
        run: |
          pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ baybe==${{  github.ref_name }}
      - name: Download packages built
        uses: actions/download-artifact@v4
        with:
          name: Dist_${{  github.ref_name }}
          path: dist
      - name: Upload package to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_TOKEN }}


Create Issue Reminder

name: 'Create Issue Reminder'

permissions:
  issues: write

on:
  issue_comment:
    types: [created, edited]

jobs:
  reminder:
    runs-on: ubuntu-latest

    steps:
      - name: check for reminder
        uses: agrc/create-reminder-action@v1


Generate & Deploy Docs

name: Generate & Deploy Docs

on:
  push:
    tags:
    branches:
      - 3.x
      - 2.x
      - 1.x
      - 0.x

jobs:
  deploy:
    runs-on: ubuntu-22.04
    steps:
      - name: Clone Source Repository
        uses: actions/checkout@v3

      - name: Fetch git tags
        run: git fetch --tags origin

      - name: Clone Generated Repository
        uses: actions/checkout@v3
        with:
          token: ${{ secrets.ACCESS_TOKEN }}
          repository: fizzadar/docs.pyinfra.com
          path: docs/public/

      - name: Set up Python
        uses: actions/setup-python@v4

      - name: Install
        run: |
          pip install '.[docs]'
          pip install 'jinja2<3.1'

      - name: Generate documentation
        run: scripts/build_public_docs.sh
        env:
          PYTHONPATH: '.'

      - name: Push & Update
        run: |
          cd docs/public/

          if [[ `git status --porcelain` ]]; then
            git config user.name github-actions
            git config user.email github-actions@github.com
            git add .
            git commit -m "Docs build $(date -I)"
            git push origin HEAD:main
          fi


Lint & Test

name: Lint & Test

on:
  push:
    branches:
      - 3.x
      - 2.x
      - 1.x
      - 0.x
  pull_request:

jobs:
  # Linting
  #

  lint:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - run: pip install '.[test]'
      - run: flake8

  type-check:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - run: pip install '.[test]'
      - run: mypy

  # Unit tests
  #

  unit-test:
    needs:
      - lint
      - type-check
    strategy:
      matrix:
        os: [ubuntu-20.04, ubuntu-latest]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        include:
          - os: macos-12
            python-version: "3.11"
          - os: windows-2022
            python-version: "3.11"
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - run: pip install '.[test]'
      - run: pytest --cov --disable-warnings
      - uses: codecov/codecov-action@v3

  # End-to-end tests
  #

  end-to-end-test:
    needs:
      - lint
      - type-check
    strategy:
      matrix:
        os: [ubuntu-20.04, ubuntu-latest]
        test-name: [local, ssh, docker]
        include:
          - os: macos-latest
            test-name: local
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - run: pip install '.[test]'
      - run: pytest -m end_to_end_${{ matrix.test-name }}

  # Finally, single jon to check if all completed, for use in protected branch
  #

  tests-done:
    if: ${{ always() }}
    needs:
      - lint
      - type-check
      - unit-test
      - end-to-end-test
    runs-on: ubuntu-latest
    steps:
      - uses: matrix-org/done-action@v2
        with:
          needs: ${{ toJSON(needs) }}


CI

name: CI
on: [pull_request, push]

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@v2

    - name: Set up Conda
      uses: conda-incubator/setup-miniconda@v2
      with:
        activate-environment: myenv
        # mamba-version: "*"
        channels: conda-forge, defaults

    - name: Create Conda environment
      shell: bash -l {0}
      run: |
        # conda upgrade --strict-channel-priority -c conda-forge --all
        conda install -c conda-forge fenics numpy=1.24 python=3.10

    - name: Install dependencies
      shell: bash -l {0}
      run: |
        pip install pytest pytest-cov

    - name: Run tests
      shell: bash -l {0}
      run: |
        pytest test/ --cov festim --cov-report xml --cov-report term

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v4
      with:
        token: ${{ secrets.CODECOV_TOKEN }}


Lint

name: Lint

on: [pull_request]

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: psf/black@stable


Upload Python Package

# This workflow will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.

name: Upload Python Package

on:
  release:
    types: [published]

jobs:
  deploy:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install build
    - name: Build package
      run: python -m build
    - name: Publish package
      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}


Check and create tag

name: Check and create tag
on:
  push:
    branches:
      - master
    paths:
      - vspreview/_metadata.py

jobs:
  new_version:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2

      - name: Get version number
        run: |
          echo "CURR_VER=v$(python -c "import runpy;runpy.run_path('vspreview/_metadata.py', None, '__github__')")" >> $GITHUB_ENV
      - name: Check if version exists
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        uses: mukunku/tag-exists-action@v1.1.0
        id: tagcheck
        with:
          tag: ${{ env.CURR_VER }}

      - name: Make tag
        uses: actions/github-script@v3
        if: steps.tagcheck.outputs.exists == 'false'
        with:
          github-token: ${{ secrets.WORKFLOW_TOKEN }}
          script: |
            github.git.createRef({
              owner: context.repo.owner,
              repo: context.repo.repo,
              ref: `refs/tags/${process.env.CURR_VER}`,
              sha: context.sha
            })
      - name: Fallback
        if: steps.tagcheck.outputs.exists == 'true'
        run: echo "Nothing to see here, move along citizen"


check if docs build

name: check if docs build

on: push
  # push:
    # paths:
    #   - "**.py"
    #   - "**.rst"

jobs:
  docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Setup Python 3.12
        uses: actions/setup-python@v3
        with:
          python-version: '3.12'
      - name: Install dependencies
        run: |
          python3 -m pip install --upgrade pip
          pip install -r docs/requirements.txt
      - name: Building sphinx
        run: python3 setup.py build_sphinx -W --keep-going


lint

name: lint

on: [push, pull_request]

jobs:
  windows:
    runs-on: windows-latest
    strategy:
      matrix:
        versions:
          - 68
        python-version:
          - '3.12'

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python3 -m pip install --upgrade pip
          pip install vapoursynth-portable==${{ matrix.versions }}
          pip install -r requirements.txt


Publish releases to PyPI

name: Publish releases to PyPI
on:
  push:
   tags:
      - v[0-9]+**

jobs:
    package_build:
        name: Build and push to PyPI
        runs-on: ubuntu-latest
        steps:
        - uses: actions/checkout@v2

        - name: Prep Python
          uses: actions/setup-python@v2
          with:
            python-version: '3.12'

        - name: Install build tools
          run: |
              python -m pip install build setuptools twine --user
          continue-on-error: false
        - name: Build wheel
          id: wheel
          run: |
              python -m build --wheel --outdir dist/
          continue-on-error: true
        - name: Build source distribution
          id: sdist
          run: |
            python -m build --sdist --outdir dist/
          continue-on-error: true
        - name: Check the output
          run: |
            python -m twine check --strict dist/*
          continue-on-error: false
        - name: Die on failure
          if: steps.wheel.outcome != 'success' && steps.sdist.outcome != 'success'
          run: exit 1
        - name: Publish to PyPI
          uses: pypa/gh-action-pypi-publish@master
          with:
            user: __token__
            password: ${{ secrets.PYPI_TOKEN }}


Test docs build

name: "Test docs build"
on:
  pull_request:
    branches: [master]

jobs:
  docs:
    runs-on: ubuntu-latest
    steps:

    - name: Clone repo
      uses: actions/checkout@v4

    - name: Install docs dependencies
      working-directory: ./docs
      run: pip3 install -r requirements.txt

    - name: Build docs
      working-directory: ./docs
      run: sphinx-build -M html . _build -W


Build and Test

name:  Build and Test

on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:

  linux:
    name: Linux with Python-${{ matrix.python }}
    runs-on: ubuntu-latest
    env:
      SOTODLIB_TEST_DISABLE_PLOTS: 1
    strategy:
      # Ensure that a test continues even if another fails.  Useful for
      # debugging multiple problems in parallel.
      fail-fast: false
      matrix:
        include:
          - python: "3.8"
            pyshort: "38"
          - python: "3.9"
            pyshort: "39"
          - python: "3.10"
            pyshort: "310"
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        name: Install Python
        with:
          python-version: ${{ matrix.python }}

      - name: Install OpenMPI for Tests
        run: |
          sudo apt update
          sudo apt install -y libopenmpi-dev

      - name: Install Dependencies
        run: |
          python3 -m pip install --upgrade pip
          python3 -m pip install --upgrade wheel
          python3 -m pip install mpi4py
          python3 -m pip install so3g
          python3 -m pip install pshmem
          python3 -m pip install --pre toast

      - name: Install sotodlib
        run: |
          python3 -m pip install -vvv .[tests]

      - name: Run Serial Tests
        run: |
          export OMP_NUM_THREADS=2
          export OPENBLAS_NUM_THREADS=2
          export MPI_DISABLE=1
          python3 setup.py test

      - name: Run MPI Tests
        run: |
          export OMP_NUM_THREADS=1
          export OPENBLAS_NUM_THREADS=1
          mpirun -np 2 python3 setup.py test

  # FIXME:  Re-enable after testing this procedure on a local
  # apple machine.
  #
  # macos-clang:
  #   name: MacOS (clang) with Python-${{ matrix.python }}
  #   runs-on: macos-latest
  #   env:
  #     SOTODLIB_TEST_DISABLE_PLOTS: 1
  #   strategy:
  #     # Ensure that a test continues even if another fails.  Useful for
  #     # debugging multiple problems in parallel.
  #     fail-fast: false
  #     matrix:
  #       include:
  #         - python: "3.9"
  #           pyshort: "39"
  #         - python: "3.10"
  #           pyshort: "310"
  #   steps:
  #     - name: Cancel Previous Runs
  #       uses: styfle/cancel-workflow-action@0.12.0
  #       with:
  #         access_token: ${{ github.token }}

  #     - name: Checkout
  #       uses: actions/checkout@v4

  #     - uses: actions/setup-python@v5
  #       name: Install Python
  #       with:
  #         python-version: ${{ matrix.python }}

  #     - name: Install Homebrew Packages
  #       run: |
  #         brew install freetype pkg-config libpng

  #     - name: Install Dependencies
  #       run: |
  #         python3 -m pip install --upgrade pip
  #         python3 -m pip install --upgrade wheel
  #         python3 -m pip install so3g
  #         python3 -m pip install --pre toast

  #     - name: Install sotodlib
  #       run: |
  #         python3 -m pip install -vvv .

  #     - name: Run Serial Tests
  #       run: |
  #         export OMP_NUM_THREADS=1
  #         export OPENBLAS_NUM_THREADS=1
  #         export MPI_DISABLE=1
  #         python3 setup.py test


Build and Publish Images

name: Build and Publish Images

on:
  workflow_dispatch:
    inputs:
      target_platforms:
        type: choice
        description: Platforms
        options:
        - linux/amd64
        - linux/arm64
        - linux/amd64,linux/arm64
  push:
    tags:
      - "[0-9]+.[0-9]+.[0-9]+"
      - "[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+"
      - "[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+"
      - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"

env:
  GO_VERSION: "1.21.4"
  BUILDKIT_VERSION: "v0.13.2"

jobs:
  publish-generic-images:
    name: Publish
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
          - image: docker-registry
          - image: pause-container
          - image: session-manager
          - image: training-portal
          - image: secrets-manager
          - image: tunnel-manager
          - image: image-cache
          - image: assets-server

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Calculate platforms
        shell: bash
        run: |
          TARGET_PLATFORMS="${{github.event.inputs.target_platforms}}"
          if [ -z "$TARGET_PLATFORMS" ]; then
            TARGET_PLATFORMS="${{secrets.TARGET_PLATFORMS}}"
          fi
          if [ -z "$TARGET_PLATFORMS" ]; then
            IS_FORK=$(if [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_EVENT_PATH ]]; then jq -r .repository.fork <<< $(cat $GITHUB_EVENT_PATH); else echo false; fi)
            if [ $IS_FORK == "true" ]; then
              TARGET_PLATFORMS="linux/amd64"
            else
              TARGET_PLATFORMS="linux/amd64,linux/arm64"
            fi
          fi
          echo "TARGET_PLATFORMS=${TARGET_PLATFORMS}" >>${GITHUB_ENV}

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=${REPOSITORY_OWNER,,}" >>${GITHUB_ENV}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: |
            image=moby/buildkit:${{env.BUILDKIT_VERSION}}

      - name: Restore Docker cache (amd64)
        if: ${{ (matrix.image == 'secrets-manager') || (matrix.image == 'session-manager') || (matrix.image == 'training-portal') || (matrix.image == 'tunnel-manager') }}
        uses: actions/cache/restore@v4
        with:
          path: /tmp/.buildx-cache-amd64-new
          key: ${{runner.os}}-buildx-cache-amd64-${{matrix.image}}-${{github.sha}}
          restore-keys: |
            ${{runner.os}}-buildx-cache-amd64-${{matrix.image}}-

      - name: Restore Docker cache (arm64)
        if: ${{ (matrix.image == 'secrets-manager') || (matrix.image == 'session-manager') || (matrix.image == 'training-portal') || (matrix.image == 'tunnel-manager') }}
        uses: actions/cache/restore@v4
        with:
          path: /tmp/.buildx-cache-arm64-new
          key: ${{runner.os}}-buildx-cache-arm64-${{matrix.image}}-${{github.sha}}
          restore-keys: |
            ${{runner.os}}-buildx-cache-arm64-${{matrix.image}}-

      - name: Rename cache directories
        run: |
          test -d /tmp/.buildx-cache-amd64-new && mv /tmp/.buildx-cache-amd64-new /tmp/.buildx-cache-amd64-old || true
          test -d /tmp/.buildx-cache-arm64-new && mv /tmp/.buildx-cache-arm64-new /tmp/.buildx-cache-arm64-old || true
          du -ks /tmp/.buildx-cache-* || true

      - name: Generate container image metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            ghcr.io/${{env.REPOSITORY_OWNER}}/educates-${{matrix.image}}
          tags: |
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to GitHub container registry
        uses: docker/login-action@v3
        with:
          username: ${{github.actor}}
          password: ${{secrets.GITHUB_TOKEN}}
          registry: ghcr.io

      - name: Build and push ${{matrix.image}} image
        uses: docker/build-push-action@v5
        with:
          context: ${{matrix.image}}
          platforms: ${{env.TARGET_PLATFORMS}}
          tags: ${{steps.meta.outputs.tags}}
          cache-from: |
            type=local,src=/tmp/.buildx-cache-arm64-old
            type=local,src=/tmp/.buildx-cache-amd64-old
          push: true

      - name: Cache build ${{matrix.image}} (amd64)
        if: contains(env.TARGET_PLATFORMS, 'linux/amd64')
        uses: docker/build-push-action@v5
        with:
          context: ${{matrix.image}}
          platforms: linux/amd64
          cache-from: type=local,src=/tmp/.buildx-cache-amd64-old
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-amd64-new
          push: false

      - name: Cache build ${{matrix.image}} (arm64)
        if: contains(env.TARGET_PLATFORMS, 'linux/arm64')
        uses: docker/build-push-action@v5
        with:
          context: ${{matrix.image}}
          platforms: linux/arm64
          cache-from: type=local,src=/tmp/.buildx-cache-arm64-old
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-arm64-new
          push: false

      - name: Dump cache directory sizes
        if: ${{ (matrix.image == 'secrets-manager') || (matrix.image == 'session-manager') || (matrix.image == 'training-portal') || (matrix.image == 'tunnel-manager') }}
        run: |
          du -ks /tmp/.buildx-cache-* || true

      - name: Save Docker cache (amd64)
        if: ${{ (matrix.image == 'secrets-manager') || (matrix.image == 'session-manager') || (matrix.image == 'training-portal') || (matrix.image == 'tunnel-manager') }}
        uses: actions/cache/save@v4
        with:
          path: /tmp/.buildx-cache-amd64-new
          key: ${{runner.os}}-buildx-cache-amd64-${{matrix.image}}-${{github.sha}}

      - name: Save Docker cache (arm64)
        if: ${{ (matrix.image == 'secrets-manager') || (matrix.image == 'session-manager') || (matrix.image == 'training-portal') || (matrix.image == 'tunnel-manager') }}
        uses: actions/cache/save@v4
        with:
          path: /tmp/.buildx-cache-arm64-new
          key: ${{runner.os}}-buildx-cache-arm64-${{matrix.image}}-${{github.sha}}

  publish-workshop-base-image:
    name: Publish (base-environment)
    runs-on: ubuntu-latest

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Calculate platforms
        shell: bash
        run: |
          TARGET_PLATFORMS="${{github.event.inputs.target_platforms}}"
          if [ -z "$TARGET_PLATFORMS" ]; then
            TARGET_PLATFORMS="${{secrets.TARGET_PLATFORMS}}"
          fi
          if [ -z "$TARGET_PLATFORMS" ]; then
            IS_FORK=$(if [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_EVENT_PATH ]]; then jq -r .repository.fork <<< $(cat $GITHUB_EVENT_PATH); else echo false; fi)
            if [ $IS_FORK == "true" ]; then
              TARGET_PLATFORMS="linux/amd64"
            else
              TARGET_PLATFORMS="linux/amd64,linux/arm64"
            fi
          fi
          echo "TARGET_PLATFORMS=${TARGET_PLATFORMS}" >>${GITHUB_ENV}

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=${REPOSITORY_OWNER,,}" >>${GITHUB_ENV}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: |
            image=moby/buildkit:${{env.BUILDKIT_VERSION}}

      - name: Verify docker daemon image contents
        run: |
          docker images

      - name: Remove unwanted images to free storage space
        run: |
          docker rmi node:14 || true
          docker rmi node:16 || true
          docker rmi node:18 || true
          docker rmi node:20 || true
          docker images prune

      - name: Restore Docker cache (amd64)
        uses: actions/cache/restore@v4
        with:
          path: /tmp/.buildx-cache-amd64-new
          key: ${{runner.os}}-buildx-cache-amd64-base-environment-${{github.sha}}
          restore-keys: |
            ${{runner.os}}-buildx-cache-amd64-base-environment-

      - name: Restore Docker cache (arm64)
        uses: actions/cache/restore@v4
        with:
          path: /tmp/.buildx-cache-arm64-new
          key: ${{runner.os}}-buildx-cache-arm64-base-environment-${{github.sha}}
          restore-keys: |
            ${{runner.os}}-buildx-cache-arm64-base-environment-

      - name: Rename cache directories
        run: |
          test -d /tmp/.buildx-cache-amd64-new && mv /tmp/.buildx-cache-amd64-new /tmp/.buildx-cache-amd64-old || true
          test -d /tmp/.buildx-cache-arm64-new && mv /tmp/.buildx-cache-arm64-new /tmp/.buildx-cache-arm64-old || true
          du -ks /tmp/.buildx-cache-* || true

      - name: Generate container image metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            ghcr.io/${{env.REPOSITORY_OWNER}}/educates-base-environment
          tags: |
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha

      - name: Login to GitHub container registry
        uses: docker/login-action@v3
        with:
          username: ${{github.actor}}
          password: ${{secrets.GITHUB_TOKEN}}
          registry: ghcr.io

      - name: Build and push base-environment image
        uses: docker/build-push-action@v5
        with:
          context: workshop-images/base-environment
          platforms: ${{env.TARGET_PLATFORMS}}
          tags: ${{steps.meta.outputs.tags}}
          cache-from: |
            type=local,src=/tmp/.buildx-cache-arm64-old
            type=local,src=/tmp/.buildx-cache-amd64-old
          push: true

      - name: Cache build ${{matrix.image}} (amd64)
        if: contains(env.TARGET_PLATFORMS, 'linux/amd64')
        uses: docker/build-push-action@v5
        with:
          context: workshop-images/base-environment
          platforms: linux/amd64
          cache-from: type=local,src=/tmp/.buildx-cache-amd64-old
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-amd64-new
          push: false

      - name: Remove old cache directories
        run: |
          rm -rf /tmp/.buildx-cache-amd64-old

      - name: Cache build ${{matrix.image}} (arm64)
        if: contains(env.TARGET_PLATFORMS, 'linux/arm64')
        uses: docker/build-push-action@v5
        with:
          context: workshop-images/base-environment
          platforms: linux/arm64
          cache-from: type=local,src=/tmp/.buildx-cache-arm64-old
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-arm64-new
          push: false

      - name: Dump cache directory sizes
        run: |
          du -ks /tmp/.buildx-cache-* || true

      - name: Save Docker cache (amd64)
        uses: actions/cache/save@v4
        with:
          path: /tmp/.buildx-cache-amd64-new
          key: ${{runner.os}}-buildx-cache-amd64-base-environment-${{github.sha}}

      - name: Save Docker cache (arm64)
        uses: actions/cache/save@v4
        with:
          path: /tmp/.buildx-cache-arm64-new
          key: ${{runner.os}}-buildx-cache-arm64-base-environment-${{github.sha}}

  publish-workshop-images:
    name: Publish
    runs-on: ubuntu-latest
    needs:
      - publish-workshop-base-image

    strategy:
      fail-fast: false
      matrix:
        include:
          - image: jdk8-environment
          - image: jdk11-environment
          - image: jdk17-environment
          - image: jdk21-environment
          - image: conda-environment

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Calculate platforms
        shell: bash
        run: |
          TARGET_PLATFORMS="${{github.event.inputs.target_platforms}}"
          if [ -z "$TARGET_PLATFORMS" ]; then
            TARGET_PLATFORMS="${{secrets.TARGET_PLATFORMS}}"
          fi
          if [ -z "$TARGET_PLATFORMS" ]; then
            IS_FORK=$(if [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_EVENT_PATH ]]; then jq -r .repository.fork <<< $(cat $GITHUB_EVENT_PATH); else echo false; fi)
            if [ $IS_FORK == "true" ]; then
              TARGET_PLATFORMS="linux/amd64"
            else
              TARGET_PLATFORMS="linux/amd64,linux/arm64"
            fi
          fi
          echo "TARGET_PLATFORMS=${TARGET_PLATFORMS}" >>${GITHUB_ENV}

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=${REPOSITORY_OWNER,,}" >>${GITHUB_ENV}
          echo "REPOSITORY_SHA_TAG=sha-${GITHUB_SHA::7}" >>${GITHUB_ENV}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: |
            image=moby/buildkit:${{env.BUILDKIT_VERSION}}

      - name: Verify docker daemon image contents
        run: |
          docker images

      - name: Remove unwanted images to free storage space
        run: |
          docker rmi node:14 || true
          docker rmi node:16 || true
          docker rmi node:18 || true
          docker rmi node:20 || true
          docker images prune

      - name: Restore Docker cache (amd64)
        if: ${{ (matrix.image == 'conda-environment') }}
        uses: actions/cache/restore@v4
        with:
          path: /tmp/.buildx-cache-amd64-new
          key: ${{runner.os}}-buildx-cache-amd64-${{matrix.image}}-${{github.sha}}
          restore-keys: |
            ${{runner.os}}-buildx-cache-amd64-${{matrix.image}}-

      - name: Restore Docker cache (arm64)
        if: ${{ (matrix.image == 'conda-environment') }}
        uses: actions/cache/restore@v4
        with:
          path: /tmp/.buildx-cache-arm64-new
          key: ${{runner.os}}-buildx-cache-arm64-${{matrix.image}}-${{github.sha}}
          restore-keys: |
            ${{runner.os}}-buildx-cache-arm64-${{matrix.image}}-

      - name: Rename cache directories
        run: |
          test -d /tmp/.buildx-cache-amd64-new && mv /tmp/.buildx-cache-amd64-new /tmp/.buildx-cache-amd64-old || true
          test -d /tmp/.buildx-cache-arm64-new && mv /tmp/.buildx-cache-arm64-new /tmp/.buildx-cache-arm64-old || true
          du -ks /tmp/.buildx-cache-* || true

      - name: Generate container image metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            ghcr.io/${{env.REPOSITORY_OWNER}}/educates-${{matrix.image}}
          tags: |
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to GitHub container registry
        uses: docker/login-action@v3
        with:
          username: ${{github.actor}}
          password: ${{secrets.GITHUB_TOKEN}}
          registry: ghcr.io

      - name: Build and push ${{matrix.image}} image
        uses: docker/build-push-action@v5
        with:
          context: workshop-images/${{matrix.image}}
          platforms: ${{env.TARGET_PLATFORMS}}
          tags: ${{steps.meta.outputs.tags}}
          build-args: |
            IMAGE_REPOSITORY=ghcr.io/${{env.REPOSITORY_OWNER}}
            BASE_IMAGE_NAME=educates-base-environment
            PACKAGE_VERSION=${{env.REPOSITORY_SHA_TAG}}
          cache-from: |
            type=local,src=/tmp/.buildx-cache-arm64-old
            type=local,src=/tmp/.buildx-cache-amd64-old
          push: true

      - name: Cache build ${{matrix.image}} (amd64)
        if: contains(env.TARGET_PLATFORMS, 'linux/amd64')
        uses: docker/build-push-action@v5
        with:
          context: workshop-images/${{matrix.image}}
          platforms: linux/amd64
          build-args: |
            IMAGE_REPOSITORY=ghcr.io/${{env.REPOSITORY_OWNER}}
            BASE_IMAGE_NAME=educates-base-environment
            PACKAGE_VERSION=${{env.REPOSITORY_SHA_TAG}}
          cache-from: type=local,src=/tmp/.buildx-cache-amd64-old
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-amd64-new
          push: false

      - name: Remove old cache directories
        run: |
          rm -rf /tmp/.buildx-cache-amd64-old

      - name: Cache build ${{matrix.image}} (arm64)
        if: contains(env.TARGET_PLATFORMS, 'linux/arm64')
        uses: docker/build-push-action@v5
        with:
          context: workshop-images/${{matrix.image}}
          platforms: linux/arm64
          build-args: |
            IMAGE_REPOSITORY=ghcr.io/${{env.REPOSITORY_OWNER}}
            BASE_IMAGE_NAME=educates-base-environment
            PACKAGE_VERSION=${{env.REPOSITORY_SHA_TAG}}
          cache-from: type=local,src=/tmp/.buildx-cache-arm64-old
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-arm64-new
          push: false

      - name: Dump cache directory sizes
        if: ${{ (matrix.image == 'conda-environment') }}
        run: |
          du -ks /tmp/.buildx-cache-* || true

      - name: Save Docker cache (amd64)
        if: ${{ (matrix.image == 'conda-environment') }}
        uses: actions/cache/save@v4
        with:
          path: /tmp/.buildx-cache-amd64-new
          key: ${{runner.os}}-buildx-cache-amd64-${{matrix.image}}-${{github.sha}}

      - name: Save Docker cache (arm64)
        if: ${{ (matrix.image == 'conda-environment') }}
        uses: actions/cache/save@v4
        with:
          path: /tmp/.buildx-cache-arm64-new
          key: ${{runner.os}}-buildx-cache-arm64-${{matrix.image}}-${{github.sha}}

  publish-carvel-bundles:
    name: Bundle
    runs-on: ubuntu-latest
    needs:
      - publish-generic-images
      - publish-workshop-images

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Install Carvel tools
        shell: bash
        run: curl -L https://carvel.dev/install.sh | bash

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=${REPOSITORY_OWNER,,}" >>${GITHUB_ENV}
          echo "REPOSITORY_TAG=${GITHUB_REF##*/}" >>${GITHUB_ENV}
          echo "REPOSITORY_SHA_TAG=sha-${GITHUB_SHA::7}" >>${GITHUB_ENV}

      - name: Create publish values file
        shell: bash
        run: |
          cat <<EOF > publish-values.yaml
          clusterInfrastructure:
            provider: "custom"
          clusterPackages:
            contour:
              enabled: true
              settings:
                infraProvider: custom
            cert-manager:
              enabled: true
            external-dns:
              enabled: true
              settings:
                infraProvider: custom
                deployment:
                  args:
                    - --provider=custom
                    - --source=custom
            certs:
              enabled: true
              settings:
                certProvider: local
                domains:
                  - "example.com"
                local:
                  caCertificate:
                    ca.crt: "AA"
                    ca.key: "BB"
            kyverno:
              enabled: true
            educates:
              enabled: true
              settings:
                clusterIngress:
                  domain: "educates.example.com"
                imageRegistry:
                  host: "ghcr.io"
                  namespace: ${{env.REPOSITORY_OWNER}}
                version: ${{env.REPOSITORY_TAG}}
          EOF

      - name: Publish educates-installer bundle
        shell: bash
        run: |
          # Create the kbld-images.yaml file with references to educates images
          ytt -f carvel-packages/installer/config/images.yaml \
              -f carvel-packages/installer/config/schema.yaml \
              -v imageRegistry.host=ghcr.io \
              -v imageRegistry.namespace=${{env.REPOSITORY_OWNER}} \
              -v version=${{env.REPOSITORY_TAG}} > carvel-packages/installer/bundle/kbld/kbld-images.yaml
          # Cat the generated file for debugging purposes
          cat carvel-packages/installer/bundle/kbld/kbld-images.yaml
          # Create images lock file. We use a sample values file to pass validations
          # We properly rewrite references to images via kbld
          ytt --data-values-file publish-values.yaml \
              -f carvel-packages/installer/bundle/config | kbld -f - \
              -f carvel-packages/installer/bundle/kbld/kbld-images.yaml \
            --imgpkg-lock-output carvel-packages/installer/bundle/.imgpkg/images.yml
          # Push the bundle to the registry
          imgpkg push \
            -b ghcr.io/${{env.REPOSITORY_OWNER}}/educates-installer:${{env.REPOSITORY_TAG}} \
            -f carvel-packages/installer/bundle \
            --registry-username=${{github.actor}} \
            --registry-password=${{secrets.GITHUB_TOKEN}}
          # Generate the schema for the bundle
          ytt -f carvel-packages/installer/bundle \
            --data-values-schema-inspect -o openapi-v3 > educates-installer-schema-openapi.yaml
          # Generate the package.yaml file
          ytt -f carvel-packages/installer/config/package.yaml \
            -f carvel-packages/installer/config/schema.yaml \
            -v version=${{env.REPOSITORY_TAG}} \
            -v releasedAt=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
            --data-value-file openapi=educates-installer-schema-openapi.yaml \
            -v imageRegistry.host=ghcr.io \
            -v imageRegistry.namespace=${{env.REPOSITORY_OWNER}} > package-repository/packages/installer.educates.dev/educates-installer-${{env.REPOSITORY_TAG}}.yaml

      - name: Save educates-installer.yaml
        uses: actions/upload-artifact@v4
        with:
          name: educates-installer-${{env.REPOSITORY_TAG}}.yaml
          path: package-repository/packages/installer.educates.dev/educates-installer-${{env.REPOSITORY_TAG}}.yaml

      - name: Publish educates-training-platform package repository
        shell: bash
        run: |
          ytt -f package-repository/packages | kbld -f - --imgpkg-lock-output package-repository/.imgpkg/images.yml
          imgpkg push --debug -b ghcr.io/${{env.REPOSITORY_OWNER}}/educates-packages:${{env.REPOSITORY_TAG}} -f package-repository --registry-username=${{github.actor}} --registry-password=${{secrets.GITHUB_TOKEN}}
          ytt -f carvel-packages/repository.yaml -v packageRepository=ghcr.io/${{env.REPOSITORY_OWNER}} -v repositoryVersion=${{env.REPOSITORY_TAG}} > educates-packagerepository.yaml

      - name: Save educates-training-platform repository.yaml
        uses: actions/upload-artifact@v4
        with:
          name: educates-packagerepository.yaml
          path: educates-packagerepository.yaml

      - name: Save educates-training-platform package repository
        uses: actions/upload-artifact@v4
        with:
          name: package-repository
          path: package-repository

  build-client-programs-linux-amd64:
    name: Build (clients) / amd64@linux
    runs-on: ubuntu-latest

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=${REPOSITORY_OWNER,,}" >>${GITHUB_ENV}

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{env.GO_VERSION}}
          cache-dependency-path: |
            client-programs/go.sum

      - name: Build educates client program
        shell: bash
        run: |
          rm -rf client-programs/pkg/renderer/files
          mkdir client-programs/pkg/renderer/files
          cp -rp workshop-images/base-environment/opt/eduk8s/etc/themes client-programs/pkg/renderer/files/
          cd client-programs
          REPOSITORY_TAG=${GITHUB_REF##*/}
          IMAGE_REPOSITORY=ghcr.io/${{env.REPOSITORY_OWNER}}
          go build -o educates-linux-amd64 -ldflags "-X 'main.projectVersion=$REPOSITORY_TAG' -X 'main.imageRepository=$IMAGE_REPOSITORY'" cmd/educates/main.go

      - uses: actions/upload-artifact@v4
        with:
          name: educates-linux-amd64
          path: client-programs/educates-linux-amd64

  build-client-programs-linux-arm64:
    name: Build (clients) / arm64@linux
    runs-on: ubuntu-latest

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=${REPOSITORY_OWNER,,}" >>${GITHUB_ENV}

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{env.GO_VERSION}}
          cache-dependency-path: |
            client-programs/go.sum

      - name: Build educates client program
        shell: bash
        run: |
          rm -rf client-programs/pkg/renderer/files
          mkdir client-programs/pkg/renderer/files
          cp -rp workshop-images/base-environment/opt/eduk8s/etc/themes client-programs/pkg/renderer/files/
          cd client-programs
          REPOSITORY_TAG=${GITHUB_REF##*/}
          IMAGE_REPOSITORY=ghcr.io/${{env.REPOSITORY_OWNER}}
          GOOS=linux GOARCH=arm64 go build -o educates-linux-arm64 -ldflags "-X 'main.projectVersion=$REPOSITORY_TAG' -X 'main.imageRepository=$IMAGE_REPOSITORY'" cmd/educates/main.go

      - uses: actions/upload-artifact@v4
        with:
          name: educates-linux-arm64
          path: client-programs/educates-linux-arm64

  build-client-programs-darwin-amd64:
    name: Build (clients) / amd64@darwin
    runs-on: macos-12

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=$(echo "$REPOSITORY_OWNER" | tr '[:upper:]' '[:lower:]')" >>${GITHUB_ENV}

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{env.GO_VERSION}}
          cache-dependency-path: |
            client-programs/go.sum

      - name: Build educates client program
        shell: bash
        run: |
          rm -rf client-programs/pkg/renderer/files
          mkdir client-programs/pkg/renderer/files
          cp -rp workshop-images/base-environment/opt/eduk8s/etc/themes client-programs/pkg/renderer/files/
          cd client-programs
          REPOSITORY_TAG=${GITHUB_REF##*/}
          IMAGE_REPOSITORY=ghcr.io/${{env.REPOSITORY_OWNER}}
          # DO NOT USE GOOS/GOARCH for native build as it appears to produce a
          # binary which is different and cannot create a Kind cluster which can
          # run both AMD and ARM images. Version with GOOS/GOARCH only retained
          # here for documentation purposes.
          # GOOS=darwin GOARCH=amd64 go build -o educates-darwin-amd64 -ldflags "-X 'main.projectVersion=$REPOSITORY_TAG' -X 'main.imageRepository=$IMAGE_REPOSITORY'" cmd/educates/main.go
          go build -o educates-darwin-amd64 -ldflags "-X 'main.projectVersion=$REPOSITORY_TAG' -X 'main.imageRepository=$IMAGE_REPOSITORY'" cmd/educates/main.go

      - uses: actions/upload-artifact@v4
        with:
          name: educates-darwin-amd64
          path: client-programs/educates-darwin-amd64

  build-client-programs-darwin-arm64:
    name: Build (clients) / arm64@darwin
    runs-on: macos-12

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=$(echo "$REPOSITORY_OWNER" | tr '[:upper:]' '[:lower:]')" >>${GITHUB_ENV}

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{env.GO_VERSION}}
          cache-dependency-path: |
            client-programs/go.sum

      - name: Build educates client program
        shell: bash
        run: |
          rm -rf client-programs/pkg/renderer/files
          mkdir client-programs/pkg/renderer/files
          cp -rp workshop-images/base-environment/opt/eduk8s/etc/themes client-programs/pkg/renderer/files/
          cd client-programs
          REPOSITORY_TAG=${GITHUB_REF##*/}
          IMAGE_REPOSITORY=ghcr.io/${{env.REPOSITORY_OWNER}}
          GOOS=darwin GOARCH=arm64 go build -o educates-darwin-arm64 -ldflags "-X 'main.projectVersion=$REPOSITORY_TAG' -X 'main.imageRepository=$IMAGE_REPOSITORY'" cmd/educates/main.go

      - uses: actions/upload-artifact@v4
        with:
          name: educates-darwin-arm64
          path: client-programs/educates-darwin-arm64

  publish-client-programs:
    name: Programs
    runs-on: ubuntu-latest
    needs:
      - build-client-programs-linux-amd64
      - build-client-programs-linux-arm64
      - build-client-programs-darwin-amd64
      - build-client-programs-darwin-arm64

    steps:
      - name: Restore educates-linux-amd64
        uses: actions/download-artifact@v4
        with:
          name: educates-linux-amd64
          path: client-programs

      - name: Restore educates-linux-arm64
        uses: actions/download-artifact@v4
        with:
          name: educates-linux-arm64
          path: client-programs

      - name: Restore educates-darwin-amd64
        uses: actions/download-artifact@v4
        with:
          name: educates-darwin-amd64
          path: client-programs

      - name: Restore educates-darwin-arm64
        uses: actions/download-artifact@v4
        with:
          name: educates-darwin-arm64
          path: client-programs

      - name: Install Carvel tools
        shell: bash
        run: curl -L https://carvel.dev/install.sh | bash

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=${REPOSITORY_OWNER,,}" >>${GITHUB_ENV}
          echo "REPOSITORY_TAG=${GITHUB_REF##*/}" >>${GITHUB_ENV}

      - name: Publish client programs
        shell: bash
        run: |
          chmod +x client-programs/*
          imgpkg push \
            -i ghcr.io/${{env.REPOSITORY_OWNER}}/educates-client-programs:${{env.REPOSITORY_TAG}} \
            -f client-programs \
            --registry-username=${{github.actor}} \
            --registry-password=${{secrets.GITHUB_TOKEN}}

  publish-docker-extension:
    name: Extension
    runs-on: ubuntu-latest
    needs:
      - publish-client-programs

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_OWNER=${{github.repository_owner}}
          echo "REPOSITORY_OWNER=${REPOSITORY_OWNER,,}" >>${GITHUB_ENV}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: |
            image=moby/buildkit:${{env.BUILDKIT_VERSION}}

      - name: Generate container image metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            ghcr.io/${{env.REPOSITORY_OWNER}}/educates-docker-extension
          tags: |
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to GitHub container registry
        uses: docker/login-action@v3
        with:
          username: ${{github.actor}}
          password: ${{secrets.GITHUB_TOKEN}}
          registry: ghcr.io

      - name: Calculate variables
        shell: bash
        run: |
          echo "REPOSITORY_TAG=${GITHUB_REF##*/}" >>${GITHUB_ENV}

      - name: Build and push docker-extension image
        uses: docker/build-push-action@v5
        with:
          context: docker-extension
          platforms: ${{env.TARGET_PLATFORMS}}
          build-args: |
            REPOSITORY=ghcr.io/${{env.REPOSITORY_OWNER}}
            TAG=${{env.REPOSITORY_TAG}}
          tags: ${{steps.meta.outputs.tags}}
          push: true

  release-artifacts:
    name: Release
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/')
    needs:
      - publish-carvel-bundles
      - build-client-programs-linux-amd64
      - build-client-programs-linux-arm64
      - build-client-programs-darwin-amd64
      - build-client-programs-darwin-arm64
      - publish-docker-extension

    steps:
      - name: Calculate variables
        shell: bash
        run: |
          REPOSITORY_TAG=${GITHUB_REF##*/}
          if [[ "$REPOSITORY_TAG" == *-* ]]; then
            PRERELEASE=true
          else
            PRERELEASE=false
          fi
          echo "REPOSITORY_TAG=${REPOSITORY_TAG}" >>${GITHUB_ENV}
          echo "PRERELEASE=${PRERELEASE}" >>${GITHUB_ENV}

      - name: Restore packagerepository.yaml
        uses: actions/download-artifact@v4
        with:
          name: educates-packagerepository.yaml

      - name: Restore educates-installer.yaml
        uses: actions/download-artifact@v4
        with:
          name: educates-installer-${{env.REPOSITORY_TAG}}.yaml

      - name: Restore educates-linux-amd64
        uses: actions/download-artifact@v4
        with:
          name: educates-linux-amd64

      - name: Restore educates-linux-arm64
        uses: actions/download-artifact@v4
        with:
          name: educates-linux-arm64

      - name: Restore educates-darwin-amd64
        uses: actions/download-artifact@v4
        with:
          name: educates-darwin-amd64

      - name: Restore educates-darwin-arm64
        uses: actions/download-artifact@v4
        with:
          name: educates-darwin-arm64

      - name: Generate file checksums for CLI binaries
        shell: bash
        run: |
          sha256sum educates-darwin-amd64 >> checksums.txt
          sha256sum educates-darwin-arm64 >> checksums.txt
          sha256sum educates-installer-${{env.REPOSITORY_TAG}}.yaml >> checksums.txt
          sha256sum educates-linux-amd64 >> checksums.txt
          sha256sum educates-linux-arm64 >> checksums.txt
          sha256sum educates-packagerepository.yaml >> checksums.txt
          echo 'File Checksums' >> release-notes.md
          echo '--------------' >> release-notes.md
          echo '```' >> release-notes.md
          cat checksums.txt >> release-notes.md
          echo '```' >> release-notes.md

      - uses: actions/upload-artifact@v4
        with:
          name: checksums.txt
          path: checksums.txt

      - uses: actions/upload-artifact@v4
        with:
          name: release-notes.md
          path: release-notes.md

      - name: Create release
        id: create_release
        uses: softprops/action-gh-release@v1
        env:
          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
        with:
          tag_name: ${{env.REPOSITORY_TAG}}
          name: "educates:${{env.REPOSITORY_TAG}}"
          draft: false
          prerelease: ${{env.PRERELEASE}}
          body_path: release-notes.md
          files: |
            checksums.txt
            educates-packagerepository.yaml
            educates-installer-${{env.REPOSITORY_TAG}}.yaml
            educates-linux-amd64
            educates-linux-arm64
            educates-darwin-amd64
            educates-darwin-arm64

  commit-packages:
    name: Commit
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/')
    needs:
      - release-artifacts

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Calculate variables
        shell: bash
        run: |
          IS_FORK=$(if [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_EVENT_PATH ]]; then jq -r .repository.fork <<< $(cat $GITHUB_EVENT_PATH); else echo false; fi)
          REPOSITORY_TAG=${GITHUB_REF##*/}
          if [[ "$REPOSITORY_TAG" == *-* ]]; then
            PRERELEASE=true
          else
            PRERELEASE=false
          fi
          echo "IS_FORK=${IS_FORK}" >>${GITHUB_ENV}
          echo "PRERELEASE=${PRERELEASE}" >>${GITHUB_ENV}
          echo "REPOSITORY_TAG=${REPOSITORY_TAG}" >>${GITHUB_ENV}

      - name: Restore educates-installer.yaml
        uses: actions/download-artifact@v4
        with:
          name: educates-installer-${{env.REPOSITORY_TAG}}.yaml
          path: package-repository/packages/installer.educates.dev/

      - name: Create pull request for package repository files
        if: ${{ (env.IS_FORK == 'false') && (env.PRERELEASE == 'false') }}
        uses: peter-evans/create-pull-request@v5
        with:
          add-paths: |
            package-repository
          commit-message: "Add package files for version ${{env.REPOSITORY_TAG}}."
          branch: release/patch
          base: develop
          title: "Add package files for version ${{env.REPOSITORY_TAG}}."
          body: >
            This is an automated pull request to add the package files for
            version ${{env.REPOSITORY_TAG}} to the repository. If the release is
            successful and you are working in the main Educates repository, this
            needs to be merged to the `develop` branch before making a further
            release. You should avoid merging this pull request if working in a
            fork of the main Educates repository as it will result in changes to
            the `develop` branch which have details specific to your fork, which
            should never end up back in the main Educates repository.


unit-tests

name: unit-tests
run-name: Run unit tests after push by @${{ github.actor }}
on:
  push:
    branches: [ test-actions ]
jobs:
  unit-test:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os : [ubuntu-latest]
        python-version: [ "3.9" ]
    env:
      DISPLAY: ':99.0'
    steps:
      - uses: actions/checkout@v3
      - uses: tlambert03/setup-qt-libs@v1
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo apt install libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 x11-utils libegl1
          /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1920x1200x24 -ac +extension GLX
          sudo apt install qt6-base-dev
          pip install -r requirements.dev.txt
      - name: Test with pytest
        run: |
          ./tests/test.sh


CI

name: CI

on:
  pull_request:
    branches:
      - main

jobs:
  linters:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ '3.11' ]
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install -e .[linters] --no-cache-dir
      - name: Run linters
        run: flake8 . && interrogate . && codespell src examples tests
  tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ '3.8', '3.9', '3.10', '3.11' ]
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        sudo apt install graphviz libgraphviz-dev -y
        python -m pip install -e .[dev] -e .[docs] --no-cache-dir
    - name: Run tests and coverage
      run: coverage run -m unittest discover && coverage report