Linux Python build

name: Linux Python build

on:
  push:
    branches: [main]
  pull_request:
  schedule:
    - cron: '13 11 * * *'
jobs:
  pythonbuild:
    runs-on: ubuntu-22.04
    steps:
      - name: Install System Dependencies
        run: |
          sudo apt update
          sudo apt install -y --no-install-recommends libssl-dev pkg-config

      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable

      - name: Emit rustc version
        run: |
          rustc --version > .rustc-version

      - uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-pythonbuild-${{ hashFiles('Cargo.lock', '.rustc-version') }}

      - name: Build
        run: |
          cargo build --release

      - name: Upload pythonbuild Executable
        uses: actions/upload-artifact@v4
        with:
          name: pythonbuild
          path: target/release/pythonbuild

  image:
    strategy:
      fail-fast: false
      matrix:
        image:
          - build
          - build.cross
          - gcc
          - xcb
          - xcb.cross
    runs-on: ubuntu-22.04
    permissions:
      packages: write
    steps:
      - uses: actions/checkout@v4

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Generate Dockerfiles
        run: |
          ./build-linux.py --make-target empty
          repo_name=$(echo "${GITHUB_REPOSITORY,,}" | sed 's|\.|_|g')
          git_ref_name=$(echo "${GITHUB_REF_NAME,,}" | sed 's|[^a-z0-9_-]|_|g')
          echo "REPO_NAME=${repo_name}" >> "${GITHUB_ENV}"
          echo "GIT_REF_NAME=${git_ref_name}" >> "${GITHUB_ENV}"

      - name: Build Image
        id: build-image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: build/${{ matrix.image }}.Dockerfile
          labels: org.opencontainers.image.source=https://github.com/${{ env.REPO_NAME }}
          # Cache from/to the current branch of the current repo as the primary cache key.
          # Cache from the default branch of the current repo so branches can have cache hits.
          # Cache from the default branch of the canonical repo so forks can have cache hits.
          # Ignore errors on cache writes so CI of forks works without a valid GHCR config.
          cache-from: |
            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }}
            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-main
            type=registry,ref=ghcr.io/indygreg/python-build-standalone:${{ matrix.image }}-main
          cache-to: |
            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }},ignore-error=true
          outputs: |
            type=docker,dest=build/image-${{ matrix.image }}.tar

      - name: Compress Image
        run: |
          echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.image }}
          zstd -v -T0 -6 --rm build/image-*.tar

      - name: Upload Docker Image
        uses: actions/upload-artifact@v4
        with:
          name: image-${{ matrix.image }}
          path: build/image-*

  build:
    strategy:
      fail-fast: false
      matrix:
        build:
          # Cross-compiles can't do PGO.
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.8'
            optimizations: 'debug'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.8'
            optimizations: 'noopt'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.8'
            optimizations: 'lto'

          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'debug'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'noopt'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'lto'

          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'debug'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'noopt'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'lto'

          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'debug'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'noopt'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'lto'

          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'debug'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'noopt'
          - target_triple: 'aarch64-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'lto'

          # Cross-compiles can't do PGO and require Python 3.9.
          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.9'
            optimizations: 'debug'
          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.9'
            optimizations: 'noopt'
          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.9'
            optimizations: 'lto'

          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.10'
            optimizations: 'debug'
          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.10'
            optimizations: 'noopt'
          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.10'
            optimizations: 'lto'

          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.11'
            optimizations: 'debug'
          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.11'
            optimizations: 'noopt'
          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.11'
            optimizations: 'lto'

          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.12'
            optimizations: 'debug'
          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.12'
            optimizations: 'noopt'
          - target_triple: 'armv7-unknown-linux-gnueabi'
            py: 'cpython-3.12'
            optimizations: 'lto'

          # Cross-compiles can't do PGO and require Python 3.9.
          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.9'
            optimizations: 'debug'
          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.9'
            optimizations: 'noopt'
          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.9'
            optimizations: 'lto'

          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.10'
            optimizations: 'debug'
          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.10'
            optimizations: 'noopt'
          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.10'
            optimizations: 'lto'

          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.11'
            optimizations: 'debug'
          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.11'
            optimizations: 'noopt'
          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.11'
            optimizations: 'lto'

          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.12'
            optimizations: 'debug'
          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.12'
            optimizations: 'noopt'
          - target_triple: 'armv7-unknown-linux-gnueabihf'
            py: 'cpython-3.12'
            optimizations: 'lto'

          # Cross-compiles can't do PGO and require Python 3.9.
          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'debug'
          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'noopt'
          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'lto'

          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'debug'
          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'noopt'
          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'lto'

          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'debug'
          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'noopt'
          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'lto'

          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'debug'
          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'noopt'
          - target_triple: 'mips-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'lto'

          # Cross-compiles can't do PGO and require Python 3.9.
          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'debug'
          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'noopt'
          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'lto'

          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'debug'
          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'noopt'
          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'lto'

          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'debug'
          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'noopt'
          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'lto'

          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'debug'
          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'noopt'
          - target_triple: 'mipsel-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'lto'

          # Cross-compiles can't do PGO and require Python 3.9.
          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'debug'
          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'noopt'
          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'lto'

          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'debug'
          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'noopt'
          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'lto'

          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'debug'
          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'noopt'
          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'lto'

          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'debug'
          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'noopt'
          - target_triple: 's390x-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'lto'

          # Cross-compiles can't do PGO and require Python 3.9.
          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'debug'
          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'noopt'
          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'lto'

          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'debug'
          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'noopt'
          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'lto'

          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'debug'
          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'noopt'
          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'lto'

          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'debug'
          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'noopt'
          - target_triple: 'ppc64le-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'lto'

          # We don't publish noopt builds when PGO is available.
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.8'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.8'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.8'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'pgo+lto'
            run: true

          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'pgo'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'pgo+lto'
            run: true

          # GitHub Actions runners don't support x86-64-v4 so we can't PGO.
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'debug'
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'noopt'
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.9'
            optimizations: 'lto'

          # GitHub Actions runners don't support x86-64-v4 so we can't PGO.
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'debug'
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'noopt'
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.10'
            optimizations: 'lto'

          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'debug'
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'noopt'
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.11'
            optimizations: 'lto'

          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'debug'
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'noopt'
          - target_triple: 'x86_64_v4-unknown-linux-gnu'
            py: 'cpython-3.12'
            optimizations: 'lto'

          # musl doesn't support PGO.
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.8'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.8'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.8'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64_v2-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'debug'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'noopt'
            run: true
          - target_triple: 'x86_64_v3-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'lto'
            run: true

          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'debug'
          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'noopt'
          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.9'
            optimizations: 'lto'

          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'debug'
          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'noopt'
          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.10'
            optimizations: 'lto'

          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'debug'
          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'noopt'
          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.11'
            optimizations: 'lto'

          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'debug'
          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'noopt'
          - target_triple: 'x86_64_v4-unknown-linux-musl'
            py: 'cpython-3.12'
            optimizations: 'lto'

    needs:
      - pythonbuild
      - image
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Download pythonbuild
        uses: actions/download-artifact@v4
        with:
          name: pythonbuild
          path: build

      - name: Download images
        uses: actions/download-artifact@v4
        with:
          pattern: image-*
          path: build
          merge-multiple: true

      - name: Load Docker Images
        run: |
          for f in build/image-*.tar.zst; do
            echo "decompressing $f"
            zstd -d --rm ${f}
          done

          for f in build/image-*.tar; do
            echo "loading $f"
            docker load --input $f
          done

      - name: Build
        run: |
          # Do empty target so all generated files are touched.
          ./build-linux.py --make-target empty

          # Touch mtimes of all images so they are newer than autogenerated files above.
          touch build/image-*

          ./build-linux.py --target-triple ${{ matrix.build.target_triple }} --python ${{ matrix.build.py }} --optimizations ${{ matrix.build.optimizations }}

      - name: Validate Distribution
        run: |
          chmod +x build/pythonbuild

          if [ -n "${{matrix.build.run}}" ]; then
            EXTRA_ARGS="--run"
          fi

          build/pythonbuild validate-distribution ${EXTRA_ARGS} dist/*.tar.zst

      - name: Upload Distribution
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.build.py }}-${{ matrix.build.target_triple }}-${{ matrix.build.optimizations }}
          path: dist/*


Windows Python build

name: Windows Python build

on:
  push:
    branches: [main]
  pull_request:
  schedule:
    - cron: '13 11 * * *'
jobs:
  pythonbuild:
    runs-on: 'windows-2019'
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable

      - name: Emit rustc version
        run: |
          rustc --version > .rustc-version

      - uses: actions/cache@v4
        with:
          path: |
            C:/Rust/.cargo/registry
            C:/Rust/.cargo/git
            target
          key: ${{ runner.os }}-pythonbuild-${{ hashFiles('Cargo.lock', '.rustc-version') }}

      - name: Build
        run: |
          cargo build --release

      - name: Upload executable
        uses: actions/upload-artifact@v4
        with:
          name: pythonbuild
          path: target/release/pythonbuild.exe

  build:
    strategy:
      fail-fast: false
      matrix:
        py:
          - 'cpython-3.8'
          - 'cpython-3.9'
          - 'cpython-3.10'
          - 'cpython-3.11'
          - 'cpython-3.12'
        vcvars:
          - 'vcvars32.bat'
          - 'vcvars64.bat'
        profile:
          - 'pgo'
    needs: pythonbuild
    runs-on: 'windows-2019'
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Cygwin Environment
        uses: cygwin/cygwin-install-action@49f298a7ebb00d4b3ddf58000c3e78eff5fbd6b9
        with:
          packages: autoconf automake libtool

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Download pythonbuild Executable
        uses: actions/download-artifact@v4
        with:
          name: pythonbuild

      # We need to do this before we activate the VC++ environment or else binary packages
      # don't get compiled properly.
      - name: Bootstrap Python environment
        run: |
          py.exe -3.9 build-windows.py --help

      - name: Build
        shell: cmd
        run: |
          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\${{ matrix.vcvars }}"
          py.exe -3.9 build-windows.py --python ${{ matrix.py }} --sh c:\cygwin\bin\sh.exe --profile ${{ matrix.profile }}

      - name: Validate Distribution
        run: |
          $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative
          .\pythonbuild.exe validate-distribution --run $Dists

      - name: Upload Distributions
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.py }}-${{ matrix.vcvars }}-${{ matrix.profile }}
          path: dist/*


bootstrap

---
# this workflow bootstraps the testing of the build the docker images
#
# - this will run the python script used to generate the workflows
#   based on a the jinja template
# - commit and push the generated workflows to the branch test_docker_build
#   where they will be executed

name: bootstrap

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

on:
    push:
        branches: [master]
        paths:
        -   .github/workflows/bootstrap.yml
        -   .github/workflows/create_workflows.py
        -   neurodocker/**

# Uses the cron schedule for github actions
#
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#scheduled-events
#
#         ┌───────────── minute (0 - 59)
#         │ ┌───────────── hour (0 - 23)
#         │ │   ┌───────────── day of the month (1 - 31)
#         │ │   │    ┌───────────── month (1 - 12 or JAN-DEC)
#         │ │   │    │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
#         │ │   │    │ │
#         │ │   │    │ │
#         │ │   │    │ │
#         * *   *    * *

    schedule:
    -   cron: 0 0 1,15 * *

  # Allows you to run this workflow manually from the Actions tab
    workflow_dispatch:
        inputs:
            software_name:
                description: software to test
                required: true
                default: all

permissions:
    contents: write
    actions: write
jobs:
    bootstrap:
        runs-on: ubuntu-22.04

        steps:
        -   uses: actions/checkout@v4
            with:
                token: ${{ secrets.CI_FLOW }}

        -   name: Set up Python
            uses: actions/setup-python@v5
            with:
                python-version: '3.11'

        -   name: Install dependencies
            run: python -m pip install jinja2 pyyaml

        -   name: Create workflows
            run: |
                software_name=${{ inputs.software_name }}
                if [ -z "$software_name" ]; then
                  software_name="all"
                fi
                if [ "$software_name" = "all" ]; then
                  echo "testing all software"
                else
                  echo "testing ${software_name}"
                fi
                git checkout -b test_docker_build
                python .github/workflows/create_workflows.py --software_name ${software_name}
                ls -l .github/workflows
                git add .
                git config --global user.email "no-reply@repronim.org"
                git config --global user.name "Repronim neurodocker bot"
                git commit -am "added new workflows"
                git push origin --force test_docker_build


---
on:
    push:
        branches:
        -   master
permissions:
    contents: write
jobs:
    build_docs:
        runs-on: ubuntu-latest
        steps:
        -   uses: actions/checkout@v4
        -   name: Set up Python
            uses: actions/setup-python@v5
            with:
                python-version: '3.10'
        -   name: Install neurodocker
            run: python -m pip install --editable .[all]
        -   name: build docs
            run: |
                make -C docs cli
                make -C docs list
                sphinx-build docs docs-build
        -   name: Deploy
            uses: JamesIves/github-pages-deploy-action@v4
            with:
                folder: docs-build # The folder the action should deploy.


Publish to pypi on Github release

---
name: Publish to pypi on Github release

on:
    release:
        types: [published]

jobs:
    pypi-release:
        runs-on: ubuntu-latest
        steps:
        -   uses: actions/checkout@v4

        -   name: Set up Python
            uses: actions/setup-python@v5
            with:
                python-version: '3.10'

        -   name: Install build & twine
            run: python -m pip install build twine

        -   name: Publish to pypi
            run: |
                python -m build
                twine upload dist/*
            env:
                TWINE_USERNAME: __token__
                TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}


CI

---
name: CI

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

on:
    pull_request:
        branches: [master]

jobs:
    run-tests:
        runs-on: ubuntu-22.04
        strategy:
            matrix:
                python-version: ['3.12', '3.11', '3.10', '3.9', '3.8']
            fail-fast: false
        steps:
        -   name: Install Apptainer
            env:
                VERSION: 1.1.5
            run: |
                sudo apt-get update
                sudo apt-get install -y wget
                wget https://github.com/apptainer/apptainer/releases/download/v${VERSION}/apptainer_${VERSION}_amd64.deb
                wget https://github.com/apptainer/apptainer/releases/download/v${VERSION}/apptainer-suid_${VERSION}_amd64.deb
                sudo apt-get install --yes ./apptainer*
        -   name: Set Apptainer/Singularity environment variables
            run: |
                # Give reproenv the full path to apptainer, so it still works with `sudo`.
                echo REPROENV_APPTAINER_PROGRAM=apptainer >> $GITHUB_ENV
                echo APPTAINER_CACHEDIR=/dev/shm/$(whoami)/singularity >> $GITHUB_ENV
        -   uses: actions/checkout@v4
        -   name: Set up Python
            uses: actions/setup-python@v5
            with:
                python-version: ${{ matrix.python-version }}
        -   name: Install neurodocker
            run: python -m pip install --editable .[dev]
        -   name: Run python tests
            run: pytest --numprocesses auto
        -   name: Get code coverage
            run: codecov


Auto-release on PR merge

---
name: Auto-release on PR merge

on:
  # ATM, this is the closest trigger to a PR merging
    push:
        branches:
        -   master

env:
    AUTO_VERSION: v11.1.6

jobs:
    auto-release:
        runs-on: ubuntu-latest
        if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')"
        steps:
        -   uses: actions/checkout@v4

        -   name: Prepare repository
        # Fetch full git history and tags
            run: git fetch --unshallow --tags

        -   name: Unset header
        # checkout@v2 adds a header that makes branch protection report errors
        # because the Github action bot is not a collaborator on the repo
            run: git config --local --unset http.https://github.com/.extraheader

        -   name: Set up Python
            uses: actions/setup-python@v5
            with:
                python-version: '3.10'

        -   name: Download auto
            run: |
                auto_download_url="$(curl -fsSL https://api.github.com/repos/intuit/auto/releases/tags/$AUTO_VERSION | jq -r '.assets[] | select(.name == "auto-linux.gz") | .browser_download_url')"
                wget -O- "$auto_download_url" | gunzip > ~/auto
                chmod a+x ~/auto

        -   name: Create release
            run: |
                ~/auto shipit -vv
            env:
                GH_TOKEN: ${{ secrets.AUTO_USER_TOKEN }}


pre-commit

---
name: pre-commit

on:
    pull_request:
    push:
        branches: [master]

jobs:
    pre-commit:
        runs-on: ubuntu-latest
        steps:
        -   uses: actions/checkout@v4
        -   uses: actions/setup-python@v5
        -   uses: pre-commit/action@v3.0.1


Close inactive issues

---
# CRON job to close inactive issues.
# https://docs.github.com/en/actions/managing-issues-and-pull-requests/closing-inactive-issues

name: Close inactive issues
on:
    schedule:
    -   cron: 30 1 * * *

jobs:
    close-issues:
        runs-on: ubuntu-latest
        permissions:
            issues: write
            pull-requests: write
        steps:
        -   uses: actions/stale@v9
            with:
                days-before-issue-stale: 60
                days-before-issue-close: 60
                stale-issue-label: stale
                stale-issue-message: This issue is stale because it has been open for 30 days with no activity.
                close-issue-message: This issue was closed because it has been inactive for 14 days since being marked as stale.
                days-before-pr-stale: -1
                days-before-pr-close: -1
                repo-token: ${{ secrets.GITHUB_TOKEN }}


test examples

---
name: test examples

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

on:
    push:
        branches: [master]
    pull_request:
        branches: [master]

    schedule:
    -   cron: 0 0 1 * *

    # Allows you to run this workflow manually from the Actions tab
    workflow_dispatch:

jobs:
    test_examples:
        runs-on: ubuntu-22.04

        steps:
        -   uses: actions/checkout@v4

        -   name: Set up Python
            uses: actions/setup-python@v5
            with:
                python-version: '3.11'

        -   name: Install neurodocker
            run: python -m pip install --editable .

        -   name: test common uses
            run: bash < docs/common_uses/conda_multiple_env.txt

        -   name: 'test nipype_tuto: failure expected'
            run: bash < docs/examples/nipype_tuto.txt
          # do not fail the workflow if the example fails
            continue-on-error: true


type checking

---
name: type checking

# runs type checks on several pythons as some errors may only show up on older versions

on:
    pull_request:
        branches: [master]

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

jobs:

    typecheck:

        runs-on: ubuntu-latest

        strategy:
            matrix:
                python-version: ['3.12', '3.11', '3.10', '3.9', '3.8']

        steps:
        -   uses: actions/checkout@v4

        -   name: Set up Python
            uses: actions/setup-python@v5
            with:
                python-version: ${{ matrix.python-version }}

        -   name: Install neurodocker
            run: python -m pip install . mypy

        -   name: Check types
            run: mypy --install-types --non-interactive neurodocker


Update precommit hooks

---
name: Update precommit hooks

on:

# Uses the cron schedule for github actions
#
# https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows#scheduled-events
#
#            ┌───────────── minute (0 - 59)
#            │ ┌───────────── hour (0 - 23)
#            │ │ ┌───────────── day of the month (1 - 31)
#            │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
#            │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
#            │ │ │ │ │
#            │ │ │ │ │
#            │ │ │ │ │
#            * * * * *
    schedule:
    -   cron: 0 0 * * 1 # every monday

  # Allows you to run this workflow manually from the Actions tab
    workflow_dispatch:

jobs:
    update_precommit_hooks:

    # only run on upstream repo
        if: github.repository_owner == 'ReproNim'

        runs-on: ubuntu-latest
        steps:
        -   name: Checkout repo
            uses: actions/checkout@v4
        -   name: Set up Python
            uses: actions/setup-python@v5
            with:
                python-version: '3.12'
                allow-prereleases: false
        -   name: Install pre-commit
            run: pip install pre-commit
        -   name: Update pre-commit hooks
            run: pre-commit autoupdate
        -   name: Create Pull Request
            uses: peter-evans/create-pull-request@v6
            with:
                commit-message: pre-commit hooks auto-update
                base: master
                token: ${{ secrets.GITHUB_TOKEN }}
                delete-branch: true
                title: '[BOT] update pre-commit hooks'
                body: done via this [GitHub Action](https://github.com/${{ github.repository_owner }}/neurodocker/blob/master/.github/workflows/update_precommit_hooks.yml)


Regression Tests

name: Regression Tests

on:
  push:
    branches: [main]
    tags: ["v*.*.*"]
  pull_request:
    branches: [main]

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
    - name: Set up Python
      uses: actions/setup-python@v4

    # Check out a *known good* version
    # For now I am assuming 'main' is known good
    - uses: actions/checkout@v3
      with:
        repository: googlefonts/glyphsLib
        ref: main

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python3 -m venv generate
        . ./generate/bin/activate && pip install -r requirements.txt -r requirements-dev.txt

    - name: Generate test files
      run: . ./generate/bin/activate && python3 tests/tools/generate_regression_test_files.py
      env:
        PYTHONPATH: Lib

    # Now check out the version we want to test
    - uses: actions/checkout@v3
      with:
        clean: false

    - name: Reinstall dependencies
      run: |
        python3 -m venv regression
        . ./regression/bin/activate && pip install -r requirements.txt -r requirements-dev.txt

    - name: Run tests
      run: . ./regression/bin/activate && pytest --run-regression-tests tests/regression_test.py -n auto
      env:
        PYTHONPATH: Lib


Generate conda-lock file

name: Generate conda-lock file

on: workflow_dispatch

jobs:
  generate_conda_lock:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Install conda-lock with Micromamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-name: ci
          create-args: conda-lock
          init-shell: bash

      - name: Run conda-lock
        run: conda-lock --mamba -f environment.yml -p osx-64 -p linux-64 -p win-64 --lockfile conda-lock.yml
        shell: bash -el {0}

      - name: Get branch name
        id: branch_name
        run: |
          GITHUB_REF=$(echo $GITHUB_REF | sed -e "s#refs/heads/##g")
          echo "name=$GITHUB_REF" >> "$GITHUB_OUTPUT"

      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v5
        with:
          add-paths: conda-lock.yml
          branch: ${{ steps.branch_name.outputs.name }}-conda-lock
          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
          title: Update conda-lock.yml of `${{ steps.branch_name.outputs.name }}`
          committer: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
          commit-message: "Update conda-lock.yml"
          assignees: ${{ github.actor }}


Docker build

name: Docker build

on:
  push:
    branches: [master]
  release:
    types: [published]

jobs:
  docker:
    runs-on: ubuntu-latest
    steps:
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract Repository Name
        run: echo "REPO_NAME=${GITHUB_REPOSITORY@L}" >>${GITHUB_ENV}

      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          push: true
          tags: |
            ${{ secrets.DOCKERHUB_USERNAME }}/cea:dev
            ghcr.io/${{ env.REPO_NAME }}/cea:dev

      - name: Push version tag on release
        if: github.event_name == 'release' && github.event.action == 'published'
        run: |
          docker buildx imagetools create \
            --tag ${{ secrets.DOCKERHUB_USERNAME }}/cea:${GITHUB_REF##*/} \
            --tag ghcr.io/${{ env.REPO_NAME }}/cea:${GITHUB_REF##*/} \
            ghcr.io/${{ env.REPO_NAME }}/cea:dev


Ruff

name: Ruff
on:
  pull_request:
  push:
    branches: [ master ]

jobs:
  ruff:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: chartboost/ruff-action@v1
        with:
          args: check --ignore E741,F841


Push to Master

name: Push to Master

on:
  push:
    branches: [master]

jobs:
  test:
    name: ${{ matrix.os }}-test
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
    steps:
      - uses: actions/checkout@v4

      - name: Install Conda environment with Micromamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: conda-lock.yml
          environment-name: cea
          cache-environment: true

      - name: Install CEA
        shell: bash -l {0}
        run: pip install .

      - name: Run tests
        shell: bash -l {0}
        run: |
          cea test --type unittest
          cea test --type integration


Pull Request Tests

name: Pull Request Tests

on:
  pull_request:
    types: [opened, synchronize]
    branches: [master]

jobs:
  check_environment_yml:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup master branch locally without switching current branch
        run: git fetch origin master:master

      - name: Check environment.yml diff
        id: environment_yml_diff
        run: |
          set +e
          git diff --quiet ..master -- environment.yml
          echo "env_diff=$?" >> "$GITHUB_OUTPUT"

      - name: Check conda-lock.yml diff
        id: conda_lock_yml_diff
        run: |
          set +e
          git diff --quiet ..master -- conda-lock.yml
          echo "lock_diff=$?" >> "$GITHUB_OUTPUT"

      - uses: actions/github-script@v7
        if: ${{ steps.environment_yml_diff.outputs.env_diff != 0 && steps.conda_lock_yml_diff.outputs.lock_diff == 0 }}
        with:
          script: |
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: 'Changes detected in `environment.yml` but `conda-lock.yml` is not updated.'
            })
            core.setFailed('conda-lock.yml needs to be updated')

  build:
    name: ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    needs: check_environment_yml
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
    steps:
      - uses: actions/checkout@v4

      - name: Install Conda environment with Micromamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: conda-lock.yml
          environment-name: cea
          cache-environment: true

      - name: Install CEA
        shell: bash -l {0}
        run: pip install .

      - name: Run tests
        shell: bash -l {0}
        run: |
          cea test --type unittest
          cea test --type integration


Build CEA Setup

name: Build CEA Setup

on:
  workflow_dispatch

jobs:
  build_win_setup:
    runs-on: "windows-latest"
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.8'

      - name: Install CEA to python environment
        run: pip install .

      - name: Get CEA version
        shell: bash
        run: echo "CEA_VERSION=$(python -c "import cea; print(cea.__version__)")" >> "$GITHUB_ENV"

      - name: Create sdist of CEA
        shell: bash
        run: |
          python -m pip install build
          python -m build
          mv dist/cityenergyanalyst-$CEA_VERSION.tar.gz setup/cityenergyanalyst.tar.gz

      - name: Cache CEA env
        id: cache-env
        uses: actions/cache@v4
        with:
          path: |
            setup/dependencies/cea-env.7z
            setup/dependencies/micromamba.exe
          key: ${{ runner.os }}-${{ hashFiles('conda-lock.yml') }}-env

      - uses: mamba-org/setup-micromamba@v1
        if: steps.cache-env.outputs.cache-hit != 'true'
        with:
          environment-file: conda-lock.yml
          environment-name: cea
          cache-environment: true
          micromamba-binary-path: setup/dependencies/micromamba.exe
          create-args: --no-pyc

      - name: Clean conda environment
        if: steps.cache-env.outputs.cache-hit != 'true'
        shell: bash -el {0}
        run: micromamba clean -afy

      - name: Compress conda environment
        if: steps.cache-env.outputs.cache-hit != 'true'
        shell: bash -el {0}
        run: 7z a setup/dependencies/cea-env.7z $MAMBA_ROOT_PREFIX

      - uses: actions/checkout@v4
        with:
          path: gui
          repository: architecture-building-systems/CityEnergyAnalyst-GUI

      - name: Package CEA GUI
        shell: bash -el {0}
        env:
          GITHUB_TOKEN: ${{ secrets.GUI_GH_TOKEN }}
        run: |
          cd $GITHUB_WORKSPACE/gui
          yarn
          yarn version $CEA_VERSION
          yarn electron:release
          mv "out/CityEnergyAnalyst-GUI Setup ${CEA_VERSION}.exe" $GITHUB_WORKSPACE/setup/gui_setup.exe

      - name: Install setup plugin
        shell: bash
        run: |
          cd "C:\Program Files (x86)\NSIS\Plugins\x86-unicode"
          curl -o Nsis7z.7z https://nsis.sourceforge.io/mediawiki/images/6/69/Nsis7z_19.00.7z
          7z e Nsis7z.7z Plugins/x86-unicode/nsis7z.dll

      - name: Make setup
        shell: bash
        run: |
          mkdir -p $GITHUB_WORKSPACE/setup/Output
          makensis $GITHUB_WORKSPACE/setup/cityenergyanalyst.nsi

      - name: Upload setup
        uses: actions/upload-artifact@v4
        with:
          name: setup
          path: ${{ github.workspace }}\setup\Output
          compression-level: 0

  build_mac_dmg:
    runs-on: "macos-12"
    steps:
      - uses: actions/checkout@v4
        with:
          path: cea

      - uses: actions/setup-python@v5
        with:
          python-version: '3.8'

      - name: Install CEA to python environment
        run: pip install ${{ github.workspace }}/cea

      - name: Get CEA version
        shell: bash
        run: echo "CEA_VERSION=$(python -c "import cea; print(cea.__version__)")" >> "$GITHUB_ENV"

      - uses: actions/checkout@v4
        with:
          path: gui
          repository: architecture-building-systems/CityEnergyAnalyst-GUI

      - name: Fetch micromamba
        run: |
          cd gui
          mkdir -p ./dependencies/arm64
          curl -Ls https://micro.mamba.pm/api/micromamba/osx-arm64/latest | tar -xvj -C ./dependencies/arm64 --strip-components=1 bin/micromamba
          mkdir -p ./dependencies/x64
          curl -Ls https://micro.mamba.pm/api/micromamba/osx-64/latest | tar -xvj -C ./dependencies/x64 --strip-components=1 bin/micromamba

      - name: Save Apple API Key secret to file
        env:
          APPLE_API_KEY_BASE64: ${{ secrets.APPLE_API_KEY_BASE64 }}
        run: |
          echo -n $APPLE_API_KEY_BASE64 | base64 -d -o ${{ github.workspace }}/gui/api_key.p8

      - uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Enable Corepack
        run: corepack enable

      - name: Package CEA GUI
        env:
          APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }}
          APPLE_API_KEY: ${{ github.workspace }}/gui/api_key.p8
          APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
          CSC_LINK: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
          CSC_KEY_PASSWORD: ${{ secrets.P12_PASSWORD }}
          GITHUB_TOKEN: ${{ secrets.GUI_GH_TOKEN }}
        run: |
          cd gui
          yarn
          yarn version $CEA_VERSION
          yarn electron:release --arm64 --x64


build

name: build

on: [push]

jobs:
  python:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
    steps:
      - uses: actions/checkout@v4
      - name: Show ref
        run: |
          echo "$GITHUB_REF"
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: Install dependencies
        env:
          DEBIAN_FRONTEND: noninteractive
        run: |
          sudo apt-get update -y -qq
          sudo apt-get install -y -qq libicu-dev
          pip install wheel pyicu setuptools
          pip install -e ".[dev]"
      - name: Run checks for default model
        run: |
          python contrib/check_model.py
          make default-model
          git diff
          if [[ -z "$(git status --porcelain)" ]];
          then
            echo "Default model is up to date 👌"
          else
            echo "⚠ Default model is not up to date. You need to run `make default-model` and commit the changes.";
            exit 1;
          fi;
      - name: Run the tests
        run: |
          make test
      - name: Partial mypy type-check
        run: |
          make typecheck
      - name: Build a followthemoney distribution
        run: |
          python setup.py sdist bdist_wheel
      - name: Publish followthemoney to PyPI
        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.pypi_password }}
          packages-dir: dist/
          skip-existing: true

  nodejs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        node-version: [16.x]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - name: Install dependencies
        run: |
          cd js
          npm install
          npm run build
      - name: Run the tests
        run: |
          cd js
          npm run test:prod && npm run lint
      - name: Publish to NPM
        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
        run: |
          cd js
          npm config set //registry.npmjs.org/:_authToken=$NPM_AUTH_TOKEN
          npm run build
          npm publish
        env:
          CI: true
          NPM_AUTH_TOKEN: ${{ secrets.npm_auth_token }}

  docker:
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags')
    steps:
      - uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          install: true
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ghcr.io/alephdata/followthemoney
          tags: |
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=sha
      - name: Set env
        run: echo "VERSION_NUMBER=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
      - name: Login to GitHub Packages
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build docker image
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          cache-from: type=gha
      - name: Build and push docker image
        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max


docs

name: docs
on:
  - push
  - workflow_dispatch

# concurrency:
#   group: pages

jobs:
  build:
    runs-on: ubuntu-latest

    permissions:
      contents: read

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          # Fetch all commits and tags
          # (required to generate version information on schema pages)
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install followthemoney Python package
        run: make dev

      - name: Build Python API docs
        run: |
          pip install pdoc
          pdoc --no-search -o docs/public/reference/python followthemoney.{model,schema,proxy,property,types.common,namespace,graph,util,exc}

      - name: Build namespace XML
        run: |
          mkdir -p docs/public/ns
          python followthemoney/ontology.py docs/public/ns

      - name: Build docs
        uses: alephdata/astro-theme-docs/.github/actions/build@main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          docs-path: ./docs

  deploy:
    if: ${{ github.ref == 'refs/heads/main' }}
    needs: ["build"]
    runs-on: ubuntu-latest

    permissions:
      pages: write
      id-token: write

    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.url }}

    steps:
      - name: Deploy docs
        id: deployment
        uses: alephdata/astro-theme-docs/.github/actions/deploy@main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
---
name: "CodeQL"

# yamllint disable-line rule:truthy
on:
  push:
    branches: [develop]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [develop]
  schedule:
    - cron: '40 23 * * 1'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        language: ['python']
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript',
        # 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v1
        with:
          languages: ${{ matrix.language }}
          # If you wish to specify custom queries, you can do so here or in a
          # config file.  By default, queries listed here will override any
          # specified in a config file.  Prefix the list here with "+" to use
          # these queries and those in the config file.  queries:
          # ./path/to/local/query, your-org/your-repo/queries@main

      # Autobuild attempts to build any compiled languages  (C/C++, C#, or
      # Java).  If this step fails, then you should remove it and run the build
      # manually (see below)
      - name: Autobuild
        uses: github/codeql-action/autobuild@v1

      # ℹ Command-line programs to run using the OS shell.
      # 📚 https://git.io/JvXDl

      # ✏ If the Autobuild fails above, remove it and uncomment the following
      # three lines
      #    and modify them (or add more) to build your code if your project
      #    uses a compiled language

      # - run: |
      #    make bootstrap
      #    make release

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v1


Python package

---
name: Python package

# yamllint disable-line rule:truthy
on:
  push:
  pull_request:
  schedule:
    # Weekly run ensuring no external dependencies cause surprises
    - cron: '0 0 * * 0'

jobs:
  test:

    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version:
          - "3.7"
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.x"

    steps:
      - uses: actions/checkout@v1
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install .[dev,test]
      - name: Test with pytest
        run: |
          pytest

  quality-control:
    runs-on: ubuntu-latest
    needs: test
    steps:
      - uses: actions/checkout@v1
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: Check black formatting
        run: |
          pip install black
          black --check puresnmp
      - name: Lint with pylint
        run: |
          pip install pylint
          pylint --reports=y --exit-zero puresnmp

  publish:

    needs: test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v1
      - name: Build distribution
        if: >
          github.event_name == 'push' &&
          startsWith(github.event.ref, 'refs/tags')
        run: |
          python -m pip install --upgrade pip wheel build
          pip install .[dev,test]
          pyproject-build
      - name: Publish package
        if: >
            github.event_name == 'push' &&
            startsWith(github.event.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@v1.5.0
        with:
          user: __token__
          password: ${{ secrets.pypi_token }}


build-and-test

name: build-and-test

on:
  push:
    branches: [ '**' ]
  pull_request:
    branches: [ '**' ]
  schedule:
    # At 23:25 on Thursday.
    - cron: "25 23 * * 4"
  workflow_dispatch:

jobs:

  # Build on all platforms
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-20.04, ubuntu-latest, macos-11, macos-latest, windows-2019, windows-latest]
        python-version: [ '3.8', '3.x' ]

    steps:
    - uses: actions/checkout@v3
    - name: Setup python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Upgrade pip
      run: python -m pip install --upgrade pip
    - name: Build and install flydra_core
      run: python -m pip install ./flydra_core
    - name: Build and install flydra_analysis
      run: python -m pip install ./flydra_analysis

  # Test on Ubuntu with system python
  test-ubuntu-system-python:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-20.04, ubuntu-latest]
    steps:
    - uses: actions/checkout@v3
    - name: Update package index
      run: sudo apt-get update
    - name: Install system packages
      run: sudo apt-get install -y libcairo2-dev octave # mayavi2 python3-sympy python3-docopt python3-cherrypy3 python3-pkgconfig python-is-python3
    - name: Upgrade pip
      run: python -m pip install --upgrade pip
    - name: Build and install flydra_core
      run: |
        cd flydra_core
        python -m pip install -e .
    - name: Test flydra_core
      run: |
        python -m pip install pytest
        cd flydra_core
        pytest

    - name: Build and install flydra_analysis
      run: |
        cd flydra_analysis
        python -m pip install -e .
    - name: Test flydra_analysis
      run: |
        python -m pip install pytest
        cd flydra_analysis
        pytest


Wikipedia Library - test and deploy

---
name: Wikipedia Library - test and deploy

on:
  push:
    branches: ["master", "staging"]
  pull_request:
    branches: ["master"]

jobs:
  cicd:
    runs-on: ubuntu-latest
    environment: cicd
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0
      - name: build & run
        run: >
          sudo .cicd/build.sh
          ${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}
          ${GITHUB_SHA}
      - name: migrate
        run: >
          docker compose exec --no-TTY twlight
          /app/bin/virtualenv_migrate.sh
      - name: translate
        run: >
          docker compose exec --no-TTY twlight
          /app/bin/virtualenv_translate.sh last-commit
      - name: test
        run: >
          docker compose exec --no-TTY twlight
          /app/bin/virtualenv_test.sh
      - name: deploy
        env:
          WIKIBOT_TOKEN: ${{secrets.WIKIBOT_TOKEN}}
          QUAYBOT_USERNAME: ${{secrets.QUAYBOT_USERNAME}}
          QUAYBOT_PASSWORD: ${{secrets.QUAYBOT_PASSWORD}}
        run: >
          sudo .cicd/deploy.sh
          ${GITHUB_REF_NAME}
          ${GITHUB_SHA}
          ${GITHUB_EVENT_NAME}
          ${GITHUB_REPOSITORY}
          ${WIKIBOT_TOKEN}
          ${QUAYBOT_USERNAME}
          ${QUAYBOT_PASSWORD}

      - name: cleanup
        if: always()
        run: sudo chown -R ${USER}:${USER} .


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ master, production ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ master ]
  schedule:
    - cron: '31 13 * * 3'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        language: [ 'javascript', 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2


Coveralls Reporting Check

name: Coveralls Reporting Check

# Controls when the workflow will run
on:
  schedule:
      - cron: "0 0 * * *"

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  coveralls_check:
    # The type of runner that the job will run on
    runs-on: ubuntu-latest

    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2

      # Runs a single command using the runners shell
      - name: Coveralls reporting check
        run: "curl -f -s -o /dev/null -w '%{http_code}' https://coveralls.io/builds/$(git rev-parse HEAD).json || curl -f -s -o /dev/null -w '%{http_code}' https://coveralls.io/builds/$(git rev-parse HEAD~).json"


Python application

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Python application

on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]

jobs:
  test:
    name: Python CI
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python 3.10
      uses: actions/setup-python@v1
      with:
        python-version: "3.10.14"
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install flake8
        python -m pip install wheel
        chmod 777 ./setup.sh
        ./setup.sh
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
    # Coveralls related steps
    - name: Coverage with coveralls
      run: |
        python -m pip install coveralls
        coverage run --source=kairon -m pytest tests/ --ignore tests/unit_test/augmentation/ --ignore stress_test/ --disable-warnings
    - env:
        COVERALLS_PARALLEL: true
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      name: Submit to coveralls.io
      run: |
        coveralls --service=github
    - name: Generate Sphinx files
      run: |
        python -m pip install -U sphinx
        python -m pip install sphinx_rtd_theme
        cd docs
        # sphinx-quickstart
        # set system_file=../system.yaml
        export system_file=../system.yaml
        export EMAIl_CONF=../email.yaml
        make html
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v1
      if: github.ref == 'refs/heads/master'
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: us-east-1
    - name: Copy files to the website
      if: github.ref == 'refs/heads/master'
      run: |
        aws s3 sync docs/_build/html s3://${{ secrets.S3_PATH }}
  finish:
    needs: test
    runs-on: ubuntu-latest
    steps:
      - name: Coveralls Finished
        uses: coverallsapp/github-action@master
        with:
          github-token: ${{ secrets.github_token }}
          parallel-finished: true


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ master ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ master ]
  schedule:
    - cron: '36 5 * * 3'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
    - name: Checkout repository
      uses: actions/checkout@v2

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v1
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v1

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v1


Vulnerability Scan

# This workflow will scan the created docker images at scheduled time and create issue if vulnerability is found
name: Vulnerability Scan

on:
  schedule:
    - cron: '30 5 * * 0'

jobs:
  scan:
    name: Daily Vulnerability Scan
    runs-on: ubuntu-latest
    steps:
      - name: Pull docker images
        run: |
          docker pull digite/kairon-api
          docker pull digite/kairon-ui
          docker pull digite/kairon-action-server
          docker pull digite/kairon-history-server
          docker pull digite/kairon-script-runner
          docker pull digite/kairon-chat
          docker pull digite/kairon-paraphrase
          docker pull digite/kairon-qg
      - uses: Azure/container-scan@v0.1
        name: Scanning api image
        with:
          image-name: digite/kairon-api
      - uses: Azure/container-scan@v0.1
        name: Scanning ui image
        with:
          image-name: digite/kairon-ui
      - uses: Azure/container-scan@v0.1
        name: Scanning action image
        with:
          image-name: digite/kairon-action-server
      - uses: Azure/container-scan@v0.1
        name: Scanning script runner image
        with:
          image-name: digite/kairon-script-runner
      - uses: Azure/container-scan@v0.1
        name: Scanning history image
        with:
          image-name: digite/kairon-history-server
      - uses: Azure/container-scan@v0.1
        name: Scanning chat image
        with:
          image-name: digite/kairon-chat
      - uses: Azure/container-scan@v0.1
        name: Scanning paraphrase image
        with:
          image-name: digite/kairon-paraphrase
      - uses: Azure/container-scan@v0.1
        name: Scanning question generation image
        with:
          image-name: digite/kairon-qg


CI

name: CI

on:
  push:
    paths-ignore:
      - '.cookiecutter/*'
      - 'docs/*'
      - 'requirements/*.in'
      - 'requirements/dev.txt'
      - '**/.gitignore'
      - '*.md'
      - 'LICENSE'
  workflow_call:

jobs:
  backend:
    name: Backend
    runs-on: ubuntu-latest
    env:
      TOX_PARALLEL_NO_SPINNER: 1

    steps:
    - name: Checkout git repo
      uses: actions/checkout@v3

    - name: Setup python
      uses: actions/setup-python@v4
      with:
        python-version-file: '.python-version'

    - name: Update pip
      run: python -m pip install --upgrade pip

    - name: Install tox
      run: python -m pip install 'tox<4'

    - name: Cache the .tox dir
      uses: actions/cache@v3
      with:
        path: .tox
        key: ${{ runner.os }}-tox-${{ hashFiles('tox.ini', 'requirements*', 'setup.py', 'setup.cfg') }}
        restore-keys: |
          ${{ runner.os }}-tox-

    - name: Run tox
      run: tox --parallel auto -e checkformatting,lint,tests,coverage,functests

  frontend:
    name: Frontend
    runs-on: ubuntu-latest

    steps:
    - name: Checkout
      uses: actions/checkout@v3

    - name: Cache the node_modules dir
      uses: actions/cache@v3
      with:
        path: node_modules
        key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}

    - name: Lint
      run: make frontend-lint

    - name: Test
      run: make frontend-test


CodeQL

name: CodeQL

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  schedule:
    #        ┌───────────── minute (0 - 59)
    #        │  ┌───────────── hour (0 - 23)
    #        │  │ ┌───────────── day of the month (1 - 31)
    #        │  │ │ ┌───────────── month (1 - 12 or JAN-DEC)
    #        │  │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
    #        │  │ │ │ │
    #        │  │ │ │ │
    #        │  │ │ │ │
    #        *  * * * *
    - cron: '30 1 * * 0'

jobs:
  CodeQL-Build:
    # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest
    runs-on: ubuntu-latest

    permissions:
      # required for all workflows
      security-events: write

      # only required for workflows in private repositories
      actions: read
      contents: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        # Override language selection by uncommenting this and choosing your languages
        # with:
        #   languages: go, javascript, csharp, python, cpp, java

      # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
      # If this step fails, then you should remove it and run the build manually (see below).
      - name: Autobuild
        uses: github/codeql-action/autobuild@v2

      # ℹ Command-line programs to run using the OS shell.
      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

      # ✏ If the Autobuild fails above, remove it and uncomment the following
      #    three lines and modify them (or add more) to build your code if your
      #    project uses a compiled language

      #- run: |
      #     make bootstrap
      #     make release

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2


Deploy

name: Deploy
concurrency:
  group: deploy
  cancel-in-progress: true
on:
  workflow_dispatch:
  push:
    branches:
      - main
    paths-ignore:
      - '.cookiecutter/*'
      - '.github/*'
      - 'docs/*'
      - 'requirements/*'
      - '!requirements/requirements.txt'
      - 'tests/*'
      - '**/.gitignore'
      - '*.md'
      - 'tox.ini'
jobs:
  docker_hub:
    name: Docker Hub
    uses: hypothesis/workflows/.github/workflows/dockerhub.yml@main
    with:
      Application: ${{ github.event.repository.name }}
    secrets: inherit
  staging:
    name: Staging
    needs: [docker_hub]
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: deploy
      github_environment_name: Staging
      github_environment_url: https://staging.hyp.is/b01DwMzhEe6X2ssmLT58kw/en.wikipedia.org/wiki/Wikipedia:Terminal_Event_Management_Policy
      aws_region: us-west-1
      elasticbeanstalk_application: bouncer
      elasticbeanstalk_environment: staging
      docker_tag: ${{ needs.Docker_Hub.outputs.docker_tag }}
    secrets: inherit
  production:
    name: Production
    needs: [docker_hub, staging]
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: deploy
      github_environment_name: Production
      github_environment_url: https://hyp.is/FiqzonGfEe2o-AfEssZXnw/en.wikipedia.org/wiki/Wikipedia:Terminal_Event_Management_Policy
      aws_region: us-west-1
      elasticbeanstalk_application: bouncer
      elasticbeanstalk_environment: prod
      docker_tag: ${{ needs.Docker_Hub.outputs.docker_tag }}
    secrets: inherit


Redeploy

name: Redeploy
concurrency:
  group: deploy
  cancel-in-progress: true
on:
  workflow_dispatch:
    inputs:
      staging:
        type: boolean
        description: Redeploy Staging
      production:
        type: boolean
        description: Redeploy Production
jobs:
  staging:
    name: Staging
    if: inputs.staging
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: redeploy
      github_environment_name: Staging
      github_environment_url: https://staging.hyp.is/FiqzonGfEe2o-AfEssZXnw/en.wikipedia.org/wiki/Wikipedia:Terminal_Event_Management_Policy
      aws_region: us-west-1
      elasticbeanstalk_application: bouncer
      elasticbeanstalk_environment: staging
    secrets: inherit
  production:
    name: Production
    if: inputs.production
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: redeploy
      github_environment_name: Production
      github_environment_url: https://hyp.is/FiqzonGfEe2o-AfEssZXnw/en.wikipedia.org/wiki/Wikipedia:Terminal_Event_Management_Policy
      aws_region: us-west-1
      elasticbeanstalk_application: bouncer
      elasticbeanstalk_environment: prod
    secrets: inherit


Slack

name: Slack
on:
  workflow_run:
    workflows: [CI]
    types: [completed]
    branches: [main]
jobs:
  on-failure:
    runs-on: ubuntu-latest
    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
    steps:
      - name: Post to Slack
        uses: slackapi/slack-github-action@v1.24.0
        with:
          channel-id: 'C4K6M7P5E'
          slack-message: "A workflow run failed\n*Repo:* `${{ github.event.repository.full_name }}` (${{ github.event.repository.html_url }})\n*Workflow:* ${{ github.event.workflow.name }} (${{ github.event.workflow.html_url }})\n*Branch:* `${{ github.event.workflow_run.head_branch }}`\n*Commit:* `${{ github.event.workflow_run.head_commit.id }}`\n*Run:* ${{ github.event.workflow_run.html_url }}\n*Conclusion:* ${{ github.event.workflow_run.conclusion }}"
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}


CI

name: CI

on:
  push:
    branches:
      - 'main'
      - '*.*'
      - '!*backport*'
    tags:
      - 'v*'
      - '!*dev*'
      - '!*pre*'
      - '!*post*'
  pull_request:
  # Allow manual runs through the web UI
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  core:
    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main
    with:
      submodules: false
      coverage: codecov
      toxdeps: tox-pypi-filter
      envs: |
        - linux: py312
    secrets:
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  sdist_verify:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - run: python -m pip install -U --user build
      - run: python -m build . --sdist
      - run: python -m pip install -U --user twine
      - run: python -m twine check dist/*

  test:
    needs: [core, sdist_verify]
    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main
    with:
      submodules: false
      coverage: codecov
      toxdeps: tox-pypi-filter
      posargs: -n auto
      envs: |
        - windows: py311
        - macos: py310
        - linux: py310-oldestdeps
    secrets:
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  docs:
    needs: [core]
    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main
    with:
      default_python: '3.10'
      submodules: false
      pytest: false
      toxdeps: tox-pypi-filter
      libraries: |
        apt:
          - graphviz
      envs: |
        - linux: build_docs
          posargs: ''
          pytest: false
          libraries:
            apt:
              - libopenjp2-7
              - graphviz

  cron:
    if: |
      github.event_name == 'workflow_dispatch' || (
        github.event_name == 'pull_request' &&
        contains(github.event.pull_request.labels.*.name, 'Run cron CI')
      )
    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main
    with:
      default_python: '3.10'
      submodules: false
      coverage: codecov
      toxdeps: tox-pypi-filter
      envs: |
        - linux: py311-devdeps
    secrets:
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  publish_pure:
      if: |
        github.event_name != 'pull_request' && (
          github.ref_name != 'main' ||
          github.event_name == 'workflow_dispatch' )
      needs: [test, docs]
      uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1
      with:
        python-version: '3.10'
        test_extras: 'tests'
        test_command: 'pytest -p no:warnings --doctest-rst --pyargs ndcube'
        submodules: false
      secrets:
        pypi_token: ${{ secrets.pypi_token }}
  notify:
    if: always() && github.event_name == 'workflow_dispatch'
    needs: [publish_pure, cron]
    runs-on: ubuntu-latest
    steps:
      - uses: Cadair/matrix-notify-action@main
        with:
          matrix_token: ${{ secrets.matrix_access_token }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          homeserver: ${{ secrets.matrix_homeserver }}
          roomid: '!TWSJXVpyflnZOzmSQz:matrix.org'
          ignore_pattern: '.*Load tox.*'


Scheduled builds

name: Scheduled builds

on:
  # Allow manual runs through the web UI
  workflow_dispatch:
  schedule:
    #        ┌───────── minute (0 - 59)
    #        │ ┌───────── hour (0 - 23)
    #        │ │ ┌───────── day of the month (1 - 31)
    #        │ │ │ ┌───────── month (1 - 12 or JAN-DEC)
    #        │ │ │ │ ┌───────── day of the week (0 - 6 or SUN-SAT)
    - cron: '0 7 * * 1'  # Every day at 07:00 UTC

jobs:
  dispatch_workflows:
    runs-on: ubuntu-latest
    steps:
      - run: gh workflow run ci.yml --repo sunpy/ndcube --ref main
      - run: gh workflow run ci.yml --repo sunpy/ndcube --ref 2.1
    env:
      GITHUB_TOKEN: ${{ secrets.WORKFLOW_TOKEN }}


Stale Bot

name: Stale Bot

on:
  schedule:
#            ┌───────── minute (0 - 59)
#            │ ┌───────── hour (0 - 23)
#            │ │ ┌───────── day of the month (1 - 31)
#            │ │ │ ┌───────── month (1 - 12 or JAN-DEC)
#            │ │ │ │ ┌───────── day of the week (0 - 6 or SUN-SAT)
    - cron: '0 5 * * *' # Every day at 05:00 UTC

jobs:
  stale:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/stale@v3
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        operations-per-run: 50
        days-before-issue-stale: 99999999 # Never mark issues as stale
        days-before-pr-stale: 150
        days-before-close: 30
        stale-pr-label: 'Stale'
        stale-pr-message: >-
          Hello :wave:,
          Thanks for your contribution to ndcube!

          I have marked this pull request as stale because there hasn't had any activity in five months.
          If you are still working on this, or if it's waiting on a maintainer to look at it then please let us know and we will keep it open.
          Please add a comment with: @sunpy/ndcube-developers to get someone's attention.

          If nobody comments on this pull request for another month, it will be closed.
        close-pr-message: >-
          Hello again :wave:,
          We want to thank you again for your contribution to ndcube!

          This pull request has had no activity since my last reminder, so I am going to close it.
          If at any time you want to come back to this please feel free to reopen it!
          If you want to discuss this, please add a comment with: @sunpy/ndcube-developers and someone will get back to you soon.
        exempt-pr-labels:  'Keep Open'


Automatic Update from package template

# This template is taken from the cruft example code, for further information please see:
# https://cruft.github.io/cruft/#automating-updates-with-github-actions
name: Automatic Update from package template
permissions:
    contents: write
    pull-requests: write

on:
  # Allow manual runs through the web UI
  workflow_dispatch:
  schedule:
    #        ┌───────── minute (0 - 59)
    #        │ ┌───────── hour (0 - 23)
    #        │ │ ┌───────── day of the month (1 - 31)
    #        │ │ │ ┌───────── month (1 - 12 or JAN-DEC)
    #        │ │ │ │ ┌───────── day of the week (0 - 6 or SUN-SAT)
    - cron: '0 7 * * 1'  # Every Monday at 7am UTC

jobs:
  update:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        include:
          - add-paths: .
            body: apply the changes to this repo.
            branch: cruft/update
            commit-message: "Automatic package template update"
            title: Updates from the package template

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Cruft
        run: python -m pip install git+https://github.com/Cadair/cruft@patch-p1

      - name: Check if update is available
        continue-on-error: false
        id: check
        run: |
          CHANGES=0
          if [ -f .cruft.json ]; then
            if ! cruft check; then
              CHANGES=1
            fi
          else
            echo "No .cruft.json file"
          fi

          echo "has_changes=$CHANGES" >> "$GITHUB_OUTPUT"

      - name: Run update if available
        if: steps.check.outputs.has_changes == '1'
        run: |
          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
          git config --global user.name "${{ github.actor }}"

          cruft update --skip-apply-ask --refresh-private-variables
          git restore --staged .

      - name: Create pull request
        if: steps.check.outputs.has_changes == '1'
        uses: peter-evans/create-pull-request@v6
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          add-paths: ${{ matrix.add-paths }}
          commit-message: ${{ matrix.commit-message }}
          branch: ${{ matrix.branch }}
          delete-branch: true
          branch-suffix: timestamp
          title: ${{ matrix.title }}
          body: |
            This is an autogenerated PR, which will ${{ matrix.body }}.
            [Cruft](https://cruft.github.io/cruft/) has detected updates from the Package Template


name: generate_db

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  generate_db:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10"]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install Perl dependencies
      uses: perl-actions/install-with-cpanm@stable
      with:
        cpanfile: "cpanfile"
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    - name: Generate database
      run: |
        chmod +x ./generate_db.sh
        ./generate_db.sh --noninteractive


languages

name: languages

on:
  push:
    branches: [main, test-me-*]
    tags: '*'
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  vars:
    runs-on: ubuntu-latest
    outputs:
      languages: ${{ steps.vars.outputs.languages }}
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - uses: actions/setup-python@v4
      with:
        python-version: 3.9
    - name: install deps
      run: python -mpip install -e . -r requirements-dev.txt
    - name: vars
      run: testing/languages ${{ github.event_name == 'push' && '--all' || '' }}
      id: vars
  language:
    needs: [vars]
    runs-on: ${{ matrix.os }}
    if: needs.vars.outputs.languages != '[]'
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.vars.outputs.languages) }}
    steps:
    - uses: asottile/workflows/.github/actions/fast-checkout@v1.4.0
    - uses: actions/setup-python@v4
      with:
        python-version: 3.9

    - run: echo "$CONDA\Scripts" >> "$GITHUB_PATH"
      shell: bash
      if: matrix.os == 'windows-latest' && matrix.language == 'conda'
    - run: testing/get-coursier.sh
      shell: bash
      if: matrix.language == 'coursier'
    - run: testing/get-dart.sh
      shell: bash
      if: matrix.language == 'dart'
    - run: |
        sudo apt-get update
        sudo apt-get install -y --no-install-recommends \
          lua5.3 \
          liblua5.3-dev \
          luarocks
      if: matrix.os == 'ubuntu-latest' && matrix.language == 'lua'
    - run: |
        echo 'C:\Strawberry\perl\bin' >> "$GITHUB_PATH"
        echo 'C:\Strawberry\perl\site\bin' >> "$GITHUB_PATH"
        echo 'C:\Strawberry\c\bin' >> "$GITHUB_PATH"
      shell: bash
      if: matrix.os == 'windows-latest' && matrix.language == 'perl'
    - uses: haskell/actions/setup@v2
      if: matrix.language == 'haskell'

    - name: install deps
      run: python -mpip install -e . -r requirements-dev.txt
    - name: run tests
      run: coverage run -m pytest tests/languages/${{ matrix.language }}_test.py
    - name: check coverage
      run: coverage report --include pre_commit/languages/${{ matrix.language }}.py,tests/languages/${{ matrix.language }}_test.py
  collector:
    needs: [language]
    if: always()
    runs-on: ubuntu-latest
    steps:
    - name: check for failures
      if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
      run: echo job failed && exit 1