Release (Part 1 of 2)

name: Release (Part 1 of 2)

on:
  workflow_dispatch:
    inputs:
      version:
        required: true
        type: string

env:
  VERSION: ${{ inputs.version }}
  AUTORELEASE_BRANCH: autorelease/${{ inputs.version }}
  # use a personal access token here which has permissions to trigger further actions
  # this is necessary for the pr checks
  GITHUB_TOKEN: ${{ secrets.AUTORELEASE_BOT_PAT }}

jobs:
  start-release:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          token: ${{ secrets.AUTORELEASE_BOT_PAT }}

      - name: Configure git
        run: |
          git config user.name "GitHub Actions"
          git config user.email "<>"

      # Members of the natcap software team can push to the autorelease branch on
      # natcap/invest; this branch is a special case for our release process.
      - name: Create autorelease branch
        run: git checkout -b "$AUTORELEASE_BRANCH"

      # Replace
      #
      # Unreleased Changes
      # ------------------
      #
      # with
      #
      # ..
      #   Unreleased Changes
      #   ------------------
      #
      # X.X.X (XXXX-XX-XX)
      # ------------------
      - name: Update HISTORY.rst
        run: |
          HEADER="$VERSION ($(date '+%Y-%m-%d'))"
          HEADER_LENGTH=${#HEADER}
          UNDERLINE=$(for i in $(seq 1 $HEADER_LENGTH); do echo -n "-"; done)
          perl -0777 -i -pe \
              "s/Unreleased Changes\n------------------/..\n  Unreleased Changes\n  ------------------\n\n${HEADER}\n${UNDERLINE}/g" \
              HISTORY.rst
          git add HISTORY.rst
          git commit -m "Committing the $VERSION release."

      - name: Tag and push
        run: |
          git tag $VERSION
          git push --atomic origin $AUTORELEASE_BRANCH $VERSION

      - name: Find actions run for the version tag
        run: |
          # wait a few seconds to make sure the actions run exists before querying it
          sleep 5
          echo "TAG_RUN_URL=$( \
            gh run list \
              --branch $VERSION \
              --limit 1 \
              --json url \
              --jq .[].url)" >> $GITHUB_ENV

      - name: Create a PR from the autorelease branch into main
        run: |
          gh pr create \
            --base main \
            --head $AUTORELEASE_BRANCH \
            --title "$VERSION release" \
            --reviewer $GITHUB_ACTOR \
            --assignee $GITHUB_ACTOR \
            --body "
            # Release $VERSION and merge into \`main\`

            This PR contains automated changes made for the $VERSION release.

            Merging this PR will trigger an action that publishes the \
            release. Closing this PR without merging will trigger an action that \
            rolls back any release steps that have completed so far.

            ## Review this PR
            - [ ] Make sure that the automated changes look correct
            - [ ] Wait for BOTH the PR checks below AND the [$VERSION tag checks]($TAG_RUN_URL) \
                  to complete. The $VERSION tag workflow is most important \
                  because it produces the artifacts that will be used in the \
                  next steps of the release process.
            - [ ] Download and try out the [tag build artifacts]($TAG_RUN_URL)

            **If everything looks good**, approve and merge this PR. This will \
            trigger a Github Action that will publish the release. Then go \
            back to the [Release Checklist](https://github.com/natcap/invest/wiki/Release-Checklist) \
            and complete any remaining tasks.

            **If there is a bug**, decline this PR. Submit a fix in a separate \
            PR into \`main\`. Once the fix has been merged, restart the release \
            process from the beginning.

            **If either workflow fails due to an intermittent problem**, \
            rerun it through the GitHub UI. Proceed to approve and merge this \
            PR once it succeeds."

      - name: Roll back on failure
        if: failure()
        uses: ./.github/actions/rollback_release
        with:
          VERSION: ${{ env.VERSION }}
          GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }}


Release (Part 2 of 2)

name: Release (Part 2 of 2)

on:
  # this workflow will run any time any PR into main is closed
  pull_request:
    types: [closed]
    branches:
      - main

  workflow_dispatch:
    inputs:
      version:
        required: true
        type: string

env:
  GITHUB_TOKEN: ${{ secrets.AUTORELEASE_BOT_PAT }}

jobs:
  roll_back_release:
    # run this job if a PR from an autorelease branch into main was closed without merging
    if: startsWith(github.head_ref, 'autorelease') && github.event.pull_request.merged != true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Extract version from autorelease branch name
        run: echo "VERSION=$(echo ${{ github.head_ref }} | cut -c 13-)" >> $GITHUB_ENV

      - name: Roll back on failure
        uses: ./.github/actions/rollback_release
        with:
          VERSION: ${{ env.VERSION }}
          GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }}

  publish_release:
    # run this job if a PR was merged from an autorelease branch into main
    # or if manually triggered, github.head_ref is undefined
    if: ${{ ! github.head_ref || (startsWith(github.head_ref, 'autorelease') && github.event.pull_request.merged == true) }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Install dependencies
        run: pip install twine

      - name: Extract version from autorelease branch name
        if: ${{ github.head_ref }}
        run: echo "VERSION=$(echo ${{ github.head_ref }} | cut -c 13-)" >> $GITHUB_ENV

      - name: Get version from manual trigger input
        if: ${{ ! github.head_ref }}
        run: echo "VERSION=${{ inputs.version }}" >> $GITHUB_ENV

      - name: Find actions run for the $VERSION tag
        run: |
          echo "RUN_ID=$( \
            gh run list \
              --branch $VERSION \
              --workflow 'Run Tests and Build' \
              --limit 1 \
              --json databaseId \
              --jq .[].databaseId)" >> $GITHUB_ENV

      - name: Download release artifacts
        run: |
          mkdir artifacts
          # this will download a folder containing each artifact
          gh run download $RUN_ID --dir artifacts --pattern "Wheel for *"
          # move the artifacts out of the folders
          mv artifacts/*/* artifacts
          rm -rf artifacts/Wheel*

          # download each artifact separately so that the command will fail if any is missing
          for artifact in Workbench-Windows-binary \
                          Workbench-macOS-binary \
                          InVEST-sample-data \
                          InVEST-user-guide \
                          "Source distribution"
          do
            gh run download $RUN_ID --dir artifacts --name "$artifact"
          done

      - name: Create Github release
        run: |
          # Copy the history notes for this version into the release message
          echo "This release includes the following fixes and features:" > notes.rst
          echo "" >> notes.rst
          sed -n "/$VERSION/,/^$/p" HISTORY.rst | tail -n +3 >> notes.rst
          gh release create $VERSION \
            --verify-tag \
            --title $VERSION \
            --notes-file notes.rst \
            artifacts/*

      - name: Create a PyPI release
        # this is the only step that can't be rolled back
        run: |
          twine upload \
            --username="__token__" \
            --password=${{ secrets.PYPI_NATCAP_INVEST_TOKEN }} \
            artifacts/natcap.invest*

      - name: Roll back on failure
        if: failure()
        uses: ./.github/actions/rollback_release
        with:
          VERSION: ${{ env.VERSION }}
          GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }}


Test Upload Python Package

# This workflow will upload a Python Package using Twine when a release is created
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.

name: Test Upload Python Package

on:
  release:
    types: [published]
  workflow_dispatch:

permissions:
  contents: read

jobs:
  deploy:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.8
        uses: actions/setup-python@v3
        with:
          python-version: "3.8"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install build
      - name: Build package
        run: python -m build
      - name: Publish package 📦 on PyPi Test
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_TEST_API_TOKEN }}
          verbose: true
          skip-existing: true
          repository-url: https://test.pypi.org/legacy/


Upload Python Package

# This workflow will upload a Python Package using Twine when a release is created
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.

name: Upload Python Package

on:
  release:
    types: [published]
  workflow_dispatch:

permissions:
  contents: read

jobs:
  deploy:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.8
        uses: actions/setup-python@v3
        with:
          python-version: "3.8"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install build
      - name: Build package
        run: python -m build
      - name: Publish package 📦 on PyPi
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}
          verbose: true
          skip-existing: true
          repository-url: https://upload.pypi.org/legacy/


run-test

name: run-test

## run for all branches

on:
  push:
  workflow_dispatch:

jobs:
  test_and_doc:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install package
        run: pip install .

      - name: Run tests
        run: |
          export PYTHONPATH=$PWD
          mkdir -p tesdata
          pytest --junitxml=tesdata/pymicro${{ matrix.python-version }}.xml

      - name: Upload test results
        uses: actions/upload-artifact@v2
        with:
          name: test-results
          path: tesdata/pymicro${{ matrix.python-version }}.xml


Build

name: Build

on:
  release:
    types: [released]
  # push:
  #   branches:
  #     - master
  # pull_request:
  workflow_dispatch:

env:
  CIBW_SKIP: pp*  # only build CPython wheels
  CIBW_ARCHS: auto64 # don't bother with 32bit wheels

jobs:
  # build_wheels:
  #   name: wheels on ${{ matrix.os }}
  #   runs-on: ${{ matrix.os }}
  #   strategy:
  #     matrix:
  #       os: [ubuntu-latest, macos-latest]
  #   steps:
  #     - name: Checkout code
  #       uses: actions/checkout@v3
  #       with:
  #         fetch-depth: 0
  #     - name: Build wheels
  #       uses: pypa/cibuildwheel@v2.11.2
  #     - uses: actions/upload-artifact@v3
  #       with:
  #         path: ./wheelhouse/*.whl

  build_sdist:
    name: source distribution
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Build sdist
        run: |
          pip install build
          python -m build --sdist
      - uses: actions/upload-artifact@v4
        with:
          path: dist/*.tar.gz

  upload_pypi:
    name: upload to PyPI
    #needs: [build_wheels, build_sdist]
    needs: [build_sdist]
    runs-on: ubuntu-latest
    if: github.event_name == 'push' &&
        github.event.ref_type == 'tag' &&
        github.ref == 'refs/heads/${{ github.event.repository.default_branch }}'
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: artifact
          path: dist
      - uses: pypa/gh-action-pypi-publish@v1.5.1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}


Python package for jwst-ero branch

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Python package for jwst-ero branch

on:
  push:
    branches: [ jwst-ero ]
  pull_request:
    branches: [ jwst-ero ]

jobs:
  build:

    #runs-on: ubuntu-latest
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: [3.8, 3.9, '3.10']
        os: [ubuntu-latest]
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install wheel
        python -m pip install flake8 pytest numpy cython

        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

        # To generate version.py and compiled utils_c
        python setup.py build
        cp build/lib*/grizli/utils_c/*so grizli/utils_c/

    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude "scripts* build* docs* dist* examples*"
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude "scripts* eazy-photoz* astropy_helpers*"
    - name: Test with pytest
      run: |
        pytest --disable-warnings


CI

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: CI

env:
  CRDS_SERVER_URL: https://jwst-crds.stsci.edu
  CRDS_PATH: $HOME/crds_cache

on:
  push:
    branches: [ master ]
  pull_request:

jobs:
  test-jwst:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: ['3.9', '3.10']
        os: [ubuntu-latest]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install package
      run: |
        pip install --upgrade pip
        pip install .[jwst,test]
    - name: Test with pytest
      run: |
        pytest --pyargs grizli --import-mode importlib


# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: CI

on:
  push:
    branches:
      - master
  pull_request:

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: [3.8, 3.9, '3.10']
        os: [ubuntu-latest]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Build check
      run: |
        pip install build twine
        python -m build
        python -m twine check --strict dist/*

  codestyle:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: ['3.10']
        os: [ubuntu-latest]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install package
      run: |
        pip install flake8
    - name: Lint with flake8
      run: |
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics

  test:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: [3.8, 3.9, '3.10']
        os: [ubuntu-latest]
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install package
      run: |
        pip install -e .[test]
    - name: Test with pytest
      run: |
        pytest


Monitor Tests

---
name: Monitor Tests
on:
  push:
    branches:
      - main
  # run the test only if the PR is to main
  # turn it on if required
  #pull_request:
  #  branches:
  #  - main
  schedule:
    - cron: '0 0 * * *'  # nightly

# Required shell entrypoint to have properly configured bash shell
defaults:
  run:
    shell: bash -l {0}

jobs:
  linux:
    runs-on: "ubuntu-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
      fail-fast: false
    name: Linux Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v3
        with:
          activate-environment: esmvalcore
          environment-file: environment.yml
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p test_linux_artifacts_python_${{ matrix.python-version }}
      - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
      - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
      - run: pip install pytest-monitor
      - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt
      - run: pytest -n 2 -m "not installation" --db ../.pymon 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt
      - run: python tests/parse_pymon.py
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: Test_Linux_python_${{ matrix.python-version }}
          path: test_linux_artifacts_python_${{ matrix.python-version }}

  osx:
    runs-on: "macos-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
      fail-fast: false
    name: OSX Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v3
        with:
          architecture: ${{ matrix.architecture }}
          activate-environment: esmvalcore
          environment-file: environment.yml
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p test_osx_artifacts_python_${{ matrix.python-version }}
      - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
      - run: python -V 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
      - run: mamba install -c conda-forge git
      - run: pip install pytest-monitor
      - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt
      - run: conda list
      - run: pytest -n 2 -m "not installation" --db ../.pymon 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt
      - run: python tests/parse_pymon.py
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: Test_OSX_python_${{ matrix.python-version }}
          path: test_osx_artifacts_python_${{ matrix.python-version }}


PyPi Build and Deploy 🐍📦

name: PyPi Build and Deploy 🐍📦

on:
  release:
    types: [published]
  # use this for testing
  push:
    branches:
      - main

jobs:
  build-n-publish:
    name: Build and publish ESMValCore on PyPi
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/project/ESMValCore/
    permissions:
      # IMPORTANT: this permission is mandatory for trusted publishing
      id-token: write
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.12
        uses: actions/setup-python@v4
        with:
          python-version: "3.12"
      - name: Install pep517
        run: >-
          python -m
          pip install
          pep517
          --user
      - name: Build a binary wheel and a source tarball
        run: >-
          python -m
          pep517.build
          --source
          --binary
          --out-dir dist/
          .
      # - name: Publish distribution to Test PyPI
      #   uses: pypa/gh-action-pypi-publish@release/v1
      #   with:
      #     repository-url: https://test.pypi.org/legacy/
      - name: Publish distribution 📦 to PyPI
        if: startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@release/v1


Install from conda-forge

# Install esmvalcore from PyPi on different OS's
# and different Python version; test locally with
# act: https://github.com/nektos/act
# Example how to setup conda workflows:
# https://github.com/marketplace/actions/setup-miniconda
# Notes:
#  - you can group commands with | delimiter (or &&) but those will be run
#    in one single call; declaring the shell variable makes the action run each
#    command separately (better for debugging);
#  - can try multiple shells eg pwsh or cmd /C CALL {0} (but overkill for now!);
# TODO: read the cron tasking documentation:
# https://www.netiq.com/documentation/cloud-manager-2-5/ncm-reference/data/bexyssf.html

name: Install from conda-forge

# runs on a push on main and at the end of every day
on:
  # triggering on push without branch name will run tests every time
  # there is a push on any branch
  # turn it on only if needed
  push:
    branches:
      - main
  # run the test only if the PR is to main
  # turn it on if required
  # pull_request:
  #   branches:
  #     - main
  schedule:
    - cron: '0 4 * * *'

# Required shell entrypoint to have properly configured bash shell
defaults:
  run:
    shell: bash -l {0}

jobs:
  linux:
    runs-on: "ubuntu-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
      # fail-fast set to False allows all other tests
      # in the workflow to run regardless of any fail
      fail-fast: false
    name: Linux Python ${{ matrix.python-version }}
    steps:
      - uses: conda-incubator/setup-miniconda@v3
        with:
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p conda_install_linux_artifacts_python_${{ matrix.python-version }}
      - name: Record versions
        run: |
          mamba --version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
          which conda 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_path.txt
          which mamba 2>&1 | tee -a conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_path.txt
          python -V 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
      - name: Install ESMValCore
        run: mamba install esmvalcore 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/install.txt
      - name: Verify installation
        run: |
          esmvaltool --help
          esmvaltool version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: Conda_Install_Linux_python_${{ matrix.python-version }}
          path: conda_install_linux_artifacts_python_${{ matrix.python-version }}

  osx:
    runs-on: "macos-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
      fail-fast: false
    name: OSX Python ${{ matrix.python-version }}
    steps:
      - uses: conda-incubator/setup-miniconda@v3
        with:
          architecture: ${{ matrix.architecture }}
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p conda_install_osx_artifacts_python_${{ matrix.python-version }}
      - name: Record versions
        run: |
          mamba --version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
          which conda 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_path.txt
          which mamba 2>&1 | tee -a conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_path.txt
          python -V 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
      - name: Install ESMValCore
        run: mamba install esmvalcore 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
      - name: Verify installation
        run: |
          esmvaltool --help
          esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: Conda_Install_OSX_python_${{ matrix.python-version }}
          path: conda_install_osx_artifacts_python_${{ matrix.python-version }}


Install from PyPi

# Install esmvalcore from PyPi on different OS's
# and different Python version; test locally with
# act: https://github.com/nektos/act
# Example how to setup conda workflows:
# https://github.com/marketplace/actions/setup-miniconda
# Notes:
#  - you can group commands with | delimiter (or &&) but those will be run
#    in one single call; declaring the shell variable makes the action run each
#    command separately (better for debugging);
#  - can try multiple shells eg pwsh or cmd /C CALL {0} (but overkill for now!);
# TODO: read the cron tasking documentation:
# https://www.netiq.com/documentation/cloud-manager-2-5/ncm-reference/data/bexyssf.html

name: Install from PyPi

# runs on a push on main and at the end of every day
on:
  # triggering on push without branch name will run tests every time
  # there is a push on any branch
  # turn it on only if needed
  push:
    branches:
      - main
  # run the test only if the PR is to main
  # turn it on if required
  #pull_request:
  #  branches:
  #  - main
  schedule:
    - cron: '0 0 * * *'

# Required shell entrypoint to have properly configured bash shell
defaults:
  run:
    shell: bash -l {0}

jobs:
  linux:
    runs-on: "ubuntu-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
      # fail-fast set to False allows all other tests
      # in the workflow to run regardless of any fail
      fail-fast: false
    name: Linux Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
      - uses: conda-incubator/setup-miniconda@v3
        with:
          activate-environment: esmvalcore
          environment-file: environment.yml
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p pip_install_linux_artifacts_python_${{ matrix.python-version }}
      - name: Record versions
        run: |
          mamba --version 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
          python -V 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
          pip -V 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/pip_version.txt
      - name: Install
        run: pip install esmvalcore 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/install.txt
      - name: Verify installation
        run: |
          esmvaltool --help
          esmvaltool version 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: PIP_Install_Linux_python_${{ matrix.python-version }}
          path: pip_install_linux_artifacts_python_${{ matrix.python-version }}

  osx:
    runs-on: "macos-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
      fail-fast: false
    name: OSX Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
      - uses: conda-incubator/setup-miniconda@v3
        with:
          architecture: ${{ matrix.architecture }}
          activate-environment: esmvalcore
          environment-file: environment.yml
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p pip_install_osx_artifacts_python_${{ matrix.python-version }}
      - name: Record versions
        run: |
          mamba --version 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
          python -V 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
          pip -V 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/pip_version.txt
      - name: Install
        run: pip install esmvalcore 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
      - name: Verify installation
        run: |
          esmvaltool --help
          esmvaltool version 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: PIP_Install_OSX_python_${{ matrix.python-version }}
          path: pip_install_osx_artifacts_python_${{ matrix.python-version }}


Install from Source

# Install esmvalcore from source on different OS's
# and different Python version; test locally with
# act: https://github.com/nektos/act
# Example how to setup conda workflows:
# https://github.com/marketplace/actions/setup-miniconda
# Notes:
#  - you can group commands with | delimiter (or &&) but those will be run
#    in one single call; declaring the shell variable makes the action run each
#    command separately (better for debugging);
#  - can try multiple shells eg pwsh or cmd /C CALL {0} (but overkill for now!);

name: Install from Source

# runs on a push on main and at the end of every day
on:
  # triggering on push without branch name will run tests every time
  # there is a push on any branch
  # turn it on only if needed
  push:
    branches:
      - main
  # run the test only if the PR is to main
  # turn it on if required
  #pull_request:
  #  branches:
  #  - main
  schedule:
    - cron: '0 0 * * *'

# Required shell entrypoint to have properly configured bash shell
defaults:
  run:
    shell: bash -l {0}

jobs:
  linux:
    runs-on: "ubuntu-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
      fail-fast: false
    name: Linux Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v3
        with:
          activate-environment: esmvalcore
          environment-file: environment.yml
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p source_install_linux_artifacts_python_${{ matrix.python-version }}
      - name: Record versions
        run: |
          mamba --version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
          python -V 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
      - name: Install
        run: pip install -e .[develop] 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/install.txt
      - name: Verify installation
        run: |
          esmvaltool --help
          esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: Source_Install_Linux_python_${{ matrix.python-version }}
          path: source_install_linux_artifacts_python_${{ matrix.python-version }}

  osx:
    runs-on: "macos-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
      fail-fast: false
    name: OSX Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v3
        with:
          architecture: ${{ matrix.architecture }}
          activate-environment: esmvalcore
          environment-file: environment.yml
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p source_install_osx_artifacts_python_${{ matrix.python-version }}
      - name: Record versions
        run: |
          mamba --version 2>&1 | tee source_install_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
          python -V 2>&1 | tee source_install_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
      - name: Force install x64 git
        run: mamba install -c conda-forge git
      - name: Install
        run: pip install -e .[develop] 2>&1 | tee source_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
      - name: Verify installation
        run: |
          esmvaltool --help
          esmvaltool version 2>&1 | tee source_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: Source_Install_OSX_python_${{ matrix.python-version }}
          path: source_install_osx_artifacts_python_${{ matrix.python-version }}


Test

# Install esmvalcore from PyPi on different OS's
# and different Python version; test locally with
# act: https://github.com/nektos/act
# Example how to setup conda workflows:
# https://github.com/marketplace/actions/setup-miniconda
# Notes:
#  - you can group commands with | delimiter (or &&) but those will be run
#    in one single call; declaring the shell variable makes the action run each
#    command separately (better for debugging);
#  - can try multiple shells eg pwsh or cmd /C CALL {0} (but overkill for now!);
# TODO: read the cron tasking documentation:
# https://www.netiq.com/documentation/cloud-manager-2-5/ncm-reference/data/bexyssf.html

name: Test

# runs on a push on main and at the end of every day
on:
  # triggering on push without branch name will run tests every time
  # there is a push on any branch
  # turn it on only if needed
  push:
    branches:
      - main
  # run the test only if the PR is to main
  # turn it on if required
  #pull_request:
  #  branches:
  #  - main
  schedule:
    - cron: '0 0 * * *'  # nightly

# Required shell entrypoint to have properly configured bash shell
defaults:
  run:
    shell: bash -l {0}

jobs:
  linux:
    runs-on: "ubuntu-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
      fail-fast: false
    name: Linux Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v3
        with:
          activate-environment: esmvalcore
          environment-file: environment.yml
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p test_linux_artifacts_python_${{ matrix.python-version }}
      - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
      - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
      - run: conda list
      - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt
      - run: conda list
      - run: flake8
      - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: Test_Linux_python_${{ matrix.python-version }}
          path: test_linux_artifacts_python_${{ matrix.python-version }}

  osx:
    runs-on: "macos-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
      fail-fast: false
    name: OSX Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v3
        with:
          architecture: ${{ matrix.architecture }}
          activate-environment: esmvalcore
          environment-file: environment.yml
          python-version: ${{ matrix.python-version }}
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - run: mkdir -p test_osx_artifacts_python_${{ matrix.python-version }}
      - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
      - run: python -V 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
      - run: mamba install -c conda-forge git
      - run: conda list
      - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt
      - run: conda list
      - run: flake8
      - run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: Test_OSX_python_${{ matrix.python-version }}
          path: test_osx_artifacts_python_${{ matrix.python-version }}


Conda lock file creation

name: Conda lock file creation

on:
  # Trigger on push on main or other branch for testing
  # NOTE that push: main will create the file very often
  # and hence lots of automated PRs
  # push:
  #   branches:
  #     - main
  schedule:
    - cron: '0 4 */10 * *'

# Required shell entrypoint to have properly configured bash shell
defaults:
  run:
    shell: bash -l {0}

jobs:
  create-verify-lock-file:
    name: Create and verify conda lock file for latest Python
    runs-on: 'ubuntu-latest'
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          activate-environment: esmvaltool-fromlock
          python-version: "3.12"
          miniforge-version: "latest"
          miniforge-variant: Mambaforge
          use-mamba: true
      - name: Update and show conda config
        run: |
          conda update -n base -c conda-forge conda
          conda info
          conda list
          conda config --show-sources
          conda config --show
          # setup-miniconda@v3 installs an old conda and mamba
          # forcing a modern mamba updates both mamba and conda
          conda install -c conda-forge "mamba>=1.4.8"
          conda --version
          mamba --version
      - name: Gather Python info
        run: |
          which python
          python --version
      - name: Install conda-lock
        run: mamba install -y -c conda-forge conda-lock
      - name: Check version of conda-lock
        run: conda-lock --version
      - name: Create conda lock file for linux-64
        run: conda-lock lock --platform linux-64 -f environment.yml --mamba --kind explicit
      - name: Creating environment from lock file
        run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
      - name: Installing pip
        run: mamba install -y pip
      - name: Gather pip info
        run: pip --version
      - name: Gather Python info again
        run: |
          which python
          python --version
      - name: Show environment contents
        run: conda list
      - name: Install esmvalcore
        run: pip install -e .[develop]
      - name: Run basic commands
        run: |
          esmvaltool --help
          esmvaltool version
      - name: Run flake8
        run: flake8
      - name: Run pytests
        run: pytest -n 2 -m "not installation"
      # Automated PR
      # see https://github.com/marketplace/actions/create-pull-request
      - name: Create Automated PR if conda lock file has changed
        uses: peter-evans/create-pull-request@v3
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          commit-message: Updating Linux condalock file
          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
          signoff: false
          branch: condalock-update
          delete-branch: true
          title: '[Condalock] Update Linux condalock file'
          body: |
            Update condalock file
            Automatic Pull Request.
          labels: |
            testing
            condalock
            automatedPR
          assignees: valeriupredoi
          reviewers: valeriupredoi
          draft: false


Conda-lock Install

# Install esmvalcore from a conda lock file
# To build a conda lock file install conda-lock first then
# run conda-lock lock --platform linux-64 -f environment.yml --mamba
# (mamba activated for speed). Change platform for osx-64 or win-64.
# Env creation then happens as per normal use with
# mamba create --name esmvaltool-fromlock --file conda-linux-64.lock
# note that pip and conda are NOT installed.

name: Conda-lock Install
on:
  push:
    branches:
      - main
  # run the test only if the PR is to main
  # turn it on if required
  #pull_request:
  #  branches:
  #  - main
  schedule:
    - cron: '0 0 * * *'

# Required shell entrypoint to have properly configured bash shell
defaults:
  run:
    shell: bash -l {0}

jobs:
  linux:
    runs-on: "ubuntu-latest"
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
      fail-fast: false
    name: Linux Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: conda-incubator/setup-miniconda@v3
        with:
          activate-environment: esmvaltool-fromlock
          python-version: ${{ matrix.python-version }}
          channels: conda-forge
      - run: mkdir -p source_install_linux_artifacts_python_${{ matrix.python-version }}
      - run: conda --version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
      - run: which python
      - run: python -V 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
      - run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
      - run: which python
      - run: pip --version
      - run: pip install -e .[develop]
      - run: esmvaltool --help
      - run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
      - run: flake8
      - run: pytest -n 2 -m "not installation"
      - name: Upload artifacts
        if: ${{ always() }}  # upload artifacts even if fail
        uses: actions/upload-artifact@v4
        with:
          name: Source_Install_Linux_python_${{ matrix.python-version }}
          path: source_install_linux_artifacts_python_${{ matrix.python-version }}


CFF File Validator

# workflow that performs a validity check of CITATION.cff file
# author: Abel Siqueira, GH: @abelsiqueira

name: CFF File Validator

on:
  push:
    paths:
      - CITATION.cff
  schedule:
    - cron: '0 0 1 * *'

jobs:
  validate:
    name: "validate"
    runs-on: ubuntu-latest
    steps:
      - name: Check out a copy of the repository
        uses: actions/checkout@v4
      - name: Check whether the citation metadata from CITATION.cff is valid
        uses: citation-file-format/cffconvert-github-action@2.0.0
        with:
          args: "--validate"


Lint

name: Lint

on:
  pull_request:
    branches:
      - master

permissions:
  checks: write
  contents: write

jobs:
  lint-python:
    name: Lint Python
    runs-on: ubuntu-latest

    steps:
      - name: Check out Git repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Check changed py files
        id: changed-files
        uses: tj-actions/changed-files@v41
        with:
          files: |
            **/*.py

      - name: Set up Python
        uses: actions/setup-python@v1
        if: ${{ steps.changed-files.outputs.all_changed_files }}
        with:
          python-version: '3.10'

      - name: Install Python dependencies
        if: ${{ steps.changed-files.outputs.all_changed_files }}
        run: pip install flake8

      - name: Run flake8 linter
        if: ${{ steps.changed-files.outputs.all_changed_files }}
        uses: wearerequired/lint-action@v2
        with:
          flake8: true
          flake8_auto_fix: false
          flake8_args: ${{ steps.changed-files.outputs.all_changed_files }}
          continue_on_error: false


commcare-cloud tests

# This workflow will install Python dependencies, run tests with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: commcare-cloud tests

on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]

permissions:
  contents: read

jobs:
  build:
    strategy:
      matrix:
        os: ["ubuntu-22.04"]
        python-version: ["3.10"]

    runs-on: ${{ matrix.os }}

    env:
      ANSIBLE_ROLES_PATH: "~/.ansible/roles"
      CI_TEST: 1
      TEST: main

    steps:
    - uses: actions/checkout@v3
    - name: Set Up Commcare-Cloud Python Version
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies
      run: |
        source control/init.sh
        # Note: the installs must be done in the following order, otherwise the installation of commcare-cloud
        # tries to install from the commcare-cloud-bootstrap.egg_info directory
        pip install -e .[test]

        # setup 3.9 for deploy-stack
        # may be removed when this file is updated to match global_vars/all.yml:python_version
        sudo add-apt-repository -y ppa:deadsnakes/ppa
        sudo apt update
        sudo apt install python3.9 python3.9-venv

        # Install virtualenv as an isolated global executable for deploy-stack in .tests/tests.sh
        python3 -m pip install --user pipx
        pipx install virtualenv

    - name: Run tests
      run: |
        .tests/tests.sh


Consistency

name: Consistency

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the "main" branch
  push:
    branches: ["main"]
  pull_request:
    branches: ["main"]
  merge_group:
  workflow_dispatch: {}

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  check-changes:
    name: Check Changes
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 ## Needed for Changesets to find `main` branch
          submodules: recursive

      - name: Install pnpm
        uses: pnpm/action-setup@v2

      - run: git pull --force --no-tags origin main:main
        name: Get main ref

      - run: pnpm install
        name: Install dependencies

      - run: pnpm change verify
        name: Check changelog
        if: ${{ !startsWith(github.head_ref, 'publish/') && !startsWith(github.head_ref, 'dependabot/') && !startsWith(github.head_ref, 'backmerge/') }}

      - run: pnpm check-version-mismatch
        name: Check version mismatch

      - run: pnpm lint
        name: Lint project

      - run: pnpm check-format
        name: Check formatting


Prepare Auto Release Branch

name: Prepare Auto Release Branch

# Disable as MS Org doesn't allow github action to create PR anymore. Can be re-added if a solution is found.
on:
  push:
    branches:
      - main

concurrency: ${{ github.workflow }}-${{ github.ref }}

permissions:
  contents: write

jobs:
  release:
    name: Release
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2

      - name: Setup Node.js 18.x
        uses: actions/setup-node@v2
        with:
          node-version: 18.x

      - name: Install pnpm
        uses: pnpm/action-setup@v2

      - run: pnpm install
        name: Install dependencies

      - name: Create release branch
        run: node ./eng/prepare-release-branch.mjs
        env:
          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}


Pipy

# Pipy upload strax after a release (or manually).
# Mostly based on https://github.com/marketplace/actions/pypi-publish
name: Pipy

on:
  workflow_dispatch:
  release:
    types: [created]
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      # Setup steps
      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - name: Checkout repo
        uses: actions/checkout@v4
      - name: Install dependencies
        run: pip install wheel
      - name: Build package
        run: python setup.py sdist bdist_wheel
      # Do the publish
      - name: Publish a Python distribution to PyPI
        # Might want to add but does not work on workflow_dispatch :
        # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@master
        with:
          user:  ${{ secrets.pipy_token }}
          password: ${{ secrets.pypi_password }}


tests

# Test strax on each PR
name: tests

# Trigger this code when a new release is published
on:
  workflow_dispatch:
  release:
    types: [ created ]
  pull_request:
    branches:
      - master
      - stable
  push:
    branches:
      - master

jobs:
  update:
    name: "${{ matrix.test }}_py${{ matrix.python-version }}"
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.9, "3.10"]
        test: ['coveralls', 'pytest']
        # Installation on py3.10 is rather slow at the moment
        exclude:
          - python-version: "3.11"
            test: coveralls
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
          cache-dependency-path: 'extra_requirements/requirements-tests.txt'
      - name: Install dependencies
        run: sudo apt-get install -y graphviz
      - name: Install requirements
        run: pip install -r extra_requirements/requirements-tests.txt
      - name: Start MongoDB
        uses: supercharge/mongodb-github-action@1.10.0
        with:
          mongodb-version: 4.2
      # Perform coveralls (if coverage is set to True) or pytest
      - name: Test package
        if: matrix.test == 'pytest'
        env:
          TEST_MONGO_URI: 'mongodb://localhost:27017/'
        run: |
          pytest -v --durations 0
      - name: Coverage run
        if: matrix.test == 'coveralls'
        env:
          NUMBA_DISABLE_JIT: 1
          TEST_MONGO_URI: 'mongodb://localhost:27017/'
        run: |
          coverage run --source=strax setup.py test -v
      - name: Coverage run - backward compatibility check with straxen
        if: matrix.test == 'coveralls'
        env:
          NUMBA_DISABLE_JIT: 1
          TEST_MONGO_URI: 'mongodb://localhost:27017/'
        run: |
          echo "clone straxen"
          straxen_dir="../straxen/"
          git clone --single-branch --branch master https://github.com/XENONnT/straxen.git $straxen_dir
          bash $straxen_dir/.github/scripts/create_pre_apply_function.sh $HOME
          pip install -e $straxen_dir  # Reinstall since tests might reflect new code.
          echo "Run straxen tests"
          coverage run --append --source=strax -m pytest $straxen_dir
          coverage report
      - name: Coveralls upload
        if: matrix.test == 'coveralls'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          coverage report
          coveralls --service=github
      - name: goodbye
        run: echo goodbye


Installation test

# Test if we can actually install strax by installing
name: Installation test

on:
  workflow_dispatch:
  release:
    types: [created]
  pull_request:
    branches:
      - master
      - stable
  push:
    branches:
      - master

jobs:
  update:
    name: "py${{ matrix.python-version }}"
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.9, "3.10"]
    steps:
      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Checkout repo
        uses: actions/checkout@v4
      - name: pre-install requirements
        run: pip install -r requirements.txt
      - name: Install strax
        run: python setup.py install
      - name: Test import
        run: python -c "import strax; print(strax.__version__)"
      - name: goodbye
        run: echo goodbye


build docs

name: build docs

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
on:
  push:
    branches:
      - 'main'
      - 'release/*'
    tags:
      - 'v*'
  pull_request:
  merge_group:
    branches: ['main']

jobs:
  builddocs:

    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        exclude:
          - os: windows-latest
            python-version: "3.10"
          - os: windows-latest
            python-version: "3.11"
          - os: windows-latest
            python-version: "3.12"
    env:
      DISPLAY: ':99.0'
      OS: ${{ matrix.os }}
      UPLOAD_TO_GHPAGES: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
    steps:
    - uses: actions/checkout@v4.1.7
      with:
        fetch-depth: '0'
        # if we upload to ghpages we need the full
        # history to generate correct version info
      if: ${{ fromJSON(env.UPLOAD_TO_GHPAGES) }}
    - uses: actions/checkout@v4.1.7
      if: ${{ !fromJSON(env.UPLOAD_TO_GHPAGES) }}
    - name: setup ubuntu-latest xvfb
      uses: ./.github/actions/setup-ubuntu-latest-xvfb
      if: runner.os == 'Linux'
    - name: install pandoc linux
      run: sudo apt install pandoc
      if: runner.os == 'Linux'
    - name: Install pandoc on windows
      uses: crazy-max/ghaction-chocolatey@v3.0.0
      with:
        args: install pandoc
      if: runner.os == 'Windows'
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5.1.0
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
        cache-dependency-path: |
          pyproject.toml
          requirements.txt
    - name: upgrade pip setuptools wheel
      run: python -m pip install --upgrade pip setuptools wheel
      shell: bash
    - name: install qcodes
      run: pip install .[docs] -c requirements.txt
    - name: Build docs on linux
      run: |
        cd docs
        export SPHINXOPTS="-W -v"
        make html
      if: runner.os == 'Linux'
    - name: Build docs on windows
      run: |
        cd docs
        $env:SPHINXOPTS = "-W -v"
        ./make.bat html
      if: runner.os == 'Windows'
    - name: Upload build docs
      uses: actions/upload-artifact@v4
      with:
        name: docs_${{ matrix.python-version }}_${{ matrix.os }}
        path: ${{ github.workspace }}/docs/_build/html
    - name: Deploy to gh pages
      uses: JamesIves/github-pages-deploy-action@v4.6.1
      with:
        branch: gh-pages
        folder: ${{ github.workspace }}/docs/_build/html
        clean: true
        single-commit: true
        git-config-email: "bot"
        git-config-name: "Documentation Bot"
      if: ${{ fromJSON(env.UPLOAD_TO_GHPAGES) }}


Run mypy and pytest

name: Run mypy and pytest

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
on:
  push:
    branches:
      - 'main'
      - 'release/*'
      - 'staging'
      - 'trying'
    tags:
      - 'v*'
  pull_request:
  merge_group:
    branches: ['main']

jobs:
  pytestmypy:

    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        exclude:
          - os: windows-latest
            python-version: "3.10"
          - os: windows-latest
            python-version: "3.11"
          - os: windows-latest
            python-version: "3.12"
    env:
      DISPLAY: ':99.0'
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python-version }}

    steps:
    - uses: actions/checkout@v4.1.7
      with:
        submodules: true
    - name: setup ubuntu-latest xvfb
      uses: ./.github/actions/setup-ubuntu-latest-xvfb
      if: runner.os == 'Linux'
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5.1.0
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
        cache-dependency-path: |
          pyproject.toml
          requirements.txt
    - name: upgrade pip setuptools wheel
      run: python -m pip install --upgrade pip setuptools wheel
      shell: bash
    - name: install package
      run: |
        pip install .[test] -c requirements.txt
    - name: Run Mypy
      run: mypy src
    - name: Run tests
      run: |
        pytest --cov=qcodes_contrib_drivers --cov-report xml --cov-config=pyproject.toml tests
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v4.5.0
      with:
        file: ./coverage.xml
        env_vars: OS,PYTHON
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


Upload Python Package

name: Upload Python Package

on:
  push:
    tags:
      - v*

jobs:
  deploy:
    runs-on: ubuntu-latest
    permissions:
      id-token: write
    environment: pypi-release-env
    steps:
    - uses: actions/checkout@v4.1.7
    - name: Set up Python
      uses: actions/setup-python@v5.1.0
      with:
        python-version: '3.10'
    - name: Install build deps
      run: pip install --upgrade pip setuptools wheel build
    - name: Build
      run: |
        python -m build
    - name: Install Twine
      run: pip install twine
    - name: Publish to pypi
      uses: pypa/gh-action-pypi-publish@release/v1


Github-CI-Release

name: Github-CI-Release

on:
  push:
    tags:
      - v[0-9]+.[0-9]+.[0-9]+

env:
  # secrets can be set in settings/secrets on github
  DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  deploy:
    name: deploy release
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    env:
      TO_TAG_PREFIX: release-github
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: find branch name
        shell: bash
        run: echo "BRANCH_NAME=$(git name-rev --refs="refs/remotes/origin/master" --refs="refs/remotes/origin/hotfix_v*" --refs="refs/remotes/origin/hotfix_staging_*" --name-only ${GITHUB_SHA})" >> $GITHUB_ENV
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
      - name: set owner variable
        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
      - name: set git tag
        run: echo "GIT_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
      - name: show system environs
        if: github.event_name == 'push'
        run: ./ci/helpers/show_system_versions.bash
      - name: source image is staging
        run: echo "FROM_TAG_PREFIX=staging-github" >> $GITHUB_ENV
      - if: contains(env.BRANCH_NAME, 'remotes/origin/hotfix_v')
        name: source image is hotfix (instead of staging)
        run: echo "FROM_TAG_PREFIX=hotfix-github" >> $GITHUB_ENV
      - name: deploy
        run: ./ci/deploy/dockerhub-tag-version.bash


Github-CI-Staging

name: Github-CI-Staging

on:
  push:
    tags:
      - staging_[a-zA-Z0-9]+[0-9]+

env:
  # secrets can be set in settings/secrets on github
  DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  deploy:
    name: deploy staging
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    env:
      TO_TAG_PREFIX: staging-github
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: find branch name
        shell: bash
        run: echo "BRANCH_NAME=$(git name-rev --refs="refs/remotes/origin/master" --refs="refs/remotes/origin/hotfix_v*" --refs="refs/remotes/origin/hotfix_staging_*" --name-only ${GITHUB_SHA})" >> $GITHUB_ENV
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
      - name: set owner variable
        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
      - name: set git tag
        run: echo "GIT_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
      - name: show system environs
        if: github.event_name == 'push'
        run: ./ci/helpers/show_system_versions.bash
      - name: source image is master
        run: echo "FROM_TAG_PREFIX=master-github" >> $GITHUB_ENV
      - if: contains(env.BRANCH_NAME, 'remotes/origin/hotfix_staging_')
        name: source image is hotfix-staging (instead of master)
        run: echo "FROM_TAG_PREFIX=hotfix-staging-github" >> $GITHUB_ENV
      - name: deploy
        run: ./ci/deploy/dockerhub-tag-version.bash