Generate Combined Inventories

# This workflow will generate inventory and combined datasets as artifact

name: Generate Combined Inventories

on:
  pull_request:
    branches: [master]
    types: [opened, reopened]   # excludes syncronize to avoid redundant trigger from commits on PRs
  workflow_dispatch:            # manual trigger only

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v3
      with:
        python-version: "3.10"

    - name: Update pip & install testing pkgs
      run: |
        python -VV
        python -m pip install --upgrade pip setuptools wheel
        pip install pytest pytest-cov

    - name: Install package and dependencies
      run: |
        pip install .

    - name: Combine inventory files
      run: |
        pytest -m combined

    - name: Upload files
      if: always() # Upload files even if some inventories fail
      uses: actions/upload-artifact@v3
      with:
        # Artifact name
        name: StEWI Combined inventory files
        # A file, directory or wildcard patter that describes what to upload
        path: | # uses local user data dir for ubuntu
          ~/.local/share/stewi/facility/*
          ~/.local/share/stewi/flow/*
          ~/.local/share/stewi/flowbyfacility/*
          ~/.local/share/stewi/flowbyprocess/*
          ~/.local/share/stewi/validation/*
          ~/.local/share/stewi/*.json
          ~/.local/share/stewicombo/*
          ~/.local/share/facilitymatcher/*.parquet
          ~/.local/share/facilitymatcher/*.json
        if-no-files-found: warn # 'warn' or 'ignore' are also available, defaults to `warn`
        # retention-days: 5 # cannot exceed the retention limit set by the repository, organization, or enterprise.


Generate single inventory

# This workflow will generate inventory files based on user input

name: Generate single inventory

on:
  workflow_dispatch:            # manual trigger only
    inputs:
        year:
            description: "Year"
            required: true
            default: "2016-2018"
        inventory:
            description: "Inventory"
            required: true
            default: "TRI"

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v3
      with:
        python-version: "3.10"

    - name: Update pip & install testing pkgs
      run: |
        python -VV
        python -m pip install --upgrade pip setuptools wheel

    - name: Install package and dependencies
      run: |
        pip install .

    - name: Generate inventory files
      env:
          YEAR: ${{ github.event.inputs.year }}
          INVENTORY: ${{ github.event.inputs.inventory }}
      run: |
        echo "Year: ${{ github.event.inputs.year }}"
        echo "Inventory: ${{ github.event.inputs.inventory }}"
        python scripts/generate_select_inventories.py --years $YEAR --inventory $INVENTORY

    - name: Upload files
      uses: actions/upload-artifact@v3
      with:
        # Artifact name
        name: "${{ github.event.inputs.inventory }}"
        # A file, directory or wildcard patter that describes what to upload
        path: | # uses local user data dir for ubuntu
          ~/.local/share/stewi/facility/*
          ~/.local/share/stewi/flow/*
          ~/.local/share/stewi/flowbyfacility/*
          ~/.local/share/stewi/flowbyprocess/*
          ~/.local/share/stewi/validation/*
          ~/.local/share/stewi/*.json
        if-no-files-found: warn # 'warn' or 'ignore' are also available, defaults to `warn`
        # retention-days: 5 # cannot exceed the retention limit set by the repository, organization, or enterprise.


Python CI/CD tests

# This workflow will install Python dependencies, run tests and lint
# across operating systems, select versions of Python, and user + dev environments
# For more info see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Python CI/CD tests

on:
  push:
    # branches: [master, develop]
    paths-ignore:               # prevents workflow execution when only these types of files are modified
      - '**.md'                 # wildcards prevent file in any repo dir from trigering workflow
      - '**.bib'
      - '**.ya?ml'              # captures both .yml and .yaml
      - 'LICENSE'
      - '.gitignore'
  pull_request:
    branches: [master, develop]
    types: [opened, reopened, ready_for_review]   # excludes syncronize to avoid redundant trigger from commits on PRs
    paths-ignore:
      - '**.md'
      - '**.bib'
      - '**.ya?ml'
      - 'LICENSE'
      - '.gitignore'
  workflow_dispatch:            # also allow manual trigger, for testing purposes

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        py-version: ['3.8', '3.9', '3.10', '3.11']

    steps:
    - uses: actions/checkout@v3

    # general Python setup
    - name: Set up Python ${{ matrix.py-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.py-version }}

    - name: Update pip & install testing pkgs
      run: |
        python -VV
        python -m pip install --upgrade pip setuptools wheel
        pip install pytest pytest-cov flake8

    # install testing
    - name: Install package and dependencies
      run: |
        pip install .

    # linting & pytest
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --extend-ignore=E12,E26,E231,E251,E701 --exclude build --exit-zero --max-line-length=127 --statistics

    - name: Test with pytest
      run: |
        pytest --doctest-modules -m "not (combined or inventory)"


name: lint_python
on:
  push:
    branches:
      - "**" # run all branches
    tags-ignore:
      - "*" # ignore all tags, release.yaml will trigger the CI
  pull_request: # run on all pull requests

concurrency:
  cancel-in-progress: true
  group: group-${{ github.ref_name }}

jobs:
  lint_python:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python: ["3.8", "3.9", "3.10", "3.11", "3.x"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - run: pip install -U pip
      - run: pip install -U bandit mypy pyupgrade safety tox setuptools
      - run: bandit --recursive --skip B105,B110,B311,B605,B607 --exclude ./.tox .
        if: ${{ matrix.python >= '3.8' }}
      - run: tox -e lint
      - run: tox -e py
      - run: shopt -s globstar && pyupgrade --py3-only **/*.py # --py36-plus
      - run: safety check -i 42559 -i 62044 -i 67599 -i 70612 # pip version issue, we don't care about it, we don't ship it
      - run: tox -e build
      - run: tox -e doc


Releases

name: Releases

on:
  push:
    tags:
    - 'v*'

jobs:

  release:
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Setup python
      uses: actions/setup-python@v5
      with:
        python-version: "3.9"
    - name: Setup tox
      run: pip install tox
    - name: Run linter
      run: tox -e lint
    - name: Run tests
      run: tox -e py
    - name: Build package
      run: tox -e build
    - name: Create release
      uses: ncipollo/release-action@v1
      with:
        artifacts: dist/gspread-*
        token: ${{ secrets.GH_TOKEN }}
        generateReleaseNotes: True
        artifactErrorsFailBuild: True
    - name: Publish to TestPyPi
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
        repository-url: https://test.pypi.org/legacy/
    - name: Publish to PyPi
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}


packages

name: packages
on:
  push:
    tags:
      - "v[0-9]+.[0-9]+.[0-9]+"
      - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+"
      - "v[0-9]+.[0-9]+.[0-9]+b[0-9]+"
      - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
  # Dry-run only
  workflow_dispatch:
  schedule:
    - cron: "0 16 * * SUN"

defaults:
  run:
    shell: bash -l {0}

env:
  PYTHON_VERSION: "3.11"
  PACKAGE: "datashader"
  MPLBACKEND: "Agg"
  DASK_DATAFRAME__QUERY_PLANNING: false

jobs:
  waiting_room:
    name: Waiting Room
    runs-on: ubuntu-latest
    needs: [conda_build, pip_install]
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
    environment:
      name: publish
    steps:
      - run: echo "All builds have finished, have been approved, and ready to publish"

  conda_build:
    name: Build Conda
    runs-on: "ubuntu-latest"
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: "100"
      - name: Fetch unshallow
        run: git fetch --prune --tags --unshallow -f
      - uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: "latest"
      - name: conda setup
        run: |
          # pyct is for running setup.py
          conda install -y conda-build build pyct -c pyviz/label/dev
      - name: conda build
        run: |
          source ./scripts/conda/build.sh
          echo "CONDA_FILE="$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.tar.bz2"" >> $GITHUB_ENV
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: conda
          path: ${{ env.CONDA_FILE }}
          if-no-files-found: error

  conda_publish:
    name: Publish Conda
    runs-on: ubuntu-latest
    needs: [conda_build, waiting_room]
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: conda
          path: dist/
      - name: Set environment variables
        run: |
          echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
          echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV
      - uses: conda-incubator/setup-miniconda@v3
        with:
          miniconda-version: "latest"
      - name: conda setup
        run: |
          conda install -y anaconda-client
      - name: conda dev upload
        if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')
        run: |
          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE
      - name: conda main upload
        if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')))
        run: |
          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE

  pip_build:
    name: Build PyPI
    runs-on: "ubuntu-latest"
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: "100"
      - name: Fetch unshallow
        run: git fetch --prune --tags --unshallow -f
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Install build
        run: |
          python -m pip install build
      - name: Build package
        run: python -m build .
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: pip
          path: dist/
          if-no-files-found: error

  pip_install:
    name: Install PyPI
    runs-on: "ubuntu-latest"
    needs: [pip_build]
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - uses: actions/download-artifact@v4
        with:
          name: pip
          path: dist/
      - name: Install package
        run: python -m pip install dist/*.whl
      - name: Test package
        run: python -c "import $PACKAGE; print($PACKAGE.__version__)"

  pip_publish:
    name: Publish PyPI
    runs-on: ubuntu-latest
    needs: [pip_build, waiting_room]
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: pip
          path: dist/
      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: ${{ secrets.PPU }}
          password: ${{ secrets.PPP }}
          repository-url: "https://upload.pypi.org/legacy/"


docs

name: docs
on:
  push:
    tags:
      - 'v[0-9]+.[0-9]+.[0-9]+'
      - 'v[0-9]+.[0-9]+.[0-9]+a[0-9]+'
      - 'v[0-9]+.[0-9]+.[0-9]+b[0-9]+'
      - 'v[0-9]+.[0-9]+.[0-9]+rc[0-9]+'
  workflow_dispatch:
    inputs:
      target:
        description: 'Site to build and deploy'
        type: choice
        options:
        - dev
        - main
        - dryrun
        required: true
        default: dryrun
  schedule:
    - cron: '0 16 * * SUN'

jobs:
  build_docs:
    name: Documentation
    runs-on: 'ubuntu-latest'
    timeout-minutes: 120
    defaults:
      run:
        shell: bash -l {0}
    env:
      DESC: "Documentation build"
      DASK_DATAFRAME__QUERY_PLANNING: false
    steps:
      - uses: holoviz-dev/holoviz_tasks/install@v0
        with:
          name: Documentation
          python-version: "3.10"
          channel-priority: strict
          channels: pyviz/label/dev,conda-forge,nodefaults
          envs: "-o doc"
          cache: true
          conda-update: true
      - name: Set output
        id: vars
        run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
      - name: fetch data
        run: |
          conda activate test-environment
          python scripts/download_data.py
      - name: generate rst
        run: |
          conda activate test-environment
          nbsite generate-rst --org holoviz --project-name datashader
      - name: build docs
        run: |
          conda activate test-environment
          nbsite build --what=html --output=builtdocs --org holoviz --project-name datashader
      - name: Deploy dev
        uses: peaceiris/actions-gh-pages@v3
        if: |
          (github.event_name == 'workflow_dispatch' && github.event.inputs.target == 'dev') ||
          (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
        with:
          personal_token: ${{ secrets.ACCESS_TOKEN }}
          external_repository: holoviz-dev/datashader
          publish_dir: ./builtdocs
          force_orphan: true
      - name: Deploy main
        if: |
          (github.event_name == 'workflow_dispatch' && github.event.inputs.target == 'main') ||
          (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./builtdocs
          cname: datashader.org
          force_orphan: true


name: downstream_tests

on:
  # Run this workflow after the build workflow has completed.
  workflow_run:
    workflows: [packages]
    types: [completed]
  # Or by triggering it manually via Github's UI
  workflow_dispatch:
    inputs:
      manual:
        description: don't change me!
        type: boolean
        required: true
        default: true

jobs:
  downstream_tests:
    uses: holoviz-dev/holoviz_tasks/.github/workflows/run_downstream_tests.yaml@main
    with:
      downstream_repos_as_json: "{\"downstream_repo\":[\"holoviews\", \"hvplot\"]}"
    secrets:
      ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}


tests

name: tests
on:
  push:
    branches:
      - main
  pull_request:
    branches:
    - '*'
  workflow_dispatch:
    inputs:
      target:
        description: "How much of the test suite to run"
        type: choice
        default: default
        options:
          - default
          - full
          - downstream
      cache:
        description: "Use cache"
        type: boolean
        default: true
  schedule:
    - cron: '0 16 * * SUN'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  OMP_NUM_THREADS: 1
  OPENBLAS_NUM_THREADS: 1
  MKL_NUM_THREADS: 1
  VECLIB_MAXIMUM_THREADS: 1
  NUMEXPR_NUM_THREADS: 1
  PYDEVD_DISABLE_FILE_VALIDATION: 1

jobs:
  pre_commit:
    name: Run pre-commit
    runs-on: 'ubuntu-latest'
    steps:
      - uses: holoviz-dev/holoviz_tasks/pre-commit@v0

  setup:
    name: Setup workflow
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ env.MATRIX }}
    steps:
      - name: Set matrix option
        run: |
          if [[ '${{ github.event_name }}' == 'workflow_dispatch' ]]; then
            OPTION=${{ github.event.inputs.target }}
          elif [[ '${{ github.event_name }}' == 'schedule' ]]; then
            OPTION="full"
          elif [[ '${{ github.event_name }}' == 'push' && '${{ github.ref_type }}' == 'tag' ]]; then
            OPTION="full"
          else
            OPTION="default"
          fi
          echo "MATRIX_OPTION=$OPTION" >> $GITHUB_ENV
      - name: Set test matrix with 'default' option
        if: env.MATRIX_OPTION == 'default'
        run: |
          MATRIX=$(jq -nsc '{
              "os": ["ubuntu-latest", "macos-latest", "windows-latest"],
              "python-version": ["3.9", "3.10", "3.11", "3.12"],
              "numpy-version": ["1"],
              "exclude": [{
                  "os": "windows-latest",
                  "python-version": "3.9"
              }],
              "include": [{
                  "os": "ubuntu-latest",
                  "python-version": "3.12",
                  "numpy-version": "2"
              },
              {
                  "os": "macos-latest",
                  "python-version": "3.12",
                  "numpy-version": "2"
              }]
          }')
          # Won't solve on Windows + Python 3.9
          echo "MATRIX=$MATRIX" >> $GITHUB_ENV
      - name: Set test matrix with 'full' option
        if: env.MATRIX_OPTION == 'full'
        run: |
          MATRIX=$(jq -nsc '{
              "os": ["ubuntu-latest", "macos-latest", "windows-latest"],
              "python-version": ["3.9", "3.10", "3.11", "3.12"],
              "numpy-version": ["1"],
              "exclude": [{
                  "os": "windows-latest",
                  "python-version": "3.9"
              }],
              "include": [{
                  "os": "ubuntu-latest",
                  "python-version": "3.12",
                  "numpy-version": "2"
              },
              {
                  "os": "macos-latest",
                  "python-version": "3.12",
                  "numpy-version": "2"
              }]
          }')
          # Won't solve on Windows + Python 3.9
          echo "MATRIX=$MATRIX" >> $GITHUB_ENV
      - name: Set test matrix with 'downstream' option
        if: env.MATRIX_OPTION == 'downstream'
        run: |
          MATRIX=$(jq -nsc '{
              "os": ["ubuntu-latest"],
              "numpy-version": ["1"],
              "python-version": ["3.11"]
          }')
          echo "MATRIX=$MATRIX" >> $GITHUB_ENV

  test_suite:
    name: Tests on ${{ matrix.os }} with Python ${{ matrix.python-version }}, numpy ${{ matrix.numpy-version }}
    needs: [pre_commit, setup]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
    timeout-minutes: 180
    defaults:
      run:
        shell: bash -l {0}
    env:
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set channels and envs
        run: |
          if [[ ${{ matrix.numpy-version }} == "2" ]]; then
            channels="pyviz/label/dev,conda-forge/label/numpy_rc,numba/label/dev,conda-forge,nodefaults"
            envs="-o numpy2"
          else
            channels="pyviz/label/dev,numba,conda-forge,nodefaults"
            envs="-o tests -o examples"
          fi
          echo "CHANNELS=$channels" >> $GITHUB_ENV
          echo "ENVS=$envs" >> $GITHUB_ENV
      - uses: holoviz-dev/holoviz_tasks/install@v0
        with:
          name: unit_test_suite_np${{ matrix.numpy-version }}
          python-version: ${{ matrix.python-version }}
          channel-priority: flexible
          channels: ${{ env.CHANNELS }}
          envs: ${{ env.ENVS }}
          cache: ${{ github.event.inputs.cache || github.event.inputs.cache == '' }}
          conda-update: true
        id: install
      - name: check version
        run: |
          conda activate test-environment
          python -c "import numba; print('Numba', numba.__version__)"
          python -c "import numpy; print('Numpy', numpy.__version__)"
      - name: download data
        run: |
          conda activate test-environment
          python scripts/download_data.py
      - name: doit test_unit
        run: |
          conda activate test-environment
          doit test_unit
      - name: doit test_unit_nojit
        run: |
          conda activate test-environment
          doit test_unit_nojit
        env:
          NUMBA_DISABLE_JIT: 1
      - name: doit test_examples
        env:
          DASK_DATAFRAME__QUERY_PLANNING: false
        run: |
          conda activate test-environment
          doit test_examples
      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


CI

# see https://github.com/karlicoss/pymplate for up-to-date reference

name: CI
on:
  push:
    branches: '*'
    tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi
    # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug:
  pull_request: # needed to trigger on others' PRs
  # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
  workflow_dispatch: # needed to trigger workflows manually
    # todo cron?
    inputs:
      debug_enabled:
        type: boolean
        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: false

jobs:
  build:
    strategy:
      fail-fast: false
      matrix:
        platform: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
        exclude: [
            # windows runners are pretty scarce, so let's only run lowest and highest python version
            {platform: windows-latest, python-version: '3.9' },
            {platform: windows-latest, python-version: '3.10'},
            {platform: windows-latest, python-version: '3.11'},

            # same, macos is a bit too slow and ubuntu covers python quirks well
            {platform: macos-latest  , python-version: '3.9' },
            {platform: macos-latest  , python-version: '3.10'},
            {platform: macos-latest  , python-version: '3.11'},
        ]

    runs-on: ${{ matrix.platform }}

    #continue-on-error: ${{ matrix.platform == 'windows-latest' }}

    steps:
    # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
    - run: echo "$HOME/.local/bin" >> $GITHUB_PATH

    - uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - uses: actions/checkout@v4
      with:
        submodules: recursive
        fetch-depth: 0  # nicer to have all git history when debugging/for tests

    - uses: mxschmitt/action-tmate@v3
      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}

    # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
    - run: bash .ci/run

    - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
      uses: actions/upload-artifact@v4
      with:
        name: .coverage.mypy-core_${{ matrix.platform }}_${{ matrix.python-version }}
        path: .coverage.mypy-core/
    - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
      uses: actions/upload-artifact@v4
      with:
        name: .coverage.mypy-misc_${{ matrix.platform }}_${{ matrix.python-version }}
        path: .coverage.mypy-misc/

  # split in two pipelines to speedup running
  end2end_tests_chrome:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: recursive
    - uses: mxschmitt/action-tmate@v3
      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
    - run: .ci/end2end/build_and_run.sh -k chrome

  end2end_tests_firefox:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: recursive
    - uses: mxschmitt/action-tmate@v3
      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
    - run: .ci/end2end/build_and_run.sh -k firefox

  install_and_run_test:
    # todo use setup-python thing?
    # todo run on macos too?
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: recursive

    - run: .ci/github-ci-compat

    - run: |
        # --use-pep517 is a work around for some breaking change in setuptools 66.0.0
        # see https://github.com/pypa/setuptools/issues/3772#issuecomment-1384671296
        python3 -m pip install --use-pep517 .
        export PATH=.ci/fake-systemd:$PATH
        tests/install_and_run

  pypi:
    runs-on: ubuntu-latest
    needs: [build, end2end_tests_chrome, end2end_tests_firefox, install_and_run_test]

    steps:
    # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
    - run: echo "$HOME/.local/bin" >> $GITHUB_PATH

    - uses: actions/setup-python@v5
      with:
        python-version: '3.10'

    - uses: actions/checkout@v4
      with:
        submodules: recursive

    - name: 'release to test pypi'
      # always deploy merged master to test pypi
      if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master'
      env:
        TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }}
      run: pip3 install --user --upgrade build twine && .ci/release --test

    - name: 'release to pypi'
      # always deploy tags to release pypi
      # NOTE: release tags are guarded by on: push: tags on the top
      if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags')
      env:
        TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
      run: pip3 install --user --upgrade build twine && .ci/release

  ###
  build_extension:
    env:
      name: 'promnesia'
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: recursive
        fetch-depth: 0  # nicer to have all git history when debugging/for tests

    - uses: actions/setup-node@v4
      with:
        node-version: '20'

    - run: extension/.ci/build --lint # debug version
    - run: extension/.ci/build --lint --release

    # TODO ugh. can't share github actions artifacts publicly...
    # TODO for fuck's sake... why does it end up named as .zip.zip ????
    - uses: actions/upload-artifact@v4
      with:
        name: '${{ env.name }}-chrome-debug-latest.zip'
        path: 'extension/dist/artifacts/chrome/${{ env.name }}_dev_-*.zip'
    - uses: actions/upload-artifact@v4
      with:
        name: '${{ env.name }}-chrome-release-latest.zip'
        path: 'extension/dist/artifacts/chrome/${{ env.name }}-*.zip'
    - uses: actions/upload-artifact@v4
      with:
        name: '${{ env.name }}-firefox-debug-latest.zip'
        path: 'extension/dist/artifacts/firefox/${{ env.name }}_dev_-*.zip'
    - uses: actions/upload-artifact@v4
      with:
        name: '${{ env.name }}-firefox-release-latest.zip'
        path: 'extension/dist/artifacts/firefox/${{ env.name }}-*.zip'


ci

name: ci

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repo
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
      - name: Run pre-commit
        uses: pre-commit/action@v3.0.0

  build:
    needs: [lint]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macOS-latest]
        python-version: ["3.8", "3.12"]
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Checkout code
        uses: nschloe/action-cached-lfs-checkout@v1
      - name: Test with tox
        run: |
          pip install tox
          tox -- --cov meshio --cov-report xml --cov-report term
      - uses: codecov/codecov-action@v3
        if: ${{ matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest' }}


Combine PRs

name: 'Combine PRs'

# Controls when the action will run - in this case triggered manually
on:
  workflow_dispatch:
    inputs:
      branchPrefix:
        description: 'Branch prefix to find combinable PRs based on'
        required: true
        default: 'dependabot'
      mustBeGreen:
        description: 'Only combine PRs that are green (status is success)'
        required: true
        default: true
      combineBranchName:
        description: 'Name of the branch to combine PRs into'
        required: true
        default: 'combine-prs-branch'
      ignoreLabel:
        description: 'Exclude PRs with this label'
        required: true
        default: 'nocombine'

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  # This workflow contains a single job called "combine-prs"
  combine-prs:
    # The type of runner that the job will run on
    runs-on: ubuntu-latest

    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      - uses: actions/github-script@v3
        id: fetch-branch-names
        name: Fetch branch names
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const pulls = await github.paginate('GET /repos/:owner/:repo/pulls', {
              owner: context.repo.owner,
              repo: context.repo.repo
            });
            branches = [];
            prs = [];
            base_branch = null;
            for (const pull of pulls) {
              const branch = pull['head']['ref'];
              console.log('Pull for branch: ' + branch);
              if (branch.startsWith('${{ github.event.inputs.branchPrefix }}')) {
                console.log('Branch matched: ' + branch);
                statusOK = true;
                if(${{ github.event.inputs.mustBeGreen }}) {
                  console.log('Checking green status: ' + branch);
                  const statuses = await github.paginate('GET /repos/{owner}/{repo}/commits/{ref}/status', {
                    owner: context.repo.owner,
                    repo: context.repo.repo,
                    ref: branch
                  });
                  if(statuses.length > 0) {
                    const latest_status = statuses[0]['state'];
                    console.log('Validating status: ' + latest_status);
                    if(latest_status != 'success') {
                      console.log('Discarding ' + branch + ' with status ' + latest_status);
                      statusOK = false;
                    }
                  }
                }
                console.log('Checking labels: ' + branch);
                const labels = pull['labels'];
                for(const label of labels) {
                  const labelName = label['name'];
                  console.log('Checking label: ' + labelName);
                  if(labelName == '${{ github.event.inputs.ignoreLabel }}') {
                    console.log('Discarding ' + branch + ' with label ' + labelName);
                    statusOK = false;
                  }
                }
                if (statusOK) {
                  console.log('Adding branch to array: ' + branch);
                  branches.push(branch);
                  prs.push('#' + pull['number'] + ' ' + pull['title']);
                  base_branch = pull['base']['ref'];
                }
              }
            }

            if (branches.length == 0) {
              core.setFailed('No PRs/branches matched criteria');
              return;
            }

            core.setOutput('base-branch', base_branch);
            core.setOutput('prs-string', prs.join('\n'));

            combined = branches.join(' ')
            console.log('Combined: ' + combined);
            return combined
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v2.3.3
        with:
          fetch-depth: 0
      # Creates a branch with other PR branches merged together
      - name: Created combined branch
        env:
          BASE_BRANCH: ${{ steps.fetch-branch-names.outputs.base-branch }}
          BRANCHES_TO_COMBINE: ${{ steps.fetch-branch-names.outputs.result }}
          COMBINE_BRANCH_NAME: ${{ github.event.inputs.combineBranchName }}
        run: |
          echo "$BRANCHES_TO_COMBINE"
          sourcebranches="${BRANCHES_TO_COMBINE%\"}"
          sourcebranches="${sourcebranches#\"}"

          basebranch="${BASE_BRANCH%\"}"
          basebranch="${basebranch#\"}"

          git config pull.rebase false
          git config user.name github-actions
          git config user.email github-actions@github.com

          git branch $COMBINE_BRANCH_NAME $basebranch
          git checkout $COMBINE_BRANCH_NAME
          git pull origin $sourcebranches --no-edit
          git push origin $COMBINE_BRANCH_NAME
      # Creates a PR with the new combined branch
      - uses: actions/github-script@v3
        name: Create Combined Pull Request
        env:
          PRS_STRING: ${{ steps.fetch-branch-names.outputs.prs-string }}
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const prString = process.env.PRS_STRING;
            const body = 'This PR was created by the Combine PRs action by combining the following PRs:\n' + prString;
            await github.pulls.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title: 'Combined PR',
              head: '${{ github.event.inputs.combineBranchName }}',
              base: '${{ steps.fetch-branch-names.outputs.base-branch }}',
              body: body
            });


Release

# This is a GitHub workflow defining a set of jobs with a set of steps.
# ref: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
#
name: Release

# Always tests wheel building, but only publish to PyPI on pushed tags.
on:
  pull_request:
    paths-ignore:
      - "docs/**"
      - ".github/workflows/*.yaml"
      - "!.github/workflows/release.yaml"
  push:
    paths-ignore:
      - "docs/**"
      - ".github/workflows/*.yaml"
      - "!.github/workflows/release.yaml"
    branches-ignore:
      - "dependabot/**"
      - "pre-commit-ci-update-config"
    tags: ["**"]
  workflow_dispatch:

jobs:
  build-release:
    runs-on: ubuntu-22.04
    permissions:
      # id-token=write is required for pypa/gh-action-pypi-publish, and the PyPI
      # project needs to be configured to trust this workflow.
      #
      # ref: https://github.com/jupyterhub/team-compass/issues/648
      #
      id-token: write

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: install build package
        run: |
          pip install --upgrade pip
          pip install build
          pip freeze

      - name: build release
        run: |
          python -m build --sdist --wheel .
          ls -l dist

      - name: publish to pypi
        uses: pypa/gh-action-pypi-publish@release/v1
        if: startsWith(github.ref, 'refs/tags/')


Test docs

# This is a GitHub workflow defining a set of jobs with a set of steps.
# ref: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
#
name: Test docs

on:
  pull_request:
    paths:
      - "docs/**"
      - "oauthenticator/**"
      - "**/test-docs.yaml"
  push:
    paths:
      - "docs/**"
      - "oauthenticator/**"
      - "**/test-docs.yaml"
    branches-ignore:
      - "dependabot/**"
      - "pre-commit-ci-update-config"
    tags: ["**"]
  workflow_dispatch:

jobs:
  linkcheck:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install deps
        run: pip install -r docs/requirements.txt

      - name: make linkcheck
        run: |
          cd docs
          make linkcheck SPHINXOPTS='--color -W --keep-going'


Tests

# This is a GitHub workflow defining a set of jobs with a set of steps.
# ref: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
#
name: Tests

on:
  pull_request:
    paths-ignore:
      - "docs/**"
      - ".github/workflows/*.yaml"
      - "!.github/workflows/test.yaml"
  push:
    paths-ignore:
      - "docs/**"
      - ".github/workflows/*.yaml"
      - "!.github/workflows/test.yaml"
    branches-ignore:
      - "dependabot/**"
      - "pre-commit-ci-update-config"
    tags: ["**"]
  workflow_dispatch:

env:
  OAUTH2_TOKEN_URL: "token_url"
  OAUTH2_USERDATA_URL: "userdata_url"

jobs:
  test:
    runs-on: ubuntu-22.04
    timeout-minutes: 10

    strategy:
      # Keep running even if one variation of the job fail
      fail-fast: false
      matrix:
        python:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        include:
          - python: "3.9"
            oldest_dependencies: oldest_dependencies

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "${{ matrix.python }}"

      - name: Install Python dependencies
        run: |
          pip install ".[test]"
          pip list

      - name: Downgrade to oldest dependencies
        if: matrix.oldest_dependencies != ''
        # take any dependencies in requirements.txt such as jupyterhub>=2.2 and
        # transform them to jupyterhub==2.2 so we can run tests with the
        # earliest-supported versions
        run: |
          cat requirements.txt | grep '>=' | sed -e 's@>=@==@g' > oldest-requirements.txt
          pip install -r oldest-requirements.txt
          pip list

      - name: Run tests
        run: |
          pytest

      # GitHub action reference: https://github.com/codecov/codecov-action
      - uses: codecov/codecov-action@v4


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches:
      - 'main'
  pull_request:
  schedule:
    - cron: '0 0 * * *'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v1
        with:
          languages: ${{ matrix.language }}
          # If you wish to specify custom queries, you can do so here or in a config file.
          # By default, queries listed here will override any specified in a config file.
          # Prefix the list here with "+" to use these queries and those in the config file.
          # queries: ./path/to/local/query, your-org/your-repo/queries@main

      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
      # If this step fails, then you should remove it and run the build manually (see below)
      - name: Autobuild
        uses: github/codeql-action/autobuild@v1

      # ℹ Command-line programs to run using the OS shell.
      # 📚 https://git.io/JvXDl

      # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
      #    and modify them (or add more) to build your code if your project
      #    uses a compiled language

      #- run: |
      #   make bootstrap
      #   make release

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v1


Test and Deploy

name: Test and Deploy

on:
  push:
  pull_request:
  release:
    types:
      - created

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - uses: pre-commit/action@v3.0.0

  test:
    needs: pre-commit
    strategy:
      fail-fast: false
      matrix:
        platform: [
          { os: 'ubuntu-latest',  python-version: '3.8' },
          { os: 'ubuntu-latest',  python-version: '3.9' },
          { os: 'ubuntu-latest',  python-version: '3.10' },
          { os: 'ubuntu-latest',  python-version: '3.11' },
          { os: 'ubuntu-latest',  python-version: '3.12' },
          { os: 'ubuntu-latest',  python-version: 'pypy3.8' },
          { os: 'ubuntu-latest',  python-version: 'pypy3.9' },
          { os: 'ubuntu-latest',  python-version: 'pypy3.10' },
          { os: 'macos-latest',  python-version: '3.8' },
          { os: 'macos-latest',  python-version: '3.9' },
          { os: 'macos-latest',  python-version: '3.10' },
          { os: 'macos-latest',  python-version: '3.11' },
          { os: 'macos-latest',  python-version: '3.12' },
          { os: 'macos-latest',  python-version: 'pypy3.8' },
          { os: 'macos-latest',  python-version: 'pypy3.9' },
          { os: 'macos-latest',  python-version: 'pypy3.10' },
          { os: 'windows-latest',  python-version: '3.8' },
          { os: 'windows-latest',  python-version: '3.9' },
          { os: 'windows-latest',  python-version: '3.10' },
          { os: 'windows-latest',  python-version: '3.11' },
          { os: 'windows-latest',  python-version: '3.12' },
        ]
        # https://endoflife.date/python
    runs-on: ${{ matrix.platform.os }}
    steps:
      - uses: actions/checkout@v3

      - name: Wake up httpbin server
        run: |
          curl https://httpbinx.fly.dev/ip

      - name: Install poetry
        run: pipx install poetry

      - name: Setup Python ${{ matrix.platform.python-version }} on ${{ matrix.platform.os }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.platform.python-version }}

      - name: Install dependencies
        run: |
          poetry install --extras 'aiohttp'

      - name: Echo installed packages
        run: |
          poetry show

      - name: Test with pytest
        run: poetry run pytest -v -rs tests --runslow --cov=./ --cov-report=xml

      - name: Upload coverage to Codecov
        if: matrix.platform.python-version == '3.11' && matrix.platform.os == 'ubuntu-latest'
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: true

  deploy:
    needs: test
    runs-on: ubuntu-latest
    if: github.event_name == 'release' && github.event.action == 'created'
    permissions:
      id-token: write
    steps:
      - uses: actions/checkout@v3

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: Install poetry
        run: pipx install poetry

      - name: Build Packages
        run: poetry build

      - name: Publish package distributions to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1


REUSE

name: REUSE

# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the master branch
on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]

jobs:
  reuse:
    # The type of runner that the job will run on
    runs-on: ubuntu-latest

    steps:
    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
    - uses: actions/checkout@v2

    # Check REUSE Compliance
    - name: REUSE Compliance Check
      uses: fsfe/reuse-action@v1.1


linux

name: linux

on: [pull_request, push]

defaults:
  run:
      shell: bash -l {0}

jobs:
  docs:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - uses: mamba-org/setup-micromamba@main
      with:
        environment-file: .github/snakePipesEnvCI.yml
        condarc: |
          channels:
            - conda-forge
            - bioconda
            - defaults
          channel_priority: 'strict'
        cache-downloads: true
    - name: Install snakePipes
      run: |
        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
    - name: docs
      run: |
        micromamba activate snakePipes_CI
        snakePipes config --tempDir /tmp
        cd docs
        make html
  flake:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - uses: mamba-org/setup-micromamba@main
      with:
        environment-file: .github/snakePipesEnvCI.yml
        condarc: |
          channels:
            - conda-forge
            - bioconda
            - defaults
          channel_priority: 'strict'
        cache-downloads: true
    - name: Install snakePipes
      run: |
        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
    - name: flake
      run: |
        micromamba activate snakePipes_CI
        snakePipes config --tempDir /tmp
        flake8 --ignore=E501,E722,E402 --exclude tests,docs/conf.py,build/lib/snakePipes/shared/tools/three_prime_seq,snakePipes/shared/tools/three_prime_seq .
  CI:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - uses: mamba-org/setup-micromamba@main
      with:
        environment-file: .github/snakePipesEnvCI.yml
        condarc: |
          channels:
            - conda-forge
            - bioconda
            - defaults
          channel_priority: 'strict'
        cache-downloads: true
    - name: Install snakePipes
      run: |
        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
    - name: CI
      run: |
        micromamba activate snakePipes_CI
        snakePipes config --tempDir /tmp
        ./.ci_stuff/test_dag.sh
    - name: pytest
      run: |
        micromamba activate snakePipes_CI
        snakePipes config --tempDir /tmp
        pytest --verbosity=2
  createEnvs:
    needs: CI
    strategy:
      fail-fast: false
      matrix:
        envs: [
          'CONDA_SHARED_ENV',
          'CONDA_CREATE_INDEX_ENV',
          'CONDA_RNASEQ_ENV',
          'CONDA_RMATS_ENV',
          'CONDA_scRNASEQ_ENV',
          'CONDA_seurat3_ENV',
          'CONDA_loompy_ENV',
          'CONDA_alevinqc_ENV',
          'CONDA_eisaR_ENV',
          'CONDA_DNA_MAPPING_ENV',
          'CONDA_CHIPSEQ_ENV',
          'CONDA_ATAC_ENV',
          'CONDA_HIC_ENV',
          'CONDA_WGBS_ENV',
          'CONDA_DSS_ENV',
          'CONDA_RMD_ENV',
          'CONDA_PREPROCESSING_ENV',
          'CONDA_NONCODING_RNASEQ_ENV',
          'CONDA_SAMBAMBA_ENV',
          'CONDA_pysam_ENV'
        ]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - uses: mamba-org/setup-micromamba@main
      with:
        environment-file: .github/snakePipesEnvCI.yml
        condarc: |
          channels:
            - conda-forge
            - bioconda
            - defaults
          channel_priority: 'strict'
        cache-downloads: true
    - name: install snakePipes
      run: |
        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
    - name: createEnvs
      run: |
        micromamba activate snakePipes_CI
        snakePipes config --tempDir /tmp
        snakePipes createEnvs --autodetectCondaEnvDir --only ${{matrix.envs}}


osx

name: osx

on: [pull_request, push]

defaults:
  run:
      shell: bash -l {0}

jobs:
  createEnvs_OSX:
    strategy:
      fail-fast: false
      matrix:
        envs: [
          'CONDA_SHARED_ENV',
          'CONDA_CREATE_INDEX_ENV',
          'CONDA_RNASEQ_ENV',
          'CONDA_RMATS_ENV',
          'CONDA_scRNASEQ_ENV',
          'CONDA_seurat3_ENV',
          'CONDA_loompy_ENV',
          'CONDA_alevinqc_ENV',
          'CONDA_eisaR_ENV',
          'CONDA_DNA_MAPPING_ENV',
          'CONDA_CHIPSEQ_ENV',
          'CONDA_ATAC_ENV',
          'CONDA_HIC_ENV',
          'CONDA_WGBS_ENV',
          'CONDA_DSS_ENV',
          'CONDA_RMD_ENV',
          'CONDA_PREPROCESSING_ENV',
          'CONDA_NONCODING_RNASEQ_ENV',
          'CONDA_SAMBAMBA_ENV',
          'CONDA_pysam_ENV'
        ]
    runs-on: macos-latest
    steps:
      - uses: actions/checkout@v3
      - uses: mamba-org/setup-micromamba@main
        with:
          environment-file: .github/snakePipesEnvCI.yml
          condarc: |
            channels:
              - conda-forge
              - bioconda
              - defaults
            channel_priority: 'strict'
          cache-downloads: true
      - name: install snakePipes_OSX
        run: |
          micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
      - name: createEnvsOSX
        run: |
          micromamba activate snakePipes_CI
          snakePipes createEnvs --autodetectCondaEnvDir --only ${{matrix.envs}}


Examples

name: Examples

on: [pull_request]

jobs:
  build:

    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash

    strategy:
      max-parallel: 1
      matrix:
        python-version: ['3.9']
        os: [ubuntu-latest]

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Cache pip
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install dependencies (pip)
        run: |
          pip install \
            -r requirements.txt \
            -r requirements/tests.txt
          pip install pypardiso

      - name: Running tests
        run:
          pytest examples/
            --nbval-lax


Clean Up Tags

name: Clean Up Tags

on: [workflow_dispatch]

jobs:
  deploy:
    name: Clean up tags
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - name: Clean up tags
      run: |
        git config --local user.email "action@github.com"
        git config --local user.name "GitHub Action"
        REPOSITORY=${INPUT_REPOSITORY:-$GITHUB_REPOSITORY}
        remote_repo="https://${GITHUB_ACTOR}:${{ secrets.GITHUB_TOKEN }}@github.com/${REPOSITORY}.git"
        git fetch --all --tags --force
        prefix="v"
        pattern="^($prefix)(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-((?:0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
        git tag | grep --invert-match -P $pattern | xargs -n 1 -I % git push "${remote_repo}" :refs/tags/%


Bump Version (release)

name: Bump Version (release)

on:
  push:
    branches:
      - release

jobs:
  build:
    name: Bump version
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        persist-credentials: false  # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
        fetch-depth: 0              # otherwise, you will failed to push refs to dest repo

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Set env variables
      run: |
        # The next line is very important, otherwise the line after triggers
        # git to track the permission change, which breaks bump2version API (needs clean git folder)
        git config core.filemode false
        chmod +x .github/workflows/utils.sh
        echo "VERSION_FILE=openpnm/__version__.py" >> $GITHUB_ENV
        echo "SETUP_CFG_FILE=setup.cfg" >> $GITHUB_ENV
        echo "${{ github.event.head_commit.message }}"

    - name: Install dependencies
      run: |
        pip install bump2version

    - name: Bump version (patch)
      if: contains(github.event.head_commit.message, '#patch')
      run: |
        source .github/workflows/utils.sh
        bump_version patch $VERSION_FILE
        echo "TAG_NEW=v$(get_version $VERSION_FILE)" >> $GITHUB_ENV

    - name: Bump version (minor)
      if: contains(github.event.head_commit.message, '#minor')
      run: |
        source .github/workflows/utils.sh
        bump_version minor $VERSION_FILE
        echo "TAG_NEW=v$(get_version $VERSION_FILE)" >> $GITHUB_ENV

    - name: Bump version (major)
      if: contains(github.event.head_commit.message, '#major')
      run: |
        source .github/workflows/utils.sh
        bump_version major $VERSION_FILE
        echo "TAG_NEW=v$(get_version $VERSION_FILE)" >> $GITHUB_ENV

    - name: Commit files
      if:
        contains(github.event.head_commit.message, '#patch') ||
        contains(github.event.head_commit.message, '#minor') ||
        contains(github.event.head_commit.message, '#major')
      run: |
        REPOSITORY=${INPUT_REPOSITORY:-$GITHUB_REPOSITORY}
        remote_repo="https://${GITHUB_ACTOR}:${{ secrets.PUSH_ACTION_TOKEN }}@github.com/${REPOSITORY}.git"

        git config --local user.email "action@github.com"
        git config --local user.name "GitHub Action"

        # Commit version bump to release
        git commit -m "Bump version number" -a
        git push "${remote_repo}" release

    - name: Create Pull Request to merge back release into dev
      uses: repo-sync/pull-request@v2
      with:
        source_branch: "release"                          # If blank, default: triggered branch
        destination_branch: "dev"                         # If blank, default: master
        pr_title: "Merge release branch back into dev"
        pr_body: "Changes made to the release branch (e.g. hotfixes), plus the version bump."
        pr_assignee: "jgostick,ma-sadeghi"                # Comma-separated list (no spaces)
        pr_label: "high priority"                         # Comma-separated list (no spaces)
        pr_draft: false                                   # Creates pull request as draft
        pr_allow_empty: true                              # Creates pull request even if there are no changes
        github_token: ${{ secrets.GITHUB_TOKEN }}

    - name: Trim the 4th digit from the tag
      run:
        echo "TAG_NEW=${TAG_NEW%.dev?}" >> $GITHUB_ENV

    - name: Create new tag
      run: |
        REPOSITORY=${INPUT_REPOSITORY:-$GITHUB_REPOSITORY}
        remote_repo="https://${GITHUB_ACTOR}:${{ secrets.PUSH_ACTION_TOKEN }}@github.com/${REPOSITORY}.git"
        if [ -z "$TAG_NEW" ]
        then
            echo "New tag not created."
        else
            git tag $TAG_NEW
            git push "${remote_repo}" $TAG_NEW
            echo "Pushed a new tag: $TAG_NEW"
        fi


Deploy to PyPI

name: Deploy to PyPI

on:
  push:
    tags:
      - '*'

jobs:
  deploy:
    name: Publish 🐍 📦 to PyPI
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        ref: release  # the production branch name (for proper version #)

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Set env variables
      run: |
        chmod +x .github/workflows/utils.sh
        source .github/workflows/utils.sh
        VERSION_FILE=openpnm/__version__.py
        echo "TAG=$(get_most_recent_tag)" >> $GITHUB_ENV
        echo "VERSION=$(get_version $VERSION_FILE)" >> $GITHUB_ENV

    - name: Set env variables (for tag mismatch)
      run: |
        echo "Tag: $TAG, Version: $VERSION"
        if [ "${TAG//v}" = "${VERSION%.dev?}" ]; then
            echo "TAG_MISMATCH=false" >> $GITHUB_ENV
        else
            echo "TAG_MISMATCH=true" >> $GITHUB_ENV
        fi

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install setuptools wheel twine
        echo "${{ github.event.ref }}"
        echo "Tag/version mismatch: $TAG_MISMATCH"

    - name: Build distribution 📦
      run: python setup.py sdist bdist_wheel

    - name: Publish distribution 📦 to PyPI
      if: startsWith(github.event.ref, 'refs/tags') && contains(env.TAG_MISMATCH, 'false')
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.PYPI_TOKEN }}
        skip-existing: true

    # - name: Publish distribution 📦 to TestPyPI
    #   if: startsWith(github.event.ref, 'refs/tags') && contains(env.TAG_MISMATCH, 'false')
    #   uses: pypa/gh-action-pypi-publish@release/v1
    #   with:
    #     user: __token__
    #     password: ${{ secrets.TESTPYPI_TOKEN }}
    #     repository_url: https://test.pypi.org/legacy/

    # Not a good idea: if a non-conforming tag is push, e.g. random_tag, it
    # first gets deleted by cleanup-tags.yml, and then publish-to-pypi.yml gets
    # tricked and deletes the most recent tag! Ouch!

    # - name: Delete tag if doesn't match with version
    #   if: contains(env.TAG_MISMATCH, 'true')
    #   run: |
    #     git config --local user.email "action@github.com"
    #     git config --local user.name "GitHub Action"
    #     REPOSITORY=${INPUT_REPOSITORY:-$GITHUB_REPOSITORY}
    #     remote_repo="https://${GITHUB_ACTOR}:${{ secrets.GITHUB_TOKEN }}@github.com/${REPOSITORY}.git"
    #     git push "${remote_repo}" :refs/tags/$TAG


Release Notes

name: Release Notes

on:
  push:
    tags:
      - 'v*'

jobs:
  build:
    name: Release notes
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v2
        with:
            fetch-depth: 0  # to retrieve entire history of refs/tags

      - name: Generate release notes
        run: |
          git fetch --all --tags --force
          chmod +x .github/workflows/logger.sh
          chmod +x .github/workflows/utils.sh
          source .github/workflows/utils.sh
          bash .github/workflows/logger.sh
          echo "TAG=$(get_most_recent_tag)" >> $GITHUB_ENV

      - name: Create GitHub release
        uses: Roang-zero1/github-create-release-action@master
        with:
          version_regex: ^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+
          create_draft: true
          created_tag: ${{ env.TAG }}
          update_existing: false
          release_title: ${{ env.TAG }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Deploy Documentation

name: Deploy Documentation

on:
  workflow_dispatch:
  pull_request:
    branches:
      - release

jobs:
  deploy-docs:

    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Cache pip
        uses: actions/cache@v3
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements file
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-

      - name: Install dependencies
        run: |
          pip install -r requirements.txt
          pip install -r requirements/docs.txt
          pip install pypardiso

      - name: Build the documentation
        run: |
          cd docs
          make html

      - name: GitHub Pages action
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./docs/_build/html
          cname: openpnm.org


Verify pip-installability

name: Verify pip-installability

on: [workflow_dispatch]

jobs:
  deploy:
    name: Verify pip-installability
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Set branch name as env variable
      run: |
        echo Current branch: ${GITHUB_REF#refs/heads/}
        echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV

    - name: Install only using pip
      run:
        pip install git+https://github.com/PMEAL/OpenPNM.git@${BRANCH_NAME}

    - name: Test the installation
      run:
        python -c "import openpnm"


Tests

name: Tests

on: [pull_request]

jobs:
  build:

    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash

    strategy:
      fail-fast: false
      max-parallel: 9
      matrix:
        python-version: ['3.9', '3.10', '3.11']
        os: [ubuntu-latest, windows-latest, macos-latest]
        include:
        - os: ubuntu-latest
          path: ~/.cache/pip
        - os: macos-latest
          path: ~/Library/Caches/pip
        - os: windows-latest
          path: ~\AppData\Local\pip\Cache

    steps:
      - name: Cache pip
        uses: actions/cache@v3
        with:
          path: ${{ matrix.path }}
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          pip install \
            -r requirements.txt \
            -r requirements/tests.txt

      - name: Install pypardiso on non-macOS
        if: (matrix.os != 'macos-latest')
        run: |
          pip install pypardiso

      - name: Disable numba JIT for codecov to include jitted methods
        if: (matrix.python-version == 3.9) && (matrix.os == 'ubuntu-latest')
        run: |
          echo "NUMBA_DISABLE_JIT=1" >> $GITHUB_ENV

      - name: Running tests
        run:
          pytest .
            --cov=.
            --cov-report=xml
            --color=yes

      - name: Upload coverage to Codecov
        if: (matrix.python-version == 3.9) && (matrix.os == 'ubuntu-latest')
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          flags: unittests
          name: codecov-umbrella

      - name: flake8 Lint
        if: (matrix.python-version == 3.9) && (matrix.os == 'ubuntu-latest')
        uses: py-actions/flake8@v2
        with:
           ignore: "D100,E226,E402,E712,E741,F401,F403,N806,W503,WPS111,WPS211,WPS237,WPS300,WPS305,WPS319,WPS347,WPS410,WPS437"
           exclude: "docs/,scripts/,tests/,openpnm/contrib/,setup.py,openpnm/models/_doctxt.py,openpnm/models/phase/_phasedocs.py,openpnm/models/geometry/_geodocs.py"
           max-line-length: "90"
           path: ".."


Bump Version (dev)

name: Bump Version (dev)

on:
  push:
    branches:
      - dev

jobs:
  build:
    if: (! contains(github.event.head_commit.message, '[no bump]'))

    name: Bump version
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Set env variables
      run: |
        # The next line is very important, otherwise the line after triggers
        # git to track the permission change, which breaks bump2version API (needs clean git folder)
        git config core.filemode false
        chmod +x .github/workflows/utils.sh
        echo "VERSION_FILE=openpnm/__version__.py" >> $GITHUB_ENV
        echo "SETUP_CFG_FILE=setup.cfg" >> $GITHUB_ENV
        echo "${{ github.event.head_commit.message }}"

    - name: Install dependencies
      run: |
        pip install bump2version

    - name: Bump version (build)
      run: |
        source .github/workflows/utils.sh
        bump_version build $VERSION_FILE
        # Note that we don't want to create a new tag for "builds"

    # - name: Commit files
    #   run: |
    #     REPOSITORY=${INPUT_REPOSITORY:-$GITHUB_REPOSITORY}
    #     remote_repo="https://${GITHUB_ACTOR}:${{ secrets.PUSH_ACTION_TOKEN }}@github.com/${REPOSITORY}.git"

    #     git config --local user.email "action@github.com"
    #     git config --local user.name "GitHub Action"

    #     # Commit version bump to dev ([no ci] to avoid infinite loop)
    #     git commit -m "Bump version number (build) [no ci]" -a
    #     git push "${remote_repo}" dev

    - name: Commit files
      uses: stefanzweifel/git-auto-commit-action@v4
      with:
        commit_message: Bump version number (build part)
        commit_author: Author <actions@github.com>


Nightly Builds

name: Nightly Builds

on:
  schedule:
    - cron: "0 0 * * *"

jobs:
  nightly:

    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash

    strategy:
      fail-fast: false
      max-parallel: 9
      matrix:
        python-version: ['3.9', '3.10', '3.11']
        # Add macos-latest to the next line once #2451 is fixed
        os: [ubuntu-latest, windows-latest, macos-latest]
        include:
        - os: ubuntu-latest
          path: ~/.cache/pip
        - os: macos-latest
          path: ~/Library/Caches/pip
        - os: windows-latest
          path: ~\AppData\Local\pip\Cache

    steps:
      - name: Cache pip
        uses: actions/cache@v3
        with:
          path: ${{ matrix.path }}
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          pip install \
            -r requirements.txt \
            -r requirements/tests.txt

      - name: Install pypardiso on non-macOS
        if: (matrix.os != 'macos-latest')
        run: |
          pip install pypardiso

      - name: Running tests
        run:
          pytest . --color=yes


flake8 Lint

name: flake8 Lint

on: [push, pull_request]

jobs:
  flake8-lint:
    runs-on: ubuntu-latest
    name: Lint
    steps:
      - name: Check out source repository
        uses: actions/checkout@v2
      - name: Set up Python environment
        uses: actions/setup-python@v2
        with:
          python-version: '3.10'
      - name: flake8 Lint
        uses: py-actions/flake8@v2
        with:
          ignore: "D100,E226,F403,N806,W503,WPS111,WPS211,WPS237,WPS300,WPS305,WPS319,WPS347,WPS410,WPS437"
          exclude: "doc/,setup.py,scripts/,tests/"
          max-line-length: "90"
          path: ".."
          plugins: "flake8-bugbear==22.1.11 flake8-black"


Cron tests

name: Cron tests

on:
  schedule:
    - cron: "0 10 * * 1"  # Mondays @ 6AM Eastern
  pull_request:
    # We also want this workflow triggered if the 'Extra CI' label is added
    # or present when PR is updated
    types:
      - synchronize
      - labeled

jobs:
  tests:
    name: Python ${{ matrix.python }} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    if: (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'Extra CI'))
    strategy:
      fail-fast: true
      matrix:
        os: [ubuntu-latest, macos-latest]  # TODO: re-enable windows-latest
        python: ['3.9', '3.10', '3.11']
        include:
          # These set options for each python version. So, for example, when any
          # python 3.11 job in the matrix runs, it should use this toxenv
          - python: '3.11'
            toxenv: py311-test

          - python: '3.10'
            toxenv: py310-test

          - python: 3.9
            toxenv: py39-test-cov

          - python: 3.11
            toxenv: py311-test-extradeps

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # For animation tests
      - uses: FedericoCarboni/setup-ffmpeg@v3
        with:
          # Not strictly necessary, but it may prevent rate limit
          # errors especially on GitHub-hosted macos machines.
          token: ${{ secrets.GITHUB_TOKEN }}
        id: setup-ffmpeg

      - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
        if: "!startsWith(matrix.os, 'windows')"
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

      # Windows:
      - uses: conda-incubator/setup-miniconda@v3
        if: startsWith(matrix.os, 'windows')
        with:
          auto-update-conda: true
          python-version: ${{ matrix.python-version }}

      - name: Install Python dependencies - Windows
        if: startsWith(matrix.os, 'windows')
        shell: bash -l {0}
        run: |
          conda install -c conda-forge -q gsl libpython
          python -m pip install -e .[test]
          python -m pip install tox

      - name: Run tests - Windows
        if: startsWith(matrix.os, 'windows')
        shell: bash -l {0}
        run: |
          tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} ${{ matrix.toxposargs }}

      # Mac:
      - name: Setup Mac - GSL
        if: startsWith(matrix.os, 'mac')
        run: |
          brew install gsl

      # Ubuntu:
      - name: Setup Linux - GSL
        if: runner.os == 'Linux'
        run: |
          sudo apt-get update
          sudo apt-get install gsl-bin libgsl0-dev build-essential

      # Any *nix:
      - name: Install Python dependencies - nix
        if: "!startsWith(matrix.os, 'windows')"
        run: python -m pip install --upgrade tox codecov

      - name: Run tests - nix
        if: "!startsWith(matrix.os, 'windows')"
        run: tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }}


Tests

name: Tests

on:
  push:
    branches:
    - main
    tags:
    - '*'
  pull_request:

jobs:
  tests:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    if: github.event.pull_request.draft == false && !contains(github.event.pull_request.labels.*.name, 'docs only')
    strategy:
      fail-fast: true
      matrix:
        include:

          - name: Code style checks
            os: ubuntu-latest
            python: 3.x
            toxenv: codestyle

          - name: Python 3.10 with minimal dependencies and coverage
            os: ubuntu-latest
            python: '3.10'
            toxenv: py310-test-cov

          - name: Python 3.9
            os: ubuntu-latest
            python: '3.9'
            toxenv: py39-test

          - name: Python 3.10
            os: ubuntu-latest
            python: '3.10'
            toxenv: py310-test

          - name: Python 3.11
            os: ubuntu-latest
            python: '3.11'
            toxenv: py311-test

          # Has to happen on ubuntu because galpy is finnicky on macOS
          - name: Python 3.10 with all optional dependencies
            os: ubuntu-latest
            python: '3.10'
            toxenv: py310-test-extradeps
            toxposargs: --durations=50

          - name: Python 3.10 without GSL
            os: ubuntu-latest
            python: '3.10'
            toxenv: nogsl

          - name: Python 3.9 with oldest supported version of all dependencies
            os: ubuntu-latest
            python: 3.9
            toxenv: py39-test-oldestdeps

          # Mac and Windows:
          - name: Python 3.10 standard tests (macOS)
            os: macos-latest
            python: '3.10'
            toxenv: py310-test

          # - name: Python 3.9 standard tests (Windows)
          #   os: windows-latest
          #   python: 3.9
          #   toxenv: py39-test

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # For animation tests
      - uses: FedericoCarboni/setup-ffmpeg@v3
        if: ${{ !startsWith(matrix.os, 'mac') }}
        with:
          # Not strictly necessary, but it may prevent rate limit
          # errors especially on GitHub-hosted macos machines.
          github-token: ${{ secrets.GITHUB_TOKEN }}
        id: setup-ffmpeg

      - name: Set up Python ${{ matrix.python }} on ${{ matrix.os }}
        if: ${{ !startsWith(matrix.os, 'windows') }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

      # Windows:
      # - uses: conda-incubator/setup-miniconda@v2
      #   if: startsWith(matrix.os, 'windows')
      #   with:
      #     auto-update-conda: true
      #     python-version: ${{ matrix.python-version }}

      # - name: Install Python dependencies - Windows
      #   if: startsWith(matrix.os, 'windows')
      #   shell: bash -l {0}
      #   run: |
      #     conda install -c conda-forge -q gsl python=3.9 libpython
      #     python -m pip install -e .[test]
      #     python -m pip install tox

      # - name: Run tests - Windows
      #   if: startsWith(matrix.os, 'windows')
      #   shell: bash -l {0}
      #   run: |
      #     tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} ${{ matrix.toxposargs }}

      # Mac:
      - name: Setup Mac - GSL
        if: startsWith(matrix.os, 'mac')
        run: |
          brew install gsl

      # Ubuntu:
      - name: Setup Linux - GSL
        if: runner.os == 'Linux'
        run: |
          sudo apt-get update
          sudo apt-get install gsl-bin libgsl0-dev build-essential
          sudo apt-get install libhdf5-serial-dev  # TODO: remove when h5py has 3.11 wheels

      # Any *nix:
      - name: Install Python dependencies - nix
        if: ${{ !startsWith(matrix.os, 'windows') }}
        run: python -m pip install --upgrade tox codecov

      - name: Run tests - nix
        if: ${{ !startsWith(matrix.os, 'windows') }}
        run: tox -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }}

      # Coverage:
      - name: Upload coverage report to codecov
        uses: codecov/codecov-action@v4
        if: steps.check_files.outputs.files_exists == 'true' && runner.os == 'Linux'
        with:
          file: ./coverage.xml # optional


Tutorials

name: Tutorials
on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  release:
    types:
      - published

jobs:
  notebooks:
    name: "Build the notebooks for the docs"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install dependencies
        run: |
          python -m pip install -U pip wheel
          python -m pip install ".[tutorials]"

      - name: Execute the notebooks
        run: |
          cd docs
          make exectutorials

      - uses: actions/upload-artifact@v3
        with:
          name: notebooks-for-${{ github.sha }}
          path: docs/tutorials

      - name: Trigger RTDs build
        if: ${{ github.event_name != 'pull_request' }}
        uses: dfm/rtds-action@v1.1.0
        with:
          webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }}
          webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }}
          commit_ref: ${{ github.ref }}


Wheels

name: Wheels
on:
  release:
    types:
      - published
      - edited
  pull_request:
    branches:
      - main
  push:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build_nix_wheels:
    name: Build wheels on ${{ matrix.os }} - ${{ matrix.python[1] }}
    runs-on: ${{ matrix.os }}
    if: github.event.pull_request.draft == false && !contains(github.event.pull_request.labels.*.name, 'docs only')
    strategy:
      fail-fast: false
      matrix:
        # Building wheels on linux takes too much time: split wheels into multiple runs
        python:
          - ["1", "cp39-* cp310-*"]
          - ["2", "cp311-* cp312-*"]
        os:
          - "macos-14"
          - "ubuntu-latest"

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # - name: Set up QEMU  # For emulation on Linux
      #   if: runner.os == 'Linux'
      #   uses: docker/setup-qemu-action@v3
      #   with:
      #     platforms: all

      # Mac:
      # NOTE: need to install pipx explicitly for macos-14
      - name: Setup Mac
        if: runner.os == 'macOS'
        run: |
          brew install gsl pipx

      # Ubuntu:
      - name: Setup Linux
        if: runner.os == 'Linux'
        run: |
          sudo apt-get install gsl-bin libgsl0-dev

      - name: Build wheels
        uses: pypa/cibuildwheel@v2.17.0
        env:
          CIBW_ARCHS_LINUX: "x86_64"
          CIBW_ARCHS_MACOS: "x86_64 arm64"
          CIBW_BUILD: ${{ matrix.python[1] }}
          CIBW_SKIP: "*-win32 *musllinux* pp* *i686"

      - uses: actions/upload-artifact@v4
        with:
          name: wheels-${{ matrix.os }}-${{ matrix.python[0] }}
          path: ./wheelhouse/*.whl

  build_sdist:
    name: Build source distribution
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        name: Install Python
        with:
          python-version: "3.11"

      - name: Build sdist
        run: |
          sudo apt-get update
          sudo apt-get install gsl-bin libgsl0-dev
          pip install build
          python -m build -s .

      - uses: actions/upload-artifact@v4
        with:
          name: wheels-sdist
          path: dist/*.tar.gz

  test_upload_pypi:
    needs: [build_nix_wheels, build_sdist]
    # Only test upload to pypi on push to main:
    if: github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    environment:
      name: release
    permissions:
      id-token: write
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: wheels*
          path: dist
          merge-multiple: true

      - name: Publish package distributions to Test PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: https://test.pypi.org/legacy/

  upload_pypi:
    needs: [build_nix_wheels, build_sdist]
    runs-on: ubuntu-latest
    if: github.event_name == 'release' && github.event.action == 'published'
    environment:
      name: release
      url: https://pypi.org/p/gala
    permissions:
      id-token: write
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: wheels*
          path: dist
          merge-multiple: true

      - name: Publish package distributions to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1


Python CI

name: Python CI

on: [push]

jobs:
  test:
    timeout-minutes: 10
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: [3.7, 3.8, 3.9, "3.10", 3.11, 3.12]
    env:
      MOLER_DEBUG_THREADS: True
      PYTHON_COVERAGE: '3.12'
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.os }} ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo apt-get -y install sshpass
          python -m pip install --upgrade pip setuptools wheel
          pip install -Ur requirements/test.txt
      - name:  Create local users required by integration tests
        run: |
          cat /etc/passwd | grep home
          sudo useradd molerssh -c MolerSshTest -s /bin/bash -l -m
          echo 'molerssh:moler_password' | sudo chpasswd
          sudo useradd sshproxy -c Proxy4SshTest -s /bin/bash -l -m
          echo 'sshproxy:proxy_password' | sudo chpasswd
          sudo useradd adbshell -c AdbShell4SshTest -s /bin/bash -l -m
          echo 'adbshell:adb_password' | sudo chpasswd
          cat /etc/passwd | grep home
          echo ---- try logging as integration tests users via ssh
          sshpass -p moler_password ssh -oStrictHostKeyChecking=no molerssh@localhost pwd
          sshpass -p proxy_password ssh -oStrictHostKeyChecking=no sshproxy@localhost pwd
          sshpass -p adb_password ssh -oStrictHostKeyChecking=no adbshell@localhost pwd
      - name:  Upload adb simulator
        run: |
          sudo cp adb_simulation.sh /usr/bin/adb
          sudo chmod +x /usr/bin/adb
          sudo ls -l /usr/bin/adb
          which adb
          echo ----------- test adb simulator
          adb devices
      - name: Test with pytest
        if: ${{ matrix.python-version != env.PYTHON_COVERAGE }}
        timeout-minutes: 9
        run: |
          python -m pytest -vvvsss test/
          # python -m pytest -c py3pytest.ini -vvvsss test/
      - name: Test with pytest and coverage
        if: ${{ matrix.python-version == env.PYTHON_COVERAGE }}
        timeout-minutes: 9
        run: |
          pip list
          python --version
          coverage --version
          coverage run -m pytest -vvvsss test/
          ls -l .coverage
      - name: Validate moler overall coverage
        if: ${{ matrix.python-version == env.PYTHON_COVERAGE }}
        run: |
          coverage report -m --fail-under=91
      - name: Validate coverage for moler commands
        if: ${{ matrix.python-version == env.PYTHON_COVERAGE }}
        run: |
          coverage report -m --fail-under=100 --include="moler/cmd/**/*.py"
      - name: Validate coverage for moler events
        if: ${{ matrix.python-version == env.PYTHON_COVERAGE }}
        run: |
          coverage report -m --fail-under=100 --include="moler/events/**/*.py"
      - name: Upload pytest test results
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: pytest-${{ matrix.os }}-${{ matrix.python-version }}
          path: |
            moler*.log
            .coverage
          retention-days: 5

  style:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: [3.12]
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.os }} ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel
          pip install -Ur requirements/test.txt
      - name: Check PyCodeStyle
        run: |
          pycodestyle --statistics --count moler
#      - name: Check PyDocStyle
#        run: |
#          pydocstyle --count moler
      - name: Check PyLint
        run: |
          pylint moler --fail-under=9.4 --rcfile=./pylint.cfg || (exit $(($? % 4)))
      - name: Check flake8
        run: |
          flake8 moler --count --select=E9,F63,F7,F82 --show-source --statistics
          flake8 moler --count --exit-zero --max-complexity=10 --max-line-length=160 --statistics


Reporters-DB PR

name: Reporters-DB PR

on:
  pull_request:

jobs:
  repo-dispatch:
    runs-on: ubuntu-latest
    steps:
      - name: Repository Dispatch
        uses: peter-evans/repository-dispatch@v2
        with:
          token: ${{ secrets.FREELAWBOT_TOKEN }}
          repository: freelawproject/eyecite
          event-type: reporters-db-pr
          client-payload: '{"ref": "${{ github.ref }}", "pr_number": "${{ github.event.number }}", "commit": "${{ github.event.pull_request.head.sha }}" }'


Lint

name: Lint

on: [push, pull_request]

jobs:
  black:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2-beta
      - uses: actions/setup-python@v2
        with:
          # Once codebase is updated, this can easily be changed to any specific version.
          python-version: "3.8"
      - name: Black Code Formatter
        uses: psf/black@stable


PyPI

name: PyPI
on:
  push:
    tags:
      - v*.*.*
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: casperdcl/deploy-pypi@v2
        with:
          password: ${{ secrets.pypi_token }}
          build: true


Semgrep

on:
  pull_request: {}
  push:
    branches:
    - main
name: Semgrep
jobs:
  semgrep:
    name: Scan
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: returntocorp/semgrep-action@v1
      with:
        publishToken: ${{ secrets.SEMGREP_APP_TOKEN }}


Tests

name: Tests

on: [push, pull_request]

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10']

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install wheel
        pip install -r requirements.txt
    - name: Run tests
      run: |
        python setup.py test


Check release notes

name: Check release notes

on:
  pull_request:
    types:
      - opened
      - edited
      - synchronize

jobs:
  check_release_notes:
    name: Notes are either written, or there are none
    uses: packit/.github/.github/workflows/check-release-notes.yml@main
    with:
      description: ${{ github.event.pull_request.body }}


name: Create a GitHub release once a release PR is merged

on:
  workflow_dispatch:
  pull_request:
    types:
      - closed

jobs:
  if_merged:
    if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release') && github.repository_owner == 'packit'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Create GitHub release
        run: |
          VERSION=$(grep -oP '^# \K[0-9.]*' CHANGELOG.md | head -n 1)
          # Take the lines between the first two headers from CHANGELOG.md,
          # and use it as a description for the new release.
          CHANGELOG=$(awk 'BEGIN { first = 0 } /^# / { if (first == 0) { first = 1 } else { exit } } /^[^#]/ { print $0 }' CHANGELOG.md)
          # Use the changelog re-calculated to create a new release
          gh release create ${VERSION} --draft --title ${VERSION} -n "${CHANGELOG}"
        env:
          GH_TOKEN: ${{ secrets.RELEASEBOT_GITHUB_TOKEN }}


Deploy documentation

name: Deploy documentation

on:
  push:
    branches:
      - main

jobs:
  build:
    runs-on: ubuntu-latest
    permissions:
      contents: write

    steps:
      - name: Checkout branch
        uses: actions/checkout@v3
      - name: Build and push docs
        run: |
          git config user.name Packit
          git config user.email user-cont-team@redhat.com

          python3 -m pip install --upgrade pip
          python3 -m pip install setuptools
          python3 -m pip install .
          python3 -m pip install pdoc3 pygments

          python3 -m pdoc --html -o built_docs ogr
          mv built_docs/ogr docs
          touch .nojekyll

          COMMIT_SHA=$(git log -1 --format=%h)
          git checkout --orphan gh-pages
          git rm --cached '*'
          git add docs .nojekyll

          git commit -m "Update docs for $COMMIT_SHA"
          git push --force origin gh-pages
        shell: bash


Move new issues into the Kanban board

name: Move new issues into the Kanban board

on:
  issues:
    types: [opened]

jobs:
  add-new-issue-to-the-kanban-board:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/add-to-project@v0.3.0
        with:
          project-url: https://github.com/orgs/packit/projects/7
          github-token: ${{ secrets.RELEASEBOT_GITHUB_TOKEN }}


Prepare a new release

name: Prepare a new release
on:
  workflow_dispatch:
    inputs:
      version:
        required: true
        description: Version number for the new release

jobs:
  prepare-release:
    # To not run in forks
    if: github.repository_owner == 'packit'
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
      contents: write

    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Prepare release content
        uses: packit/prepare-release@v1
        with:
          version: ${{ inputs.version }}
          specfiles: fedora/python-ogr.spec
      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v4
        with:
          labels: release
          commit-message: Release ${{ inputs.version }}
          title: Release ${{ inputs.version }}
          body: Update the changelog and the specfile for release ${{ inputs.version }}.
          delete-branch: true


# Upload a Python package when a release is created
# https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows

name: Publish Python 🐍 distributions 📦 to PyPI

on:
  release:
    types: [released]

jobs:
  build-n-publish:
    runs-on: ubuntu-latest
    permissions:
      id-token: write # for trusted publishing

    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4

      - name: Build a source tarball and a binary wheel
        # https://pypa-build.readthedocs.io
        run: |
          python -m pip install build
          python -m build

      - name: Publish 📦 to PyPI
        # https://github.com/pypa/gh-action-pypi-publish
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          verbose: true


Setup & build calm-dsl

name: Setup & build calm-dsl
on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - name: black
      if: ${{matrix.python-version != '3.7'}}
      uses: psf/black@stable # Exclude list is not honored - https://github.com/psf/black/issues/1584
      with:
        version: "22.6.0"
        options: "--check --extend-exclude tests/escript/scripts/"
    - name: Setup env
      run: |
        python -m pip install --upgrade pip
        pip install virtualenv
    - name: Install requirements
      run: |
        make dev
    - name: Build python package
      run: |
        make dist
    - name: Build docker image
      run: |
        make docker
    - name: Execute container
      uses: addnab/docker-run-action@v3
      with:
        image: ntnx/calm-dsl
        run: calm
    - name: Publish docker image
      if: ${{ github.event_name == 'push' && github.repository == 'nutanix/calm-dsl' && matrix.python-version == '3.9'}}
      run: |
        docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
        docker push ntnx/calm-dsl


Setup & Build release containers

name: Setup & Build release containers
on:
  push:
    branches:
      - 'release/**'
  pull_request:
    branches:
      - 'release/**'

jobs:
  build:
    runs-on: self-hosted
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ['3.7']
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Setup env
      run: |
        python -m pip install --upgrade pip
        pip install virtualenv
    - name: Build docker image
      run: |
        make docker
    - name: Read VERSION file
      id: getversion
      run: echo "::set-output name=version::$(cat CalmVersion)"
    - name: Execute container
      uses: addnab/docker-run-action@v3
      with:
        image: ntnx/calm-dsl
        run: calm
    - name: Publish docker image to jfrog-artifactory
      if: ${{ github.repository == 'nutanix/calm-dsl' && matrix.python-version == '3.7'}}
      run: |
        docker login ${{ secrets.JFROG_REGISTRY }} -u ${{ secrets.JFROG_USERNAME }} -p ${{ secrets.JFROG_PASSWORD }}
        docker tag ntnx/calm-dsl:latest ${{ secrets.JFROG_REGISTRY }}/calm-dsl/calm-dsl:v${{ steps.getversion.outputs.version }}
        docker push ${{ secrets.JFROG_REGISTRY }}/calm-dsl/calm-dsl:v${{ steps.getversion.outputs.version }}


Sync Private Repo

name: Sync Private Repo
on:
  push:
    branches:
      - master

jobs:
  build:
    if: ${{ github.repository == 'nutanix/calm-dsl' }}
    runs-on: [ubuntu-latest]
    steps:
    - uses: actions/checkout@v2
      with:
        repository: ideadevice/calm-dsl
        ref: master
        token: ${{ secrets.DSL_AUTH_TOKEN }}
    - name: Git remote add
      run: git remote add upstream https://github.com/nutanix/calm-dsl
    - name: Git push disable upstream
      run: git remote set-url --push upstream DISABLE
    - name: Print remote
      run: git remote -v
    - name: Git Fetch upstream
      run: git fetch --all --prune --unshallow
    - name: Git Merge
      run:  |
        git merge upstream/master --allow-unrelated-histories
        git fetch --unshallow origin
    - name: Push changes
      uses: ad-m/github-push-action@master
      with:
        github_token: ${{ secrets.DSL_AUTH_TOKEN }}
        repository: ideadevice/calm-dsl
        branch: master