Upload Python Package

name: Upload Python Package

on:
  release:
    types: [created]
  workflow_dispatch:

jobs:
  deploy:

    runs-on: ubuntu-latest
    permissions:
      # IMPORTANT: this permission is mandatory for trusted publishing
      id-token: write

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install build wheel twine
    - name: Install modelskill
      run: pip install .[test]
    - name: Test
      run: pytest
    - name: Build
      run: python -m build
    - name: Publish package distributions to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1


Scheduled test

name: Scheduled test

on:
  workflow_dispatch:

  schedule:
    # * is a special character in YAML so you have to quote this string
    - cron:  '0 5 */3 * *'

jobs:
  build:

    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest]
        python-version: [3.8, 3.9, "3.10"]

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install modelskill
      run: |
        pip install .[test]
    - name: Test with pytest
      run: |
        pytest --ignore tests/notebooks/


formatting

name: formatting

on: [push, pull_request]

jobs:
  formatting:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install poetry
        run: pipx install poetry

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
          cache: "poetry"

      - name: Install dependencies
        run: poetry install --no-interaction

      - name: Check format with black
        run: poetry run black --check .


publish

name: publish

on:
  release:
    types: [published] # Runs both for releases and prereleases

jobs:
  test:
    uses: google/temporian/.github/workflows/test.yaml@main
    secrets: inherit

  build-linux:
    runs-on: ubuntu-latest
    container: gcr.io/tfx-oss-public/manylinux2014-bazel:bazel-5.3.0
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
      - name: Checkout
        uses: actions/checkout@v3  # v4 doesn't work on the container

      - name: Build package
        run: |
          version="${{ matrix.python-version }}"
          versionstr=${version//./}
          PYTHON_VERSION=$versionstr ./tools/build_manylinux.sh

      - name: Save package for following jobs
        uses: actions/upload-artifact@v3
        with:
          name: dist-ubuntu-${{ matrix.python-version }}
          path: dist

  # macos-XXX-large are Intel runners, macos-XXX-xlarge are arm64 runners
  # https://docs.github.com/en/actions/using-github-hosted-runners/about-larger-runners/about-larger-runners
  build-macos-intel:
    runs-on: ${{ matrix.platform.runner }}
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        platform:
          [
            { runner: macos-12-large, artifact: macos12-intel },
            { runner: macos-13-large, artifact: macos13-intel },
            { runner: macos-14-large, artifact: macos14-intel },
          ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install poetry
        run: pip install poetry

      - name: Build package
        run: poetry build

      - name: Save package for following jobs
        uses: actions/upload-artifact@v3
        with:
          name: dist-${{ matrix.platform.artifact }}-${{ matrix.python-version }}
          path: dist

  build-macos-arm:
    runs-on: ${{ matrix.platform.runner }}
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        platform:
          [
            # there is not runner for macos12 on arm
            { runner: macos-13-xlarge, artifact: macos-13-arm },
            { runner: macos-14-xlarge, artifact: macos-14-arm }
          ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      # actions/setup-python doesn't support py < 3.10 on macos arm
      # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760

      - name: Install Python
        run: |
          brew update
          brew install python@${{ matrix.python-version }}

      - name: Install pip
        run: |
          curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
          python${{ matrix.python-version }} get-pip.py

      - name: Install poetry
        run: pip install poetry

      - name: Build package
        run: poetry build

      - name: Save package for following jobs
        uses: actions/upload-artifact@v3
        with:
          name: dist-${{ matrix.platform.artifact }}-${{ matrix.python-version }}
          path: dist

  test-install:
    needs: [build-linux, build-macos-intel]
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        platform:
          [
            { runner: ubuntu-latest, artifact: ubuntu },
            { runner: macos-12-large, artifact: macos12-intel },
            { runner: macos-13-large, artifact: macos13-intel },
            { runner: macos-14-large, artifact: macos14-intel },
          ]
    runs-on: ${{ matrix.platform.runner }}
    env:
      # Workaround for macos pip compatibility issue
      # See https://github.com/actions/setup-python/issues/469
      SYSTEM_VERSION_COMPAT: 0
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Download build
        uses: actions/download-artifact@v3
        with:
          name: dist-${{ matrix.platform.artifact }}-${{ matrix.python-version }}
          path: dist

      - name: Install from wheel
        run: pip install dist/*.whl

      - name: Test installation
        run: python tools/check_install.py

  test-macos-arm-install:
    needs: [build-macos-arm]
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        platform:
          [
            { runner: macos-13-xlarge, artifact: macos-13-arm },
            { runner: macos-14-xlarge, artifact: macos-14-arm }
          ]
    runs-on: ${{ matrix.platform.runner }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install Python
        run: |
          brew update
          brew install python@${{ matrix.python-version }}

      - name: Install pip
        run: |
          curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
          python${{ matrix.python-version }} get-pip.py

      - name: Download build
        uses: actions/download-artifact@v3
        with:
          name: dist-${{ matrix.platform.artifact }}-${{ matrix.python-version }}
          path: dist

      - name: Install from wheel
        run: pip install dist/*.whl

      - name: Test installation
        run: python${{ matrix.python-version }} tools/check_install.py

  publish:
    needs: [test, test-install, test-macos-arm-install]
    runs-on: ubuntu-latest
    environment: pypi
    env:
      POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install poetry
        run: pip install poetry

      - name: Download builds for all Python versions and platforms
        uses: actions/download-artifact@v3
        with:
          # Skipping `name` key to download all artifacts from this workflow
          # Artifacts are downloaded to dist/<artifact-name>
          path: dist

      - name: Move wheels and sources to dist/
        run: |
          mv dist/*/*.whl dist/
          mv -n dist/*/*.tar.gz dist/

      - name: Publish to PyPI
        run: poetry publish --no-interaction --skip-existing


tests

name: tests

on:
  push:
    branches: # Specifying branches makes it not run on pushed tags
      - "*"
  workflow_call: # Allow being ran from other actions
  pull_request: # Run on PRs from forks (push doesn't cover those)

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Bazel cache
        id: bazel-cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/bazel
          key: ${{ runner.os }}-${{ matrix.python-version }}-bazel-cache-${{ hashFiles('**/BUILD', '.bazelrc', '.bazelversion', 'WORKSPACE') }}
          restore-keys: ${{ runner.os }}-${{ matrix.python-version }}-bazel-cache-

      - name: Install poetry
        run: pipx install poetry

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: poetry

      - name: Install dependencies
        run: poetry install --no-interaction --extras polars

      - name: Run tests
        run: poetry run bazel test //temporian/...:all //docs/...:all //tools/...:all --test_output=errors --compilation_mode=dbg --config=linux
        # Note: Since mid-August 23, using "//...:all" incorrectly try to
        # compile the directory "build" created during the installation of one
        # of the dependency during "poetry install".


name: test_notebooks

on: [pull_request]

jobs:
  run_notebooks:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # Paths relative to docs/src/, without the .ipynb extension
        # * can be used and all .ipynb files in that dir will be tested sequentially
        path:
          - user_guide
          - getting_started
          - recipes/*
          - tutorials/anomaly_detection_supervised
          - tutorials/anomaly_detection_unsupervised
          - tutorials/bank_fraud_detection_with_tfdf
          - tutorials/heart_rate_analysis
          # - tutorials/loan_outcomes_prediction # TODO: re enable this notebook
          # - tutorials/m5_competition # Note: Github cannot download dataset anymore.
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Bazel cache
        id: bazel-cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/bazel
          key: ${{ runner.os }}-3.9-bazel-cache-${{ hashFiles('**/BUILD', '.bazelrc', '.bazelversion', 'WORKSPACE') }}
          restore-keys: ${{ runner.os }}-3.9-bazel-cache-

      - name: Notebooks cache
        id: notebooks-cache
        uses: actions/cache@v3
        with:
          path: ./docs/src/tutorials/tmp
          key: ${{ runner.os }}-notebooks-${{ matrix.path }}-${{ hashFiles(format('docs/src/{0}.ipynb', matrix.path)) }}
          restore-keys: |
            ${{ runner.os }}-notebooks-${{ matrix.path }}-

      - name: Install poetry
        run: pipx install poetry

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
          cache: poetry
          cache-dependency-path: |
            poetry.lock
            ./docs/src/tutorials/requirements.txt

      - name: Install poetry dependencies
        run: poetry install --no-interaction

      - name: Install notebook dependencies
        run: poetry run pip install -r ./docs/src/tutorials/requirements.txt

      - name: Execute notebook(s)
        run: poetry run ./tools/run_notebooks.sh `ls docs/src/${{ matrix.path }}.ipynb`


Add new issues to the team project

name: Add new issues to the team project

on:
  issues:
    types:
      - opened

jobs:
  add-to-project:
    name: Add issue to project
    runs-on: ubuntu-latest
    steps:
      - uses: actions/add-to-project@v0.5.0
        with:
          project-url: https://github.com/orgs/mozilla/projects/214
          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}


Documentation on the wiki page

name: Documentation on the wiki page

on:
  push:
    branches:
      - "master"

permissions:
  contents: read

jobs:
  check-wiki:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v3
        with:
          python-version: 3.9

      - name: Check rules on the wiki page
        run: python scripts/check_rules_on_wiki.py -ci


Check formatting

name: Check formatting

on: [push, pull_request]

jobs:
  skip_check:
    runs-on: ubuntu-latest
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
        - id: skip_check
          uses: fkirc/skip-duplicate-actions@v5
          with:
              concurrent_skipping: 'same_content_newer'
              skip_after_successful_duplicate: 'true'

  ts_format:
    name: Check TypeScript formatting
    runs-on: ubuntu-latest

    needs: skip_check
    if: needs.skip_check.outputs.should_skip != 'true'

    steps:
      - uses: actions/checkout@v3
      - name: Install packages
        run: |
          cd timApp
          rm package-lock.json
          npm install --unsafe-perm --legacy-peer-deps
          npm install --no-package-lock --legacy-peer-deps --no-save prettier@2.7.1
      - name: Check formatting
        run: cd timApp && npm run checkformat

  py_format:
    name: Check Python formatting
    runs-on: ubuntu-latest
    env:
      PYTHON_VERSION: "3.11"

    steps:
      - uses: actions/checkout@v3
      - name: Install Poetry
        run: pipx install poetry
      - name: Setup Python
        uses: actions/setup-python@v3
        with:
          python-version: "${{ env.PYTHON_VERSION }}"
          cache: poetry
      - name: Install dependencies
        run: |
          poetry env use "${{ env.PYTHON_VERSION }}"
          poetry install --only=dev
      - name: Run format
        run: .venv/bin/black --check .


Lint code

name: Lint code

on: [push, pull_request]

jobs:
  skip_check:
    runs-on: ubuntu-latest
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
        - id: skip_check
          uses: fkirc/skip-duplicate-actions@v5
          with:
              concurrent_skipping: 'same_content_newer'
              skip_after_successful_duplicate: 'true'

  ts_lint:
    name: Lint TypeScript
    runs-on: ubuntu-latest

    needs: skip_check
    if: needs.skip_check.outputs.should_skip != 'true'

    steps:
      - uses: actions/checkout@v3
      - name: Cache node modules
        uses: actions/cache@v3
        with:
          path: |
            timApp/node_modules
            timApp/modules/jsrunner/server/node_modules
          key: ${{ runner.os }}-eslint-npm-${{ hashFiles('**/package-lock.json') }}
      - name: Install packages
        run: |
          npm i --global npm@6
          cd timApp
          npm install
          cd modules/jsrunner/server
          npm install
      - name: Lint main code
        run: cd timApp && npm run lint
      - name: Lint JSRunner
        run: cd timApp/modules/jsrunner/server && npm run lint

  py_lint:
    name: Lint Python
    runs-on: ubuntu-latest
    env:
      PYTHON_VERSION: "3.11"

    needs: skip_check
    if: needs.skip_check.outputs.should_skip != 'true'

    steps:
      - uses: actions/checkout@v3
      - name: Install APT Packages
        uses: awalsh128/cache-apt-pkgs-action@v1.0.1
        with:
          packages: libxmlsec1-dev libxmlsec1-openssl
          version: "1.1"
      - name: Install Poetry
        run: pipx install poetry
      - name: Setup Python
        uses: actions/setup-python@v3
        with:
          python-version: "${{ env.PYTHON_VERSION }}"
          cache: poetry
      - name: Install dependencies
        run: |
          poetry env use "${{ env.PYTHON_VERSION }}"
          poetry install --only=dev
      - name: Run MyPy
        run: .venv/bin/mypy -p timApp -p tim_common


Check spelling

name: Check spelling

on: [push, pull_request]

jobs:
  skip_check:
    runs-on: ubuntu-latest
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
        - id: skip_check
          uses: fkirc/skip-duplicate-actions@v5
          with:
              concurrent_skipping: 'same_content_newer'
              skip_after_successful_duplicate: 'true'

  check_spelling:
    name: Spell Check with Typos
    runs-on: ubuntu-latest

    needs: skip_check
    if: needs.skip_check.outputs.should_skip != 'true'

    steps:
      - uses: actions/checkout@v3
      - name: Check spelling of repository files
        uses: crate-ci/typos@v1.13.6


Generate code statistics

name: Generate code statistics

on: [push, pull_request]

jobs:
  skip_check:
    runs-on: ubuntu-latest
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
        - id: skip_check
          uses: fkirc/skip-duplicate-actions@v5
          with:
              concurrent_skipping: 'same_content_newer'
              skip_after_successful_duplicate: 'true'

  ts_bundle_stats:
    name: Bundle stats for TypeScript
    runs-on: ubuntu-latest

    needs: skip_check
    if: needs.skip_check.outputs.should_skip != 'true'

    steps:
      - uses: actions/checkout@v3
      - name: Cache Angular build assets
        uses: actions/cache@v3
        with:
          path: timApp/.angular/cache
          key: ${{ runner.os }}-angular-cache-stats-${{ hashFiles('**/*.ts') }}
          restore-keys: |
            ${{ runner.os }}-angular-cache-stats-
      - name: Cache node modules
        uses: actions/cache@v3
        with:
          path: |
            timApp/node_modules
          key: ${{ runner.os }}-ts-stats-npm-${{ hashFiles('timApp/package-lock.json') }}
      - name: Install packages
        run: |
          npm i --global npm@6
          cd timApp
          npm install
          npm install -g webpack-bundle-analyzer
      - name: Generate bundle statistics
        run: cd timApp && npm run statshtml
      - name: Upload stats
        uses: actions/upload-artifact@v3.0.0
        with:
          name: Bundle Report
          path: timApp/report.html


Run tests

name: Run tests

on: [push, pull_request]

jobs:
  skip_check:
    runs-on: ubuntu-latest
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
        - id: skip_check
          uses: fkirc/skip-duplicate-actions@v5
          with:
              concurrent_skipping: 'same_content_newer'
              skip_after_successful_duplicate: 'true'

  ci_tests:
    name: CI tests (${{ matrix.test_module }})
    runs-on: ubuntu-latest

    needs: skip_check
    if: needs.skip_check.outputs.should_skip != 'true'

    strategy:
      matrix:
        test_module:
          - browser
          - unit
          - db
          - server
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          submodules: true
      - name: Cache Angular build assets
        uses: actions/cache@v3
        if: "${{ matrix.test_module == 'browser' }}"
        with:
          path: timApp/.angular/cache
          key: ${{ runner.os }}-angular-cache-${{ hashFiles('**/*.ts') }}
          restore-keys: |
            ${{ runner.os }}-angular-cache-
      - name: Cache node modules
        uses: actions/cache@v3
        if: "${{ matrix.test_module == 'browser' || matrix.test_module == 'server'  }}"
        with:
          path: |
            timApp/node_modules
            timApp/modules/jsrunner/server/node_modules
          key: ${{ runner.os }}-ci-test-npm-${{ hashFiles('**/package-lock.json') }}
      - name: Cache Rust crates
        uses: actions/cache@v3
        with:
          path: |
            tim_rust/target
          key: ${{ runner.os }}-tim-rust-cache-${{ hashFiles('tim_rust/Cargo.lock') }}
      - name: Log info
        run: |
          uname -a
          docker info
          docker-compose --version
      - name: Set up
        run: ./tim setup --no-interactive --no-install --profile test
      - name: Download images
        run: ./tim dc pull --quiet
      - name: Install dependencies
        if: "${{ matrix.test_module == 'browser' || matrix.test_module == 'server'  }}"
        run: ./tim npmi
      - name: Compile JS
        if: "${{ matrix.test_module == 'browser' || matrix.test_module == 'server'  }}"
        run: ./tim js --target ${{ matrix.test_module }}
      - name: Install TIM Rust library
        run: ./tim rust
      - name: Run tests
        run: ./tim test --dc-up ${{ matrix.test_module }}
        timeout-minutes: 40
        env:
          SKIP_JSRUNNER_START: ${{ matrix.test_module == 'unit' || matrix.test_module == 'db' }}
      - name: Upload screenshots
        if: "${{ matrix.test_module == 'browser' && always() }}"
        uses: actions/upload-artifact@v3.0.0
        with:
          name: Browser Screenshots
          path: screenshots


Add Issue or Pull Request to Dashboard

name: Add Issue or Pull Request to Dashboard

on:
  issues:
    types:
      - opened
  pull_request:
    types:
      - opened

jobs:
  add-to-project:
    name: Add issue or pull request to project
    runs-on: ubuntu-latest
    steps:
      - uses: actions/add-to-project@v0.5.0
        with:
          project-url: https://github.com/orgs/catalystneuro/projects/3
          github-token: ${{ secrets.PROJECT_TOKEN }}


Assess file changes

name: Assess file changes

on:
  workflow_call:
    # Map the workflow outputs to job outputs
    outputs:
      SOURCE_CHANGED:
        description: "Whether the files under /src/ were changed."
        value: ${{ jobs.build.outputs.SOURCE_CHANGED }}
      CHANGELOG_UPDATED:
        description: "Whether or the CHANGELOG.md file was updated."
        value: ${{ jobs.build.outputs.CHANGELOG_UPDATED }}

jobs:
  build:
    runs-on: ubuntu-latest
    # Map the job outputs to step outputs
    outputs:
      SOURCE_CHANGED: ${{ steps.assess-changes.outputs.SOURCE_CHANGED }}
      CHANGELOG_UPDATED: ${{ steps.assess-changes.outputs.CHANGELOG_UPDATED }}

    name: Test changed-files
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v41.0.0

      - name: Assess Source Code Changes
        id: assess-changes
        run: |
          echo "SOURCE_CHANGED=false" >> $GITHUB_OUTPUT
          echo "CHANGELOG_UPDATED=false" >> $GITHUB_OUTPUT
          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
            echo $file
            if [[ $file == "src/"* || $file == "tests/"* || $file == "requirements-minimal.txt" || $file == "requirements-testing.txt" || $file == ".github/"* && $file != ".github/workflows/assess-file-changes.yml" ]]
            then
              echo "Source changed"
              echo "SOURCE_CHANGED=true" >> $GITHUB_OUTPUT
            else
              echo "Source not changed"
            fi
            if [[ $file == "CHANGELOG.md" ]]
            then
              echo "Changelog updated"
              echo "CHANGELOG_UPDATED=true" >> $GITHUB_OUTPUT
            else
              echo "Changelog not updated"
            fi
          done


Upload Package to PyPI

# This workflow will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Upload Package to PyPI

on:
  release:
    types: [published]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.9"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install wheel
      - name: Build package
        run: |
          python setup.py sdist bdist_wheel
      - name: pypi-publish
        uses: pypa/gh-action-pypi-publish@v1.4.2
        with:
          verbose: true
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}


Check Docstrings

name: Check Docstrings
on:
  workflow_dispatch:
  pull_request:

jobs:
  check-docstrings:
    uses: catalystneuro/.github/.github/workflows/check_docstrings.yaml@main
    with:
      python-version: '3.10'
      repository: 'catalystneuro/roiextractors'
      package-name: 'roiextractors'


Daily Workflows

name: Daily Workflows

on:
  workflow_dispatch:
  schedule:
    - cron: "0 14 * * *"  # Daily at 10am EST

jobs:
    update-testing-data:
        uses: ./.github/workflows/update-testing-data.yml
        secrets:
            AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
            AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
            S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }}

    run-tests:
        needs: update-testing-data
        uses: ./.github/workflows/run-tests.yml
        secrets:
            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

    notify:
        runs-on: ubuntu-latest
        needs: [update-testing-data, run-tests]
        if: ${{ always() && contains(needs.*.result, 'failure') }}
        steps:
        - uses: dawidd6/action-send-mail@v3
          with:
            server_address: smtp.gmail.com
            server_port: 465  # TSL
            username: ${{ secrets.MAIL_USERNAME }}
            password: ${{ secrets.MAIL_PASSWORD }}
            subject: ROIExtractors Daily Failure
            to: ${{ secrets.DAILY_FAILURE_EMAIL_LIST }}  # add more with commas, no separation
            from: ROIExtractors
            body: "The daily workflow for ROIExtractors failed: please check status at https://github.com/catalystneuro/roiextractors/actions/workflows/dailies.yaml"


Deploy PR Tests

name: Deploy PR Tests

on:
  workflow_dispatch:
  pull_request:

concurrency:
    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
    cancel-in-progress: true

jobs:
    update-testing-data:
        uses: ./.github/workflows/update-testing-data.yml
        secrets:
            AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
            AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
            S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }}

    run-tests:
        needs: update-testing-data
        uses: ./.github/workflows/run-tests.yml
        secrets:
            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

    dev-tests:
        needs: update-testing-data
        uses: ./.github/workflows/dev-testing.yaml


Detect Changelog Updates

name: Detect Changelog Updates
on:
  pull_request:
  workflow_dispatch:

jobs:
  assess-file-changes:
    uses: ./.github/workflows/assess-file-changes.yml

  detect-changelog-updates:
    needs: assess-file-changes
    if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }}
    name: Auto-detecting CHANGELOG.md updates
    runs-on: ubuntu-latest
    steps:
      - if:  ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'true' }}
        run: echo "CHANGELOG.md has been updated."
      - if:  ${{ needs.assess-file-changes.outputs.CHANGELOG_UPDATED == 'false' }}
        run: |
          echo "CHANGELOG.md has not been updated."
          0


Dev Testing

name: Dev Testing
on:
  workflow_dispatch:
  workflow_call:

jobs:
  run:
    name: Ubuntu tests with Python ${{ matrix.python-version }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
    steps:
      - uses: actions/checkout@v4
      - run: git fetch --prune --unshallow --tags
      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Global Setup
        run: |
            pip install -U pip
            git config --global user.email "CI@example.com"
            git config --global user.name "CI Almighty"

      - name: Install full requirements
        run: |
            pip install .[test,full]

      - name: Clone and Install NeuroConv
        run: |
            git clone https://github.com/catalystneuro/neuroconv.git
            cd neuroconv
            pip install --no-cache-dir .[full,test]
            cd ../
      - name: Get ophys_testing_data current head hash
        id: ophys
        run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)"
      - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}
        uses: actions/cache@v2
        id: cache-ophys-datasets
        with:
          path: ./ophys_testing_data
          key: ophys-datasets-042023-ubuntu-latest-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}

      - name: Run NeuroConv tests
        run: |
            pytest -n auto --dist loadscope neuroconv/tests/test_ophys neuroconv/tests/test_on_data/test_imaging_interfaces.py neuroconv/tests/test_on_data/test_segmentation_interfaces.py


Full Tests

name: Full Tests
on:
  workflow_dispatch:
  workflow_call:
    secrets:
      CODECOV_TOKEN:
        required: true

jobs:
  run:
    name: Full tests on ${{ matrix.os }} with Python ${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        os: [ubuntu-latest, windows-latest, macos-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip
    steps:
      - uses: actions/checkout@v4
      - run: git fetch --prune --unshallow --tags
      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Global Setup
        run: |
          pip install -U pip
          git config --global user.email "CI@example.com"
          git config --global user.name "CI Almighty"

      - name: Install roiextractors with minimal requirements
        run: pip install .
      - name: Test initial import of all non-lazy dependencies
        run: python -c "import roiextractors"

      - name: Install roiextractors with testing requirements
        run: pip install .[test]
      - name: Run minimal tests
        run: pytest tests/test_internals -n auto --dist loadscope

      - name: Test full installation
        run: pip install .[full]

      - name: Get ophys_testing_data current head hash
        id: ophys
        run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)"
      - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}
        uses: actions/cache@v2
        id: cache-ophys-datasets
        with:
          path: ./ophys_testing_data
          key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}

      - name: Run full pytest with coverage
        run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml
      - if: ${{ matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest'}}
        name: Upload full coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./codecov.xml
          flags: unittests
          name: codecov-umbrella
          codecov_yml_path: ./codecov.yml
          verbose: true


update-testing-data

name: update-testing-data
on:
  workflow_dispatch:
  workflow_call:
    secrets:
      AWS_ACCESS_KEY_ID:
        required: true
      AWS_SECRET_ACCESS_KEY:
        required: true
      S3_GIN_BUCKET:
        required: true

jobs:
  run:
    name: Update testing data on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip
    steps:
      - uses: actions/checkout@v2
      - run: git fetch --prune --unshallow --tags
      - name: Setup Python 3.11
        uses: actions/setup-python@v2
        with:
          python-version: 3.11

      - name: Global Setup
        run: |
          pip install -U pip
          git config --global user.email "CI@example.com"
          git config --global user.name "CI Almighty"
          pip install wheel==0.41.2  # needed for scanimage

      - name: Get ophys_testing_data current head hash
        id: ophys
        run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)"
      - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}
        uses: actions/cache@v2
        id: cache-ophys-datasets
        with:
          path: ./ophys_testing_data
          key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}
      - if: steps.cache-ophys-datasets.outputs.cache-hit == false
        name: Install and configure AWS CLI
        run: |
          pip install awscli
          aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      - if: steps.cache-ophys-datasets.outputs.cache-hit == false
        name: Download data from S3
        run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data


Update Dev Version on Release

name: Update Dev Version on Release

on:
    workflow_dispatch:
    workflow_run:
        workflows: [Upload Package to PyPI]
        types: [completed]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Checkout New Branch
        run: |
          git checkout -b update_dev_version
      - name: Update dev version
        run: |
          git fetch --prune --unshallow --tags
          tags="$(git tag --list)"
          latest_tag=${tags: -6 : 6}
          old_version=${latest_tag:1:5}
          echo "Old Version: $old_version"
          old_major_version=${old_version:0:1}
          old_minor_version=${old_version:2:1}
          old_patch_version=${old_version:4:1}
          new_patch_version=`expr $old_patch_version + 1`
          new_version="$old_major_version.$old_minor_version.$new_patch_version"
          echo "New Version: $new_version"
          sed -i -e "s/version=\"$old_version\"/version=\"$new_version\"/g" setup.py
          echo "old_version=$old_version" >> "$GITHUB_ENV"
          echo "new_version=$new_version" >> "$GITHUB_ENV"
      - name: Commit Changes and Create Pull Request
        run: |
          git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com
          git config --global user.name github-actions[bot]
          git commit . -m "Update dev version from $old_version to $new_version"
          git push origin update_dev_version
          gh pr create --title "[Github.CI] Update dev version from $old_version to $new_version" --body "version: $old_version --> $new_version"
        env:
          GITHUB_TOKEN: ${{ secrets.MY_GITHUB_TOKEN }}


CI Test

name: CI Test

on: [push, pull_request]

jobs:

  ci-test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ '3.8', '3.9', '3.10' ]

    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 1

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          version: 1.4.2
      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: |
          sudo apt-get update && sudo apt-get install -y --no-install-recommends gcc git g++ libev-dev libyaml-dev
          cd lib/poseidon_api && poetry install --no-interaction && cd ../../
          cd lib/poseidon_cli && poetry install --no-interaction && cd ../../
          cd lib/poseidon_core && poetry install --no-interaction && cd ../../

      - name: Code Quality - Black
        run: |
          cd lib/poseidon_api && poetry run black . --check && cd ../../
          cd lib/poseidon_cli && poetry run black . --check && cd ../../
          cd lib/poseidon_core && poetry run black . --check && cd ../../

      - name: Code Quality - Pytype
        run: |
          cd lib/poseidon_api && poetry run pytype poseidon_api/ && cd ../../
          cd lib/poseidon_cli && poetry run pytype poseidon_cli/ && cd ../../
          cd lib/poseidon_core && poetry run pytype poseidon_core/ && cd ../../

      - name: Code Quality - Pylint
        run: |
          cd lib/poseidon_api && poetry run pylint --fail-under=4 poseidon_api/ && cd ../../
          cd lib/poseidon_cli && poetry run pylint --fail-under=4 poseidon_cli/ && cd ../../
          cd lib/poseidon_core && poetry run pylint --fail-under=4 poseidon_core/ && cd ../../

      - name: Test with pytest
        # https://github.com/python-poetry/poetry/issues/4511
        # TODO: setuptools needed by c65faucet, but poetry < 1.2.0 does not allow adding setuptools.
        run: |
          export POSEIDON_CONFIG=$PWD/config/poseidon.config
          cd lib/poseidon_api && poetry run pip install 'setuptools==64.0.3' && poetry run pytest --cov-report term-missing --cov=. --cov-report=xml && cd ../../
          cd lib/poseidon_cli && poetry run pip install 'setuptools==64.0.3' && poetry run pytest --cov-report term-missing --cov=. --cov-report=xml && cd ../../
          cd lib/poseidon_core && poetry run pip install 'setuptools==64.0.3' && poetry run pytest --cov-report term-missing --cov=. --cov-report=xml && cd ../../
      - name: Upload coverage
        uses: codecov/codecov-action@v4
        if: github.repository == 'faucetsdn/poseidon' && github.ref_name == 'main'
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: /home/runner/work/poseidon/poseidon/lib/poseidon_api/coverage.xml,/home/runner/work/poseidon/poseidon/lib/poseidon_cli/coverage.xml,/home/runner/work/poseidon/poseidon/lib/poseidon_core/coverage.xml
          fail_ci_if_error: true


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ main ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ main ]
  schedule:
    - cron: '15 9 * * 3'

jobs:
  analyze:
    permissions:
      actions: read
      contents: write
      security-events: write
    name: Analyze
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v3

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3


buildx-extras

name: buildx-extras

on:
  push:
    branches: main
    tags: 'v*'

jobs:
  buildx-extras:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Get the version
      id: get_version
      run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | cut -d / -f 3)
    - name: Change for main
      id: change_version
      run: if [ "${{ steps.get_version.outputs.VERSION }}" == "main" ]; then echo ::set-output name=VERSION::latest; else echo ::set-output name=VERSION::${{ steps.get_version.outputs.VERSION }}; fi
    - name: Set up qemu
      uses: docker/setup-qemu-action@v2
      with:
        platforms: all
    - name: Set up Docker Buildx
      id: buildx
      uses: docker/setup-buildx-action@v2
      with:
        version: latest
    - name: Docker Login
      env:
        DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
      run: |
        echo "${DOCKER_PASSWORD}" | docker login --username "${{ secrets.DOCKER_USERNAME }}" --password-stdin
      if: github.repository == 'iqtlabs/poseidon' && github.event_name == 'push'

    - name: Build and push platforms
      env:
        DOCKER_CLI_EXPERIMENTAL: enabled
      run: |
        docker buildx build \
          --platform linux/amd64,linux/arm64 \
          --push \
          -t iqtlabs/rabbitmq:${{ steps.change_version.outputs.VERSION }} helpers/rabbitmq && \
        docker buildx build \
          --platform linux/amd64,linux/arm64 \
          --push \
          -t iqtlabs/poseidon-api:${{ steps.change_version.outputs.VERSION }} helpers/api
      if: github.repository == 'iqtlabs/poseidon' && github.event_name == 'push'


buildx

name: buildx

on:
  push:
    branches: main
    tags: 'v*'

jobs:
  buildx:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Get the version
      id: get_version
      run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | cut -d / -f 3)
    - name: Change for main
      id: change_version
      run: if [ "${{ steps.get_version.outputs.VERSION }}" == "main" ]; then echo ::set-output name=VERSION::latest; else echo ::set-output name=VERSION::${{ steps.get_version.outputs.VERSION }}; fi
    - name: Set up qemu
      uses: docker/setup-qemu-action@v2
      with:
        platforms: all
    - name: Set up Docker Buildx
      id: buildx
      uses: docker/setup-buildx-action@v2
      with:
        version: latest
    - name: Docker Login
      env:
        DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
      run: |
        echo "${DOCKER_PASSWORD}" | docker login --username "${{ secrets.DOCKER_USERNAME }}" --password-stdin
      if: github.repository == 'iqtlabs/poseidon' && github.event_name == 'push'

    - name: Build and push platforms
      env:
        DOCKER_CLI_EXPERIMENTAL: enabled
      run: |
        docker buildx build \
          --platform linux/amd64,linux/arm64 \
          --push \
          -t iqtlabs/poseidon:${{ steps.change_version.outputs.VERSION }} . && \
        docker buildx build \
          --platform linux/amd64,linux/arm64 \
          --push \
          -t iqtlabs/poseidon-workers:${{ steps.change_version.outputs.VERSION }} workers
      if: github.repository == 'iqtlabs/poseidon' && github.event_name == 'push'


release

name: release

on:
  push:
    branches: main
    tags: 'v*.*.*'

jobs:
  release:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Install Poetry
      uses: snok/install-poetry@v1
      with:
        virtualenvs-create: true
        virtualenvs-in-project: true
    - name: Build and publish packages
      id: build_and_publish_packages
      run: |
        sudo apt-get update && \
        sudo apt-get install -yq --no-install-recommends curl gcc g++ libev-dev libyaml-dev python3-pip python3.8 python3.8-dev && \
        cd lib/poseidon_api && poetry build && poetry publish -u ${{ secrets.PYPI_USERNAME }} -p ${{ secrets.PYPI_TOKEN }} && cd ../../ && \
        cd lib/poseidon_cli && poetry build && poetry publish -u ${{ secrets.PYPI_USERNAME }} -p ${{ secrets.PYPI_TOKEN }} && cd ../../ && \
        cd lib/poseidon_core && poetry build && poetry publish -u ${{ secrets.PYPI_USERNAME }} -p ${{ secrets.PYPI_TOKEN }} && cd ../../
      if: github.repository == 'iqtlabs/poseidon' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')


secrets

name: secrets

on: [push, pull_request]

jobs:
  scan:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: scan
      run: |
        export DEBIAN_FRONTEND=noninteractive && \
        echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections && \
        sudo apt-get update && \
        python3 -m pip install --upgrade pip && \
        pip3 install whispers && \
        mkdir /home/runner/reports/ && \
        whispers --severity BLOCKER,CRITICAL -o /home/runner/reports/whispers.json -c ${GITHUB_WORKSPACE}/.github/workflows/config/whispers-config.yml ${GITHUB_WORKSPACE} && \
        echo "::set-output name=found-count::$(wc -l /home/runner/reports/whispers.json | cut -d' ' -f1)"
    - name: Fail if found
      if: steps.scan.outputs.found-count != 0
      uses: actions/github-script@v6
      with:
        script: |
            echo {{steps.scan.outputs.found-count}} && \
            core.setFailed('Secrets found. Please check the uploaded report')
    - name: Upload scan reports
      uses: actions/upload-artifact@v3.1.2
      if: failure()
      with:
        name: whispers-report
        path: /home/runner/reports/whispers.json


Close stale issues and PRs

name: 'Close stale issues and PRs'
on:
  schedule:
    - cron: '30 1 * * *'

jobs:
  stale:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/stale@v8
        with:
          stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
          stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.'
          close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.'
          close-pr-message: 'This PR was closed because it has been stalled for 10 days with no activity.'
          days-before-issue-stale: 30
          days-before-pr-stale: 45
          days-before-issue-close: 5
          days-before-pr-close: 10
          exempt-issue-labels: 'dependencies'


YAML Lint

name: YAML Lint
on: [push, pull_request]
jobs:
  yamllint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: yaml-lint
      uses: ibiqlik/action-yamllint@v3
      with:
        config_data: "{extends: default, ignore: .github}"


test

name: test

on: [push, pull_request]

jobs:
  test:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: shell_test
      run: |
        wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" && \
        tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" && \
        shellcheck() { "shellcheck-stable/shellcheck" "$@"; } && \
        shellcheck --version && \
        /bin/bash --version && /bin/bash -n bin/poseidon && shellcheck -x -e SC2016 -e SC2119 -e SC2129 -e SC2001 -e SC2038 -e SC2044 bin/poseidon
    - name: e2e_test
      run: |
        export DEBIAN_FRONTEND=noninteractive && \
        echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections
        sudo apt-get update && sudo apt-get install -y libev-dev tshark wget jq tcpreplay tcpdump
        docker system prune -a -f --volumes && ./tests/test_e2e.sh


Build package / publish

name: Build package / publish

on:
  push:
    branches: [ main ]
    tags: [ "v*" ]
  release:
    types: [ published ]
  # Check that package can be built even on PRs
  pull_request:
    branches: [ main ]

jobs:
  publish:
    uses: iiasa/actions/.github/workflows/publish.yaml@main
    secrets:
      PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
      TESTPYPI_TOKEN: ${{ secrets.TESTPYPI_TOKEN }}


Test snapshots

name: Test snapshots

on:
  # Please enable this if your PR targets snapshot data or functionality for that
  # pull_request:
  #   branches: [ main ]
  schedule:
  # 05:00 UTC = 06:00 CET = 07:00 CEST
  - cron: "0 5 * * *"

# Cancel previous runs that have not completed
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  snapshots:
    runs-on: ubuntu-latest
    name: Test snapshots ubuntu-latest
    env:
      upstream-version: main
      python-version: "3.12"
      extra-deps: 'dask[dataframe]'

    steps:
    - name: Cache test data
      uses: actions/cache@v4
      with:
        path: message-local-data
        key: ubuntu-latest

    - name: Check out message-ix-models
      uses: actions/checkout@v4
      with:
        lfs: true
        fetch-depth: ${{ env.depth }}

    - uses: actions/setup-python@v5
      with:
        python-version: ${{ env.python-version }}
        cache: pip
        cache-dependency-path: "**/pyproject.toml"

    - uses: iiasa/actions/setup-gams@main
      with:
        version: 25.1.1
        license: ${{ secrets.GAMS_LICENSE }}

    - uses: ts-graphviz/setup-graphviz@v2
      with:
        macos-skip-brew-update: true

    - name: Install packages and dependencies
      # By default, install:
      # - ixmp, message_ix: from GitHub branches/tags per env.upstream-version (above)
      # - other dependencies including genno: from PyPI.
      #
      # To test against unreleased code (on `main`, or other branches
      # for open PRs), temporarily uncomment, add, or edit lines below
      # as needed. DO NOT merge such changes to `main`.
      run: |
        # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main"
        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ env.upstream-version }}"
        # pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ env.upstream-version }}"
        pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@issue/723"

        pip install .[docs,tests] ${{ env.extra-deps }}

    - name: Configure local data path
      run: |
        mkdir -p message-local-data/cache
        mix-models config set "message local data" "$(realpath message-local-data)"
        mix-models config show
      shell: bash

    - name: Run test suite using pytest
      run: |
        pytest message_ix_models \
          -m "snapshot" \
          -rA --verbose --color=yes --durations=20 \
          --cov-report=term-missing --cov-report=xml \
          --numprocesses=auto \
          --local-cache --jvmargs="-Xmx6G"
      shell: bash

    - name: Upload test coverage to Codecov.io
      uses: codecov/codecov-action@v4
      with:
        token: ${{ secrets.CODECOV_TOKEN }} # required


Test

name: Test

on:
  pull_request:
    branches: [ main , "migrate-*"]
  schedule:
  # 05:00 UTC = 06:00 CET = 07:00 CEST
  - cron: "0 5 * * *"

# Cancel previous runs that have not completed
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  warm-lfs-cache:
    strategy:
      matrix:
        os: [ macos-13, ubuntu-latest, windows-latest ]
    runs-on: ${{ matrix.os }}
    steps:
    - uses: nschloe/action-cached-lfs-checkout@v1

  pytest:
    needs: warm-lfs-cache

    strategy:
      matrix:
        os:
        - macos-13
        - ubuntu-latest
        - windows-latest
        upstream:
        # In each group:
        # - Versions of ixmp and message_ix to test.
        # - Latest supported Python version for those or other dependencies.
        # - Extra dependencies, in particular fixed/maximum versions to resolve conficts.
        #   - dask[dataframe] >= 2024.3.0 requires dask-expr and in turn pandas >= 2.0.
        #     https://github.com/iiasa/message-ix-models/pull/156#issuecomment-2020152360
        #   - genno: upstream versions < 3.8.0 import genno.computations, removed in 1.25.0.
        #     https://github.com/iiasa/message-ix-models/pull/156
        #   - pytest: upstream versions < 3.9.0 use a hook argument removed in pytest 8.1.0.
        #     https://github.com/iiasa/message-ix-models/pull/155
        #
        # Minimum version given in pyproject.toml
        - version: v3.4.0
          python-version: "3.11"
          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
        - version: v3.5.0
          python-version: "3.11"
          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
        - version: v3.6.0
          python-version: "3.11"
          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
        - version: v3.7.0
          python-version: "3.11"
          extra-deps: 'dask[dataframe]   "genno < 1.25"                "pytest == 8.0.0"' #
        # Latest released version
        - version: v3.8.0
          python-version: "3.12"
          extra-deps: 'dask[dataframe]                                 "pytest == 8.0.0"' #
        # Development version
        - version: main
          python-version: "3.12"
          extra-deps: 'dask[dataframe]' #

      fail-fast: false

    runs-on: ${{ matrix.os }}
    name: ${{ matrix.os }}-py${{ matrix.upstream.python-version }}-upstream-${{ matrix.upstream.version }}

    steps:
    - name: Cache test data
      uses: actions/cache@v4
      with:
        path: message-local-data
        key: ${{ matrix.os }}

    - name: Check out message-ix-models
      uses: nschloe/action-cached-lfs-checkout@v1

    - uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.upstream.python-version }}
        cache: pip
        cache-dependency-path: "**/pyproject.toml"

    - uses: iiasa/actions/setup-gams@main
      with:
        version: 25.1.1
        license: ${{ secrets.GAMS_LICENSE }}

    - uses: ts-graphviz/setup-graphviz@v2
      # TEMPORARY Work around ts-graphviz/setup-graphviz#630
      if: ${{ ! startswith(matrix.os, 'macos-') }}

    - name: Install packages and dependencies
      # By default, install:
      # - ixmp, message_ix: from GitHub branches/tags per matrix.upstream-version (above)
      # - other dependencies including genno: from PyPI.
      #
      # To test against unreleased code (on `main`, or other branches
      # for open PRs), temporarily uncomment, add, or edit lines below
      # as needed. DO NOT merge such changes to `main`.
      run: |
        # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main"
        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.upstream.version }}"
        pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.upstream.version }}"

        pip install .[docs,tests] ${{ matrix.upstream.extra-deps }}

        # TEMPORARY Work around hgrecco/pint#2007, unionai-oss/pandera#1685;
        # see https://github.com/khaeru/genno/issues/140
        pip install "pint != 0.24.0" "numpy < 2"

    - name: Configure local data path
      run: |
        mkdir -p message-local-data/cache
        mix-models config set "message local data" "$(realpath message-local-data)"
        mix-models config show
      shell: bash

    - name: Run test suite using pytest
      run: |
        pytest message_ix_models \
          -m "not snapshot" \
          -rA --verbose --color=yes --durations=20 \
          --cov-report=term-missing --cov-report=xml \
          --numprocesses=auto \
          --local-cache --jvmargs="-Xmx6G"
      shell: bash

    - name: Upload test coverage to Codecov.io
      uses: codecov/codecov-action@v4
      with:
        token: ${{ secrets.CODECOV_TOKEN }} # required

  pre-commit:
    name: Code quality

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with: { python-version: "3.12" }

    - name: Force recreation of pre-commit virtual environment for mypy
      if: github.event_name == 'schedule'  # Comment this line to run on a PR
      run: gh cache list -L 999 | cut -f2 | grep pre-commit | xargs -I{} gh cache delete "{}" || true
      env: { GH_TOKEN: "${{ github.token }}" }

    - uses: pre-commit/action@v3.0.1


Indices

name: Indices

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  workflow_dispatch:

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

jobs:
  update-indices:
    runs-on: ubuntu-latest
    steps:
    - name: checkout repo content
      uses: actions/checkout@v3

    - name: setup python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'

    - name: update platforms/index.js
      run: cd platforms && python generate_index.py

    - name: update platform_wallpapers_packs/index.js
      run: cd themes/platform_wallpapers_packs && python generate_index.py

    - name: update platform_thumbnails_packs/index.js
      run: cd themes/platform_thumbnails_packs && python generate_index.py

    - name: commit files
      run: |
        git config --local user.email "action@github.com"
        git config --local user.name "GitHub Action"
        git add -A
        git diff-index --quiet HEAD || (git commit -a -m "updated indices by GitHub Actions" --allow-empty)

    - name: push changes
      uses: ad-m/github-push-action@v0.6.0
      with:
        github_token: ${{ secrets.GITHUB_TOKEN }}
        branch: main


Docs Publish

---
name: Docs Publish
on:
  push:
    branches: [main]

jobs:
  deploy:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip==23.0
          pip install -U tox==4.11.0 setuptools==67.3.1 virtualenv==20.24.3 wheel==0.38.4
          sudo apt update
          sudo apt install graphviz=2.42.2-6 pandoc=2.9.2.1-3ubuntu2 qtbase5-dev=5.15.3+dfsg-2ubuntu0.2 qt5-qmake=5.15.3+dfsg-2ubuntu0.2
      - name: Build docs
        run: tox -edocs -- -j auto
      - name: Bypass Jekyll Processing # Necessary for setting the correct css path
        run: touch docs/_build/html/.nojekyll
      - name: Deploy
        uses: JamesIves/github-pages-deploy-action@v4
        with:
          folder: docs/_build/html/


Tests

name: Tests
on:
  # Run this workflow every time a new commit is pushed to or a pull request is created from one of these branches
  push:
    branches: [main, "stable/*"]
  pull_request:
    branches: [main, "stable/*"]
jobs:
  # NOTE: The ubuntu-22.04 and macos-latest tests have been separated out because ubuntu
  # requires installing some additional libraries (libglu1-mesa) for Gmsh to run. Please
  # consider this aspect before combining the tests for both the OS platforms.
  ubuntu-tests:
    # Name the Job
    name: tests-python${{ matrix.python-version }}-ubuntu-latest
    # Set the type of machine to run on
    runs-on: ubuntu-22.04
    # Set matrix for runs-on
    strategy:
      matrix:
        python-version: ["3.9", "3.10"]
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Pip cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ubuntu-latest-${{ matrix.python-version }}-pip-tests-${{ hashFiles('setup.py','requirements-dev.txt','requirements.txt') }}
          restore-keys: |
            ubuntu-latest-${{ matrix.python-version }}-pip-tests-
            ubuntu-latest-${{ matrix.python-version }}-pip-
            ubuntu-latest-${{ matrix.python-version }}
      - name: Install Deps
        run: |
          python -m pip install -U tox==4.11.0 setuptools==67.3.1 virtualenv==20.24.3 wheel==0.38.4
          sudo apt install libglu1-mesa=9.0.2-1 libglu1-mesa-dev=9.0.2-1
      - name: Install and Run Tests
        run: tox -e py
  macos-tests:
    # Name the Job
    name: tests-python${{ matrix.python-version }}-macOS-latest
    # Set the type of machine to run on
    runs-on: macos-12
    # Set matrix for runs-on
    strategy:
      matrix:
        python-version: ["3.9", "3.10"]
    steps:
      - name: Chekout code
        uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Pip cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: macOS-latest-${{ matrix.python-version }}-pip-tests-${{ hashFiles('setup.py','requirements-dev.txt','requirements.txt') }}
          restore-keys: |
            macOS-latest-${{ matrix.python-version }}-pip-tests-
            macOS-latest-${{ matrix.python-version }}-pip-
            macOS-latest-${{ matrix.python-version }}
      - name: Install Deps
        run: python -m pip install -U tox==4.11.0 setuptools==67.3.1 virtualenv==20.24.3 wheel==0.38.4
      - name: Install and Run Tests
        run: tox -e py
  windows-tests:
    name: tests-python${{ matrix.python-version }}-windows
    runs-on: windows-2022
    strategy:
      matrix:
        python-version: ["3.9", "3.10"]
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Pip cache
        uses: actions/cache@v3
        with:
          path: ~\AppData\Local\pip\Cache
          key: ${{ runner.os }}-${{ matrix.python-version }}-pip-tests-${{ hashFiles('setup.py','requirements-dev.txt','requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.python-version }}-pip-tests-
            ${{ runner.os }}-${{ matrix.python-version }}-pip-
            ${{ runner.os }}-${{ matrix.python-version }}-
      - name: Install deps
        run: |
          python -m pip install -U tox==4.11.0 cvxopt==1.3.0 setuptools==67.3.1 virtualenv==20.24.3 wheel==0.38.4
        shell: pwsh
      - name: Install and Run Tests
        run: tox --sitepackages -e py
        shell: bash -l {0}
  lint:
    name: lint
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - name: Pip cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-${{ matrix.python-version }}-pip-lint-${{ hashFiles('setup.py','requirements-dev.txt','requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.python-version }}-pip-lint-
            ${{ runner.os }}-${{ matrix.python-version }}-pip-
            ${{ runner.os }}-${{ matrix.python-version }}-
      - name: Install Deps
        run: python -m pip install -U tox==4.11.0
      - name: Run lint
        run: tox -elint
  docs:
    name: docs
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - name: Pip cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-${{ matrix.python-version }}-pip-docs-${{ hashFiles('setup.py','requirements-dev.txt','requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-docs-
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: Install Deps
        run: |
          python -m pip install -U tox==4.11.0
          sudo apt update
          sudo apt install -y graphviz=2.42.2-6 pandoc=2.9.2.1-3ubuntu2 qtbase5-dev=5.15.3+dfsg-2ubuntu0.2 qt5-qmake=5.15.3+dfsg-2ubuntu0.2
      - name: Build Docs
        run: tox -edocs
      - uses: actions/upload-artifact@v2
        with:
          name: html_docs
          path: docs/_build/html


Release Artifacts

name: Release Artifacts
on:
  push:
    tags:
      - "*"
jobs:
  wheel-build:
    name: Build and Publish Release Artifacts
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        name: Install Python
        with:
          python-version: "3.10"
      - name: Install Deps
        run: pip install -U twine==4.0.2 wheel==0.38.4
      - name: Build Artifacts
        run: |
          python setup.py sdist
          python setup.py bdist_wheel
        shell: bash
      - uses: actions/upload-artifact@v2
        with:
          path: ./dist/qiskit*
      - name: Publish to PyPi
        env:
          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
          TWINE_USERNAME: qiskit
        run: twine upload dist/qiskit*