Main

name: Main

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}

on:
  pull_request:
    branches:
      - "*"
  push:
    branches:
      - main
    tags:
      - "v*.*.*"

env:
  CACHE_PREFIX: v5 # Change this to invalidate existing cache.
  PYTHON_PATH: ./
  DEFAULT_PYTHON: 3.9
  WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
  BEAKER_TOKEN: ${{ secrets.BEAKER_TOKEN }}
  BEAKER_WORKSPACE: ai2/tango-testing
  BEAKER_IMAGE: petew/tango-testing
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
  checks:
    name: python ${{ matrix.python }} - ${{ matrix.task.name }}
    runs-on: [ubuntu-latest]
    timeout-minutes: 30
    permissions:
      contents: "read"
      id-token: "write"
    strategy:
      fail-fast: false
      matrix:
        python: ["3.9"]
        task:
          - name: Lint
            extras: dev,all
            requires_torch: true
            run: |
              ruff check .

          - name: Type check
            extras: dev,all
            requires_torch: true
            run: |
              mypy --check-untyped-defs .

          - name: Build
            extras: dev,all
            requires_torch: true
            run: |
              tango --version
              python -m build

          - name: Style
            extras: dev
            requires_torch: false
            run: |
              isort --check .
              black --check .

          - name: Docs
            extras: dev,all
            requires_torch: true
            run: |
              cd docs && make html SPHINXOPTS="-W --keep-going"

          - name: Test
            extras: dev
            requires_torch: false
            run: |
              pytest -v --durations=10 --color=yes --doctest-modules --ignore=tests/integrations --ignore=tango/integrations tests/ tango/

          - name: Datasets integration
            extras: dev,datasets
            requires_torch: false
            run: |
              pytest -v --color=yes --doctest-modules tango/integrations/datasets tests/integrations/datasets

          - name: PyTorch integration
            extras: dev,torch
            requires_torch: true
            run: |
              pytest -v --color=yes --doctest-modules tango/integrations/torch tests/integrations/torch

          - name: Transformers integration
            extras: dev,flax,transformers
            requires_torch: true
            run: |
              pytest -v --color=yes --doctest-modules tango/integrations/transformers tests/integrations/transformers

          - name: FairScale integration
            extras: dev,fairscale
            requires_torch: true
            run: |
              pytest -v --color=yes --doctest-modules tango/integrations/fairscale tests/integrations/fairscale

          - name: W&B integration
            extras: dev,torch,flax,wandb
            requires_torch: true
            run: |
              pytest -v --color=yes --doctest-modules tango/integrations/wandb tests/integrations/wandb

          - name: Beaker integration
            extras: dev,beaker
            requires_torch: false
            run: |
              pytest -v --color=yes --doctest-modules tango/integrations/beaker tests/integrations/beaker

          - name: Flax integration
            extras: dev,flax,transformers
            requires_torch: false
            run: |
              pytest -v --color=yes --doctest-modules tango/integrations/flax tests/integrations/flax

          - name: GS integration
            extras: dev,gs
            requires_torch: false
            run: |
              pytest -v --color=yes --doctest-modules tango/integrations/gs tests/integrations/gs

          - name: Example - train_lm
            extras: dev,all
            requires_torch: true
            run: |
              cd examples/train_lm
              pytest -v --color=yes test.py

        include:
          # Run the core tests on other Python versions as well.
          - task:
              name: Test
              extras: dev
              requires_torch: false
              run: |
                pytest -v --durations=10 --color=yes --doctest-modules --ignore=tests/integrations --ignore=tango/integrations tests/ tango/
            python: "3.8"

          - task:
              name: Test
              extras: dev
              requires_torch: false
              run: |
                pytest -v --durations=10 --color=yes --doctest-modules --ignore=tests/integrations --ignore=tango/integrations tests/ tango/
            python: "3.10"

    steps:
      - uses: "actions/checkout@v3"
      - name: Checkout
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v3

      # For pull requests we need to checkout the HEAD commit instead of the merge
      # commit since some tests depend on having an existing commit.
      - name: Checkout (pull request)
        if: github.event_name == 'pull_request'
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}

      - name: Install prerequisites
        run: |
          pip install --upgrade pip setuptools wheel virtualenv

      - name: Set build variables
        shell: bash
        run: |
          set -e
          # Get the exact Python version to use in the cache key.
          echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
          echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
          # Use week number in cache key so we can refresh the cache weekly.
          echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV
          echo "EXTRAS_HASH=$(python scripts/hash_extras.py ${{ matrix.task.extras }})" >> $GITHUB_ENV

      - uses: actions/cache@v3
        id: virtualenv-cache
        with:
          path: .venv
          key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ env.EXTRAS_HASH }}-${{ hashFiles('pyproject.toml') }}

      - name: Setup virtual environment (no cache hit)
        if: steps.virtualenv-cache.outputs.cache-hit != 'true'
        run: |
          test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv

      # Reference: https://github.com/marketplace/actions/authenticate-to-google-cloud#setup
      - name: Authenticate to Google Cloud
        if: matrix.task.name == 'GS integration'
        uses: "google-github-actions/auth@v1"
        with:
          workload_identity_provider: "projects/10554368204/locations/global/workloadIdentityPools/tango-ci-pool/providers/tango-ci-provider"
          service_account: "tango-service@ai2-allennlp.iam.gserviceaccount.com"

      - name: Pre-install torch
        if: steps.virtualenv-cache.outputs.cache-hit != 'true' && (contains(matrix.task.extras, 'torch') || contains(matrix.task.extras, 'all') || matrix.task.requires_torch)
        run: |
          . .venv/bin/activate
          pip install torch==2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu

      - name: Pre-install flax
        if: steps.virtualenv-cache.outputs.cache-hit != 'true' && (contains(matrix.task.extras, 'flax') || contains(matrix.task.extras, 'all'))
        run: |
          . .venv/bin/activate
          pip install flax jax jaxlib "tensorflow-cpu>=2.9.1" optax

      - name: Install editable (no cache hit)
        if: steps.virtualenv-cache.outputs.cache-hit != 'true'
        run: |
          . .venv/bin/activate
          pip install -e .[${{ matrix.task.extras }}]

      - name: Install editable (cache hit)
        if: steps.virtualenv-cache.outputs.cache-hit == 'true'
        run: |
          . .venv/bin/activate
          pip install --no-deps -e .[${{ matrix.task.extras }}]

      - name: Show environment info
        run: |
          . .venv/bin/activate
          echo "========= Python location ==========="
          which python
          echo "========= Python version ============"
          python --version
          echo "========= Python packages ==========="
          pip freeze
          echo "========= Tango installation ========"
          tango info

      - name: ${{ matrix.task.name }}
        run: |
          . .venv/bin/activate
          ${{ matrix.task.run }}

      - name: Upload package distribution files
        if: matrix.task.name == 'Build' && matrix.python == env.DEFAULT_PYTHON
        uses: actions/upload-artifact@v3
        with:
          name: package
          path: dist

      - name: Upload docs build
        if: matrix.task.name == 'Docs' && matrix.python == env.DEFAULT_PYTHON
        uses: actions/upload-artifact@v3
        with:
          name: docs
          path: docs/build

      - name: Clean up
        if: always()
        run: |
          . .venv/bin/activate
          pip uninstall -y ai2-tango

  gpu_tests:
    name: GPU Tests
    runs-on: ubuntu-latest
    steps:
      - name: Determine current commit SHA (pull request)
        if: github.event_name == 'pull_request'
        run: |
          echo "COMMIT_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV

      - name: Determine current commit SHA (push)
        if: github.event_name != 'pull_request'
        run: |
          echo "COMMIT_SHA=$GITHUB_SHA" >> $GITHUB_ENV

      - name: GPU Tests
        uses: allenai/beaker-run-action@v1.2
        with:
          spec: |
            version: v2
            description: GPU Tests
            budget: ai2/oe-training
            tasks:
              - name: tests
                image:
                  beaker: ${{ env.BEAKER_IMAGE }}
                context:
                  preemptible: true
                resources:
                  gpuCount: 2
                envVars:
                  - name: COMMIT_SHA
                    value: ${{ env.COMMIT_SHA }}
                command: ["/entrypoint.sh", "pytest", "-v", "-m", "gpu", "tests/"]
                result:
                  path: /unused
          token: ${{ secrets.BEAKER_TOKEN }}
          workspace: ${{ env.BEAKER_WORKSPACE }}
          clusters: ai2/general-cirrascale,ai2/allennlp-cirrascale,ai2/aristo-cirrascale,ai2/mosaic-cirrascale,ai2/s2-cirrascale

  release:
    name: Release
    runs-on: ubuntu-latest
    needs: [gpu_tests, checks]
    if: startsWith(github.ref, 'refs/tags/')
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}

      - name: Install requirements
        run: |
          pip install -e .[dev]

      - name: Prepare environment
        run: |
          echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
          echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV

      - name: Download package distribution files
        uses: actions/download-artifact@v3
        with:
          name: package
          path: dist

      - name: Generate release notes
        run: |
          python scripts/release_notes.py > ${{ github.workspace }}-RELEASE_NOTES.md

      - name: Publish package to PyPI
        run: |
          twine upload -u __token__ -p ${{ secrets.PYPI_PASSWORD }} dist/*

      - name: Publish GitHub release
        uses: softprops/action-gh-release@v1
        with:
          body_path: ${{ github.workspace }}-RELEASE_NOTES.md
          prerelease: ${{ contains(env.TAG, 'rc') }}
          files: |
            dist/*


Update dependency PR

name: Update dependency PR

on:
  pull_request:
    types:
      - opened
    paths:
      - "pyproject.toml"

permissions:
  pull-requests: write

jobs:
  torch:
    name: torch
    runs-on: ubuntu-latest
    if: startsWith(github.head_ref, 'dependabot/pip/torch-')
    steps:
      - uses: actions/github-script@v6
        with:
          script: |
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: 'Hello! This is a [PyTorch](https://pytorch.org/) upgrade, which means you will also need to update:\n- [ ] The base image in `Dockerfile`\n- [ ] The base image in `Dockerfile.test`\n- [ ] The torch version hard-coded in `.github/workflows/main.yml`'
            })


C Actions

# Copyright (c) 2020 The University of Manchester
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: C Actions
on: [push]
jobs:
  build:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Checkout SupportScripts
      uses: actions/checkout@v4
      with:
        repository: SpiNNakerManchester/SupportScripts
        path: support
    - name: Install Ubuntu packages
      uses: ./support/actions/apt-get-install
      with:
        packages: doxygen gcc-arm-none-eabi
    - name: Set up Python 3.12
      uses: actions/setup-python@v5
      with:
        python-version: 3.12
    - name: Checkout SpiNNaker Dependencies
      uses: ./support/actions/install-spinn-deps
      with:
        # Note: SpiNNUtils needed for spinn_utilities.make_tools
        repositories: spinnaker_tools spinn_common SpiNNFrontEndCommon

    - name: "Prepare: Install SpiNNUtils"
      uses: ./support/actions/install-spinn-deps
      with:
        # Note: SpiNNUtils needed for spinn_utilities.make_tools
        repositories: SpiNNUtils
        install: true

    # Note: run these before building to avoid linting modified_src directories
    - name: Lint C code using Vera++
      uses: ./support/actions/vera
      with:
        base-dir: spinnaker_graph_front_end
        options: -P max-dirname-length=40
    - name: Lint C code using Vera++
      uses: ./support/actions/vera
      with:
        base-dir: gfe_integration_tests

    - name: Build C dependencies
      run: |
        export SPINN_DIRS=$PWD/spinnaker_tools
        echo "SPINN_DIRS=$PWD/spinnaker_tools" >> $GITHUB_ENV
        make -C spinnaker_tools
        make -C spinn_common install
        make -C SpiNNFrontEndCommon/c_common install

    - name: Build Example code
      run: make
      env:
        CFLAGS: -fdiagnostics-color=always
        GCC_COLORS: error=01;31:warning=01;35:note=01;36:range1=32:range2=34:locus=01:quote=01:fixit-insert=32:fixit-delete=31:diff-filename=01:diff-hunk=32:diff-delete=31:diff-insert=32
      working-directory: gfe_examples

    - name: Build Integration Test code (gfe_integration_tests)
      run: make
      env:
        CFLAGS: -fdiagnostics-color=always
        GCC_COLORS: error=01;31:warning=01;35:note=01;36:range1=32:range2=34:locus=01:quote=01:fixit-insert=32:fixit-delete=31:diff-filename=01:diff-hunk=32:diff-delete=31:diff-insert=32
      working-directory: gfe_integration_tests


Python Actions

# Copyright (c) 2020 The University of Manchester
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This workflow will install Python dependencies, run tests, lint and rat with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Python Actions
on: [push]
jobs:
  build:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    strategy:
      matrix:
        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]

    env:
      BASE_PKG: spinnaker_graph_front_end
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Checkout SupportScripts
      uses: actions/checkout@v4
      with:
        repository: SpiNNakerManchester/SupportScripts
        path: support

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install pip, etc
      uses: ./support/actions/python-tools
    - name: Install mypy
      run: pip install mypy

    - name: Install Spinnaker Dependencies
      uses: ./support/actions/install-spinn-deps
      with:
        repositories: >
          SpiNNUtils SpiNNMachine SpiNNMan PACMAN spalloc
          SpiNNFrontEndCommon TestBase
        install: true
    - name: Install matplotlib
      uses: ./support/actions/install-matplotlib
    - name: Setup
      uses: ./support/actions/run-install

    - name: Create a configuration file
      uses: ./support/actions/configure-spynnaker
      with:
        board-address: spinn-4.cs.man.ac.uk
    - run: mv ~/.spynnaker.cfg ~/.spiNNakerGraphFrontEnd.cfg
    - name: Test with pytest
      uses: ./support/actions/pytest
      with:
        tests: unittests
        coverage: ${{ matrix.python-version == 3.12 }}
        cover-packages: ${{ env.BASE_PKG }}
        coveralls-token: ${{ secrets.GITHUB_TOKEN }}

    - name: Lint with flake8
      run: flake8 ${{ env.BASE_PKG }} unittests gfe_examples
    - name: Lint with pylint
      uses: ./support/actions/pylint
      with:
        package: ${{ env.BASE_PKG }} gfe_examples
        exitcheck: 31 # Action fails on any message
        language: en_GB

    - name: Lint with mypy
      run: mypy $BASE_PKG

  validate:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.12]

    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Checkout SupportScripts
      uses: actions/checkout@v4
      with:
        repository: SpiNNakerManchester/SupportScripts
        path: support

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install pip, etc
      uses: ./support/actions/python-tools
    - name: Install Spinnaker Dependencies
      uses: ./support/actions/install-spinn-deps
      with:
        repositories: >
          SpiNNUtils SpiNNMachine SpiNNMan PACMAN spalloc
          SpiNNFrontEndCommon TestBase
        install: true
    - name: Install matplotlib
      uses: ./support/actions/install-matplotlib
    - name: Setup
      uses: ./support/actions/run-install

    - name: Create a configuration file
      uses: ./support/actions/configure-spynnaker
      with:
        board-address: spinn-4.cs.man.ac.uk
    - run: mv ~/.spynnaker.cfg ~/.spiNNakerGraphFrontEnd.cfg
    - name: Run rat copyright enforcement
      uses: ./support/actions/check-copyrights
      with:
        config_file: rat_asl20.xml

    - name: Build documentation with sphinx
      uses: ./support/actions/sphinx
      with:
        directory: doc/source

    - name: Validate CITATION.cff
      uses: dieghernan/cff-validator@main


C++ network

name: C++ network

on: [pull_request, workflow_dispatch]
jobs:
  compilation:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: "pip"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Install pynucastro
        run: pip install .

      - name: Get AMReX
        run: |
          mkdir external
          cd external
          git clone https://github.com/AMReX-Codes/amrex.git
          cd amrex
          git checkout development
          echo 'AMREX_HOME=$(GITHUB_WORKSPACE)/external/amrex' >> $GITHUB_ENV
          echo $AMREX_HOME
          if [[ -n "${AMREX_HOME}" ]]; then exit 1; fi
          cd ../..

      - name: Get Microphysics
        run: |
          cd external
          git clone https://github.com/AMReX-Astro/Microphysics.git
          cd Microphysics
          git checkout development
          cd ../..

      - name: Install dependencies
        run: |
          sudo apt-get update -y -qq
          sudo apt-get -qq -y install curl cmake jq clang g++>=9.3.0

      - name: Regenerate subch_simple network
        run: |
          cd external/Microphysics/networks/subch_simple
          python subch_simple.py

      - name: Compile burn_cell (subch_simple)
        run: |
          cd external/Microphysics/unit_test/burn_cell
          make realclean
          make NETWORK_DIR=subch_simple -j 4

      - name: Run burn_cell (subch_simple)
        run: |
          cd external/Microphysics/unit_test/burn_cell
          ./main3d.gnu.ex inputs_subch_simple > test.out

      - name: Compare to stored output (subch_simple)
        run: |
          cd external/Microphysics/unit_test/burn_cell
          diff -I "^Initializing AMReX" -I "^AMReX" -I "^reading in reaclib rates" test.out ${GITHUB_WORKSPACE}/.github/workflows/microphysics-benchmarks/subch_simple_unit_test.out

      - name: Regenerate ECSN network
        run: |
          cd external/Microphysics/networks/ECSN
          python ecsn_network_generation.py

      - name: Compile burn_cell (ECSN)
        run: |
          cd external/Microphysics/unit_test/burn_cell
          make realclean
          make NETWORK_DIR=ECSN -j 4

      - name: Run burn_cell (ECSN)
        run: |
          cd external/Microphysics/unit_test/burn_cell
          ./main3d.gnu.ex inputs_ecsn > test.out

      - name: Compare to stored output (ECSN)
        run: |
          cd external/Microphysics/unit_test/burn_cell
          diff -I "^Initializing AMReX" -I "^AMReX" -I "^reading in reaclib rates" test.out ${GITHUB_WORKSPACE}/.github/workflows/microphysics-benchmarks/ecsn_unit_test.out

      - name: Regenerate ase network
        run: |
          cd external/Microphysics/networks/ase
          python ase.py

      - name: Compile test_ase (NSE_NET, ase)
        run: |
          cd external/Microphysics/unit_test/test_ase
          make realclean
          make -j 4

      - name: Run test_ase (NSE_NET, ase)
        run: |
          cd external/Microphysics/unit_test/test_ase
          ./main3d.gnu.ex inputs_ase > test.out

      - name: Compare to stored output (NSE_NET, ase)
        run: |
          cd external/Microphysics/unit_test/test_ase
          diff -I "^Initializing AMReX" -I "^AMReX" -I "^reading in reaclib rates" test.out ${GITHUB_WORKSPACE}/.github/workflows/microphysics-benchmarks/ase_nse_net_unit_test.out


docs build

name: docs build

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  workflow_dispatch:

jobs:
  docs:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Install pandoc and doxygen
        run: |
          sudo apt install pandoc doxygen

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: "pip"

      - name: Install dependencies
        run: pip install -r ./requirements.txt

      - name: Install pynucastro
        run: pip install .

      - name: Build docs
        run: |
          cd docs/
          make SPHINXOPTS='-v -W --keep-going -n' html

      - name: Check links
        run: |
          cd docs/
          make SPHINXOPTS=-v linkcheck


name: flake8

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  workflow_dispatch:

jobs:
  flake8:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Install Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
          cache: "pip"

      - name: Install pylint
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install flake8

      - name: Validate
        run: flake8 pynucastro


github pages

name: github pages

on:
  push:
    branches:
      - main

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Install pandoc and doxygen
        run: |
          sudo apt install pandoc doxygen

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: "pip"

      - name: Install dependencies
        run: pip install -r ./requirements.txt

      - name: Install pynucastro
        run: pip install .

      - name: Build docs
        run: ./deploy_docs_action.sh

      - name: Deploy
        uses: peaceiris/actions-gh-pages@v4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./out
          keep_files: true


isort

name: isort

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  workflow_dispatch:

jobs:
  isort:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: "pip"

      - name: Install dependencies + isort
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install isort

      - name: Install pynucastro
        run: pip install .

      - name: Validate
        run: isort --check --diff pynucastro


pylint

name: pylint

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  workflow_dispatch:

jobs:
  pylint:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: "pip"

      - name: Install dependencies + pylint
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install 'pylint>=3.2.0'

      - name: Install pynucastro
        run: pip install .

      - name: Validate
        run: pylint --output-format=github pynucastro pynucastro/**/tests


pytest-all

name: pytest-all

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
      - development
  workflow_dispatch:

jobs:
  test:
    runs-on: ubuntu-latest

    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    steps:
      - uses: actions/checkout@v4

      - name: Install Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Install pynucastro
        run: pip install --editable .

      - name: Print system info
        run: |
          echo '::group::NumPy'
          pip install threadpoolctl &>/dev/null
          python -c "import sys, numpy; print(numpy.__version__); print(sys.version); print(numpy.show_runtime())"
          echo '::endgroup::'
          echo '::group::CPU info'
          lscpu
          echo '::endgroup::'
          echo '::group::Package versions'
          pip list --format=freeze
          echo '::endgroup::'

      - name: Run tests with pytest
        run: pytest -v -s --nbval --cov=pynucastro --color=yes

      - name: Upload output from failed write_network tests
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: failed_networks_${{ matrix.python-version }}
          path: _test_*/
          if-no-files-found: ignore
          retention-days: 5


Simple C++ network

name: Simple C++ network

on: [pull_request, workflow_dispatch]
jobs:
  compilation:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: "pip"

      - name: Install dependencies
        run: |
          sudo apt-get update -y -qq
          sudo apt-get -qq -y install curl cmake jq clang g++>=9.3.0
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Install pynucastro
        run: pip install .

      - name: Build the network
        run: |
          mkdir work
          cd work
          cp ../.github/workflows/simple_cxx_network/* .
          python test_simple_cxx.py
          make

      - name: Run and compare
        run: |
          cd work
          ./main > test.out
          diff test.out simple_cxx_net.out


Build and test cunumeric (with CPU legate) on GH


Build and test cunumeric (with GPU legate) on GH


Build and test cunumeric on GH

name: Build and test cunumeric on GH

concurrency:
  group: ci-build-and-test-on-${{ github.event_name }}-from-${{ github.ref_name }}
  cancel-in-progress: true

on:
  push:
    branches:
      - "pull-request/[0-9]+"
      - "branch-*"

jobs:
  build-and-test:
    strategy:
      fail-fast: false
      matrix:
        include:
          - device: "gpu"
            image: "rapidsai/devcontainers:23.06-cpp-mambaforge-ubuntu22.04"

          - device: "cpu"
            image: "rapidsai/devcontainers:23.06-cpp-mambaforge-ubuntu22.04"
    uses:
      ./.github/workflows/gh-build-and-test.yml
    with:
      device: ${{ matrix.device }}
      image: ${{ matrix.image }}


on:
  workflow_call:
    inputs:
      image:
        type: string
        required: true
      device:
        type: string
        required: true

jobs:
  build:
    name: "Build cunumeric (with ${{ inputs.device }} legate) on GH"
    uses:
      ./.github/workflows/gh-build.yml
    with:
      device: ${{ inputs.device }}
      image: ${{ inputs.image }}
      runs-on: ${{ github.repository_owner == 'nv-legate' && 'linux-amd64-cpu32' || 'ubuntu-latest' }}

  test:
    needs:
      - build
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: 1 CPU test
            options: test --cpus 1 --unit --debug
            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-cpu4' }}
            has-gpu: false
            enabled: true

          - name: 2 CPUs test
            options: test --cpus 2 --debug
            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-cpu8' }}
            has-gpu: false
            enabled: true

          - name: GPU test
            options: test --use cuda --gpus 1 --debug
            runner: linux-amd64-gpu-v100-latest-1
            has-gpu: true
            enabled: ${{ inputs.device == 'gpu' }}

          - name: 2 GPUs test
            options: test --use cuda --gpus 2 --debug
            runner: linux-amd64-2gpu
            has-gpu: true
            enabled: ${{ inputs.device == 'gpu' }}

          - name: OpenMP test
            options: test --use openmp --omps 1 --ompthreads 2 --debug
            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-cpu32' }}
            has-gpu: ${{ inputs.device == 'gpu' }}
            enabled: true

          - name: 2 NUMA OpenMPs test
            options: test --use openmp --omps 2 --ompthreads 2 --numamem 2048 --debug
            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-cpu32' }}
            has-gpu: ${{ inputs.device == 'gpu' }}
            enabled: true

          - name: Eager execution test
            options: test --use eager --debug
            runner: ${{ inputs.device == 'gpu' && 'linux-amd64-gpu-v100-latest-1' || 'linux-amd64-cpu4' }}
            has-gpu: ${{ inputs.device == 'gpu' }}
            enabled: true

          - name: mypy
            options: mypy
            runner: linux-amd64-cpu4
            has-gpu: false
            enabled: true

          - name: documentation
            options: docs
            runner: linux-amd64-cpu4
            has-gpu: false
            enabled: ${{ inputs.device == 'gpu' }}

    uses:
      ./.github/workflows/gh-test.yml
    with:
      name: ${{ matrix.name }}
      device: ${{ inputs.device }}
      image: ${{ inputs.image }}
      runs-on: ${{ matrix.runner }}
      has-gpu: ${{ matrix.has-gpu }}
      test-options: ${{ matrix.options }}
      enabled: ${{ matrix.enabled }}


Build

name: Build

on:
  workflow_call:
    inputs:
      image:
        type: string
        required: true
      device:
        required: true
        type: string
      runs-on:
        required: true
        type: string

jobs:
  build:
    name: build-${{ inputs.device }}-sub-workflow

    permissions:
      id-token: write # This is required for configure-aws-credentials
      contents: read  # This is required for actions/checkout

    runs-on: ${{ inputs.runs-on }}

    container:
      options: -u root
      image: "${{ inputs.image }}"
      env:
        CUDA_VERSION: "12.2"
        CUDA_VERSION_MAJOR: "12"
        CUDA_VERSION_MINOR: "2"
        SCCACHE_REGION: "us-east-2"
        SCCACHE_BUCKET: "rapids-sccache-devs"
        SCCACHE_S3_KEY_PREFIX: "legate-cunumeric-dev"
        USE_CUDA: "${{ inputs.device == 'gpu' && 'ON' || 'OFF' }}"
        GH_TOKEN: "${{ env.GH_TOKEN }}"
        GITHUB_TOKEN: "${{ env.GITHUB_TOKEN }}"
        VAULT_HOST: "${{ github.repository_owner != 'nv-legate' && 'https://vault.ops.k8s.rapids.ai' || '' }}"
    defaults:
      run:
        shell: su coder {0}
        working-directory: /home/coder

    steps:
      - name: Checkout cunumeric (= this repo)
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          path: cunumeric
          persist-credentials: false

      - name: Dump environment
        run: |
          env

      - name: Copy source folder
        run: |
          set -x
          pwd
          cp -r $GITHUB_WORKSPACE/cunumeric .
          chown -R coder:coder cunumeric;
          ls -R

      - name: Copy .gitconfig
        run: cp ~/cunumeric/continuous_integration/dot-gitconfig ~/.gitconfig

      - id: legate_core_info
        name: Read legate.core SHA
        shell: bash --noprofile --norc -xeo pipefail {0}
        run: |
          git_tag="$(jq -r '.packages.legate_core.git_tag' cunumeric/cmake/versions.json)";

          echo "git_tag=$git_tag" | tee -a "${GITHUB_OUTPUT}";

      - name: Download dependencies (artifacts)
        uses: ./cunumeric/.github/actions/download-artifacts
        with:
          device: "${{ inputs.device }}"
          git_sha: "${{ steps.legate_core_info.outputs.git_tag }}"

      - if: github.repository_owner == 'nv-legate'
        name: Get AWS credentials for sccache bucket
        uses: aws-actions/configure-aws-credentials@v2
        with:
          aws-region: us-east-2
          role-duration-seconds: 28800 # 8 hours
          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate

      - name: Build cunumeric
        run: |
          export PATH="/home/coder/cunumeric/continuous_integration/scripts:$PATH"
          build-cunumeric-all

      - name: Upload build artifacts
        uses: actions/upload-artifact@v3
        with:
          name: "cunumeric-${{ inputs.device }}-${{ github.sha }}"
          path: |
            /tmp/out
            /tmp/conda-build


Test cunumeric on GH

name: Test cunumeric on GH

on:
  workflow_call:
    inputs:
      name:
        required: true
        type: string
      image:
        type: string
        required: true
      device:
        required: true
        type: string
      runs-on:
        required: true
        type: string
      has-gpu:
        required: true
        type: boolean
        description: "The runner has GPU(s)."
      test-options:
        required: true
        type: string
      enabled:
        required: true
        type: boolean

env:
  build_artifact_name: "cunumeric-${{ inputs.device }}-${{ github.sha }}"

jobs:
  test:
    name: ${{ inputs.name }}
    if: inputs.enabled && github.repository_owner == 'nv-legate'
    runs-on: ${{ inputs.runs-on }}

    container:
      options: -u root
      image: "${{ inputs.image }}"
      env:
        # CUDA_VERSION: "${{ inputs.CUDA }}"
        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}

    defaults:
      run:
        shell: su coder {0}
        working-directory: /home/coder

    steps:
      - if: inputs.has-gpu
        name: Run nvidia-smi to make sure GPU is working
        run: nvidia-smi

      - name: Install numactl
        run: |
          export DEBIAN_FRONTEND=noninteractive && \
            sudo apt-get update && \
            sudo apt-get install -y numactl

      - name: Checkout cunumeric
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          path: cunumeric
          persist-credentials: false

      - name: Copy source folder
        run: |
          set -x
          pwd
          cp -r $GITHUB_WORKSPACE/cunumeric .
          chown -R coder:coder cunumeric;
          ls -R

      - name: Download build artifacts
        uses: actions/download-artifact@v3
        with:
          name: ${{ env.build_artifact_name }}
          path: /home/coder/.artifacts

      - name: Run cunumeric test / analysis
        shell: su coder {0}
        run: |
          set -x
          sudo chown -R coder:coder /home/coder/.artifacts

          export PATH="/home/coder/cunumeric/continuous_integration/scripts:$PATH"

          set -eo pipefail
          test-cunumeric ${{ inputs.test-options }}


Pull Request Labels

name: Pull Request Labels
on:
  pull_request:
    types: [opened, labeled, unlabeled, synchronize]
jobs:
  label:
    runs-on: ubuntu-latest
    steps:
      - name: Check Labels
        uses: mheap/github-action-required-labels@v3
        with:
          mode: exactly
          count: 1
          labels: "category:new-feature, category:improvement, category:bug-fix, category:task, category:documentation"


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ master ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ master ]

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://git.io/codeql-language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v3

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3


Docs CI

name: Docs CI

on:
  push:
    branches:
      - master
      - '[0-9].[0-9]'
  pull_request:
    branches:
      - master
      - '[0-9].[0-9]'
  schedule:
    - cron: '0 1 * * *' # nightly build

concurrency:
  group: ${{ github.event.pull_request.number || github.ref }}-docs
  cancel-in-progress: true

permissions:
  contents: read  #  to fetch code (actions/checkout)

jobs:

   build-docs:
     name: Build docs
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
           cache: 'pip'
       - name: install deps
         run: |
           sudo apt-get update -yqq
           sudo apt-get install -yqq pandoc make
       - name: run code linters
         run: |
           pip install -r requirements.txt -r dev_requirements.txt -r docs/requirements.txt
           invoke build-docs

       - name: upload docs
         uses: actions/upload-artifact@v4
         with:
           name: redis-py-docs
           path: |
             docs/_build/html


CI

name: CI

on:
  push:
    paths-ignore:
      - 'docs/**'
      - '**/*.rst'
      - '**/*.md'
    branches:
      - master
      - '[0-9].[0-9]'
  pull_request:
    branches:
      - master
      - '[0-9].[0-9]'
  schedule:
    - cron: '0 1 * * *' # nightly build

concurrency:
  group: ${{ github.event.pull_request.number || github.ref }}-integration
  cancel-in-progress: true

permissions:
  contents: read  #  to fetch code (actions/checkout)

env:
  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
  REDIS_IMAGE: redis/redis-stack-server:7.4.0-rc1
  REDIS_STACK_IMAGE: redis/redis-stack-server:7.4.0-rc1

jobs:
  dependency-audit:
    name: Dependency audit
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: pypa/gh-action-pip-audit@v1.0.8
        with:
          inputs: requirements.txt dev_requirements.txt
          ignore-vulns: |
            GHSA-w596-4wvx-j9j6  # subversion related git pull, dependency for pytest. There is no impact here.

  lint:
    name: Code linters
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
          cache: 'pip'
      - name: run code linters
        run: |
          pip install -r dev_requirements.txt
          invoke linters

  run-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      max-parallel: 15
      fail-fast: false
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', 'pypy-3.8', 'pypy-3.9']
        test-type: ['standalone', 'cluster']
        connection-type: ['hiredis', 'plain']
    env:
      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
    name: Python ${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.connection-type}} tests
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
      - name: run tests
        run: |
          pip install -U setuptools wheel
          pip install -r requirements.txt
          pip install -r dev_requirements.txt
          if [ "${{matrix.connection-type}}" == "hiredis" ]; then
           pip install hiredis
          fi
          invoke devenv
          sleep 10 # time to settle
          invoke ${{matrix.test-type}}-tests

      - uses: actions/upload-artifact@v4
        if: success() || failure()
        with:
          name: pytest-results-${{matrix.test-type}}-${{matrix.connection-type}}-${{matrix.python-version}}
          path: '${{matrix.test-type}}*results.xml'

      - name: Upload codecov coverage
        uses: codecov/codecov-action@v4
        with:
          fail_ci_if_error: false

      - name: View Test Results
        uses: dorny/test-reporter@v1
        if: success() || failure()
        continue-on-error: true
        with:
          name: Test Results ${{matrix.python-version}} ${{matrix.test-type}}-${{matrix.connection-type}}
          path: '*.xml'
          reporter: java-junit
          list-suites: all
          list-tests: all
          max-annotations: 10
          fail-on-error: 'false'

  resp3_tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.8', '3.11']
        test-type: ['standalone', 'cluster']
        connection-type: ['hiredis', 'plain']
        exclude:
          - test-type: 'cluster'
            connection-type: 'hiredis'
    env:
      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
    name: RESP3 [${{ matrix.python-version }} ${{matrix.test-type}}-${{matrix.connection-type}}]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
      - name: run tests
        run: |
          pip install -U setuptools wheel
          pip install -r requirements.txt
          pip install -r dev_requirements.txt
          if [ "${{matrix.connection-type}}" == "hiredis" ]; then
            pip install hiredis
          fi
          invoke devenv
          sleep 10 # time to settle
          invoke ${{matrix.test-type}}-tests --protocol=3
          invoke ${{matrix.test-type}}-tests --uvloop --protocol=3

      - uses: actions/upload-artifact@v4
        if: success() || failure()
        with:
          name: pytest-results-${{matrix.test-type}}-${{matrix.connection-type}}-${{matrix.python-version}}-resp3
          path: '${{matrix.test-type}}*results.xml'

      - name: Upload codecov coverage
        uses: codecov/codecov-action@v4
        with:
          fail_ci_if_error: false

      - name: View Test Results
        uses: dorny/test-reporter@v1
        if: success() || failure()
        continue-on-error: true
        with:
          name: Test Results ${{matrix.python-version}} ${{matrix.test-type}}-${{matrix.connection-type}}-resp3
          path: '*.xml'
          reporter: java-junit
          list-suites: all
          list-tests: all
          max-annotations: 10
          fail-on-error: 'false'

  build_and_test_package:
    name: Validate building and installing the package
    runs-on: ubuntu-latest
    needs: [run-tests]
    strategy:
      fail-fast: false
      matrix:
        extension: ['tar.gz', 'whl']
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Run installed unit tests
        run: |
          bash .github/workflows/install_and_test.sh ${{ matrix.extension }}

  install_package_from_commit:
    name: Install package from commit hash
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', 'pypy-3.8', 'pypy-3.9']
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
      - name: install from pip
        run: |
          pip install --quiet git+${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}


Publish tag to Pypi

name: Publish tag to Pypi

on:
  release:
    types: [published]

permissions:
  contents: read  #  to fetch code (actions/checkout)

jobs:

  build_and_package:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: install python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dev tools
        run: |
          pip install -r dev_requirements.txt
          pip install twine wheel

      - name: Build package
        run: |
          python setup.py build
          python setup.py sdist bdist_wheel

      - name: Basic package test prior to upload
        run: |
          twine check dist/*

      - name: Publish to Pypi
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}


Release Drafter

name: Release Drafter

on:
  push:
    # branches to consider in the event; optional, defaults to all
    branches:
      - master

permissions: {}
jobs:
  update_release_draft:
    permissions:
      pull-requests: write  #  to add label to PR (release-drafter/release-drafter)
      contents: write  #  to create a github release (release-drafter/release-drafter)

    runs-on: ubuntu-latest
    steps:
      # Drafts your next Release notes as Pull Requests are merged into "master"
      - uses: release-drafter/release-drafter@v6
        with:
          # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml
           config-name: release-drafter-config.yml
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


spellcheck

name: spellcheck
on:
  pull_request:
jobs:
  check-spelling:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Check Spelling
        uses: rojopolis/spellcheck-github-actions@0.37.0
        with:
          config_path: .github/spellcheck-settings.yml
          task_name: Markdown


Close stale issues

name: "Close stale issues"
on:
  schedule:
  - cron: "0 0 * * *"

permissions: {}
jobs:
  stale:
    permissions:
      issues: write  #  to close stale issues (actions/stale)
      pull-requests: write  #  to close stale PRs (actions/stale)

    runs-on: ubuntu-latest
    steps:
    - uses: actions/stale@v9
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        stale-issue-message: 'This issue is marked stale. It will be closed in 30 days if it is not updated.'
        stale-pr-message: 'This pull request is marked stale. It will be closed in 30 days if it is not updated.'
        days-before-stale: 365
        days-before-close: 30
        stale-issue-label: "Stale"
        stale-pr-label: "Stale"
        operations-per-run: 10
        remove-stale-when-updated: true


Close inactive issues

name: Close inactive issues

on:
  schedule:
    - cron: "0 0 * * *"

jobs:
  close-issues:
    if: github.event.pull_request.draft == false && github.base_ref == 'main' && github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI'
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    steps:
      - uses: actions/stale@v3
        with:
          days-before-issue-stale: 14
          days-before-issue-close: -1
          stale-issue-label: "stale"
          stale-issue-message: "This issue is stale because it has been open for 14 days with no activity."
#           close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
          days-before-pr-stale: 14
          days-before-pr-close: -1
          stale-pr-message: "This PR is stale because it has been open for 14 days with no activity."
#           close-pr-message: "This PR was closed because it has been inactive for 14 days since being marked as stale."
          repo-token: ${{ secrets.GITHUB_TOKEN }}


Synchronize Submodule

name: Synchronize Submodule

on:
  workflow_dispatch:
  schedule:
    - cron: "0 0 * * *"

jobs:
  sync-submodule:
    runs-on: ubuntu-latest
    if: github.repository == 'hpcaitech/ColossalAI'
    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          ref: 'main'
          submodules: true

      - name: echo
        run: |
          echo ${{github}}

      - name: Git Sumbodule Update
        run: |
          git pull --recurse-submodules
          git submodule update --remote --recursive

      - name: Commit update
        run: |
          git config --global user.name 'github-actions'
          git config --global user.email 'github-actions@github.com'
          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
          git commit -am "Automated submodule synchronization"

      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v3
        with:
          title: '[Bot] Synchronize Submodule References'
          body: |
            Automated PR to update submodule commits
          committer: GitHub <noreply@github.com>
          author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
          assignees: ${{ github.actor }}
          delete-branch: true
          branch: create-pull-request/patch-sync-submodule


issue-translator

name: 'issue-translator'
on:
  issue_comment:
    types: [created]
  issues:
    types: [opened]

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: usthe/issues-translate-action@v2.7
        with:
          IS_MODIFY_TITLE: false
          # not require, default false, . Decide whether to modify the issue title
          # if true, the robot account @Issues-translate-bot must have modification permissions, invite @Issues-translate-bot to your project or use your custom bot.
          CUSTOM_BOT_NOTE: Bot detected the issue body's language is not English, translate it automatically. 👯👭🏻🧑🤝🧑👫🧑🏿🤝🧑🏻👩🏾🤝👨🏿👬🏿
          # not require. Customize the translation robot prefix message.


Report Test Coverage

name: Report Test Coverage

on:
  workflow_run:
    workflows: [Build on PR]
    types:
      - completed

jobs:
  report-test-coverage:
    runs-on: ubuntu-latest
    if: ${{ github.event.workflow_run.conclusion == 'success' }}
    steps:
      - name: "Download artifact"
        uses: actions/github-script@v6
        with:
          script: |
            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
               owner: context.repo.owner,
               repo: context.repo.repo,
               run_id: context.payload.workflow_run.id,
            });
            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
              return artifact.name == "report"
            })[0];
            let download = await github.rest.actions.downloadArtifact({
               owner: context.repo.owner,
               repo: context.repo.repo,
               artifact_id: matchArtifact.id,
               archive_format: 'zip',
            });
            let fs = require('fs');
            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/report.zip`, Buffer.from(download.data));

      - name: "Unzip artifact"
        id: unzip
        run: |
          unzip report.zip
          if [ -f "coverage.txt" ]; then
            echo "hasReport=true" >> $GITHUB_OUTPUT
          else
            echo "hasReport=false" >> $GITHUB_OUTPUT
          fi

      - name: Make Coverage Report Collapsable
        if: steps.unzip.outputs.hasReport == 'true'
        run: |
          covNum=$(cat cov_number)
          title="The code coverage for the changed files is ${covNum}%."
          touch coverage_report.txt
          echo $title >> coverage_report.txt
          echo " " >> coverage_report.txt
          echo "" >> coverage_report.txt
          echo "Click me to view the complete report" >> coverage_report.txt
          echo " " >> coverage_report.txt
          echo "\`\`\`" >> coverage_report.txt
          cat coverage.txt >> coverage_report.txt
          echo "\`\`\`" >> coverage_report.txt
          echo "" >> coverage_report.txt
          mv coverage_report.txt coverage.txt

      - name: "Comment on PR"
        if: steps.unzip.outputs.hasReport == 'true'
        uses: actions/github-script@v6
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            let fs = require('fs');
            let issue_number = Number(fs.readFileSync('./pr_number'));
            let owner = context.repo.owner;
            let repo = context.repo.repo;
            let run_id = context.payload.workflow_run.id;
            let run_url = `https://github.com/${owner}/${repo}/actions/runs/${run_id}`
            let body = fs.readFileSync('./coverage.txt', {encoding:'utf8', flag:'r'})

            await github.rest.issues.createComment({
              owner: owner,
              repo: repo,
              issue_number: issue_number,
              body: body
            });


Build on PR

name: Build on PR

on:
  pull_request:
    types: [synchronize, opened, reopened, ready_for_review, closed]
    branches:
      - "main"
      - "develop"
      - "feature/**"
    paths:
      - ".github/workflows/build_on_pr.yml" # run command & env variables change
      - "colossalai/**" # source code change
      - "!colossalai/**.md" # ignore doc change
      - "op_builder/**" # cuda extension change
      - "!op_builder/**.md" # ignore doc change
      - "requirements/**" # requirements change
      - "tests/**" # test change
      - "!tests/**.md" # ignore doc change
      - "pytest.ini" # test config change
      - "setup.py" # install command change
  create:
  delete:

jobs:
  detect:
    name: Detect file change
    if: |
      github.event_name == 'pull_request' &&
      (github.event.action == 'synchronize' || github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'ready_for_review') &&
      github.event.pull_request.draft == false &&
      github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI'
    outputs:
      changedExtenisonFiles: ${{ steps.find-extension-change.outputs.all_changed_files }}
      anyExtensionFileChanged: ${{ steps.find-extension-change.outputs.any_changed }}
      changedLibraryFiles: ${{ steps.find-lib-change.outputs.all_changed_files }}
      anyLibraryFileChanged: ${{ steps.find-lib-change.outputs.any_changed }}
    runs-on: ubuntu-latest
    concurrency:
      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-detect-change
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Locate base commit
        id: locate-base-sha
        run: |
          curBranch=$(git rev-parse --abbrev-ref HEAD)
          commonCommit=$(git merge-base origin/main $curBranch)
          echo $commonCommit
          echo "baseSHA=$commonCommit" >> $GITHUB_OUTPUT

      - name: Find the changed extension-related files
        id: find-extension-change
        uses: tj-actions/changed-files@v35
        with:
          base_sha: ${{ steps.locate-base-sha.outputs.baseSHA }}
          files: |
            op_builder/**
            colossalai/kernel/**
            setup.py

      - name: Find the changed library-related files
        id: find-lib-change
        uses: tj-actions/changed-files@v35
        with:
          base_sha: ${{ steps.locate-base-sha.outputs.baseSHA }}
          files: |
            **/*.py
            **/*.h
            **/*.cpp
            **/*.cu
            **/*.txt

      - name: List changed files
        run: |
          for file in ${{ steps.find-extension-change.outputs.all_changed_files }}; do
            echo "$file was changed"
          done
          for file in ${{ steps.find-lib-change.outputs.all_changed_files }}; do
            echo "$file was changed"
          done

  build:
    name: Build and Test Colossal-AI
    needs: detect
    if: needs.detect.outputs.anyLibraryFileChanged == 'true'
    runs-on: [self-hosted, gpu]
    container:
      image: hpcaitech/pytorch-cuda:2.1.0-12.1.0
      options: --gpus all --rm -v /dev/shm -v /data/scratch/llama-tiny:/data/scratch/llama-tiny
    timeout-minutes: 90
    defaults:
      run:
        shell: bash
    concurrency:
      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-run-test
      cancel-in-progress: true
    steps:
      - name: Checkout TensorNVMe
        uses: actions/checkout@v2
        with:
          repository: hpcaitech/TensorNVMe
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}
          path: TensorNVMe

      - name: Restore TensorNVMe Cache
        run: |
          if [ -d /github/home/tensornvme_cache ] && [ ! -z "$(ls -A /github/home/tensornvme_cache/)" ]; then
            cp -p -r /github/home/tensornvme_cache/* /__w/ColossalAI/ColossalAI/TensorNVMe
          fi

      - name: Install TensorNVMe
        run: |
          cd TensorNVMe
          conda install cmake
          pip install -r requirements.txt
          DISABLE_URING=1 pip install -v .

      - name: Store TensorNVMe Cache
        run: |
          cd TensorNVMe
          cp -p -r ./build /github/home/tensornvme_cache/
          cp -p -r ./cmake-build /github/home/tensornvme_cache/

      - name: Checkout Colossal-AI
        uses: actions/checkout@v2
        with:
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}

      - name: Restore Colossal-AI Cache
        if: needs.detect.outputs.anyExtensionFileChanged != 'true'
        run: |
          # -p flag is required to preserve the file timestamp to avoid ninja rebuild
          if [ -d /github/home/cuda_ext_cache ] && [ ! -z "$(ls -A /github/home/cuda_ext_cache/)" ]; then
            cp -p -r /github/home/cuda_ext_cache/* /__w/ColossalAI/ColossalAI/
          fi

      - name: Install Colossal-AI
        run: |
          BUILD_EXT=1 pip install -v -e .
          pip install -r requirements/requirements-test.txt

      - name: Store Colossal-AI Cache
        run: |
          # -p flag is required to preserve the file timestamp to avoid ninja rebuild
          cp -p -r /__w/ColossalAI/ColossalAI/build /github/home/cuda_ext_cache/

      - name: Execute Unit Testing
        run: |
          CURL_CA_BUNDLE="" PYTHONPATH=$PWD FAST_TEST=1 pytest \
          -m "not largedist" \
          --durations=0 \
          --ignore tests/test_analyzer \
          --ignore tests/test_auto_parallel \
          --ignore tests/test_fx \
          --ignore tests/test_autochunk \
          --ignore tests/test_gptq \
          --ignore tests/test_infer_ops \
          --ignore tests/test_legacy \
          --ignore tests/test_smoothquant \
          tests/
        env:
          LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
          LLAMA_PATH: /data/scratch/llama-tiny

      - name: Collate artifact
        env:
          PR_NUMBER: ${{ github.event.number }}
          changedLibraryFiles: ${{ needs.detect.outputs.changedLibraryFiles }}
          anyLibraryFileChanged: ${{ needs.detect.outputs.anyLibraryFileChanged }}
          changedExtenisonFiles: ${{ needs.detect.outputs.changedExtenisonFiles }}
        run: |
          mkdir report
          echo $PR_NUMBER > ./report/pr_number

          # generate coverage.xml if any
          if [ "$anyLibraryFileChanged" == "true" ] && [ -e .coverage ]; then
            allFiles=""
            for file in $changedLibraryFiles; do
              if [ "$allFiles" == "" ]; then
                allFiles=$file
              else
                allFiles=$allFiles,$file
              fi
            done

            coverage report --data-file .coverage --include $allFiles > ./coverage.txt

            covPercentage=$(tail -n 1 coverage.txt  | grep -o '[1-9]*%$')
            covNum=${covPercentage::-1}
            mv coverage.txt ./report
            echo $covNum > ./report/cov_number
          else
            echo "No coverage report is generated"
          fi

      - name: Upload test coverage artifact
        uses: actions/upload-artifact@v3
        with:
          name: report
          path: report/


name: Generate Community Report and Send to Lark

on:
  workflow_dispatch:
  schedule:
    # release on every Friday 09:00 UTC time, 17:00 Beijing/Singapore time
    - cron:  '0 9 * * 5'

jobs:
  generate-and-publish:
    if: github.repository == 'hpcaitech/ColossalAI'
    name: Generate leaderboard report and publish to Lark
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v2

    - uses: actions/setup-python@v2
      with:
        python-version: '3.8.14'

    - run: pip install requests matplotlib seaborn requests_toolbelt pytz

    - run: python .github/workflows/scripts/generate_leaderboard_and_send_to_lark.py
      env:
        LARK_APP_ID: ${{ secrets.LARK_LEADERBOARD_APP_ID }}
        LARK_APP_SECRET: ${{ secrets.LARK_LEADERBOARD_APP_SECRET }}
        LARK_WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
        GITHUB_TOKEN: ${{ github.token }}


Test Example on Dispatch

name: Test Example on Dispatch
on:
  workflow_dispatch:
    inputs:
      example_directory:
        type: string
        description: example directory, separated by space. For example, language/gpt, images/vit. Simply input language or simply gpt does not work.
        required: true

jobs:
  matrix_preparation:
    if: |
        github.event.pull_request.draft == false &&
        github.base_ref == 'main' &&
        github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI'
    name: Check the examples user want
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
    - name: 📚 Checkout
      uses: actions/checkout@v3
    - name: Set up matrix
      id: set-matrix
      env:
        check_dir: ${{ inputs.example_directory }}
      run: |
        res=`python .github/workflows/scripts/example_checks/check_dispatch_inputs.py --fileNameList $check_dir`
        if [ res == "failure" ];then
          exit -1
        fi
        dirs="[${check_dir}]"
        echo "Testing examples in $dirs"
        echo "matrix={\"directory\":$(echo "$dirs")}" >> $GITHUB_OUTPUT

  test_example:
    if: |
        github.event.pull_request.draft == false &&
        github.base_ref == 'main' &&
        github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI'
    name: Manually check example files
    needs: manual_check_matrix_preparation
    runs-on: [self-hosted, gpu]
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.manual_check_matrix_preparation.outputs.matrix)}}
    container:
      image: hpcaitech/pytorch-cuda:2.1.0-12.1.0
      options: --gpus all --rm -v /data/scratch/examples-data:/data/ -v /dev/shm
    timeout-minutes: 15
    steps:
      - name: 📚 Checkout
        uses: actions/checkout@v3
      - name: Install Colossal-AI
        run: |
          BUILD_EXT=1 pip install -v .
      - name: Test the example
        run: |
          dir=${{ matrix.directory }}
          echo "Testing ${dir} now"
          cd "${PWD}/examples/${dir}"
          bash test_ci.sh


Test Example on PR

name: Test Example on PR
on:
  pull_request:
    branches:
      - "main"
      - "develop"
      - "feature/**"
    # any change in the examples folder will trigger check for the corresponding example.
    paths:
      - "examples/**"
      - "!examples/**.md"

jobs:
  # This is for changed example files detect and output a matrix containing all the corresponding directory name.
  detect-changed-example:
    if: |
      github.event.pull_request.draft == false &&
      github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI' && github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.setup-matrix.outputs.matrix }}
      anyChanged: ${{ steps.setup-matrix.outputs.anyChanged }}
      anyExtensionFileChanged: ${{ steps.find-extension-change.outputs.any_changed }}
    name: Detect changed example files
    concurrency:
      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-detect-change
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Locate base commit
        id: locate-base-sha
        run: |
          curBranch=$(git rev-parse --abbrev-ref HEAD)
          commonCommit=$(git merge-base origin/main $curBranch)
          echo $commonCommit
          echo "baseSHA=$commonCommit" >> $GITHUB_OUTPUT

      - name: Find the changed extension-related files
        id: find-extension-change
        uses: tj-actions/changed-files@v35
        with:
          base_sha: ${{ steps.locate-base-sha.outputs.baseSHA }}
          files: |
            op_builder/**
            colossalai/kernel/**
            setup.py

      - name: Get all changed example files
        id: changed-files
        uses: tj-actions/changed-files@v35
        with:
          base_sha: ${{ steps.locate-base-sha.outputs.baseSHA }}

      - name: setup matrix
        id: setup-matrix
        run: |
          changedFileName=""
          for file in ${{ steps.changed-files.outputs.all_changed_files  }}; do
            changedFileName="${file}:${changedFileName}"
          done
          echo "$changedFileName was changed"
          res=`python .github/workflows/scripts/example_checks/detect_changed_example.py --fileNameList $changedFileName`
          echo "All changed examples are $res"

          if [ "$res" == "[]" ]; then
            echo "anyChanged=false" >> $GITHUB_OUTPUT
            echo "matrix=null" >> $GITHUB_OUTPUT
          else
            dirs=$( IFS=',' ; echo "${res[*]}" )
            echo "anyChanged=true" >> $GITHUB_OUTPUT
            echo "matrix={\"directory\":$(echo "$dirs")}" >> $GITHUB_OUTPUT
          fi

  # If no file is changed, it will prompt an error and shows the matrix do not have value.
  check-changed-example:
    # Add this condition to avoid executing this job if the trigger event is workflow_dispatch.
    if: |
      github.event.pull_request.draft == false &&
      github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI' && github.event_name == 'pull_request' &&
      needs.detect-changed-example.outputs.anyChanged == 'true'
    name: Test the changed example
    needs: detect-changed-example
    runs-on: [self-hosted, gpu]
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.detect-changed-example.outputs.matrix)}}
    container:
      image: hpcaitech/pytorch-cuda:2.1.0-12.1.0
      options: --gpus all --rm -v /data/scratch/examples-data:/data/ -v /dev/shm
    timeout-minutes: 30
    concurrency:
      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-run-example-${{ matrix.directory }}
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v3

      - name: Restore Colossal-AI Cache
        if: needs.detect.outputs.anyExtensionFileChanged != 'true'
        run: |
          if [ -d /github/home/cuda_ext_cache ] && [ ! -z "$(ls -A /github/home/cuda_ext_cache/)" ]; then
            cp -p -r /github/home/cuda_ext_cache/* /__w/ColossalAI/ColossalAI/
          fi

      - name: Install Colossal-AI
        run: |
          BUILD_EXT=1 pip install -v .

      - name: Store Colossal-AI Cache
        run: |
          cp -p -r /__w/ColossalAI/ColossalAI/build /github/home/cuda_ext_cache/

      - name: Test the example
        run: |
          example_dir=${{ matrix.directory }}
          cd "${PWD}/examples/${example_dir}"
          bash test_ci.sh


Test Example on Schedule

name: Test Example on Schedule
on:
  # run at 00:00 of every Sunday(singapore time) so here is UTC time Saturday 16:00
  schedule:
    - cron:  '0 16 * * 6'
  workflow_dispatch:

jobs:
  # This is for all files' weekly check. Specifically, this job is to find all the directories.
  matrix_preparation:
    if: github.repository == 'hpcaitech/ColossalAI'
    name: Prepare matrix for weekly check
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.setup-matrix.outputs.matrix }}
    steps:
    - name: 📚 Checkout
      uses: actions/checkout@v3

    - name: setup matrix
      id: setup-matrix
      run: |
        res=`python .github/workflows/scripts/example_checks/check_example_weekly.py`
        all_loc=$( IFS=',' ; echo "${res[*]}" )
        echo "Found the examples: $all_loc"
        echo "matrix={\"directory\":$(echo "$all_loc")}" >> $GITHUB_OUTPUT

  weekly_check:
    if: github.repository == 'hpcaitech/ColossalAI'
    name: Weekly check all examples
    needs: matrix_preparation
    runs-on: [self-hosted, gpu]
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
    container:
      image: hpcaitech/pytorch-cuda:2.1.0-12.1.0
      options: --gpus all --rm -v /data/scratch/examples-data:/data/ -v /dev/shm
    timeout-minutes: 30
    steps:
      - name: 📚 Checkout
        uses: actions/checkout@v3

      - name: Install Colossal-AI
        run: |
          BUILD_EXT=1 pip install -v .

      - name: Traverse all files
        run: |
          example_dir=${{ matrix.directory }}
          echo "Testing ${example_dir} now"
          cd "${PWD}/examples/${example_dir}"
          bash test_ci.sh

      - name: Notify Lark
        id: message-preparation
        if: ${{ failure() }}
        run: |
          url=$SERVER_URL/$REPO/actions/runs/$RUN_ID
          msg="Example tests failed for $EXAMPLE_DIR, please visit $url for details"
          echo $msg
          python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
        env:
          SERVER_URL: ${{github.server_url }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
          EXAMPLE_DIR: ${{ matrix.directory }}


Build on Schedule

name: Build on Schedule

on:
  schedule:
    # run at 00:00 of every Sunday
    - cron: "0 0 * * *"
  workflow_dispatch:

jobs:
  build:
    name: Build and Test Colossal-AI
    if: github.repository == 'hpcaitech/ColossalAI'
    runs-on: [self-hosted, gpu]
    container:
      image: hpcaitech/pytorch-cuda:2.1.0-12.1.0
      options: --gpus all --rm -v /dev/shm -v /data/scratch/llama-tiny:/data/scratch/llama-tiny
    timeout-minutes: 90
    steps:
      - name: Check GPU Availability # ensure all GPUs have enough memory
        id: check-avai
        run: |
          avai=true
          ngpu=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
          endIndex=$(($ngpu-1))
          for i in $(seq 0 $endIndex);
          do
            gpu_used=$(nvidia-smi -i $i --query-gpu=memory.used --format=csv,noheader,nounits)
            [ "$gpu_used" -gt "2000" ] && avai=false
          done

          echo "GPU is available: $avai"
          echo "avai=$avai" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v2
        if: steps.check-avai.outputs.avai == 'true'
        with:
          repository: hpcaitech/TensorNVMe
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}
          path: TensorNVMe

      - name: Install tensornvme
        if: steps.check-avai.outputs.avai == 'true'
        run: |
          cd TensorNVMe
          conda install cmake
          pip install -r requirements.txt
          DISABLE_URING=1 pip install -v .

      - uses: actions/checkout@v2
        if: steps.check-avai.outputs.avai == 'true'
        with:
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}

      - name: Install Colossal-AI
        if: steps.check-avai.outputs.avai == 'true'
        run: |
          [ ! -z "$(ls -A /github/home/cuda_ext_cache/)" ] && cp -r /github/home/cuda_ext_cache/* /__w/ColossalAI/ColossalAI/
          BUILD_EXT=1 pip install -v -e .
          cp -r /__w/ColossalAI/ColossalAI/build /github/home/cuda_ext_cache/
          pip install -r requirements/requirements-test.txt

      - name: Unit Testing
        if: steps.check-avai.outputs.avai == 'true'
        run: |
          PYTHONPATH=$PWD pytest \
          -m "not largedist" \
          --durations=0 \
          tests/
        env:
          LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
          LLAMA_PATH: /data/scratch/llama-tiny

      - name: Notify Lark
        id: message-preparation
        if: ${{ failure() }}
        run: |
          url=$SERVER_URL/$REPO/actions/runs/$RUN_ID
          msg="Scheduled Build and Test failed, please visit $url for details"
          echo $msg
          python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
        env:
          SERVER_URL: ${{github.server_url }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}


Compatibility Test on Dispatch

name: Compatibility Test on Dispatch

on:
  workflow_dispatch:
    inputs:
      torch_version:
        type: string
        description: torch version, separated by comma
        required: true
      cuda_version:
        type: string
        description: cuda version, separated by comma
        required: true

jobs:
  matrix_preparation:
    name: Prepare Container List
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - id: set-matrix
        env:
          TORCH_VERSIONS: ${{ inputs.torch_version }}
          CUDA_VERSIONS: ${{ inputs.cuda_version }}
        run: |
          IFS=','
          DOCKER_IMAGE=()

          for tv in $TORCH_VERSIONS
          do
              for cv in $CUDA_VERSIONS
              do
                  DOCKER_IMAGE+=("\"hpcaitech/pytorch-cuda:${tv}-${cv}\"")
              done
          done

          container=$( IFS=',' ; echo "${DOCKER_IMAGE[*]}" )
          container="[${container}]"
          echo "$container"
          echo "::set-output name=matrix::{\"container\":$(echo "$container")}"

  build:
    name: Test for PyTorch Compatibility
    needs: matrix_preparation
    if: github.repository == 'hpcaitech/ColossalAI'
    runs-on: [self-hosted, 8-gpu]
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
    container:
      image: ${{ matrix.container }}
      options: --gpus all --rm -v /dev/shm -v /data/scratch/cifar-10:/data/scratch/cifar-10 -v /data/scratch/llama-tiny:/data/scratch/llama-tiny
    timeout-minutes: 200
    steps:
      - name: Install dependencies
        run: |
          pip install -U pip setuptools==68.2.2 wheel --user
      - uses: actions/checkout@v2
        with:
          repository: hpcaitech/TensorNVMe
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}
          path: TensorNVMe
      - name: Install tensornvme
        run: |
          cd TensorNVMe
          apt update && apt install -y cmake
          pip install -r requirements.txt
          DISABLE_URING=1 pip install -v .
      - uses: actions/checkout@v2
        with:
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}
      - name: Download cub for CUDA 10.2
        run: |
          CUDA_VERSION=$(nvcc -V | awk -F ',| ' '/release/{print $6}')

          # check if it is CUDA 10.2
          # download cub
          if [ "$CUDA_VERSION" = "10.2" ]; then
            wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip
            unzip 1.8.0.zip
            cp -r cub-1.8.0/cub/ colossalai/kernel/cuda_native/csrc/kernels/include/
          fi
      - name: Install Colossal-AI
        run: |
          BUILD_EXT=1 pip install -v .
          pip install -r requirements/requirements-test.txt
      - name: Unit Testing
        run: |
          PYTHONPATH=$PWD pytest --durations=0 tests
        env:
          DATA: /data/scratch/cifar-10
          LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
          LLAMA_PATH: /data/scratch/llama-tiny


Compatibility Test on PR

name: Compatibility Test on PR

on:
  pull_request:
    paths:
      - "version.txt"
      - ".compatibility"

jobs:
  matrix_preparation:
    name: Prepare Container List
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    concurrency:
      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-prepare-matrix
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v3
      - id: set-matrix
        run: |
          IFS=','
          DOCKER_IMAGE=()

          while read tag; do
            DOCKER_IMAGE+=("\"hpcaitech/pytorch-cuda:${tag}\"")
          done <.compatibility

          container=$( IFS=',' ; echo "${DOCKER_IMAGE[*]}" )
          container="[${container}]"
          echo "$container"
          echo "::set-output name=matrix::{\"container\":$(echo "$container")}"

  build:
    name: Test for PyTorch Compatibility
    needs: matrix_preparation
    if: github.repository == 'hpcaitech/ColossalAI'
    runs-on: [self-hosted, 8-gpu]
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
    container:
      image: ${{ matrix.container }}
      options: --gpus all --rm -v /dev/shm -v /data/scratch/cifar-10:/data/scratch/cifar-10 -v /data/scratch/llama-tiny:/data/scratch/llama-tiny
    timeout-minutes: 200
    concurrency:
      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-run-test-${{ matrix.container }}
      cancel-in-progress: true
    steps:
      - name: Install dependencies
        run: |
          pip install -U pip setuptools==68.2.2 wheel --user
      - uses: actions/checkout@v2
        with:
          repository: hpcaitech/TensorNVMe
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}
          path: TensorNVMe
      - name: Install tensornvme
        run: |
          cd TensorNVMe
          apt update && apt install -y cmake
          pip install -r requirements.txt
          DISABLE_URING=1 pip install -v .
      - uses: actions/checkout@v2
        with:
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}
      - name: Download cub for CUDA 10.2
        run: |
          CUDA_VERSION=$(nvcc -V | awk -F ',| ' '/release/{print $6}')

          # check if it is CUDA 10.2
          # download cub
          if [ "$CUDA_VERSION" = "10.2" ]; then
            wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip
            unzip 1.8.0.zip
            cp -r cub-1.8.0/cub/ colossalai/kernel/cuda_native/csrc/kernels/include/
          fi

      - name: Install Colossal-AI
        run: |
          BUILD_EXT=1 pip install -v .
          pip install -r requirements/requirements-test.txt
      - name: Unit Testing
        run: |
          PYTHONPATH=$PWD pytest --durations=0 tests
        env:
          DATA: /data/scratch/cifar-10
          LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
          LLAMA_PATH: /data/scratch/llama-tiny


Compatibility Test on Schedule

name: Compatibility Test on Schedule

on:
  # run at 03:00 of every Sunday(singapore time) so here is UTC time Saturday 16:00
  schedule:
    - cron:  '0 19 * * 6'
  workflow_dispatch:

jobs:
  matrix_preparation:
    name: Prepare Container List
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3
      - id: set-matrix
        run: |
          IFS=','
          DOCKER_IMAGE=()

          while read tag; do
            DOCKER_IMAGE+=("\"hpcaitech/pytorch-cuda:${tag}\"")
          done <.compatibility

          container=$( IFS=',' ; echo "${DOCKER_IMAGE[*]}" )
          container="[${container}]"
          echo "$container"
          echo "::set-output name=matrix::{\"container\":$(echo "$container")}"

  build:
    name: Test for PyTorch Compatibility
    needs: matrix_preparation
    if: github.repository == 'hpcaitech/ColossalAI'
    runs-on: [self-hosted, 8-gpu]
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
    container:
      image: ${{ matrix.container }}
      options: --gpus all --rm -v /dev/shm -v /data/scratch/cifar-10:/data/scratch/cifar-10 -v /data/scratch/llama-tiny:/data/scratch/llama-tiny
    timeout-minutes: 200
    steps:
      - name: Install dependencies
        run: |
          pip install -U pip setuptools==68.2.2 wheel --user

      - uses: actions/checkout@v2
        with:
          repository: hpcaitech/TensorNVMe
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}
          path: TensorNVMe

      - name: Install tensornvme
        run: |
          cd TensorNVMe
          apt update && apt install -y cmake
          pip install -r requirements.txt
          DISABLE_URING=1 pip install -v .
      - uses: actions/checkout@v2
        with:
          ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}

      - name: Download cub for CUDA 10.2
        run: |
          CUDA_VERSION=$(nvcc -V | awk -F ',| ' '/release/{print $6}')

          # check if it is CUDA 10.2
          # download cub
          if [ "$CUDA_VERSION" = "10.2" ]; then
            wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip
            unzip 1.8.0.zip
            cp -r cub-1.8.0/cub/ colossalai/kernel/cuda_native/csrc/kernels/include/
          fi

      - name: Install Colossal-AI
        run: |
          BUILD_EXT=1 pip install -v .
          pip install -r requirements/requirements-test.txt

      - name: Unit Testing
        run: |
          PYTHONPATH=$PWD pytest --durations=0 tests
        env:
          DATA: /data/scratch/cifar-10
          LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
          LLAMA_PATH: /data/scratch/llama-tiny

      - name: Notify Lark
        id: message-preparation
        if: ${{ failure() }}
        run: |
          url=$SERVER_URL/$REPO/actions/runs/$RUN_ID
          msg="Compatibility test failed with $container, please visit $url for details"
          echo $msg
          python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
        env:
          SERVER_URL: ${{github.server_url }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
          container: ${{ matrix.container }}


Publish to Test-PyPI Before Merge

name: Publish to Test-PyPI Before Merge

on:
  pull_request:
    paths:
      - 'version.txt'

jobs:
  build-n-publish:
    if: github.event_name == 'workflow_dispatch' || github.repository == 'hpcaitech/ColossalAI'
    name: Build and publish Python 🐍 distributions 📦 to Test PyPI
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v2

    - uses: actions/setup-python@v2
      with:
        python-version: '3.8.14'

    - name: add timestamp to the version
      id: prep-version
      run: |
        version=$(cat version.txt)
        timestamp=$(date +%s)
        new_version="${version}.post${timestamp}"
        echo $new_version > ./version.txt
        echo "version=$new_version" >> $GITHUB_OUTPUT

    - run: |
        pip install --upgrade pip
        python setup.py sdist build

    # publish to PyPI if executed on the main branch
    - name: Publish package to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
        repository_url: https://test.pypi.org/legacy/
        verbose: true

    - name: Wait for Test-PyPI refresh
      run: sleep 300s
      shell: bash

    - name: Try installation
      run: |
        # we need to install the requirements.txt first
        # as test-pypi may not contain the distributions for libs listed in the txt file
        pip install -r requirements/requirements.txt
        pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.python.org/pypi colossalai==$VERSION
      env:
        VERSION: ${{ steps.prep-version.outputs.version }}


Publish Nightly Version to PyPI

name: Publish Nightly Version to PyPI

on:
  workflow_dispatch:
  schedule:
    - cron:  '0 0 * * 6' # release on every Sunday 00:00 UTC time

jobs:
  publish:
    if: github.repository == 'hpcaitech/ColossalAI'
    name: Build and publish Python 🐍 distributions 📦 to PyPI
    runs-on: ubuntu-latest
    timeout-minutes: 20
    outputs:
      status: ${{ steps.publish.outcome }}
    steps:
    - uses: actions/checkout@v2

    - uses: actions/setup-python@v2
      with:
        python-version: '3.8.14'

    - run: |
        python .github/workflows/scripts/update_setup_for_nightly.py
        python setup.py sdist build

    # publish to PyPI if executed on the main branch
    - name: Publish package to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      id: publish
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}
        verbose: true

  notify:
    name: Notify Lark via webhook
    needs: publish
    runs-on: ubuntu-latest
    if: ${{ always() }} && github.repository == 'hpcaitech/ColossalAI'
    steps:
      - uses: actions/checkout@v2

      - uses: actions/setup-python@v2
        with:
          python-version: '3.8.14'

      - name: Install requests
        run: pip install requests

      - name: Notify Lark
        id: message-preparation
        run: |
          url=$SERVER_URL/$REPO/actions/runs/$RUN_ID

          if [ $STATUS == 'success' ]
          then
            msg="The Colossal-AI nightly version has been successfully released to PyPI."
          else
            msg="Failed to release Colossal-AI nightly version to PyPI, please visit $url for details."
          fi
          echo $msg
          python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
        env:
          SERVER_URL: ${{github.server_url }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
          STATUS: ${{ needs.publish.outputs.status }}


Publish to PyPI

name: Publish to PyPI

on:
  workflow_dispatch:
  pull_request:
    paths:
      - 'version.txt'
    types:
      - closed

jobs:
  build-n-publish:
    if: github.event_name == 'workflow_dispatch' || github.repository == 'hpcaitech/ColossalAI' && github.event.pull_request.merged == true && github.base_ref == 'main'
    name: Build and publish Python 🐍 distributions 📦 to PyPI
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v2

    - uses: actions/setup-python@v2
      with:
        python-version: '3.8.14'

    - run: python setup.py sdist build

    # publish to PyPI if executed on the main branch
    - name: Publish package to PyPI
      id: publish
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}
        verbose: true

  notify:
    name: Notify Lark via webhook
    needs: build-n-publish
    runs-on: ubuntu-latest
    if: ${{ always() }}
    steps:
      - uses: actions/checkout@v2

      - uses: actions/setup-python@v2
        with:
          python-version: '3.8.14'

      - name: Install requests
        run: pip install requests

      - name: Notify Lark
        id: message-preparation
        run: |
          url=$SERVER_URL/$REPO/actions/runs/$RUN_ID

          if [ "$STATUS" == 'success' ]
          then
            msg="The Colossal-AI latest version has been successfully released to PyPI."
          else
            msg="Failed to release Colossal-AI to PyPI, please visit $url for details."
          fi
          echo $msg
          python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
        env:
          SERVER_URL: ${{github.server_url }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
          STATUS: ${{ needs.build-n-publish.result }}


Draft GitHub Release Post

name: Draft GitHub Release Post

on:
  workflow_dispatch:
  pull_request:
    paths:
      - 'version.txt'
    types:
      - closed

jobs:
  release:
    name: Draft Release Post
    if: ( github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true ) && github.repository == 'hpcaitech/ColossalAI'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v2
        with:
          python-version: '3.8.14'
      - name: generate draft
        id: generate_draft
        run: |
          version=v$(cat version.txt)
          pip install requests
          python ./.github/workflows/scripts/generate_release_draft.py --out $PWD/release_draft.md --version $version
          echo "::set-output name=version::$version"
          echo "::set-output name=path::$PWD/release_draft.md"
        env:
          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Create Release
        id: create_release
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: ${{ steps.generate_draft.outputs.version }}
          release_name: Version ${{ steps.generate_draft.outputs.version }} Release Today!
          body_path: ${{ steps.generate_draft.outputs.path }}
          draft: True
          prerelease: false


Check CUDA Extension Build Before Merge

name: Check CUDA Extension Build Before Merge

on:
  workflow_dispatch:
  pull_request:
    paths:
      - 'version.txt'

jobs:
  matrix_preparation:
    name: Prepare Container List
    if: github.repository == 'hpcaitech/ColossalAI'
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3

      - id: set-matrix
        run: |
          cuda_ext=$(cat .cuda_ext.json | tr '\n' ' ')
          echo "matrix=${cuda_ext}" >> $GITHUB_OUTPUT

  build:
    name: Release bdist wheels
    needs: matrix_preparation
    runs-on: [self-hosted, gpu]
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
    container:
      image: ${{ matrix.build.cuda_image }}
      options: --gpus all --rm
    steps:
      - uses: actions/checkout@v2

      - name: Install PyTorch
        run: eval ${{ matrix.build.torch_command }}

      - name: Download cub for CUDA 10.2
        run: |
          CUDA_VERSION=$(nvcc -V | awk -F ',| ' '/release/{print $6}')

          # check if it is CUDA 10.2
          # download cub
          if [ "$CUDA_VERSION" = "10.2" ]; then
            wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip
            unzip 1.8.0.zip
            cp -r cub-1.8.0/cub/ colossalai/kernel/cuda_native/csrc/kernels/include/
          fi

      - name: Build
        run: |
          BUILD_EXT=1 pip install -v .


Check Documentation on PR

name: Check Documentation on PR

on:
  pull_request:
    branches:
      - "main"
      - "develop"
      - "feature/**"
    paths:
      - "docs/**"

jobs:
  check-i18n:
    name: Check docs in diff languages
    if: |
      github.event.pull_request.draft == false &&
      github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI'
    runs-on: ubuntu-latest
    concurrency:
      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-check-i18n
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v2

      - uses: actions/setup-python@v2
        with:
          python-version: "3.8.14"

      - run: python .github/workflows/scripts/check_doc_i18n.py -d docs/source

  check-doc-build:
    name: Test if the docs can be built
    if: |
      github.event.pull_request.draft == false &&
      github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI'
    runs-on: ubuntu-latest
    concurrency:
      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-check-doc
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v2
        with:
          path: "./ColossalAI"
          fetch-depth: 0

      - uses: actions/checkout@v2
        with:
          path: "./ColossalAI-Documentation"
          repository: "hpcaitech/ColossalAI-Documentation"

      - uses: actions/setup-python@v2
        with:
          python-version: "3.8.14"

      # we use the versions in the main branch as the guide for versions to display
      # checkout will give your merged branch
      # therefore, we need to make the merged branch as the main branch
      # there is no main branch, so it's safe to checkout the main branch from the merged branch
      # docer will rebase the remote main branch to the merged branch, so we have to config user
      - name: Make the merged branch main
        run: |
          cd ColossalAI
          git checkout -b main
          git branch -u origin/main
          git config user.name 'github-actions'
          git config user.email 'github-actions@github.com'

      - name: Build docs
        run: |
          cache_dir=ColossalAI-Documentation/doc-build/.cache
          mkdir $cache_dir
          mv ColossalAI $cache_dir
          cd ColossalAI-Documentation
          pip install -v ./doc-build/third_party/hf-doc-builder
          pip install -v ./doc-build
          bash ./scripts/build.sh