Update Contributors

name: Update Contributors

on:
  push:
    branches-ignore:
      - main
    paths:
      - '.all-contributorsrc'

jobs:
  generate-markdown-and-commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Set up tool
        run: npm install -g all-contributors-cli@6.24.0
      - name: Generate file
        id: generate
        run: npx all-contributors generate
      - name: commit-and-push
        id: candp
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          commit_message: '[AUTOMATED] update CONTRIBUTORS.md'
          file_pattern: 'CONTRIBUTORS.md'
          commit_user_name: github-actions[bot]
      - name: Echo Results
        if: steps.candp.outputs.changes_detected == 'true'
        run: echo "changes detected and committed."


base

name: base
on:
  workflow_call:
jobs:
  detect:
    name: detect
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      base: ${{ steps.filter.outputs.base }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            base:
              - pyproject.toml
              - sktime/base/**
  test:
    needs: detect
    name:
    if: ${{ needs.detect.outputs.base == 'true' }}
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        operating-system:
          - macos-13
          - ubuntu-latest
          - windows-latest
    runs-on: ${{ matrix.operating-system }}
    steps:
      - name: checkout pull request branch
        uses: actions/checkout@v4

      - name: run tests on python ${{ matrix.python-version }}
        uses: ./.github/actions/test-base
        with:
          python-version-identifier: ${{ matrix.python-version }}
          sub-sample-estimators: "True"
          test-affected-estimators: "True"

      - name: upload coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},base,incomplete


test all

name: test all
on:
  schedule:
    - cron: 0 0 * * 0
  workflow_dispatch:

jobs:
  code_quality:
    name: code-quality
    runs-on: ubuntu-latest
    steps:
      - name: repository checkout step
        uses: actions/checkout@v4
      - name: python environment step
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: check missing __init__ files
        run: build_tools/fail_on_missing_init_files.sh
        shell: bash

  test_base:
    needs: code_quality
    name: base
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        operating-system:
          - macos-13
          - ubuntu-latest
          - windows-latest
    runs-on: ${{ matrix.operating-system }}
    steps:
      - name: checkout pull request branch
        uses: actions/checkout@v4
      - name: run tests on python ${{ matrix.python-version }}
        uses: ./.github/actions/test-base
        with:
          python-version-identifier: ${{ matrix.python-version }}
          sub-sample-estimators: "False"
          test-affected-estimators: "False"
      - name: upload coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},base,complete

  test_module:
    needs: code_quality
    name: module
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        operating-system:
          - macos-13
          - ubuntu-latest
          - windows-latest
        sktime-component:
          - alignment
          - annotation
          - classification
          - clustering
          - forecasting
          - networks
          - param_est
          - regression
          - transformations
    runs-on: ${{ matrix.operating-system }}
    steps:
      - name: checkout pull request branch
        uses: actions/checkout@v4
      - name: run tests for component ${{ matrix.sktime-component }} on python ${{ matrix.python-version }}
        uses: ./.github/actions/test-component
        with:
          sktime-component-identifier: ${{ matrix.sktime-component }}
          python-version-identifier: ${{ matrix.python-version }}
          sub-sample-estimators: "False"
          test-affected-estimators: "False"
      - name: upload coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},${{ matrix.sktime-component }},complete

  test_other:
    needs: code_quality
    name: other
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        operating-system:
          - macos-13
          - ubuntu-latest
          - windows-latest
    runs-on: ${{ matrix.operating-system }}
    steps:
      - name: checkout pull request branch
        uses: actions/checkout@v4
      - name: update local git tracking reference
        run: git remote set-branches origin main
      - name: update local shallow clone
        run: git fetch --depth 1
      - name: create python virtual environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: install core, test and all soft dependencies
        run: python3 -m pip install .[all_extras_pandas2,tests]
      - name: run unit tests
        run: >-
          python3
          -m pytest
          sktime
          --ignore sktime/base
          --ignore sktime/datasets
          --ignore sktime/alignment
          --ignore sktime/annotation
          --ignore sktime/classification
          --ignore sktime/clustering
          --ignore sktime/forecasting
          --ignore sktime/networks
          --ignore sktime/param_est
          --ignore sktime/regression
          --ignore sktime/transformations
          --matrixdesign False
          --only_changed_modules False
      - name: upload coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},components_without_extras,complete


datasets

name: datasets
on:
  schedule:
    - cron: 0 0 1 * *
  workflow_call:
  workflow_dispatch:
jobs:
  test_internal_data:
    name: onboard
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        operating-system:
          - macos-13
          - ubuntu-latest
          - windows-latest
    runs-on: ${{ matrix.operating-system }}
    steps:
      - name: repository checkout step
        uses: actions/checkout@v4
      - name: update tracking reference step
        run: git remote set-branches origin main
        shell: bash
      - name: shallow clone update step
        run: git fetch --depth 1
        shell: bash
      - name: python environment step
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: dependencies installation step
        run: python3 -m pip install .[tests]
        shell: bash
      - name: unit test step
        run: python3 -m pytest -m "not datadownload" sktime/datasets
        shell: bash
      - name: upload coverage step
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},datasets
  test_external_data:
    name: downloads
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        operating-system:
          - macos-13
          - ubuntu-latest
          - windows-latest
    runs-on: ${{ matrix.operating-system }}
    steps:
      - name: repository checkout step
        uses: actions/checkout@v4
      - name: update tracking reference step
        run: git remote set-branches origin main
        shell: bash
      - name: shallow clone update step
        run: git fetch --depth 1
        shell: bash
      - name: python environment step
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: dependencies installation step
        run: python3 -m pip install .[tests,datasets]
        shell: bash
      - name: unit test step
        run: python3 -m pytest -m "datadownload" sktime/datasets
        shell: bash
      - name: upload coverage step
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},datasets


test module

name: test module
on:
  workflow_call:
jobs:
  detect:
    name: detect
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      module_changes: ${{ steps.filter.outputs.changes }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            alignment:
              - pyproject.toml
              - sktime/base/**
              - sktime/alignment/**
            annotation:
              - pyproject.toml
              - sktime/base/**
              - sktime/annotation/**
            classification:
              - pyproject.toml
              - sktime/base/**
              - sktime/classification/**
            clustering:
              - pyproject.toml
              - sktime/base/**
              - sktime/clustering/**
            forecasting:
              - pyproject.toml
              - sktime/base/**
              - sktime/forecasting/**
            networks:
              - pyproject.toml
              - sktime/base/**
              - sktime/networks/**
            param_est:
              - pyproject.toml
              - sktime/base/**
              - sktime/param_est/**
            regression:
              - pyproject.toml
              - sktime/base/**
              - sktime/regression/**
            transformations:
              - pyproject.toml
              - sktime/base/**
              - sktime/transformations/**
  test:
    needs: detect
    name:
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        operating-system:
          - macos-13
          - ubuntu-latest
          - windows-latest
        sktime-component: ${{ fromJSON(needs.detect.outputs.module_changes) }}
    runs-on: ${{ matrix.operating-system }}
    steps:
      - name: checkout pull request branch
        uses: actions/checkout@v4

      - name: ${{ matrix.sktime-component }}, ${{ matrix.operating-system }}, ${{ matrix.python-version }}
        uses: ./.github/actions/test-component
        with:
          sktime-component-identifier: ${{ matrix.sktime-component }}
          python-version-identifier: ${{ matrix.python-version }}
          sub-sample-estimators: "True"
          test-affected-estimators: "True"

      - name: upload coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},${{ matrix.sktime-component }},incomplete


test other

name: test other
on:
  workflow_call:
jobs:
  detect:
    name: detect
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      other_change: ${{ steps.filter.outputs.other_change }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            other_change:
              - pyproject.toml
              - sktime/base/**
              - sktime/_contrib/**
              - sktime/benchmarking/**
              - sktime/datatypes/**
              - sktime/distances/**
              - sktime/dists_kernels/**
              - sktime/performance_metrics/**
              - sktime/pipeline/**
              - sktime/proba/**
              - sktime/registry/**
              - sktime/series_as_features/**
              - sktime/split/**
              - sktime/tests/**
              - sktime/utils/**
  test_module:
    needs: detect
    name:
    if: ${{ needs.detect.outputs.other_change == 'true' }}
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
        operating-system:
          - macos-13
          - ubuntu-latest
          - windows-latest
    runs-on: ${{ matrix.operating-system }}
    steps:
      - name: checkout pull request branch
        uses: actions/checkout@v4
      - name: update local git tracking reference
        run: git remote set-branches origin main
      - name: update local shallow clone
        run: git fetch --depth 1
      - name: create python virtual environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: install core, test and all soft dependencies
        run: python3 -m pip install .[all_extras_pandas2,tests]
      - name: run unit tests
        run: >-
          python3
          -m pytest
          sktime
          --ignore sktime/base
          --ignore sktime/datasets
          --ignore sktime/alignment
          --ignore sktime/annotation
          --ignore sktime/classification
          --ignore sktime/clustering
          --ignore sktime/forecasting
          --ignore sktime/networks
          --ignore sktime/param_est
          --ignore sktime/regression
          --ignore sktime/transformations
          --matrixdesign True
          --only_changed_modules True
      - name: upload coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},components_without_extras,incomplete


Core Tests.

name: Core Tests.

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

permissions:
  contents: read

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.8.13
        uses: actions/setup-python@v4
        with:
          python-version: '3.8.13'
      - uses: actions/cache@v2
        with:
          path: ${{ env.pythonLocation }}
          key: ${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
      - name: Install dependencies
        run: |
          pip install uv
          uv pip install --system --upgrade -e .[dev]
      - name: Run license checks
        run: |
          ./nerfstudio/scripts/licensing/license_headers.sh --check
      - name: Check notebook cell metadata
        run: |
          python ./nerfstudio/scripts/docs/add_nb_tags.py --check
      - name: Run Ruff Linter
        run: |
          ruff check docs/ nerfstudio/ tests/ --output-format=github
      - name: Run Ruff Formatter
        run: |
          ruff format docs/ nerfstudio/ tests/ --diff
      - name: Run Pyright
        run: |
          pyright
      - name: Test with pytest
        run: |
          pytest


Upload Python Package

# This workflows will upload a Python Package using twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Upload Python Package

on:
  release:
    types: [created]

jobs:
  deploy:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: '3.8'
      - name: Install dependencies
        run: |
          python -m pip install build twine
      - name: Strip unsupported tags in README
        run: |
          sed -i '/<!-- pypi-strip -->/,/<!-- \/pypi-strip -->/d' README.md
      - name: Build and publish
        env:
          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
        run: |
          python -m build
          twine upload --username __token__ --password $PYPI_TOKEN dist/*


Docs

name: Docs
on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

permissions:
  contents: write
jobs:
  docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: '3.9'
      - name: Install dependencies
        run: |
          pip install uv
          uv pip install --system .[docs]
      - name: Sphinx build
        # fail on warnings
        run: |
          sphinx-build docs _build -W --keep-going
      - name: Deploy
        uses: peaceiris/actions-gh-pages@v3
        with:
          publish_branch: gh-pages
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: _build/
          force_orphan: true
          cname: docs.nerf.studio
        if: github.event_name != 'pull_request'


Tests And Linting

name: Tests And Linting

on:
  pull_request:
  merge_group:
  push:
    branches:
      - main
      - v1.51

jobs:
  validate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.8"

      - name: Install Pre-Commit
        run: python -m pip install pre-commit

      - name: Load cached Pre-Commit Dependencies
        id: cached-pre-commit-dependencies
        uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit/
          key: pre-commit-4|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}

      - name: Execute Pre-Commit
        run: pre-commit run --show-diff-on-failure --color=always --all-files

  mypy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.8"
          allow-prereleases: true

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: "3.8"
          allow-python-prereleases: false
          cache: true
          cache-dependency-path: |
            ./pdm.lock

      - name: Install dependencies
        run: pdm install -G:all

      - name: Run mypy
        run: pdm run mypy

  pyright:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.8"
          allow-prereleases: true

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: "3.8"
          allow-python-prereleases: false
          cache: true
          cache-dependency-path: |
            ./pdm.lock

      - name: Install dependencies
        run: pdm install -G:all

      - name: Run pyright
        run: pdm run pyright

  slotscheck:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.8"
          allow-prereleases: false

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: "3.8"
          allow-python-prereleases: false
          cache: true
          cache-dependency-path: |
            ./pdm.lock

      - name: Install dependencies
        run: pdm install -G:all

      - name: Run slotscheck
        run: pdm run slotscheck litestar

  test:
    name: "test (${{ matrix.python-version }})"
    strategy:
      fail-fast: true
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
    uses: ./.github/workflows/test.yml
    with:
      coverage: ${{ (matrix.python-version == '3.12' || matrix.python-version == '3.8') }}
      python-version: ${{ matrix.python-version }}

  test_integration:
    name: Test server integration
    runs-on: ubuntu-latest
    strategy:
      matrix:
        uvicorn-version: ["uvicorn<0.27.0", "uvicorn>=0.27.0"]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: 3.11
          allow-python-prereleases: false
          cache: true
          cache-dependency-path: |
            ./pdm.lock

      - name: Install dependencies
        run: |
          pdm install -G:all
          pip install -U "${{ matrix.uvicorn-version }}"

      - name: Set PYTHONPATH
        run: echo "PYTHONPATH=$PWD" >> $GITHUB_ENV

      - name: Test
        run: pdm run pytest tests -m server_integration

  test-platform-compat:
    if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'test platform compat')
    strategy:
      fail-fast: false
      matrix:
        os: ["macos-latest", "windows-latest"]
    uses: ./.github/workflows/test.yml
    with:
      python-version: "3.12"
      os: ${{ matrix.os }}
      timeout: 30

  sonar:
    needs:
      - test
      - validate
    if: github.event.pull_request.head.repo.fork == false && github.repository_owner == 'litestar-org'
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: SonarCloud Scan
        uses: sonarsource/sonarcloud-github-action@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

  codeql:
    needs:
      - test
      - validate
    runs-on: ubuntu-latest
    permissions:
      security-events: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Initialize CodeQL Without Dependencies
        uses: github/codeql-action/init@v3
        with:
          setup-python-dependencies: false

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3

  build-docs:
    needs:
      - validate
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          allow-prereleases: true

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: "3.12"
          allow-python-prereleases: false
          cache: true
          cache-dependency-path: |
            ./pdm.lock

      - name: Install dependencies
        run: pdm install -G:all

      - name: Build docs
        run: pdm run make docs

      - name: Check docs links
        env:
          LITESTAR_DOCS_IGNORE_MISSING_EXAMPLE_OUTPUT: 1
        run: pdm run make docs-linkcheck

      - name: Save PR number
        env:
          PR_NUMBER: ${{ github.event.number }}
        run: echo $PR_NUMBER > .pr_number

      - name: Upload artifact
        uses: actions/upload-artifact@v3
        with:
          name: docs-preview
          path: |
            docs/_build/html
            .pr_number

  test_minimal_app:
    name: Test Minimal Application with Base Dependencies
    runs-on: ubuntu-latest
    env:
      python_version: "3.12"
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: "3.12"
          allow-python-prereleases: false
          cache: true
          cache-dependency-path: |
            ./pdm.lock

      - name: Install dependencies
        run: pdm install

      - name: Set pythonpath
        run: echo "PYTHONPATH=$PWD" >> $GITHUB_ENV

      - name: Test
        run: mv tests/examples/test_hello_world.py test_hello_world.py && pdm run pytest test_hello_world.py

  test_pydantic_1_app:
    name: Test Minimal Pydantic 1 application
    runs-on: ubuntu-latest
    env:
      python_version: "3.12"
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: "3.12"
          allow-python-prereleases: false
          cache: true
          cache-dependency-path: |
            ./pdm.lock

      - name: Install dependencies
        run: |
          pdm install
          pdm run python -m ensurepip
          pdm run python -m pip install "pydantic==1.*"

      - name: Set pythonpath
        run: echo "PYTHONPATH=$PWD" >> $GITHUB_ENV

      - name: Test
        run: pdm run coverage run --branch -m unittest test_apps/pydantic_1_app.py

      - name: Rename coverage file
        run: mv .coverage* .coverage.pydantic_v1

      - uses: actions/upload-artifact@v3
        with:
          name: coverage-data
          path: .coverage.pydantic_v1

  upload-test-coverage:
    runs-on: ubuntu-latest
    needs:
      - test
      - test_pydantic_1_app
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Download Artifacts
        uses: actions/download-artifact@v3
        with:
          name: coverage-data

      - name: Combine coverage files
        run: |
          python -Im pip install coverage covdefaults
          python -Im coverage combine
          python -Im coverage xml -i

      - name: Fix coverage file name
        run: sed -i "s/home\/runner\/work\/litestar\/litestar/github\/workspace/g" coverage.xml

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v4
        with:
          files: coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}


CodeQL scheduled

name: CodeQL scheduled
on:
  schedule:
    - cron: "0 4 * * *"
jobs:
  codeql:
    runs-on: ubuntu-latest
    permissions:
      security-events: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: "main"
      - name: Initialize CodeQL With Dependencies
        uses: github/codeql-action/init@v3
      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3


Deploy documentation preview

name: Deploy documentation preview
on:
  workflow_run:
    workflows: [Tests And Linting]
    types: [completed]

jobs:
  deploy:
    if: ${{ github.event.workflow_run.conclusion == 'success' &&  github.event.workflow_run.event == 'pull_request' }}
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write

    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Download artifact
        uses: dawidd6/action-download-artifact@v6
        with:
          workflow_conclusion: success
          run_id: ${{ github.event.workflow_run.id }}
          path: docs-preview
          name: docs-preview

      - name: Set PR number
        run: echo "PR_NUMBER=$(cat docs-preview/.pr_number)" >> $GITHUB_ENV

      - name: Deploy docs preview
        uses: JamesIves/github-pages-deploy-action@v4
        with:
          folder: docs-preview/docs/_build/html
          token: ${{ secrets.DOCS_PREVIEW_DEPLOY_TOKEN }}
          repository-name: litestar-org/litestar-docs-preview
          clean: false
          target-folder: ${{ env.PR_NUMBER }}
          branch: gh-pages

      - uses: actions/github-script@v7
        env:
          PR_NUMBER: ${{ env.PR_NUMBER }}
        with:
          script: |
            const issue_number = process.env.PR_NUMBER
            const body = "Documentation preview will be available shortly at https://litestar-org.github.io/litestar-docs-preview/" + issue_number

            const opts = github.rest.issues.listComments.endpoint.merge({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: issue_number,
            });

            const comments = await github.paginate(opts)

            for (const comment of comments) {
              if (comment.user.id === 41898282 && comment.body === body) {
                await github.rest.issues.deleteComment({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  comment_id: comment.id
                })
              }
            }

            await github.rest.issues.createComment({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  issue_number: issue_number,
                  body: body,
            })


Documentation Building

name: Documentation Building

on:
  release:
    types: [published]
  push:
    branches:
      - main
      - develop
      - v3.0

jobs:
  docs:
    permissions:
      contents: write
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: "3.12"
          allow-python-prereleases: false
          cache: true
          cache-dependency-path: |
            ./pdm.lock

      - name: Install dependencies
        run: pdm sync -G:all

      - name: Fetch gh pages
        run: git fetch origin gh-pages --depth=1

      - name: Build release docs
        run: pdm run python tools/build_docs.py docs-build
        if: github.event_name == 'release'

      - name: Build docs (main branch)
        run: pdm run python tools/build_docs.py docs-build --version main
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'

      - name: Build docs (develop branch)
        run: pdm run python tools/build_docs.py docs-build --version develop
        if: github.event_name == 'push' && github.ref == 'refs/heads/develop'

      - name: Build docs (v3.0 branch)
        run: pdm run python tools/build_docs.py docs-build --version 3-dev
        if: github.event_name == 'push' && github.ref == 'refs/heads/v3.0'

      - name: Deploy
        uses: JamesIves/github-pages-deploy-action@v4
        with:
          folder: docs-build


Notify released issues

name: Notify released issues
on:
  workflow_call:
    inputs:
      release_tag:
        type: string
        required: true

jobs:
  notify:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Get released issues
        id: get-released-issues
        run:
          echo "issues=$(python ./.github/workflows/notify_released_issues/get_closed_issues.py ${{ inputs.release_tag }})" >> "$GITHUB_OUTPUT"

      - uses: actions/github-script@v7
        env:
          CLOSED_ISSUES: ${{ steps.get-released-issues.outputs.issues }}
        with:
          script: |
            const script = require('./.github/workflows/notify_released_issues/notify.js')
            await script({github, context, core})


Pull Request Labeler

name: "Pull Request Labeler"

on:
  pull_request_target:

jobs:
  apply-labels:
    permissions:
      contents: read
      pull-requests: write
      checks: write
      statuses: write
    runs-on: ubuntu-latest
    steps:
    - uses: fuxingloh/multi-labeler@v4
      with:
        github-token: "${{ secrets.GITHUB_TOKEN }}"

  distinguish-pr-origin:
    needs: apply-labels
    if: ${{ always() }}
    permissions:
      pull-requests: write
    runs-on: ubuntu-latest
    steps:
      - uses: actions/github-script@v7
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const maintainers = [
              'JacobCoffee', 'provinzkraut', 'cofin',
              'peterschutt', 'Alc-Alc', 'guacs',
              'dependabot[bot]', 'all-contributors[bot]'
            ]
            if (maintainers.includes(context.payload.sender.login)) {
              github.rest.issues.addLabels({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                labels: ['pr/internal']
              })
            } else {
              github.rest.issues.addLabels({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                labels: ['pr/external', 'Triage Required :hospital:']
              })
            }


PR merged

name: "PR merged"

on:
  pull_request:
    types:
      - closed
    branches:
      - develop
      - main

jobs:
  close_and_notify:
    name: Close issues and notify
    if: github.event.pull_request.merged == true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/github-script@v7
        with:
          script: |
            const prNumber = context.payload.number
            const branch = context.baseRef
            const isDevelop = branch === "develop"
            const commentBody = `<!--closing-comment-->\nThis issue has been closed in #${prNumber}. The change will be included in the upcoming ${isDevelop ? "minor" : "patch"} release.`

            const query = `query($number: Int!, $owner: String!, $name: String!) { repository(owner: $owner, name: $name) {
                pullRequest(number: $number) {
                      id
                      closingIssuesReferences (first: 10) { edges { node { number } } }
                      }
                  }
              }`
            const res = await github.graphql(query, {number: prNumber, owner: context.repo.owner, name: context.repo.repo})
            const linkedIssues = res.repository.pullRequest.closingIssuesReferences.edges.map(
              edge => edge.node.number
            )

            for (const issueNumber of linkedIssues) {
              const res = await github.rest.issues.update({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  issue_number: issueNumber,
                  state: "closed",
                  state_reason: "completed"
              })
              if (res.status === 200) {
                await github.rest.issues.createComment({
                      owner: context.repo.owner,
                      repo: context.repo.repo,
                      issue_number: issueNumber,
                      body: commentBody,
                })
              }
            }


Lint PR Title

name: "Lint PR Title"

on:
  pull_request_target:
    types:
      - opened
      - edited
      - synchronize

permissions:
  pull-requests: read

jobs:
  main:
    name: Validate PR title
    runs-on: ubuntu-latest
    steps:
      - uses: amannn/action-semantic-pull-request@v5
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Latest Release

name: Latest Release

on:
  release:
    types: [published]
  workflow_dispatch:

jobs:

  publish-release:
    name: upload release to PyPI
    runs-on: ubuntu-latest
    permissions:
      id-token: write
    environment: release
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: "3.12"
          cache: true

      - name: Build package
        run: pdm build

      - name: Publish package distributions to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1

  notify-issues:
    needs: publish-release
    name: Notify issues
    uses: ./.github/workflows/notify-released-issues.yml
    with:
      release_tag: ${{ github.event.release.tag_name }}


Test

name: Test

on:
  workflow_call:
    inputs:
      python-version:
        required: true
        type: string
      coverage:
        required: false
        type: boolean
        default: false
      os:
        required: false
        type: string
        default: "ubuntu-latest"
      timeout:
        required: false
        type: number
        default: 10

jobs:
  test:
    runs-on: ${{ inputs.os }}
    timeout-minutes: ${{ inputs.timeout }}
    defaults:
      run:
        shell: bash
    steps:
      - name: Check out repository
        uses: actions/checkout@v4

      - name: Set up python ${{ inputs.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ inputs.python-version }}

      - uses: pdm-project/setup-pdm@v4
        name: Set up PDM
        with:
          python-version: ${{ inputs.python-version }}
          allow-python-prereleases: false
          cache: true
          cache-dependency-path: |
            ./pdm.lock

      - name: Install dependencies
        run: pdm install -G:all

      - name: Set PYTHONPATH
        run: echo "PYTHONPATH=$PWD" >> $GITHUB_ENV

      - name: Test
        if: ${{ !inputs.coverage }}
        run: pdm run pytest docs/examples tests -n auto

      - name: Test with coverage
        if: inputs.coverage
        run: pdm run pytest docs/examples tests -n auto --cov

      - name: Rename coverage file
        if: inputs.coverage
        run: mv .coverage .coverage.${{ inputs.python-version }}

      - uses: actions/upload-artifact@v3
        if: inputs.coverage
        with:
          name: coverage-data
          path: .coverage.${{ inputs.python-version }}


Build-and-test

name: Build-and-test

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  build-and-test:
    runs-on: ubuntu-latest

    steps:
      - id: checkout
        uses: actions/checkout@v2

      - id: dockerx
        name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v2

      - id: build
        name: Build dev docker
        uses: docker/build-push-action@v2
        with:
          context: .
          file: ./docker/Dockerfile
          push: false
          load: true
          tags: codiumai/pr-agent:test
          cache-from: type=gha,scope=dev
          cache-to: type=gha,mode=max,scope=dev
          target: test

      - id: test
        name: Test dev docker
        run: |
          docker run --rm codiumai/pr-agent:test pytest -v


docs-ci

name: docs-ci
on:
  push:
    branches:
      - main
      - add-docs-portal
    paths:
      - docs/**
permissions:
  contents: write
jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Configure Git Credentials
        run: |
          git config user.name github-actions[bot]
          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
      - uses: actions/setup-python@v5
        with:
          python-version: 3.x
      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
      - uses: actions/cache@v4
        with:
          key: mkdocs-material-${{ env.cache_id }}
          path: .cache
          restore-keys: |
            mkdocs-material-
      - run: pip install mkdocs-material
      - run: pip install "mkdocs-material[imaging]"
      - run: pip install mkdocs-glightbox
      - run: mkdocs gh-deploy -f docs/mkdocs.yml --force


PR-Agent

# This workflow enables developers to call PR-Agents `/[actions]` in PR's comments and upon PR creation.
# Learn more at https://www.codium.ai/pr-agent/
# This is v0.2 of this workflow file

name: PR-Agent

on:
# pull_request:
# issue_comment:
  workflow_dispatch:

permissions:
  issues: write
  pull-requests: write

jobs:
  pr_agent_job:
    runs-on: ubuntu-latest
    name: Run pr agent on every pull request
    steps:
      - name: PR Agent action step
        id: pragent
        uses: Codium-ai/pr-agent@main
        env:
          OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
          OPENAI_ORG: ${{ secrets.OPENAI_ORG }} # optional
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PINECONE.API_KEY: ${{ secrets.PINECONE_API_KEY }}
          PINECONE.ENVIRONMENT: ${{ secrets.PINECONE_ENVIRONMENT }}
          GITHUB_ACTION_CONFIG.AUTO_DESCRIBE: true
          GITHUB_ACTION_CONFIG.AUTO_REVIEW: true
          GITHUB_ACTION_CONFIG.AUTO_IMPROVE: true


Update Contributors

name: Update Contributors

on:
  push:
    branches:
      - main
    paths:
      - '.all-contributorsrc'

jobs:
  generate-markdown-and-commit:
    runs-on: ubuntu-20.04

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: 18

      - name: Setup all-contributors-cli
        run: npm install -g all-contributors-cli

      - name: Generate CONTRIBUTORS.md
        id: generate
        run: npx all-contributors generate

      - uses: actions/create-github-app-token@v1
        id: app-token
        with:
          app-id: ${{ vars.PR_APP_ID }}
          private-key: ${{ secrets.PR_APP_KEY }}

      - uses: peter-evans/create-pull-request@v6
        with:
          token: ${{ steps.app-token.outputs.token }}
          commit-message: "Automated `CONTRIBUTORS.md` update"
          branch: update_contributors
          title: "[MNT] Automated `CONTRIBUTORS.md` update"
          body: "Automated update to CONTRIBUTORS.md caused by an update to the `.all-contributorsrc` file."
          labels: maintenance, no changelog


Release

name: Release

on:
  release:
    types:
      - published
  workflow_dispatch:

jobs:
  check-manifest:
    runs-on: ubuntu-20.04

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - uses: pre-commit/action@v3.0.1
        with:
          extra_args: check-manifest --hook-stage manual

  build-project:
    needs: check-manifest
    runs-on: ubuntu-20.04

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Build project
        run: |
          python -m pip install build
          python -m build

      - name: Store build files
        uses: actions/upload-artifact@v4
        with:
          name: dist
          path: dist/*
          retention-days: 5

  test-wheels:
    needs: build-project
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-20.04, macOS-13, windows-2022 ]
        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - uses: actions/download-artifact@v4
        with:
          name: dist
          path: dist

      - if: runner.os == 'Windows'
        name: Windows wheel filename
        run: echo "WHEELNAME=$(ls ./dist/aeon-*-none-any.whl)" >> $env:GITHUB_ENV
      - if: runner.os != 'Windows'
        name: Unix wheel filename
        run: echo "WHEELNAME=$(ls ./dist/aeon-*-none-any.whl)" >> $GITHUB_ENV

      - if: runner.os == 'Windows'
        name: Windows install
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install "${env:WHEELNAME}[all_extras,dev]"
      - if: runner.os != 'Windows'
        name: Unix install
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install "${{ env.WHEELNAME }}[all_extras,dev]"

      - name: Tests
        run: python -m pytest -n logical

  upload-wheels:
    needs: test-wheels
    runs-on: ubuntu-20.04

    steps:
      - uses: actions/download-artifact@v4
        with:
          name: dist
          path: dist

      - name: Publish package to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_TOKEN }}


Periodic Tests

name: Periodic Tests

on:
  schedule:
    # every day at 1:30 AM UTC
    - cron:  "30 1 * * *"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  check-manifest:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Run check-manifest
        uses: pre-commit/action@v3.0.1
        with:
          extra_args: check-manifest --hook-stage manual

  pre-commit:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Run pre-commit
        uses: pre-commit/action@v3.0.1
        with:
          extra_args: --all-files

  run-notebook-examples:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Use numba cache to set env variables but not restore cache
        uses: ./.github/actions/numba_cache
        with:
          cache_name: "run-notebook-examples"
          runner_os: ${{ runner.os }}
          python_version: "3.10"
          restore_cache: "false"

      - name: Install dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install .[all_extras,binder,dev]

      - name: Run example notebooks
        run: build_tools/run_examples.sh false
        shell: bash

      - name: Save new cache
        uses: actions/cache/save@v4
        with:
          path: ${{ github.workspace }}/.numba_cache
          # Save cache with the current date (ENV set in numba_cache action)
          key: numba-run-notebook-examples-${{ runner.os }}-3.10-${{ env.CURRENT_DATE }}

  test-no-soft-deps:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Use numba cache to set env variables but not restore cache
        uses: ./.github/actions/numba_cache
        with:
          cache_name: "test-no-soft-deps"
          runner_os: ${{ runner.os }}
          python_version: "3.10"
          restore_cache: "false"

      - name: Install aeon and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install .[dev]

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        run: python -m pytest -n logical

      - name: Save new cache
        uses: actions/cache/save@v4
        with:
          path: ${{ github.workspace }}/.numba_cache
          # Save cache with the current date (ENV set in numba_cache action)
          key: numba-test-no-soft-deps-${{ runner.os }}-3.10-${{ env.CURRENT_DATE }}

  pytest:
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-20.04, macOS-13, windows-2022 ]
        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Use numba cache to set env variables but not restore cache
        uses: ./.github/actions/numba_cache
        with:
          cache_name: "pytest"
          runner_os: ${{ runner.os }}
          python_version: ${{ matrix.python-version }}
          restore_cache: "false"

      - name: Install aeon and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install .[all_extras,dev]

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        run: python -m pytest -n logical

      - name: Save new cache
        uses: actions/cache/save@v4
        with:
          path: ${{ github.workspace }}/.numba_cache
          # Save cache with the current date (ENV set in numba_cache action)
          key: numba-pytest-${{ runner.os }}-${{ matrix.python-version}}-${{ env.CURRENT_DATE }}

  codecov:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Disable Numba JIT
        run: echo "NUMBA_DISABLE_JIT=1" >> $GITHUB_ENV

      - name: Install aeon and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install .[all_extras,unstable_extras,dev]

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        run: python -m pytest -n logical --cov=aeon --cov-report=xml --timeout 1800

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


PR Examples

name: PR Examples

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
    paths:
      - "examples/**"
      - "aeon/**"
      - ".github/workflows/**"
      - "pyproject.toml"

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  run-notebook-examples:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Restore numba cache
        uses: ./.github/actions/numba_cache
        with:
          cache_name: "run-notebook-examples"
          runner_os: ${{ runner.os }}
          python_version: "3.10"

      - name: Install aeon and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install .[all_extras,binder,dev]

      - name: Run example notebooks
        run: build_tools/run_examples.sh ${{ github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'full examples run') }}
        shell: bash


PR pytest

name: PR pytest

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
    paths:
      - "aeon/**"
      - ".github/workflows/**"
      - "pyproject.toml"

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  test-no-soft-deps:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Restore numba cache
        uses: ./.github/actions/numba_cache
        with:
          cache_name: "test-no-soft-deps"
          runner_os: ${{ runner.os }}
          python_version: "3.10"

      - name: Install aeon and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install .[dev]

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        run: python -m pytest -n logical -k 'not TestAll'

  pytest:
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-20.04, macOS-13, windows-2022 ]
        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
        # skip python versions unless the PR has the 'full pytest actions' label
        pr-testing:
          - ${{ (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'full pytest actions')) }}
        exclude:
          - pr-testing: true
            python-version: "3.9"
          - pr-testing: true
            python-version: "3.11"

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Restore numba cache
        uses: ./.github/actions/numba_cache
        with:
          cache_name: "pytest"
          runner_os: ${{ runner.os }}
          python_version: ${{ matrix.python-version }}

      - name: Install aeon and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install .[all_extras,dev]

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        # run the full test suit if a PR has the 'full pytest actions' label
        run: python -m pytest -n logical --prtesting ${{ github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'full pytest actions') }}

  codecov:
    # run the code coverage job if a PR has the 'codecov actions' label
    if: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'codecov actions') }}

    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Disable Numba JIT
        run: echo "NUMBA_DISABLE_JIT=1" >> $GITHUB_ENV

      - name: Install aeon and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 30
          max_attempts: 3
          command: python -m pip install .[all_extras,unstable_extras,dev]

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        run: python -m pytest -n logical --cov=aeon --cov-report=xml --timeout 1800

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


PR pre-commit

name: PR pre-commit

on:
  pull_request_target:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  pre-commit:
    runs-on: ubuntu-20.04

    steps:
      - name: Create app token
        uses: actions/create-github-app-token@v1
        id: app-token
        with:
          app-id: ${{ vars.PR_APP_ID }}
          private-key: ${{ secrets.PR_APP_KEY }}

      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          token: ${{ steps.app-token.outputs.token }}

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Get changed files
        uses: tj-actions/changed-files@v44
        id: changed-files

      - name: List changed files
        run: echo '${{ steps.changed-files.outputs.all_changed_files }}'

      - if: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'full pre-commit') }}
        name: Full pre-commit
        uses: pre-commit/action@v3.0.1
        with:
          extra_args: --all-files
      - if: ${{ github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'full pre-commit') }}
        name: Local pre-commit
        uses: pre-commit/action@v3.0.1
        with:
          extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }}

      - if: ${{ failure() && github.event_name == 'pull_request' && github.event.pull_request.draft == false && !contains(github.event.pull_request.labels.*.name, 'stop pre-commit fixes')}}
        name: Push pre-commit fixes
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          commit_message: Automatic `pre-commit` fixes
          commit_user_name: aeon-actions-bot[bot]


PR Opened

name: PR Opened

on:
  pull_request_target:
    types: [opened]

permissions:
  contents: read
  pull-requests: write
  issues: write

jobs:
  # based on the scikit-learn 1.3.1 PR labelers
  labeler:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          sparse-checkout: build_tools

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install PyGithub
        run: pip install -Uq PyGithub

      - name: Create app token
        uses: actions/create-github-app-token@v1
        id: app-token
        with:
          app-id: ${{ vars.PR_APP_ID }}
          private-key: ${{ secrets.PR_APP_KEY }}

      - name: Label pull request
        id: label-pr
        run: python build_tools/pr_labeler.py
        env:
          CONTEXT_GITHUB: ${{ toJson(github) }}
          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}

      - name: Write pull request comment
        run: python build_tools/pr_open_commenter.py
        env:
          CONTEXT_GITHUB: ${{ toJson(github) }}
          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
          TITLE_LABELS: ${{ steps.label-pr.outputs.title-labels }}
          TITLE_LABELS_NEW: ${{ steps.label-pr.outputs.title-labels-new }}
          CONTENT_LABELS: ${{ steps.label-pr.outputs.content-labels }}
          CONTENT_LABELS_STATUS: ${{ steps.label-pr.outputs.content-labels-status }}


Update pre-commit Hooks

name: Update pre-commit Hooks

on:
  schedule:
    # every Monday at 12:30 AM UTC
    - cron:  "30 0 * * 1"
  workflow_dispatch:

jobs:
  pre-commit-auto-update:
    runs-on: ubuntu-20.04

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - uses: browniebroke/pre-commit-autoupdate-action@v1

      - if: always()
        name: Create app token
        uses: actions/create-github-app-token@v1
        id: app-token
        with:
          app-id: ${{ vars.PR_APP_ID }}
          private-key: ${{ secrets.PR_APP_KEY }}

      - if: always()
        uses: peter-evans/create-pull-request@v6
        with:
          token: ${{ steps.app-token.outputs.token }}
          commit-message: "Automated `pre-commit` hook update"
          branch: pre-commit-hooks-update
          title: "[MNT] Automated `pre-commit` hook update"
          body: "Automated weekly update to `.pre-commit-config.yaml` hook versions."
          labels: maintenance, full pre-commit, no changelog


GitHub Maintenance

name: GitHub Maintenance

on:
  schedule:
    # every Monday at 01:00 AM UTC
    - cron:  "0 1 * * 1"
  workflow_dispatch:

permissions:
  issues: write
  contents: write

jobs:
  stale_branches:
    runs-on: ubuntu-20.04

    steps:
      - name: Create app token
        uses: actions/create-github-app-token@v1
        id: app-token
        with:
          app-id: ${{ vars.PR_APP_ID }}
          private-key: ${{ secrets.PR_APP_KEY }}

      - name: Stale Branches
        uses: crs-k/stale-branches@v5
        with:
          repo-token: ${{ steps.app-token.outputs.token }}
          days-before-stale: 140
          days-before-delete: 175
          comment-updates: true
          tag-committer: true
          stale-branch-label: "stale branch"
          compare-branches: "info"
          pr-check: true


Issue Comment Posted

name: Issue Comment Posted

on:
  issue_comment:
    types: [created]

jobs:
  self-assign:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          sparse-checkout: build_tools

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install PyGithub
        run: pip install -Uq PyGithub

      - name: Create app token
        uses: actions/create-github-app-token@v1
        id: app-token
        with:
          app-id: ${{ vars.PR_APP_ID }}
          private-key: ${{ secrets.PR_APP_KEY }}

      - name: Assign issue
        run: python build_tools/issue_assign.py
        env:
          CONTEXT_GITHUB: ${{ toJson(github) }}
          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}


Issue Comment Edited

name: Issue Comment Posted

on:
  issue_comment:
    types: [edited]

jobs:
  check-pr-labels:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          sparse-checkout: build_tools

      - name: Setup Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install PyGithub
        run: pip install -Uq PyGithub

      - name: Create app token
        uses: actions/create-github-app-token@v1
        id: app-token
        with:
          app-id: ${{ vars.PR_APP_ID }}
          private-key: ${{ secrets.PR_APP_KEY }}

      - name: Assign issue
        run: python build_tools/comment_pr_labeler.py
        env:
          CONTEXT_GITHUB: ${{ toJson(github) }}
          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}


ci-cd-base

name: ci-cd-base

on:
  workflow_dispatch:

jobs:
  build-base:
    if: github.repository_owner == 'OpenAccess-AI-Collective'
    # this job needs to be run on self-hosted GPU runners...
    runs-on: axolotl-gpu-runner
    strategy:
      fail-fast: false
      matrix:
        include:
          - cuda: "118"
            cuda_version: 11.8.0
            python_version: "3.10"
            pytorch: 2.1.2
            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
          - cuda: "121"
            cuda_version: 12.1.0
            python_version: "3.10"
            pytorch: 2.1.2
            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
          - cuda: "121"
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.1.2
            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
          - cuda: "121"
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.2.2
            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
          - cuda: "121"
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.3.0
            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Docker metadata
        id: metadata
        uses: docker/metadata-action@v3
        with:
          images: winglian/axolotl-base
      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build
        uses: docker/build-push-action@v4
        with:
          context: .
          file: ./docker/Dockerfile-base
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.metadata.outputs.tags }}-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
          labels: ${{ steps.metadata.outputs.labels }}
          build-args: |
            CUDA_VERSION=${{ matrix.cuda_version }}
            CUDA=${{ matrix.cuda }}
            PYTHON_VERSION=${{ matrix.python_version }}
            PYTORCH_VERSION=${{ matrix.pytorch }}
            TORCH_CUDA_ARCH_LIST=${{ matrix.torch_cuda_arch_list }}


Publish Docs

name: Publish Docs
on:
  push:
    branches:
      - main

permissions:
    contents: write
    pages: write

jobs:
    build-deploy:
        runs-on: ubuntu-latest
        steps:
        - name: Check out repository
          uses: actions/checkout@v4
        - name: Set up Quarto
          uses: quarto-dev/quarto-actions/setup@v2
        - name: Setup Python
          uses: actions/setup-python@v3
          with:
            python-version: '3.10'
        - name: install dependencies
          run: |
            python3 -m pip install jupyter
        - name: Publish to GitHub Pages (and render)
          uses: quarto-dev/quarto-actions/publish@v2
          with:
            target: gh-pages
          env:
            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


lint

name: lint
on:
  # check on PRs, and manual triggers
  pull_request:
      paths:
       - '**.py'
       - 'requirements.txt'
       - '.github/workflows/*.yml'
       - "*.md"
       - "examples/**/*.y[a]?ml"
  workflow_dispatch:

jobs:
  pre-commit:
    name: pre-commit
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: 'pip' # caching pip dependencies
      - uses: pre-commit/action@v3.0.0


ci-cd

name: ci-cd

on:
  push:
    branches:
      - "main"
  workflow_dispatch:

jobs:
  build-axolotl:
    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'OpenAccess-AI-Collective' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - cuda: 118
            cuda_version: 11.8.0
            python_version: "3.10"
            pytorch: 2.1.2
            axolotl_extras:
            axolotl_args: "--extra-index-url https://download.pytorch.org/whl/cu118"
            is_latest: true
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.10"
            pytorch: 2.1.2
            axolotl_extras:
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.2.2
            axolotl_extras:
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.3.0
            axolotl_extras:
    runs-on: axolotl-gpu-runner
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Docker metadata
        id: metadata
        uses: docker/metadata-action@v5
        with:
          images: winglian/axolotl
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      # guidance for testing before pushing: https://docs.docker.com/build/ci/github-actions/test-before-push/
      - name: Build and export to Docker
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            BASE_TAG=${{ github.ref_name }}-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}
            CUDA=${{ matrix.cuda }}
            PYTORCH_VERSION=${{ matrix.pytorch }}
            AXOLOTL_ARGS=${{ matrix.axolotl_args }}
          file: ./docker/Dockerfile
          push: ${{ github.event_name != 'pull_request' }}
          tags: |
            ${{ steps.metadata.outputs.tags }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
            ${{ (matrix.is_latest) && format('{0}-latest', steps.metadata.outputs.tags) || '' }}
          labels: ${{ steps.metadata.outputs.labels }}

  build-axolotl-cloud:
    needs: build-axolotl
    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'OpenAccess-AI-Collective' }}
    # this job needs to be run on self-hosted GPU runners...
    strategy:
      matrix:
        include:
          - cuda: 118
            cuda_version: 11.8.0
            python_version: "3.10"
            pytorch: 2.1.2
            axolotl_extras:
            is_latest: true
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.10"
            pytorch: 2.1.2
            axolotl_extras:
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.2.2
            axolotl_extras:
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.3.0
            axolotl_extras:
    runs-on: axolotl-gpu-runner
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Docker metadata
        id: metadata
        uses: docker/metadata-action@v5
        with:
          images: winglian/axolotl-cloud
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            BASE_TAG=${{ github.ref_name }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
            CUDA=${{ matrix.cuda }}
          file: ./docker/Dockerfile-cloud
          push: ${{ github.event_name != 'pull_request' }}
          tags: |
             ${{ steps.metadata.outputs.tags }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
             ${{ (matrix.is_latest) && format('{0}-latest', steps.metadata.outputs.tags) || '' }}
          labels: ${{ steps.metadata.outputs.labels }}

  build-axolotl-cloud-no-tmux:
    needs: build-axolotl
    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'OpenAccess-AI-Collective' }}
    # this job needs to be run on self-hosted GPU runners...
    strategy:
      matrix:
        include:
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.3.0
            axolotl_extras:
    runs-on: axolotl-gpu-runner
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Docker metadata
        id: metadata
        uses: docker/metadata-action@v5
        with:
          images: winglian/axolotl-cloud-term
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            BASE_TAG=${{ github.ref_name }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
            CUDA=${{ matrix.cuda }}
          file: ./docker/Dockerfile-cloud-no-tmux
          push: ${{ github.event_name != 'pull_request' }}
          tags: |
             ${{ steps.metadata.outputs.tags }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
             ${{ (matrix.is_latest) && format('{0}-latest', steps.metadata.outputs.tags) || '' }}
          labels: ${{ steps.metadata.outputs.labels }}


docker-nightlies

name: docker-nightlies

on:
  workflow_dispatch:
  schedule:
    - cron: '0 0 * * *'  # Runs at 00:00 UTC every day

jobs:
  build-axolotl:
    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'OpenAccess-AI-Collective' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - cuda: 118
            cuda_version: 11.8.0
            python_version: "3.10"
            pytorch: 2.1.2
            axolotl_extras:
            axolotl_args: "--extra-index-url https://download.pytorch.org/whl/cu118"
            is_latest: true
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.10"
            pytorch: 2.1.2
            axolotl_extras:
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.2.2
            axolotl_extras:
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.3.0
            axolotl_extras:
    runs-on: axolotl-gpu-runner
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Docker metadata
        id: metadata
        uses: docker/metadata-action@v5
        with:
          images: winglian/axolotl
          tags: |
            type=raw,value={{ branch }}-{{ date 'YYYYMMDD' }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      # guidance for testing before pushing: https://docs.docker.com/build/ci/github-actions/test-before-push/
      - name: Build and export to Docker
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            BASE_TAG=${{ github.ref_name }}-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}
            CUDA=${{ matrix.cuda }}
            PYTORCH_VERSION=${{ matrix.pytorch }}
            AXOLOTL_ARGS=${{ matrix.axolotl_args }}
          file: ./docker/Dockerfile
          push: ${{ github.event_name != 'pull_request' }}
          tags: |
            ${{ steps.metadata.outputs.tags }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
          labels: ${{ steps.metadata.outputs.labels }}

  build-axolotl-cloud:
    needs: build-axolotl
    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'OpenAccess-AI-Collective' }}
    # this job needs to be run on self-hosted GPU runners...
    strategy:
      matrix:
        include:
          - cuda: 118
            cuda_version: 11.8.0
            python_version: "3.10"
            pytorch: 2.1.2
            axolotl_extras:
            is_latest: true
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.10"
            pytorch: 2.1.2
            axolotl_extras:
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.2.2
            axolotl_extras:
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.3.0
            axolotl_extras:
    runs-on: axolotl-gpu-runner
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Docker metadata
        id: metadata
        uses: docker/metadata-action@v5
        with:
          images: winglian/axolotl-cloud
          tags: |
            type=raw,value={{ branch }}-{{ date 'YYYYMMDD' }}
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Build
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            BASE_TAG=${{ github.ref_name }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
            CUDA=${{ matrix.cuda }}
          file: ./docker/Dockerfile-cloud
          push: ${{ github.event_name != 'pull_request' }}
          tags: |
             ${{ steps.metadata.outputs.tags }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
          labels: ${{ steps.metadata.outputs.labels }}


publish pypi

name: publish pypi

on:
  push:
    tags:
      - '*'

jobs:
  pypi-publish:
    name: Upload release to PyPI
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/p/axolotl
    permissions:
      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
    steps:
      - name: Check out repository code
        uses: actions/checkout@v3

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Install dependencies
        run: |
          pip3 install wheel packaging
          pip3 install -e .
          pip3 install -r requirements-tests.txt

      - name: Extract tag name
        id: tag
        run: echo ::set-output name=TAG_NAME::$(echo $GITHUB_REF | cut -d / -f 3)

      - name: Update version in setup.py
        run: |
          sed -i -E 's/version="([0-9.]+)",/version="${{ steps.tag.outputs.TAG_NAME }}",/g' setup.py

      - name: Build a binary wheel
        run: |
          python setup.py sdist bdist_wheel

      - name: Publish package distributions to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1


Tests

name: Tests
on:
  # check on push/merge to main, PRs, and manual triggers
  push:
    branches:
      - "main"
    paths:
      - '**.py'
      - 'requirements.txt'
      - '.github/workflows/*.yml'
  pull_request:
      paths:
       - '**.py'
       - 'requirements.txt'
       - '.github/workflows/*.yml'
  workflow_dispatch:

jobs:
  pre-commit:
    name: pre-commit
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: 'pip' # caching pip dependencies
      - uses: pre-commit/action@v3.0.0

  pytest:
    name: PyTest
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python_version: ["3.10", "3.11"]
    timeout-minutes: 20

    steps:
      - name: Check out repository code
        uses: actions/checkout@v3

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}
          cache: 'pip' # caching pip dependencies

      - name: Install dependencies
        run: |
          pip3 install --upgrade pip
          pip3 install --upgrade packaging
          pip3 install -U -e .
          pip3 install -r requirements-tests.txt

      - name: Run tests
        run: |
          pytest --ignore=tests/e2e/ tests/

  docker-e2e-tests:
    if: github.repository_owner == 'OpenAccess-AI-Collective'
    # this job needs to be run on self-hosted GPU runners...
    runs-on: [self-hosted, modal]
    timeout-minutes: 60
    needs: [pre-commit, pytest]

    strategy:
      fail-fast: false
      matrix:
        include:
          - cuda: 118
            cuda_version: 11.8.0
            python_version: "3.10"
            pytorch: 2.1.2
            axolotl_args: "--extra-index-url https://download.pytorch.org/whl/cu118"
            num_gpus: 1
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.10"
            pytorch: 2.1.2
            num_gpus: 1
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.2.2
            num_gpus: 1
          - cuda: 121
            cuda_version: 12.1.0
            python_version: "3.11"
            pytorch: 2.3.0
            num_gpus: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Install Modal
        run: |
          python -m pip install --upgrade pip
          pip install modal jinja2
      - name: Update env vars
        run: |
          echo "BASE_TAG=main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}" >> $GITHUB_ENV
          echo "PYTORCH_VERSION=${{ matrix.pytorch}}" >> $GITHUB_ENV
          echo "AXOLOTL_ARGS=${{ matrix.axolotl_args}}" >> $GITHUB_ENV
          echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV
          echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV
      - name: Run tests job on Modal
        run: |
          modal run cicd.tests


Docker Publish

name: Docker Publish

on:
  push:
    paths:
      - "langroid/**"
    branches:
      - main

jobs:
  build_amd64:
    name: Build Docker image for amd64
    runs-on: ubuntu-latest

    steps:
      - name: Check out the repo
        uses: actions/checkout@v2

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v1

      - name: Login to Docker Hub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Build and push amd64
        uses: docker/build-push-action@v2
        with:
          context: .
          push: true
          platforms: linux/amd64
          tags: langroid/langroid:latest-amd64

  build_arm64:
    name: Build Docker image for arm64
    runs-on: ubuntu-latest

    steps:
      - name: Check out the repo
        uses: actions/checkout@v2

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v1

      - name: Login to Docker Hub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Build and push arm64
        uses: docker/build-push-action@v2
        with:
          context: .
          push: true
          platforms: linux/arm64
          tags: langroid/langroid:latest-arm64


Doc pages

name: Doc pages

on:
  push:
    paths:
      - "docs/**"
      - "mkdocs.yml"
      - ".github/workflows/mkdocs-deploy.yml"
    branches:
      - main
      - simpler_doc_linking

permissions:
  contents: write

jobs:
  deploy:
    runs-on: ubuntu-latest

    steps:
      - name: Check out repository
        uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11  # Replace with your desired Python version

      - name: Install Poetry
        uses: Gr1N/setup-poetry@v8

      - name: Install Poetry dependencies
        run: |
          poetry --version
          echo "$HOME/.local/bin" >> $GITHUB_PATH
          python -m pip install --upgrade pip

          poetry install --only docs

      - name: Build MkDocs documentation
        # change this to -b gh-pages when ready to make docs public
        run: poetry run mkdocs gh-deploy -b gh-pages --force

#      - name: Deploy to GitHub Pages
#        uses: peaceiris/actions-gh-pages@v3
#        with:
#          deploy_key: ${{ secrets.GITHUB_TOKEN }}
#          publish_branch: gh-pages
#          publish_dir: ./site


Pytest

name: Pytest

on:
  push:
    paths:
      - "langroid/**"
      - "tests/**"
      - ".github/workflows/pytest.yml"
    branches:
      - main  # Replace with your default branch name
  pull_request:
    branches:
      - main  # Replace with your default branch name

jobs:
  cancel:
    if: "contains(github.event.head_commit.message, 'notest') == false"
    name: Cancel Previous Runs
    runs-on: ubuntu-latest
    timeout-minutes: 3
    steps:
      - name: Cancel Previous Workflow Runs
        uses: styfle/cancel-workflow-action@0.9.1
        with:
          access_token: ${{ github.token }}
  pytest:
    if: "contains(github.event.head_commit.message, 'notest') == false"
    runs-on: ubuntu-latest
    services:
      meilisearch:
        image: getmeili/meilisearch:latest
        ports:
          - 7700:7700
        options: --env MEILI_MASTER_KEY=masterKey
      neo4j:
        image: neo4j:latest
        ports:
          - 7474:7474 # HTTP
          - 7687:7687 # Bolt
        env:
          options: --env NEO4J_AUTH=neo4j/password
    env:
      CI: true # allows conditional skipping of tests based on this env var
      OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }}
      GITHUB_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
      REDIS_HOST: ${{ secrets.REDIS_HOST }}
      REDIS_PORT: ${{ secrets.REDIS_PORT }}
      MOMENTO_AUTH_TOKEN: ${{ secrets.MOMENTO_AUTH_TOKEN }}
      MOMENTO_API_KEY: ${{ secrets.MOMENTO_API_KEY }}
      QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY }}
      QDRANT_API_URL: ${{ secrets.QDRANT_API_URL }}
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
      GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}

      # Azure OpenAI env vars
      AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
      AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
      AZURE_OPENAI_API_VERSION: "2023-05-15"
      AZURE_OPENAI_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_DEPLOYMENT_NAME }}
      AZURE_OPENAI_MODEL_NAME: ${{ secrets.AZURE_GPT_MODEL_NAME }}
      AZURE_OPENAI_MODEL_VERSION: ${{ secrets.AZURE_OPENAI_MODEL_VERSION }}
    steps:
    - name: Check out repository
      uses: actions/checkout@v3
    - name: Increase ulimit
      run: ulimit -n 4096
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: 3.11  # Replace with your desired Python version

    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y libmagic1
        sudo apt-get install -y poppler-utils tesseract-ocr

    - name: Install Poetry
      uses: Gr1N/setup-poetry@v8

    - name: Install Poetry dependencies
      run: |
        poetry --version
        echo "$HOME/.local/bin" >> $GITHUB_PATH
        python -m pip install --upgrade pip

        poetry install \
          -E "lancedb momento meilisearch chromadb hf-embeddings unstructured pdf-parsers docx sql" \
          --with dev

    - name: Dump dependencies to file
      run: pip freeze > requirements_github.txt

    - name: Upload dependencies file
      uses: actions/upload-artifact@v3
      with:
        name: github-dependencies
        path: requirements_github.txt

    - name: Run ALL tests with coverage (GPT-4o)
      id: first_test
      env:
        PYTEST_ADDOPTS: "-p no:logging"
      run: |
        poetry run coverage run --source=langroid \
          -m pytest -rf --tb=no --show-capture=no \
          --first-test-file=tests/main/test_task_inf_loop.py \
          --first-test-file=tests/main/test_task.py \
          --first-test-file=tests/main/test_lance_doc_chat_agent.py \
          --first-test-file=tests/main/test_llm.py \
          --first-test-file=tests/main/test_llm_async.py \
          tests/main tests/extras/test_hf_embeddings.py
      continue-on-error: true

    - name: Retry FAILED tests ONLY with gpt-4 instead of gpt-4o (no coverage)
      id: second_test
      if: steps.first_test.outcome == 'failure'
      env:
        PYTEST_ADDOPTS: "-p no:logging"
      run: |
        poetry run coverage run --append --source=langroid \
          -m pytest --m gpt-4  -rf --tb=no --show-capture=no \
          --first-test-file=tests/main/test_task_inf_loop.py \
          --first-test-file=tests/main/test_task.py \
          --first-test-file=tests/main/test_lance_doc_chat_agent.py \
          --first-test-file=tests/main/test_llm.py \
          --first-test-file=tests/main/test_llm_async.py \
          --lf --last-failed-no-failures none \
          tests/main tests/extras/test_hf_embeddings.py
      continue-on-error: true

    - name: Install Pydantic 1.x to pass lancedb tests
      run: poetry add "pydantic<2.0.0"
    - name: Retry FAILED tests ONLY with pydantic 1.x, gpt-4
      if: steps.second_test.outcome == 'failure'
      run: |
        poetry run coverage run --append --source=langroid \
          -m pytest --m gpt-4 \
          --first-test-file=tests/main/test_task_inf_loop.py \
          --first-test-file=tests/main/test_task.py \
          --first-test-file=tests/main/test_lance_doc_chat_agent.py \
          --first-test-file=tests/main/test_llm.py \
          --first-test-file=tests/main/test_llm_async.py \
          --lf --last-failed-no-failures none \
          tests/main tests/extras/test_hf_embeddings.py

    - name: Generate final coverage report from all runs
      run: poetry run coverage report

    - name: Generate XML coverage report
      run: poetry run coverage xml
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v2
      with:
        file: ./coverage.xml
        flags: unittests
        name: codecov-umbrella
        fail_ci_if_error: false
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


Lint

name: Lint

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11  # Replace with your desired Python version

      - name: Install Poetry
        uses: Gr1N/setup-poetry@v8

      - name: Install Poetry dependencies
        run: |
          poetry --version
          echo "$HOME/.local/bin" >> $GITHUB_PATH
          python -m pip install --upgrade pip

          poetry install

      - name: Run black
        run: |
          poetry run black --check .

      - name: Install ruff
        uses: chartboost/ruff-action@v1

      - name: Run mypy
        run: |
          poetry run mypy -p langroid

  # Any additional jobs...


Codacy Security Scan

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.

# This workflow checks out code, performs a Codacy security scan
# and integrates the results with the
# GitHub Advanced Security code scanning feature.  For more information on
# the Codacy security scan action usage and parameters, see
# https://github.com/codacy/codacy-analysis-cli-action.
# For more information on Codacy Analysis CLI in general, see
# https://github.com/codacy/codacy-analysis-cli.

name: Codacy Security Scan

on:
  push:
    branches: [development, master]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [development]
  schedule:
    - cron: "38 5 * * 3"

permissions:
  contents: read

jobs:
  codacy-security-scan:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    name: Codacy Security Scan
    runs-on: ubuntu-latest
    steps:
      # Checkout the repository to the GitHub Actions runner
      - name: Checkout code
        uses: actions/checkout@v4

      # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis
      - name: Run Codacy Analysis CLI
        uses: codacy/codacy-analysis-cli-action@v4.4.1
        with:
          # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository
          # You can also omit the token and run the tools that support default configurations
          project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
          verbose: true
          output: results.sarif
          format: sarif
          # Adjust severity of non-security issues
          gh-code-scanning-compat: true
          # Force 0 exit code to allow SARIF file generation
          # This will handover control about PR rejection to the GitHub side
          max-allowed-issues: 2147483647

      # Upload the SARIF file generated in the previous step
      - name: Upload SARIF results file
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: results.sarif


SonarCloud analysis

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.

# This workflow helps you trigger a SonarCloud analysis of your code and populates
# GitHub Code Scanning alerts with the vulnerabilities found.
# Free for open source project.

# 1. Login to SonarCloud.io using your GitHub account

# 2. Import your project on SonarCloud
#     * Add your GitHub organization first, then add your repository as a new project.
#     * Please note that many languages are eligible for automatic analysis,
#       which means that the analysis will start automatically without the need to set up GitHub Actions.
#     * This behavior can be changed in Administration > Analysis Method.
#
# 3. Follow the SonarCloud in-product tutorial
#     * a. Copy/paste the Project Key and the Organization Key into the args parameter below
#          (You'll find this information in SonarCloud. Click on "Information" at the bottom left)
#
#     * b. Generate a new token and add it to your Github repository's secrets using the name SONAR_TOKEN
#          (On SonarCloud, click on your avatar on top-right > My account > Security
#           or go directly to https://sonarcloud.io/account/security/)

# Feel free to take a look at our documentation (https://docs.sonarcloud.io/getting-started/github/)
# or reach out to our community forum if you need some help (https://community.sonarsource.com/c/help/sc/9)

name: SonarCloud analysis

on:
  push:
    branches: [development, master]
  pull_request:
    branches: [development]
  workflow_dispatch:

permissions:
  pull-requests: read # allows SonarCloud to decorate PRs with analysis results

jobs:
  Analysis:
    runs-on: ubuntu-latest

    steps:
      - name: Analyze with SonarCloud

        uses: SonarSource/sonarcloud-github-action@v2.3.0
        env:
          GITHUB_TOKEN: ${{ github.token }} # Needed to get PR information
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} # Generate a token on Sonarcloud.io, add it to the secrets of this repo with the name SONAR_TOKEN (Settings > Secrets > Actions > add new repository secret)
        with:
          # Additional arguments for the sonarcloud scanner
          args:
            # Unique keys of your project and organization. You can find them in SonarCloud > Information (bottom-left menu)
            # mandatory
            -Dsonar.projectKey=didx-xyz_aries-cloudapi-python
            -Dsonar.organization=didx-xyz
            # Comma-separated paths to directories containing main source files.
            #-Dsonar.sources= # optional, default is project base directory
            # When you need the analysis to take place in a directory other than the one from which it was launched
            #-Dsonar.projectBaseDir= # optional, default is .
            # Comma-separated paths to directories containing test source files.
            #-Dsonar.tests= # optional. For more info about Code Coverage, please refer to https://docs.sonarcloud.io/enriching/test-coverage/overview/
            # Adds more detail to both client and server-side analysis logs, activating DEBUG mode for the scanner, and adding client-side environment variables and system properties to the server-side log of analysis report processing.
            #-Dsonar.verbose= # optional, default is false