Check examples run

name: Check examples run
on:
  push:
    branches:
      - "master"
  pull_request:
    branches:
      - "master"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}

jobs:
  tests:
    runs-on: ubuntu-latest
    timeout-minutes: 10

    steps:
      - name: Checkout repository code
        uses: actions/checkout@v4

      # Setup Python (faster than using Python container)
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.11

      - name: Install Python Kernel
        run: |
          pip install ipykernel
          python -m ipykernel install --user --name python3

      - name: Install dev dependencies
        run: pip install -r requirements-dev.txt

      - name: Install dependencies
        run: pip install -r requirements.txt

      - name: Build package
        run: |
          pip install build
          python -m build

      - name: Install built package
        run: pip install --force-reinstall dist/*.whl

      - name: Remove source
        run: rm -rf ./src

      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - name: Checkout PS
        uses: actions/checkout@v4
        with:
          repository: smogon/pokemon-showdown
          path: pokemon-showdown
          submodules: recursive

      - name: Get last showdown commit hash
        id: showdown-hash
        run: |
          cd pokemon-showdown/
          export hash=`git log -1 --pretty=format:%H`
          echo "hash=$hash" >> $GITHUB_OUTPUT

      - name: Restore server cache
        uses: actions/cache@v4
        with:
          path: pokemon-showdown/node_modules
          key: showdown-python${{ matrix.version }}-${{ steps.showdown-hash.outputs.hash }}
          restore-keys: showdown-python${{ matrix.version }}-

      - name: Install PS dependencies & setup config
        run: |
            cd pokemon-showdown
            npm install
            cp config/config-example.js config/config.js
            sed  -i 's/backdoor = true/backdoor = false/g' config/config.js
            sed  -i 's/simulatorprocesses = 1/simulatorprocesses = 2/g' config/config.js
            sed  -i 's/.workers = 1 = 1/.workers = 2/g' config/config.js

      - name: Start PS
        run: cd pokemon-showdown;node pokemon-showdown start --no-security --max-old-space-size=3000 &

      - name: Wait for server to be up
        run: |
            until $(curl --output /dev/null --silent --head --fail http://localhost:8000); do
              sleep .01
            done
            sleep 1

      - name: Run example notebooks
        run: |
            for notebook in $(find examples -name '*.ipynb'); do
              jupyter nbconvert --to notebook --execute "$notebook" --output temp.ipynb
              if [ $? -ne 0 ]; then
              echo "Execution of $notebook failed"
              exit 1
              fi
              rm examples/temp.ipynb
            done


Lint

name: Lint
on:
  push:
    branches:
      - "master"
  pull_request:
    branches:
      - "master"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}

jobs:
  lint:
    runs-on: ubuntu-latest
    timeout-minutes: 10

    steps:
      - name: Checkout repository code
        uses: actions/checkout@v4

      # Setup Python (faster than using Python container)
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install dev dependencies
        run: pip install -r requirements-dev.txt

      - name: Install dependencies
        run: pip install -r requirements.txt

      - name: Download Pandoc
        run: |
          wget https://github.com/jgm/pandoc/releases/download/3.1.9/pandoc-3.1.9-linux-amd64.tar.gz
          tar xvzf pandoc-3.1.9-linux-amd64.tar.gz --strip-components 2 -C /usr/local/bin

      - name: Run black
        run: black --check src/ unit_tests/ integration_tests/

      - name: Run flake8
        run: flake8 src/ unit_tests/ integration_tests/

      - name: Run isort
        run: isort --check .

      - name: Run pyright
        run: pyright src/

      - name: Build docs
        run: |
            cd docs
            make html


Upload Python Package

# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Upload Python Package

on:
  release:
    types: [created]

jobs:
  deploy:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install setuptools wheel twine build
    - name: Build and publish
      env:
        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
      run: |
        python -m build
        twine upload dist/*


Run tests

name: Run tests
on:
  push:
    branches:
      - "master"
  pull_request:
    branches:
      - "master"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}

jobs:
  tests:
    runs-on: ubuntu-latest
    timeout-minutes: 10

    strategy:
      matrix:
        version: ['3.8', '3.9', '3.10', '3.11']

    steps:
      - name: Checkout repository code
        uses: actions/checkout@v4

      # Setup Python (faster than using Python container)
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.version }}

      - name: Install dev dependencies
        run: pip install -r requirements-dev.txt

      - name: Install dependencies
        run: pip install -r requirements.txt

      - name: Run tests
        run: PYTHONPATH=src pytest -x --cov=src/poke_env unit_tests/

      - name: Convert coverage to XML
        run: coverage xml

      - name: Upload coverage report to Codecov
        uses: codecov/codecov-action@v4
        with:
          file: coverage.xml

      - name: Build package
        run: |
          pip install build
          python -m build

      - name: Install built package
        run: pip install --force-reinstall dist/*.whl

      - name: Remove source
        run: rm -rf ./src

      - name: Run package-based tests
        run: pytest -x unit_tests/

      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - name: Checkout PS
        uses: actions/checkout@v4
        with:
          repository: smogon/pokemon-showdown
          path: pokemon-showdown
          submodules: recursive

      - name: Get last showdown commit hash
        id: showdown-hash
        run: |
          cd pokemon-showdown/
          export hash=`git log -1 --pretty=format:%H`
          echo "hash=$hash" >> $GITHUB_OUTPUT

      - name: Restore server cache
        uses: actions/cache@v4
        with:
          path: pokemon-showdown/node_modules
          key: showdown-python${{ matrix.version }}-${{ steps.showdown-hash.outputs.hash }}
          restore-keys: showdown-python${{ matrix.version }}-

      - name: Install PS dependencies & setup config
        run: |
            cd pokemon-showdown
            npm install
            cp config/config-example.js config/config.js
            sed  -i 's/backdoor = true/backdoor = false/g' config/config.js
            sed  -i 's/simulatorprocesses = 1/simulatorprocesses = 2/g' config/config.js
            sed  -i 's/.workers = 1 = 1/.workers = 2/g' config/config.js

      - name: Start PS
        run: cd pokemon-showdown;node pokemon-showdown start --no-security --max-old-space-size=3000 &

      - name: Wait for server to be up
        run: |
            until $(curl --output /dev/null --silent --head --fail http://localhost:8000); do
              sleep .01
            done
            sleep 1

      - name: Run integration tests
        run: pytest -x integration_tests/


Update data files from smogon database

name: Update data files from smogon database
on:
  workflow_dispatch:
  schedule:
    - cron: 0 0 * * *
jobs:
  Update-data-files:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: master
      - name: Run update script
        run: |
          chmod +x ./scripts/update_all_data.sh
          ./scripts/update_all_data.sh
          chmod -x ./scripts/update_all_data.sh
      - name: Create or update pull request
        uses: peter-evans/create-pull-request@v6
        with:
          commit-message: JSON data update from smogon
          branch: create-pull-request/update-data
          delete-branch: true
          title: Automatic data update from smogon
          body: |
            Automated data update by [update-data.yml](https://github.com/hsahovic/poke-env/tree/master/.github/workflows/update-data.yml)
          assignees: ${{ github.repository_owner }}
          reviewers: ${{ github.repository_owner }}
          author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>


Run Tests

name: Run Tests
on:
  pull_request:
  push:
    branches:
      - main

jobs:
  unit-tests:
    name: Run Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # Select the Python versions to test against
        os: ["ubuntu-latest", "macos-latest"]
        python-version: ["3.10", "3.11"]
      fail-fast: true
    steps:
      - name: Check out the code
        uses: actions/checkout@v3.5.2
        with:
          fetch-depth: 1
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      # Install Poetry
      - name: Install Poetry
        uses: snok/install-poetry@v1.3.3
        with:
          version: 1.4.0

      # Configure Poetry to use the virtual environment in the project
      - name: Setup Poetry
        run: |
          poetry config virtualenvs.in-project true

      # Install the dependencies
      - name: Install Package
        run: |
          poetry install --with tests

      - name: Check docstrings
        run: |
          poetry run xdoctest ./gpjax

      # Run the unit tests and build the coverage report
      - name: Run Tests
        run: poetry run pytest -v --cov=./gpjax --cov-report=xml:./coverage.xml

      - name: Upload code coverage
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          name: gpjax
          flags: unittests
          env_vars: OS,PYTHON


PR Greetings

---
name: PR Greetings

on: [pull_request_target]

permissions:
  pull-requests: write

jobs:
  greeting:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/first-interaction@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          pr-message: >+
            Thank you for opening your first PR into GPJax!

            If you have not heard from us in a while, please feel free to ping
            `@gpjax/developers` or anyone who has commented on the PR.
            Most of our reviewers are volunteers and sometimes things fall
            through the cracks.

            You can also join us [on
            Slack](https://join.slack.com/t/gpjax/shared_invite/zt-1da57pmjn-rdBCVg9kApirEEn2E5Q2Zw) for real-time
            discussion.

            For details on testing, writing docs, and our review process,
            please see [the developer
            guide](https://docs.jaxgaussianprocesses.com/contributing/)

            We strive to be a welcoming and open project. Please follow our
            [Code of
            Conduct](https://github.com/JaxGaussianProcesses/GPJax/blob/main/.github/CODE_OF_CONDUCT.md).


Check linting

name: Check linting
on:
  pull_request:
  push:
    branches:
      - main
jobs:
  ruff:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3.5.2
      - uses: chartboost/ruff-action@v1


Build the documentation

name: Build the documentation

on:
  push:
    branches:
      - main

permissions:
  contents: write

jobs:
  build-docs:
    concurrency: ci-${{ github.ref }}
    name: Build docs (${{ matrix.python-version }}, ${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        os: ["ubuntu-latest"]
        python-version: ["3.10"]

    steps:
      # Grap the latest commit from the branch
      - name: Checkout the branch
        uses: actions/checkout@v3.5.2
        with:
          persist-credentials: false

      # Create a virtual environment
      - name: create Conda environment
        uses: conda-incubator/setup-miniconda@v2
        with:
          auto-update-conda: true
          python-version: ${{ matrix.python-version }}

      # Install katex for math support
      - name: Install NPM
        uses: actions/setup-node@v3
        with:
          node-version: 16
      - name: Install KaTeX
        run: |
          npm install katex
      # Install Poetry and build the documentation
      - name: Install and configure Poetry
        uses: snok/install-poetry@v1
        with:
          version: 1.2.2
          virtualenvs-create: false
          virtualenvs-in-project: false
          installer-parallel: true

      - name: Install LaTex
        run: |
          sudo apt-get update
          sudo apt-get install texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra dvipng cm-super

      - name: Build the documentation with MKDocs
        run: |
          cp docs/examples/gpjax.mplstyle .
          poetry install --all-extras --with docs
          conda install pandoc
          poetry run mkdocs build

      - name: Deploy Page 🚀
        uses: JamesIves/github-pages-deploy-action@v4.4.1
        with:
          branch: gh-pages
          folder: site


Test documentation

name: Test documentation

on:
  pull_request:

jobs:
  test-docs:
    # Functionality for testing documentation builds on multiple OSes and Python versions
    name: Build docs (${{ matrix.python-version }}, ${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    concurrency:
      group: ${{ github.head_ref }}
      cancel-in-progress: true
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        os: ["ubuntu-latest"]
        python-version: ["3.10"]

    steps:
      # Grap the latest commit from the branch
      - name: Checkout the branch
        uses: actions/checkout@v3.5.2
        with:
          persist-credentials: false

      # Create a virtual environment
      - name: create Conda environment
        uses: conda-incubator/setup-miniconda@v2
        with:
          auto-update-conda: true
          python-version: ${{ matrix.python-version }}

      # Install katex for math support
      - name: Install NPM
        uses: actions/setup-node@v3
        with:
          node-version: 16
      - name: Install KaTeX
        run: |
          npm install katex

      - name: Install LaTex
        run: |
          sudo apt-get update
          sudo apt-get install texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra dvipng cm-super

      # Install Poetry and build the documentation
      - name: Install and configure Poetry
        uses: snok/install-poetry@v1
        with:
          version: 1.2.2
          virtualenvs-create: false
          virtualenvs-in-project: false
          installer-parallel: true

      - name: Build the documentation with MKDocs
        run: |
          cp docs/examples/gpjax.mplstyle .
          poetry install --all-extras --with docs
          conda install pandoc
          poetry run mkdocs build


Labeler

name: Labeler

on:
  push:
    branches:
      - main

jobs:
  labeler:
    runs-on: ubuntu-latest
    steps:
      - name: Check out the repository
        uses: actions/checkout@v3.5.2

      - name: Run Labeler
        uses: crazy-max/ghaction-github-labeler@v4.1.0
        with:
          skip-delete: true


Integration Tests

name: Integration Tests
on:
  pull_request:
  push:
    branches:
      - main

jobs:
  integration-tests:
    name: Run Integration Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # Select the Python versions to test against
        os: ["ubuntu-latest", "macos-latest"]
        python-version: ["3.10", "3.11"]
      fail-fast: true
    steps:
      - name: Check out the code
        uses: actions/checkout@v3.5.2
        with:
          fetch-depth: 1
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      # Install Poetry
      - name: Install Poetry
        uses: snok/install-poetry@v1.3.3
        with:
          version: 1.4.0

      # Configure Poetry to use the virtual environment in the project
      - name: Setup Poetry
        run: |
          poetry config virtualenvs.in-project true

      # Install the dependencies
      - name: Install Package
        run: |
          poetry install --all-extras --with docs

      # Run the unit tests and build the coverage report
      - name: Run Integration Tests
        run: poetry run python tests/integration_tests.py


Deploy Docs

name: Deploy Docs

on:
  push:
    branches:
      - master
    paths:
      - docs/**
      - .github/workflows/deploy-docs.yml

defaults:
  run:
    working-directory: docs

jobs:
  deploy:
    name: Deploy docs to GitHub Pages
    runs-on: ubuntu-latest

    steps:
      - name: Clone the repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0 # History of commits needed for lastUpdate author/time

      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version-file: docs/.nvmrc
          cache: npm
          cache-dependency-path: docs/package-lock.json

      - name: Install dependencies
        run: npm ci --omit=dev

      - name: Build website
        run: npm run build

      - name: Deploy to GitHub Pages
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./docs/build
          user_name: nantralplatform-bot
          user_email: nantralplatform@gmail.com


Deploy Server

name: Deploy Server

on:
  push:
    branches:
      - master
    paths:
      - backend/**
      - deployment/**
      - frontend/**
      - email-templates-generator/**
      - .github/workflows/deploy-server.yml

jobs:
  changes:
    name: Check changes on Front end
    runs-on: ubuntu-latest
    outputs:
      backend: ${{ steps.filter.outputs.backend}}
      frontend: ${{ steps.filter.outputs.frontend }}
      email-templates: ${{ steps.filter.outputs.email-templates }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          base: ${{ github.ref }}
          filters: |
            backend:
              - 'backend/**'
            frontend:
              - 'frontend/**'
            email-templates:
              - 'email-templates-generator/**'

  frontend-quality:
    name: Test front end quality
    needs: changes
    if: ${{ needs.changes.outputs.frontend == 'true' }}
    uses: ./.github/workflows/quality-frontend.yml
    secrets: inherit

  backend-quality:
    name: Test Back End Quality
    needs: changes
    if: ${{ needs.changes.outputs.backend == 'true' }}
    uses: ./.github/workflows/quality-backend.yml
    secrets: inherit

  email-templates-quality:
    name: Test Email Templates Quality
    needs: changes
    if: ${{ needs.changes.outputs.email-templates == 'true' }}
    uses: ./.github/workflows/quality-email-templates.yml
    secrets: inherit

  frontend-build:
    name: Build front end
    needs: frontend-quality
    uses: ./.github/workflows/build-frontend.yml
    with:
      target-directory: "nantralPlatform"
    secrets: inherit

  email-templates-build:
    name: Build email templates
    needs: email-templates-quality
    uses: ./.github/workflows/build-email-templates.yml
    with:
      target-directory: "nantralPlatform"
    secrets: inherit

  backup-database:
    name: Create a backup of the database
    runs-on: ubuntu-latest
    steps:
      - name: Backup before deploy
        uses: appleboy/ssh-action@v1.0.0
        with:
          host: ${{ secrets.INSTANCE_ADDRESS }}
          username: "ubuntu"
          key: ${{ secrets.SSH_KEY_DEPLOY }}
          script: |
            cd nantralPlatform/deployment/scripts
            ./db_backup.sh

  deploy:
    name: Deploy on VPS
    needs:
      [frontend-build, email-templates-build, backend-quality, backup-database]
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - name: Deploy to the VPS
        uses: appleboy/ssh-action@master
        with:
          host: ${{ secrets.INSTANCE_ADDRESS }}
          username: "ubuntu"
          key: ${{ secrets.SSH_KEY_DEPLOY }}
          script: |
            cd nantralPlatform/deployment
            git pull
            sudo cp /etc/letsencrypt/live/nantral-platform.fr/privkey.pem /home/ubuntu/nantralPlatform/deployment/certs/key.pem
            sudo cp /etc/letsencrypt/live/nantral-platform.fr/fullchain.pem /home/ubuntu/nantralPlatform/deployment/certs/cert.pem
            sudo docker-compose -f docker-compose.yml -f docker-compose.prod.yml build --no-cache nginx backend celery celery-beat
            docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d


Deploy Staging

name: Deploy Staging

on: workflow_dispatch

jobs:
  frontend-quality:
    name: Test front end quality
    uses: ./.github/workflows/quality-frontend.yml
    secrets: inherit

  backend-quality:
    name: Test Back End Quality
    uses: ./.github/workflows/quality-backend.yml
    secrets: inherit

  email-templates-quality:
    name: Test Email Templates Quality
    uses: ./.github/workflows/quality-email-templates.yml
    secrets: inherit

  frontend-build:
    name: Build front end
    needs: frontend-quality
    uses: ./.github/workflows/build-frontend.yml
    with:
      target-directory: "nantralPlatform-staging"
    secrets: inherit

  email-templates-build:
    name: Build email templates
    needs: email-templates-quality
    uses: ./.github/workflows/build-email-templates.yml
    with:
      target-directory: "nantralPlatform-staging"
    secrets: inherit

  clone-database:
    name: Make a copy of the database
    runs-on: ubuntu-latest
    steps:
      - name: Clone the prod database to staging database
        uses: appleboy/ssh-action@v1.0.0
        with:
          host: ${{ secrets.INSTANCE_ADDRESS }}
          username: "ubuntu"
          key: ${{ secrets.SSH_KEY_DEPLOY }}
          script: |
            cd nantralPlatform/deployment/scripts
            sudo apt-get install python3-psycopg2 -y
            python3 -m venv env
            source env/bin/activate
            pip install -r requirements.txt
            python3 db_staging.py
            deactivate
            rm -r env

  deploy:
    name: Deploy to staging
    needs:
      [frontend-build, email-templates-build, backend-quality, clone-database]
    runs-on: ubuntu-latest
    steps:
      - name: Deploy to the VPS
        uses: appleboy/ssh-action@master
        with:
          host: ${{ secrets.INSTANCE_ADDRESS }}
          username: "ubuntu"
          key: ${{ secrets.SSH_KEY_DEPLOY }}
          script: |
            cd nantralPlatform-staging
            git fetch origin ${{ github.ref_name }}:${{ github.ref_name }}
            git switch ${{ github.ref_name }}
            git reset --hard origin/${{ github.ref_name }}
            cd ../nantralPlatform/deployment
            sudo docker-compose -f docker-compose.yml -f docker-compose.prod.yml build backend-staging
            docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d


Quality Tests

name: Quality Tests

on: pull_request

jobs:
  changes:
    name: Check changes
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      backend: ${{ steps.filter.outputs.backend }}
      frontend: ${{ steps.filter.outputs.frontend }}
      docs: ${{ steps.filter.outputs.docs }}
      email_templates: ${{ steps.filter.outputs.email_templates }}
    steps:
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            backend:
              - 'backend/**'
            frontend:
              - 'frontend/**'
            docs:
              - 'docs/**'
            email_templates:
              - 'email-templates-generator/**'

  backend:
    name: Back End Quality
    needs: changes
    if: ${{ needs.changes.outputs.backend == 'true' }}
    uses: ./.github/workflows/quality-backend.yml
    secrets: inherit

  frontend:
    name: Front End Quality
    needs: changes
    if: ${{ needs.changes.outputs.frontend == 'true' }}
    uses: ./.github/workflows/quality-frontend.yml
    secrets: inherit

  email_templates:
    name: Email Templates Quality
    needs: changes
    if: ${{ needs.changes.outputs.email_templates == 'true' }}
    uses: ./.github/workflows/quality-email-templates.yml
    secrets: inherit

  docs:
    name: Docs Quality
    needs: changes
    if: ${{ needs.changes.outputs.docs == 'true' }}
    uses: ./.github/workflows/quality-docs.yml
    secrets: inherit

  final-check:
    name: Final Check
    needs: [backend, frontend, docs, email_templates]
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - name: Check previous jobs status
        run: |
          status=$(echo "${{ needs.backend.result }}" | tr '[:upper:]' '[:lower:]')
          status+=$(echo " ${{ needs.frontend.result }}" | tr '[:upper:]' '[:lower:]')
          status+=$(echo " ${{ needs.docs.result }}" | tr '[:upper:]' '[:lower:]')
          status+=$(echo " ${{ needs.email_templates.result }}" | tr '[:upper:]' '[:lower:]')
          if [[ "$status" == *"failure"* ]]; then
            echo "One or more previous jobs failed."
            exit 1
          else
            echo "All previous jobs succeeded or were skipped."
            exit 0
          fi


Test Back End Quality

name: Test Back End Quality

on: workflow_call

jobs:
  backend-quality:
    name: Back End Quality
    runs-on: ubuntu-latest

    defaults:
      run:
        # default directory for 'run' actions only in steps
        working-directory: backend

    steps:
      - name: Pull latest commit
        uses: actions/checkout@v3
        with:
          fetch-depth: 0 # History of commits needed for SonarCloud

      - name: Setup Node for email templates
        uses: actions/setup-node@v3
        with:
          node-version-file: email-templates-generator/.nvmrc
          cache: npm
          cache-dependency-path: email-templates-generator/package-lock.json

      - name: Build email templates
        run: |
          cd ../email-templates-generator
          npm ci
          npm run build

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version-file: backend/.python-version
          cache: pipenv
          cache-dependency-path: backend/Pipfile.lock

      - name: Install dependencies
        run: |
          curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
          pipenv verify
          pipenv sync --dev

      - name: Linter
        run: |
          pipenv run lint

      - name: Run tests and collect coverage
        run: |
          pipenv run test:coverage
          pipenv run coverage xml

      - name: SonarCloud Scan
        uses: SonarSource/sonarcloud-github-action@master
        with:
          projectBaseDir: backend
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN_BACKEND }}


Test Docs Quality

name: Test Docs Quality

on: workflow_call

jobs:
  docs-quality:
    name: Docs Quality
    runs-on: ubuntu-latest

    defaults:
      run:
        # default directory for 'run' actions only in steps
        working-directory: docs

    steps:
      - name: Pull latest commit
        uses: actions/checkout@v3

      - name: Set up Node
        uses: actions/setup-node@v3
        with:
          node-version-file: docs/.nvmrc
          cache: npm
          cache-dependency-path: docs/package-lock.json

      - name: Install dependencies
        run: npm ci

      - name: Types test
        run: npm run typecheck

      - name: Lint test
        run: npm run lint

      - name: Try to build website
        run: npm run build


Test Front End Quality

name: Test Front End Quality

on: workflow_call

jobs:
  frontend-quality:
    name: Front End Quality
    runs-on: ubuntu-latest

    defaults:
      run:
        # default directory for 'run' actions only in steps
        working-directory: frontend

    steps:
      - name: Clone the repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0 # History of commits needed for SonarCloud

      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version-file: frontend/.nvmrc
          cache: npm
          cache-dependency-path: frontend/package-lock.json

      - name: Install dependencies
        run: npm ci

      - name: Types test
        run: npm run types

      - name: Lint test
        run: npm run lint

      - name: Unit test
        run: npm run jest -- --coverage

      - name: Run SonarCloud scan
        uses: SonarSource/sonarcloud-github-action@master
        with:
          projectBaseDir: frontend/
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN_FRONTEND }}


Test Email Templates Quality

name: Test Email Templates Quality

on: workflow_call

jobs:
  email-templates-quality:
    name: Email Templates Quality
    runs-on: ubuntu-latest

    defaults:
      run:
        # default directory for 'run' actions only in steps
        working-directory: email-templates-generator

    steps:
      - name: Clone the repository
        uses: actions/checkout@v3

      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version-file: email-templates-generator/.nvmrc
          cache: npm
          cache-dependency-path: email-templates-generator/package-lock.json

      - name: Install dependencies
        run: npm ci

      - name: Types test
        run: npm run types

      - name: Lint test
        run: npm run lint


Build Email Templates

name: Build Email Templates

on:
  workflow_call:
    inputs:
      target-directory:
        required: true
        type: string

jobs:
  email-templates-build:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: email-templates-generator

    steps:
      - name: Clone the repository
        uses: actions/checkout@v3

      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version-file: email-templates-generator/.nvmrc
          cache: npm
          cache-dependency-path: email-templates-generator/package-lock.json

      - name: Install dependencies
        run: npm ci --omit=dev

      - name: Build email templates
        run: npm run build

      - name: Upload files to the VPS
        uses: appleboy/scp-action@master
        with:
          host: ${{ secrets.INSTANCE_ADDRESS }}
          username: "ubuntu"
          key: ${{ secrets.SSH_KEY_DEPLOY }}
          source: "backend/templates/emails/*"
          target: ${{ inputs.target-directory }}


Build Frontend

name: Build Frontend

on:
  workflow_call:
    inputs:
      target-directory:
        required: true
        type: string

jobs:
  frontend-build:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: frontend

    steps:
      - name: Clone the repository
        uses: actions/checkout@v3

      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version-file: frontend/.nvmrc
          cache: npm
          cache-dependency-path: frontend/package-lock.json

      - name: Install dependencies
        run: npm ci --omit=dev

      - name: Build React
        run: npm run build

      - name: Upload files to the VPS
        uses: appleboy/scp-action@master
        with:
          host: ${{ secrets.INSTANCE_ADDRESS }}
          username: "ubuntu"
          key: ${{ secrets.SSH_KEY_DEPLOY }}
          source: "backend/static/front/*"
          target: ${{ inputs.target-directory }}


CI

name: CI

on: [push, pull_request]

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-13] # macos-latest broken atm
        python-version: ['3.9', '3.11']

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip setuptools wheel
    - name: Build Morph-KGC
      run: |
        pip install .[test]
    - name: Test with pytest
      run: |
        pytest


Publish to PyPi

# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Publish to PyPi

on:
  release:
    types: [created]

jobs:
  deploy:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install setuptools wheel twine build
    - name: Build and publish
      env:
        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
      run: |
        python -m build --sdist --wheel
        twine upload dist/*


Contributor License Agreement

name: Contributor License Agreement

on:
  issue_comment:
    types: [created]
  pull_request_target:
    types: [opened, closed, synchronize]

jobs:
  cla:
    runs-on: ubuntu-latest

    steps:
    - name: Get CLA Manager access token
      id: cla-token
      uses: getsentry/action-github-app-token@v2.0.0
      with:
        app_id: ${{ secrets.CLA_MANAGER_APP_ID }}
        private_key: ${{ secrets.CLA_MANAGER_PRIVATE_KEY}}

    - name: cla
      if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
      uses: contributor-assistant/github-action@v2.2.0
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        PERSONAL_ACCESS_TOKEN: ${{ steps.cla-token.outputs.token }}
      with:
        path-to-document: 'https://cla.qunasys.com'
        remote-organization-name: 'QunaSys'
        remote-repository-name: 'cla'
        branch: 'main'
        path-to-signatures: 'signatures/v1/cla.json'
        custom-notsigned-prcomment: '
Thank you for your submission, we really appreciate it. We ask that $you sign our [Contributor License Agreement](https://cla.qunasys.com) before we can accept your contribution. This CLA will also cover your future contributions submitted to QunaSys. You can sign the CLA by just posting a pull request comment same as the below format.
'
        allowlist: dependabot*


Document

name: Document

on:
  pull_request:
    paths:
      - packages/**/*.py
      - docs
      - poetry.lock
      - .github/workflows/doc.yml
  push:
    branches: [main]

jobs:
  build-deploy:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - uses: ./.github/actions/python-poetry-install
      with:
        poetry-version: '1.4.0'
        python-version: '3.9'

    - uses: actions/cache@v3
      with:
        path: |
          ~/.julia/
        key: ci-${{ runner.os }}-julia-${{ hashFiles('poetry.lock') }}-cache
        restore-keys: ci-${{ runner.os }}-julia-

    - name: Download pandoc
      if: steps.cache-pandoc.outputs.cache-hit != 'true'
      run: |
        wget https://github.com/jgm/pandoc/releases/download/2.18/pandoc-2.18-1-amd64.deb

    - name: Install pandoc
      run: |
        yes | sudo dpkg -i pandoc-2.18-1-amd64.deb

    - run: |
        poetry env use 3.9
        poetry install --only main,dev,doc

    - run: poetry run julia -e 'using Pkg; Pkg.add("PythonCall")'

    - run: poetry run make html
      working-directory: docs

    - name: deploy to Netlify
      uses: nwtgck/actions-netlify@v2.0
      with:
        publish-dir: './docs/_build'
        production-branch: main
        alias: deploy-preview-${{ github.event.number }}
        github-token: ${{ secrets.GITHUB_TOKEN }}
        enable-pull-request-comment: true
      env:
        NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
        NETLIFY_SITE_ID: c809dd00-1b20-41de-adc9-ad4c882d452d


Check licenses of dependencies

name: Check licenses of dependencies

on:
  pull_request:
    paths:
      - .licenses/**
      - packages/*/pyproject.toml
  push:
    branches: [main]
    paths:
      - packages/*/pyproject.toml
  schedule:
    - cron: '0 0 * * MON-FRI'
  workflow_dispatch:
    branches: [main]

jobs:
  check-licenses:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - uses: actions/setup-python@v4
      with:
        python-version: '3.9'
        cache: 'pip'
        cache-dependency-path: 'packages/*/pyproject.toml'

    - run: python -m venv venv
    - run: . venv/bin/activate
    - run: pip install -e packages/*

    - uses: jonabc/setup-licensed@v1.1.2
      with:
        version: '3.x'
        github_token: ${{ secrets.GITHUB_TOKEN }}

    - run: licensed cache
    - run: licensed status

    - uses: slackapi/slack-github-action@v1.22.0
      if: failure() && github.event_name == 'schedule'
      with:
        payload: |
          {
            "message": "License check for QURI Parts failed.",
            "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          }
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}


Lint

name: Lint

on:
  pull_request:
    paths:
      - packages/**/*.py
      - pyproject.toml
      - poetry.lock
      - .flake8
      - .github/workflows/lint.yml
  push:
    branches: [main]

jobs:
  lint:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - uses: ./.github/actions/python-poetry-install
      with:
        poetry-version: '1.4.0'
        python-version: '3.9'

    - run: |
        poetry env use 3.9
        poetry install --only lint

    - run: poetry run isort packages --check --diff
      if: success() || failure()

    - run: poetry run black packages --check
      if: success() || failure()

    - run: poetry run flake8 packages
      if: success() || failure()

    - run: poetry run docformatter -c -r packages
      if: success() || failure()


Package and release

name: Package and release

on:
  pull_request:
  push:
    branches: [main]
  release:
    types: [published]

jobs:
  package:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        # Necessary to get tags for correct versioning
        fetch-depth: 0

    - uses: ./.github/actions/python-poetry-install
      with:
        poetry-version: '1.4.0'
        python-version: '3.9'

    - run: |
        poetry self add "poetry-dynamic-versioning[plugin]"

    - run: |
        poetry env use 3.9
        mkdir -p dist
        cp -f NOTICE quri_parts/
        poetry build
        cd packages
        for pkg in *
        do
          cd $pkg
          cp -f NOTICE quri_parts/$pkg/
          poetry build
          mv dist/* ../../dist
          cd ..
        done

    - uses: actions/upload-artifact@v3
      with:
        name: QURI parts packages
        path: dist/*
        if-no-files-found: error

  release:
    needs: [package]
    if: github.event_name == 'release'
    runs-on: ubuntu-latest
    steps:
    - uses: actions/download-artifact@v3
      with:
        name: QURI parts packages

    - name: Install Twine
      run: python -m pip install twine

    - name: Release to PyPI
      run: python -m twine upload *
      env:
        TWINE_USERNAME: "__token__"
        TWINE_PASSWORD: ${{ secrets.TWINE_API_TOKEN }}


Test

name: Test

on:
  pull_request:
    paths:
      - packages/**/*.py
      - poetry.lock
      - .github/workflows/test.yml
  push:
    branches: [main]

jobs:
  test:

    strategy:
      matrix:
        python_version:
          - 3.9

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - uses: ./.github/actions/python-poetry-install
      with:
        poetry-version: '1.4.0'
        python-version: ${{ matrix.python_version }}

    - uses: actions/cache@v3
      with:
        path: |
          ~/.julia/
        key: ci-${{ runner.os }}-julia-${{ hashFiles('poetry.lock') }}-cache
        restore-keys: ci-${{ runner.os }}-julia-

    - run: |
        poetry env use ${{ matrix.python_version }}
        poetry install --only main,dev

    - run: poetry run python -c "import juliapkg; print('PROJECT={}'.format(juliapkg.project()))" | sed -ne '/^PROJECT=/p' >> "$GITHUB_OUTPUT"
      id: julia_project

    - run: poetry run julia  "--project=${{ steps.julia_project.outputs.PROJECT }}" -e 'using Pkg; Pkg.add("PythonCall"); Pkg.instantiate()'

    - run: poetry run pytest


Type check

name: Type check

on:
  pull_request:
    paths:
      - packages/**/*.py
      - poetry.lock
      - mypy.ini
      - .github/workflows/typecheck.yml
  push:
    branches: [main]

jobs:
  typecheck:

    strategy:
      matrix:
        python_version:
          - 3.9

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - uses: ./.github/actions/python-poetry-install
      with:
        poetry-version: '1.4.0'
        python-version: ${{ matrix.python_version }}

    - run: |
        poetry env use ${{ matrix.python_version }}
        poetry install --only main,dev,typecheck

    - name: Cache mypy cache
      uses: AustinScola/mypy-cache-github-action@v1.0.0

    - run: poetry run mypy .
      if: success() || failure()


build-docs

name: "build-docs"
on:
  push:
    branches: ["main"]
  pull_request:
    branches: ["main"]
  workflow_dispatch:

defaults:
  run:
    shell: bash

jobs:
  build-docs:
    runs-on: ubuntu-latest
    steps:
      - name: Clone repo
        uses: actions/checkout@v3
      - name: Clone docs repo
        uses: actions/checkout@v3
        with:
          repository: Nixtla/docs
          ref: scripts
          path: docs-scripts
      - uses: actions/setup-python@v4
        with:
          cache: "pip"
          python-version: '3.10'
          cache-dependency-path: settings.ini
      - name: Build docs
        run: |
          set -ux
          python -m pip install --upgrade pip
          pip install -Uq nbdev
          pip install -e ".[dev]"
          mkdir nbs/_extensions
          cp -r docs-scripts/mintlify/ nbs/_extensions/
          python docs-scripts/update-quarto.py
          echo "procs = nbdev_plotly.plotly:PlotlyProc" >> settings.ini
          nbdev_docs
      - name: Apply final formats
        run: bash ./docs-scripts/docs-final-formatting.bash
      - name: Copy over necessary assets
        run: |
          cp nbs/mint.json _docs/mint.json
          cp docs-scripts/imgs/* _docs/
      - name: Deploy to Mintlify Docs
        if: github.event_name == 'push'
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_branch: docs
          publish_dir: ./_docs
          user_name: github-actions[bot]
          user_email: 41898282+github-actions[bot]@users.noreply.github.com
      - name: Trigger mintlify workflow
        if: github.event_name == 'push'
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.DOCS_WORKFLOW_TOKEN }}
          script: |
            await github.rest.actions.createWorkflowDispatch({
              owner: 'nixtla',
              repo: 'docs',
              workflow_id: 'mintlify-action.yml',
              ref: 'main',
            });
      - name: Configure redirects for gh-pages
        run: python docs-scripts/configure-redirects.py statsforecast
      - name: Deploy to Github Pages
        if: github.event_name == 'push'
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_branch: gh-pages
          publish_dir: ./gh-pages
          user_name: github-actions[bot]
          user_email: 41898282+github-actions[bot]@users.noreply.github.com


CI

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

defaults:
  run:
    shell: bash -l {0}

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  NIXTLA_NUMBA_CACHE: '1'

jobs:
  nb-sync:
    runs-on: ubuntu-latest
    steps:
      - name: Clone repo
        uses: actions/checkout@v2

      - name: Set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install nbdev
        run: pip install nbdev

      - name: Check if all notebooks are cleaned
        run: |
          echo "Check we are starting with clean git checkout"
          if [ -n "$(git status -uno -s)" ]; then echo "git status is not clean"; false; fi
          echo "Trying to strip out notebooks"
          ./action_files/clean_nbs
          echo "Check that strip out was unnecessary"
          git status -s # display the status to see which nbs need cleaning up
          if [ -n "$(git status -uno -s)" ]; then echo -e "!!! Detected unstripped out notebooks\n!!!Remember to run nbdev_install_git_hooks and running ./action_files/clean_nbs before committing"; false; fi

  run-local-tests:
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
    steps:
      - name: Clone repo
        uses: actions/checkout@v3

      - name: Set up environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: dev/local_environment.yml
          create-args: python=${{ matrix.python-version }}
          cache-environment: true

      - name: Install pip requirements
        run: pip install ./

      - name: Run local tests
        run: nbdev_test --skip_file_re '(models|distributed|ets).*.ipynb' --pause 1.0 --do_print --timing

  run-tests:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-13]
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
    steps:
      - name: Clone repo
        uses: actions/checkout@v3

      - name: Set up environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: dev/environment.yml
          create-args: python=${{ matrix.python-version }}
          cache-environment: true

      - name: Install pip requirements
        run: pip install ./

      - name: Run tests
        run: nbdev_test --skip_file_re '(distributed).*.ipynb' --pause 1.0 --do_print --timing

      - name: Run integration tests
        run: |
          pip install ".[dev]" pytest
          pytest --durations=0 action_files

  test-m3-performance:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
    steps:
      - name: Clone repo
        uses: actions/checkout@v3

      - name: Set up environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: dev/environment.yml
          create-args: python=3.10
          cache-environment: true

      - name: Install library and extra deps
        run: pip install ".[dev]" fire

      - name: Run M3 experiment
        run: |
          python -m src.experiment
          python -m src.evaluation --test
        working-directory: ./experiments/m3


Lint

name: Lint

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Clone repo
        uses: actions/checkout@v3

      - name: Set up python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install linters
        run: pip install black nbdev pre-commit

      - name: Run pre-commit
        run: pre-commit run --files statsforecast/*


No Response Bot

name: No Response Bot

on:
  issue_comment:
    types: [created]
  schedule:
    - cron: '0 4 * * *'

jobs:
  noResponse:
    runs-on: ubuntu-latest
    steps:
      - uses: lee-dohm/no-response@v0.5.0
        with:
          closeComment: >
              This issue has been automatically closed because it has been awaiting a response for too long.
              When you have time to to work with the maintainers to resolve this issue, please post a new comment and it will be re-opened.
              If the issue has been locked for editing by the time you return to it, please open a new issue and reference this one.
          daysUntilClose: 30
          responseRequiredLabel: awaiting response
          token: ${{ github.token }}


Upload Python Package

name: Upload Python Package

on:
  release:
    types: [published]

jobs:
  deploy:

    runs-on: ubuntu-latest
    permissions:
      id-token: write

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v3
      with:
        python-version: '3.10'
    - name: Install dependencies
      run: python -m pip install build wheel
    - name: Build package
      run: python -m build -sw
    - name: Publish package
      uses: pypa/gh-action-pypi-publish@release/v1


Release Drafter

name: Release Drafter

on:
  push:
    branches:
      - main

permissions:
  contents: read

jobs:
  update_release_draft:
    permissions:
      contents: write
      pull-requests: read
    runs-on: ubuntu-latest
    steps:
      - uses: release-drafter/release-drafter@v5
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


build

name: build

on:
  push:
    branches:
      - 'main'
  pull_request:

env:
  SRC_DIR: discopy
  TEST_DIR: test
  DOCS_DIR: docs

jobs:
  lint:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.9 ]
    steps:
    - uses: actions/checkout@v3
    - name: Setup Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install linter
      run:
        python -m pip install pyproject-flake8 coverage pylint
    - name: Check for errors
      run:
        # stop the build if there are Python syntax errors or undefined names
        pflake8 discopy
    - name: pylint
      run:
        pylint discopy
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.9, "3.10", "3.11", "3.12" ]
    steps:
    - uses: actions/checkout@v3
    - name: Setup Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'  # caching pip dependencies
    - name: Install
      run: |
        pip install coverage pytest
        pip install torch --index-url https://download.pytorch.org/whl/cpu  # Install PyTorch without CUDA
        pip install .[test]
    - name: Test
      run:
        coverage run -m pytest
    - name: Coverage
      run:
        coverage report --fail-under=98 --show-missing
  docs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.9 ]
    steps:
    - uses: actions/checkout@v3
    - name: Setup Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    - name: Cache
      id: cache
      uses: actions/cache@v3
      with:
        path: |
          ${{ env.DOCS_DIR }}/_api
          ${{ env.DOCS_DIR }}/_build
        key: ${{ runner.os }}-docs-${{ hashFiles('discopy/**', 'docs/**') }}  # TODO: refactor to use env.SRC_DIR and DOCS_DIR
        restore-keys: |
          ${{ runner.os }}-docs
    - name: Install
      if: steps.cache.outputs.cache-hit != 'true'
      run: |
        sudo apt-get update
        sudo apt-get install -y pandoc
        pip install .[docs]
    - name: Build
      if: steps.cache.outputs.cache-hit != 'true'
      run: |
        sphinx-build ${{ env.DOCS_DIR }} ${{ env.DOCS_DIR }}/_build/html
    - name: Upload
      uses: actions/upload-artifact@v4
      with:
        name: docs
        path: ${{ env.DOCS_DIR }}/_build/html # or path/to/artifact


Build Linux package v1.0.2

name: 'Build Linux package v1.0.2'

on:
  workflow_call:
    inputs:
      TAG:
        default: 'testing'
        required: true
        type: string
  workflow_dispatch:
    inputs:
      TAG:
        default: 'testing'
        required: true
        type: string

jobs:
  linux:
    name: Build Linux BIN
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r res/requirements.txt
        pip install pyinstaller
        pip install pyinstaller-hooks-contrib
    - name: Build with Pyinstaller
      run: |
        pyinstaller --clean --noconfirm .github/workflows/JWLManager.bin.spec
        mv dist/JWLManager dist/JWLManager_${{ inputs.TAG }}
    - name: Upload BIN
      uses: actions/upload-artifact@v4
      with:
        name: LinuxBIN
        path: dist/JWLManager_${{ inputs.TAG }}


Build Mac package v1.0.2

name: 'Build Mac package v1.0.2'

on:
  workflow_call:
    inputs:
      TAG:
        default: 'testing'
        required: true
        type: string
  workflow_dispatch:
    inputs:
      TAG:
        default: 'testing'
        required: true
        type: string

jobs:
  macos:
    name: Build macOS APP
    runs-on: macos-12
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r res/requirements.txt
        pip install pyinstaller==5.13
        pip install pyinstaller-hooks-contrib
    - name: Build with Pyinstaller
      run: pyinstaller --clean --noconfirm .github/workflows/JWLManager.app.spec
    - name: Zip app
      run: |
        cd dist/
        zip -r9 JWLManager_${{ inputs.TAG }}.app.zip JWLManager.app/
    - name: Upload APP
      uses: actions/upload-artifact@v4
      with:
        name: MacAPP
        path: dist/JWLManager_${{ inputs.TAG }}.app.zip


Build Windows packages v1.0.1

name: 'Build Windows packages v1.0.1'

on:
  workflow_call:
    inputs:
      TAG:
        default: 'testing'
        required: true
        type: string
  workflow_dispatch:
    inputs:
      TAG:
        default: 'testing'
        required: true
        type: string

jobs:
  windows:
    name: Build Windows EXE & ZIP
    runs-on: windows-2019
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r res/requirements.txt
        pip install pyinstaller
        pip install pyinstaller-hooks-contrib
    - name: Build EXE with Pyinstaller
      run: |
        pyinstaller --noconfirm .github/workflows/JWLManager.exe.spec
    - name: Rename EXE
      run: |
        cd dist
        ren JWLManager.exe JWLManager_${{ inputs.TAG }}.exe
    - name: Build with Pyinstaller
      run: |
        pyinstaller --clean --noconfirm .github/workflows/JWLManager.zip.spec
    - name: Rename & zip folder
      run: |
        cd dist
        tar.exe -a -c -f JWLManager_${{ inputs.TAG }}.zip JWLManager
    - name: Upload EXE
      uses: actions/upload-artifact@v4
      with:
        name: WindowsEXE
        path: dist/JWLManager_${{ inputs.TAG }}.exe
    - name: Upload ZIP
      uses: actions/upload-artifact@v4
      with:
        name: WindowsZIP
        path: dist/JWLManager_${{ inputs.TAG }}.zip


Draft Release v1.0.0

name: 'Draft Release v1.0.0'

on:
  push:
    tags:
      - 'v*'

jobs:
  linux:
    uses: ./.github/workflows/build_lin.yml
    with:
      TAG:  ${{ github.ref_name }}
  windows:
    uses: ./.github/workflows/build_win.yml
    with:
      TAG:  ${{ github.ref_name }}
  macOS:
    uses: ./.github/workflows/build_mac.yml
    with:
      TAG:  ${{ github.ref_name }}
  release:
    name: Create Release Draft
    needs: [linux, windows, macos]
    runs-on: ubuntu-latest
    env:
      TAG: ${{ github.ref_name }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Download LinuxBIN
      uses: actions/download-artifact@v4
      with:
        name: LinuxBIN
    - name: Download WindowsEXE
      uses: actions/download-artifact@v4
      with:
        name: WindowsEXE
    - name: Download WindowsZIP
      uses: actions/download-artifact@v4
      with:
        name: WindowsZIP
    - name: Download MacAPP
      uses: actions/download-artifact@v4
      with:
        name: MacAPP
    - name: Extract release notes
      id: extract-release-notes
      uses: ffurrer2/extract-release-notes@v2
    - name: Create release
      uses: softprops/action-gh-release@v2
      with:
        tag_name: ${{ github.ref_name }}
        name: ${{ github.ref_name }}
        draft: true
        prerelease: false
        body: |
          ${{ steps.extract-release-notes.outputs.release_notes }}
          ____
          You can download the latest source script and execute it from the commandline, or try these self-contained packages, which don't require the installation of Python or dependencies:
          - Linux:
            - Stand-alone: [JWLManager_${{ github.ref_name }}](https://github.com/${{ github.repository }}/releases/download/${{ github.ref_name }}/JWLManager_${{ github.ref_name }})
          - Windows 10/11:
            - Stand-alone: [JWLManager_${{ github.ref_name }}.exe](https://github.com/${{ github.repository }}/releases/download/${{ github.ref_name }}/JWLManager_${{ github.ref_name }}.exe)
            - Zipped folder: [JWLManager_${{ github.ref_name }}.zip](https://github.com/${{ github.repository }}/releases/download/${{ github.ref_name }}/JWLManager_${{ github.ref_name }}.zip)
          - macOS 11/12/13:
            - Zipped app: [JWLManager_${{ github.ref_name }}.app.zip](https://github.com/${{ github.repository }}/releases/download/${{ github.ref_name }}/JWLManager_${{ github.ref_name }}.app.zip)
        files: |
          JWLManager_${{ github.ref_name }}
          JWLManager_${{ github.ref_name }}.exe
          JWLManager_${{ github.ref_name }}.zip
          JWLManager_${{ github.ref_name }}.app.zip


Publish

name: Publish

on:
  workflow_dispatch:
  release:
    types: [created]

jobs:
  publish:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v3
    - name: Install and configure Poetry
      uses: snok/install-poetry@v1
      with:
        virtualenvs-create: false
        virtualenvs-in-project: false
    - name: Publish to pypi
      run: |
        poetry config pypi-token.pypi ${{secrets.PYPI_TOKEN}}
        poetry --build publish


Build docs

name: Build docs
on:
  workflow_dispatch:
  release:
    types: [published]

jobs:
  update-docs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install python
        uses: actions/setup-python@v3
        with:
          python-version: 3.x

      - name: Install and configure Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: false
          virtualenvs-in-project: false

      - name: Install requirements
        run: |
          poetry lock
          poetry install --with docs

      - name: Build docs
        run: mkdocs build --clean

      - name: Push back changes
        uses: stefanzweifel/git-auto-commit-action@v4
        with:
          commit_message: "update docs"
          commit_user_name: "documentation-bot"
          commit_user_email: "documentation-bot@users.noreply.github.com"


Run tests

name: Run tests

on:
    workflow_dispatch:
    push:
        branches: [master]
        paths: ["**.py", "pyproject.toml"]
    pull_request:
        branches: [master]

jobs:
    build:
        runs-on: ubuntu-20.04
        strategy:
            matrix:
                python-versions: ["3.8", "3.9", "3.10", "3.11"]

        steps:
            - name: Checkout master
              uses: actions/checkout@v4

            - name: Install Python ${{ matrix.python-versions }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python-versions }}

            - name: Install poetry
              run: curl -sSL https://install.python-poetry.org | python3 -

            - name: Install dependencies
              run: |
                poetry lock
                poetry install --with dev

            - uses: actions/setup-java@v3
              with:
                distribution: 'oracle'
                java-version: '17'

            - name: Download test files
              run: |
                  git clone https://github.com/roshan-research/hazm-test.git tests/files/

            - name: Run tests
              run: poetry run poe test


Run lints

name: Run lints

on:
    workflow_dispatch:
    push:
        paths: ["**.py"]
jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
        - name: Checkout master
          uses: actions/checkout@v3

        - name: Get changed files
          id: changed-files
          uses: tj-actions/changed-files@v41
          with:
            files: |
              **/*.py

        - name: Upgrade pip
          run: pip install --upgrade pip

        - name: Install ruff
          run: pip install ruff

        - name: Ruff check
          run: ruff check ${{ steps.changed-files.outputs.all_changed_files }}

        - name: Push back changes
          uses: stefanzweifel/git-auto-commit-action@v4
          with:
            commit_message: "fix auto-fixable linting errors"
            commit_user_name: "linter-bot"
            commit_user_email: "linter-bot@users.noreply.github.com"


Main

name: Main

on:
  push:
    branches:
      - main
    tags:
      - '**'
    paths-ignore:
      - docs/**
      - README.rst
      - LICENSE.md
      - publishing.md
  pull_request:

env:
  WEAVIATE_123: 1.23.14
  WEAVIATE_124: 1.24.10
  WEAVIATE_125: preview--c57fecf

jobs:
  lint-and-format:
    name: Run Linter and Formatter
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: psf/black@stable
        with:
            version: "23.9.1"
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: 'pip' # caching pip dependencies
      - run: pip install -r requirements-devel.txt
      - name: "Flake 8"
        run: flake8 weaviate test mock_tests integration
      - name: "Check release for pypi"
        run: |
          python -m build
          python -m twine check dist/*

  type-checking:
    name: Run Type Checking
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        folder: ["weaviate"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: 'pip' # caching pip dependencies
      - run: pip install -r requirements-devel.txt
      - name: Run mypy
        run: mypy --warn-unused-ignores --python-version ${{matrix.version}} ${{ matrix.folder }}
      - uses: jakebailey/pyright-action@v2
        with:
          version: 1.1.347

  unit-tests:
    name: Run Unit Tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        folder: ["test", "mock_tests"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.version }}
          cache: 'pip' # caching pip dependencies
      - run: pip install -r requirements-devel.txt
      - name: Run unittests
        run: pytest --cov -v --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-${{ matrix.folder }}.xml ${{ matrix.folder }}
      - name: Archive code coverage results
        if: matrix.version == '3.10' && (github.ref_name != 'main')
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report-${{ matrix.folder }}
          path: coverage-${{ matrix.folder }}.xml

  integration-tests-embedded:
    name: Run Integration Tests Embedded
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        optional_dependencies: [false]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.version }}
          cache: 'pip' # caching pip dependencies
      - run: |
          pip install -r requirements-devel.txt
          pip install .
      - name: Run integration tests
        if: ${{ !github.event.pull_request.head.repo.fork }}
        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-embedded.xml integration_embedded
      - name: Archive code coverage results
        if: matrix.version == '3.10' && (github.ref_name != 'main')
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report-integration-embedded
          path: coverage-integration-embedded.xml

  integration-tests-v3:
    name: Run Integration Tests v3
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        versions: [
          { py: "3.8", weaviate: $WEAVIATE_125},
          { py: "3.9", weaviate: $WEAVIATE_125},
          { py: "3.10", weaviate: $WEAVIATE_125},
          { py: "3.11", weaviate: $WEAVIATE_123},
          { py: "3.11", weaviate: $WEAVIATE_124},
          { py: "3.11", weaviate: $WEAVIATE_125},
          { py: "3.12", weaviate: $WEAVIATE_125}
        ]
        optional_dependencies: [false]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.versions.py }}
          cache: 'pip' # caching pip dependencies
      - run: |
          pip install -r requirements-devel.txt
          pip install .
      - name: free space
        run: sudo rm -rf /usr/local/lib/android
      - name: start weaviate
        run: /bin/bash ci/start_weaviate.sh ${{ matrix.versions.weaviate }}
      - name: Run integration tests with auth secrets
        if: ${{ !github.event.pull_request.head.repo.fork }}
        env:
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }}
          WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
          OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
#          OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}  disabled until we have a working key
        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-v3.xml integration_v3
      - name: Run integration tests without auth secrets (for forks)
        if: ${{ github.event.pull_request.head.repo.fork }}
        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-v3.xml integration_v3
      - name: Archive code coverage results
        if: matrix.versions.py == '3.10' && (github.ref_name != 'main')
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report-integration-v3
          path: coverage-integration-v3.xml

  integration-tests:
    name: Run Integration Tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        versions: [
          { py: "3.8", weaviate: $WEAVIATE_125},
          { py: "3.9", weaviate: $WEAVIATE_125},
          { py: "3.10", weaviate: $WEAVIATE_125},
          { py: "3.11", weaviate: $WEAVIATE_123},
          { py: "3.11", weaviate: $WEAVIATE_124},
          { py: "3.11", weaviate: $WEAVIATE_125},
          { py: "3.12", weaviate: $WEAVIATE_125}
        ]
        optional_dependencies: [false]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.versions.py }}
          cache: 'pip' # caching pip dependencies
      - run: |
          pip install -r requirements-devel.txt
          pip install .
      - name: free space
        run: sudo rm -rf /usr/local/lib/android
      - name: start weaviate
        run: /bin/bash ci/start_weaviate.sh ${{ matrix.versions.weaviate }}
      - name: Run integration tests with auth secrets
        if: ${{ !github.event.pull_request.head.repo.fork }}
        env:
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }}
          WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
          OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
#          OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}  disabled until we have a working key
        run: pytest -n auto -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
      - name: Run integration tests without auth secrets (for forks)
        if: ${{ github.event.pull_request.head.repo.fork }}
        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
      - name: Archive code coverage results
        if: matrix.versions.py == '3.10' && (github.ref_name != 'main')
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report-integration
          path: coverage-integration.xml

  Codecov:
    needs: [Unit-Tests, Integration-Tests, Integration-Tests-v3]
    runs-on: ubuntu-latest
    if: github.ref_name != 'main'
    steps:
      - uses: actions/checkout@v4
      - name: Download coverage artifacts mock
        uses: actions/download-artifact@v4
        with:
          name: coverage-report-mock_tests
      - name: Download coverage artifacts unit
        uses: actions/download-artifact@v4
        with:
          name: coverage-report-test
      - name: Download coverage integration
        uses: actions/download-artifact@v4
        with:
          name: coverage-report-integration
      - name: Download coverage integration v3
        uses: actions/download-artifact@v4
        with:
          name: coverage-report-integration-v3
      - name: Download coverage integration embedded
        uses: actions/download-artifact@v4
        with:
          name: coverage-report-integration-embedded
      - name: Codecov
        uses: codecov/codecov-action@v4
        with:
          fail_ci_if_error: true
          files: ./coverage-integration.xml, ./coverage-integration-v3.xml, ./coverage-integration-embedded.xml, ./coverage-test.xml, ./coverage-mock_tests.xml
          verbose: true
          token: ${{ secrets.CODECOV_TOKEN }}

  build-package:
    name: Build package
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: 'pip' # caching pip dependencies
      - name: Install dependencies
        run: pip install -r requirements-devel.txt
      - name: Build a binary wheel
        run: python -m build
      - name: Create Wheel Artifacts
        uses: actions/upload-artifact@v4
        with:
          path: "dist/*.whl"
          name: weaviate-python-client-wheel
          retention-days: 30

  test-package:
    needs: [build-package]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        server: [
          $WEAVIATE_123,
          $WEAVIATE_124,
          $WEAVIATE_125
        ]
    steps:
      - name: Download build artifact to append to release
        uses: actions/download-artifact@v4
        with:
          name: weaviate-python-client-wheel
      - run: |
          pip install weaviate_client-*.whl
          pip install pytest pytest-benchmark pytest-profiling grpcio grpcio-tools pytest-xdist
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: free space
        run: sudo rm -rf /usr/local/lib/android
      - run: rm -r weaviate
      - name: start weaviate
        run: /bin/bash ci/start_weaviate.sh ${{ matrix.server }}
      - name: Run integration tests with auth secrets
        if: ${{ !github.event.pull_request.head.repo.fork }}
        env:
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }}
          WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }}
          OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }}
        run: pytest -v -n auto integration
      - name: Run integration tests without auth secrets (for forks)
        if: ${{ github.event.pull_request.head.repo.fork }}
        run: pytest -v -n auto integration

  build-and-publish:
    name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
    needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: 'pip' # caching pip dependencies
      - name: Install dependencies
        run: pip install -r requirements-devel.txt
      - name: Build a binary wheel
        run: python -m build
      - name: Publish distribution 📦 to PyPI on new tags
        if: startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          verbose: true
          password: ${{ secrets.PYPI_API_TOKEN }}

  gh-release:
    name: Create a GitHub Release on new tags
    if: startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-latest
    needs: [build-and-publish]
    steps:
      - name: Download build artifact to append to release
        uses: actions/download-artifact@v4
        with:
          name: weaviate-python-client-wheel
          path: dist
      - name: Release
        uses: softprops/action-gh-release@v1
        with:
          generate_release_notes: true
          draft: true
          files: dist/*.whl


Add issues to Backlog

name: Add issues to Backlog

on:
  issues:
    types:
      - opened

jobs:
  add-to-project:
    name: Add issue to project
    runs-on: ubuntu-latest
    steps:
      - uses: actions/add-to-project@v0.4.0
        with:
          project-url: https://github.com/orgs/IFRCGo/projects/12
          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}


Builds and pushes Docker Images and Helm charts to Github Registry


Python check

name: Python check

on:
  push:
    branches:
      - develop
  pull_request:

jobs:
  pre_commit_checks:
    name: 🚴 Pre-Commit checks 🚴
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@main
      - name: Install poetry
        run: pipx install poetry
      - uses: actions/setup-python@main
        with:
          cache: 'poetry'
      - run: poetry install
      - uses: pre-commit/action@main


CI

name: CI
on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
    steps:
      - uses: actions/checkout@v2
      - name: Install Python 3
        uses: actions/setup-python@v1
      - name: Uninstall meshtastic
        run: |
          pip3 uninstall meshtastic
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip3 install -r requirements.txt
      - name: Install meshtastic from local
        run: |
          pip3 install .
          which meshtastic
          meshtastic --version
      - name: Run pylint
        run: pylint meshtastic examples/ --ignore-patterns ".*_pb2.pyi?$"
      - name: Check types with mypy
        run: mypy meshtastic/
      - name: Run tests with pytest
        run: pytest --cov=meshtastic
      - name: Generate coverage report
        run: |
          pytest --cov=meshtastic --cov-report=xml
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          env_vars: OS, PYTHON
          files: ./coverage.xml
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: true
          verbose: true
  validate:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
    steps:
      - uses: actions/checkout@v2
      - name: Install Python 3
        uses: actions/setup-python@v1
      - name: Install meshtastic from local
        run: |
          pip3 install .
          which meshtastic
          meshtastic --version


Update protobufs

name: "Update protobufs"
on: workflow_dispatch

jobs:
  update-protobufs:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v2
        with:
          submodules: true

      - name: Update Submodule
        run: |
          git pull --recurse-submodules
          git submodule update --remote --recursive

      - name: Download nanopb
        run: |
          wget https://jpa.kapsi.fi/nanopb/download/nanopb-0.4.6-linux-x86.tar.gz
          tar xvzf nanopb-0.4.6-linux-x86.tar.gz
          mv nanopb-0.4.6-linux-x86 nanopb-0.4.6

      - name: Re-generate protocol buffers
        run: |
          ./bin/regen-protobufs.sh

      - name: Commit update
        run: |
          git config --global user.name 'github-actions'
          git config --global user.email 'bot@noreply.github.com'
          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
          git add protobufs
          git add meshtastic
          git commit -m "Update protobuf submodule" && git push || echo "No changes to commit"


Make Release

name: Make Release
on: workflow_dispatch

jobs:
  release_create:
    runs-on: ubuntu-latest

    outputs:
      version: ${{ steps.get_version.outputs.version }}
      upload_url: ${{ steps.create_release.outputs.upload_url }}
      new_sha: ${{ steps.commit_updated.outputs.sha }}

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Bump version
        run: >-
          bin/bump_version.py

      - name: Commit updated version.py
        id: commit_updated
        run: |
          git config --global user.name 'github-actions'
          git config --global user.email 'bot@noreply.github.com'
          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
          git add setup.py
          git commit -m "bump version" && git push || echo "No changes to commit"
          git log -n 1 --pretty=format:"%H" | tail -n 1 | awk '{print "::set-output name=sha::"$0}'

      - name: Get version
        id: get_version
        run: >-
          bin/show_version.py

      - name: Create GitHub release
        uses: actions/create-release@v1
        id: create_release

        with:
          draft: true
          prerelease: true
          release_name: Meshtastic Python ${{ steps.get_version.outputs.version }}
          tag_name: ${{ steps.get_version.outputs.version }}
          body: |
            Autogenerated by github action, developer should edit as required before publishing...
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Python 3.9
        uses: actions/setup-python@v2
        with:
          python-version: 3.9

      - name: Install pypa/build
        run: >-
          python -m
          pip install
          build
          --user

      - name: Build a binary wheel and a source tarball
        run: >-
          python -m
          build
          --sdist
          --wheel
          --outdir dist/
          .

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@master
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}

  # build-and-publish-mac:
  #   runs-on: macos-latest
  #   needs: release_create
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #       with:
  #         ref: ${{ needs.release_create.outputs.new_sha }}

  #     - name: Set up Python 3.9
  #       uses: actions/setup-python@v2
  #       with:
  #         python-version: 3.9

  #     - name: Setup code signing
  #       env:
  #         MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }}
  #         MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
  #         MACOS_KEYCHAIN_PASSWORD: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }}
  #       run: |
  #         echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
  #         security create-keychain -p "$MACOS_KEYCHAIN_PASSWORD" meshtastic.keychain
  #         security default-keychain -s meshtastic.keychain
  #         security unlock-keychain -p "$MACOS_KEYCHAIN_PASSWORD" meshtastic.keychain
  #         security import certificate.p12 -k meshtastic.keychain -P "$MACOS_CERTIFICATE_PWD" -T /usr/bin/codesign
  #         security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$MACOS_KEYCHAIN_PASSWORD" meshtastic.keychain

  #     - name: Build
  #       env:
  #         MACOS_SIGNING_IDENTITY: ${{ secrets.MACOS_SIGNING_IDENTITY }}
  #       run: |
  #         pip install pyinstaller
  #         pip install -r requirements.txt
  #         pip install .
  #         pyinstaller -F -n meshtastic --collect-all meshtastic --codesign-identity "$MACOS_SIGNING_IDENTITY" meshtastic/__main__.py

  #     - name: Add mac to release
  #       uses: actions/upload-release-asset@v1
  #       env:
  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  #       with:
  #         upload_url: ${{ needs.release_create.outputs.upload_url }}
  #         asset_path: dist/meshtastic
  #         asset_name: meshtastic_mac
  #         asset_content_type: application/zip

  build-and-publish-ubuntu:
    runs-on: ubuntu-latest
    needs: release_create
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{ needs.release_create.outputs.new_sha }}

      - name: Set up Python 3.9
        uses: actions/setup-python@v2
        with:
          python-version: 3.9

      - name: Build
        run: |
          pip install pyinstaller
          pip install -r requirements.txt
          pip install .
          pyinstaller -F -n meshtastic --collect-all meshtastic meshtastic/__main__.py

      - name: Add ubuntu to release
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ needs.release_create.outputs.upload_url }}
          asset_path: dist/meshtastic
          asset_name: meshtastic_ubuntu
          asset_content_type: application/zip

      - name: Add readme.txt to release
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ needs.release_create.outputs.upload_url }}
          asset_path: standalone_readme.txt
          asset_name: readme.txt
          asset_content_type: text/plain

  build-and-publish-windows:
    runs-on: windows-latest
    needs: release_create
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{ needs.release_create.outputs.new_sha }}

      - name: Set up Python 3.9
        uses: actions/setup-python@v2
        with:
          python-version: 3.9

      - name: Build
        run: |
          pip install pyinstaller
          pip install -r requirements.txt
          pip install .
          pyinstaller -F -n meshtastic --collect-all meshtastic meshtastic/__main__.py

      - name: Add windows to release
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ needs.release_create.outputs.upload_url }}
          asset_path: dist/meshtastic.exe
          asset_name: meshtastic_windows
          asset_content_type: application/zip


Remove old artifacts

name: Remove old artifacts

on:
  schedule:
    # Every day at 1am
    - cron: "0 1 * * *"

  workflow_dispatch:

jobs:
  remove-old-artifacts:
    runs-on: ubuntu-latest
    timeout-minutes: 10

    steps:
      - name: Remove old artifacts
        uses: c-hive/gha-remove-artifacts@v1
        with:
          age: "1 month"
          skip-tags: true


Documentation

name: Documentation
on:
  push:
    branches:
      - develop
      - develop-ref
      - feature_*
      - main_*
      - bugfix_*
    paths:
      - docs/**
  pull_request:
    types: [opened, reopened, synchronize]

jobs:
  documentation:
    name: Build Documentation
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade python-dateutil requests sphinx \
          sphinx-gallery matplotlib Pillow sphinx_rtd_theme
          python -m pip install -r docs/requirements.txt
      - name: Build docs
        run: ./.github/jobs/build_documentation.sh
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: documentation
          path: artifact/documentation
      - uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: documentation_warnings.log
          path: artifact/doc_warnings.log
          if-no-files-found: ignore


Add checksum to release

name: Add checksum to release

on:
  release:
    types:
      - published

jobs:
  add-checksum:
     runs-on: ubuntu-latest
     steps:
        - uses: dtcenter/metplus-action-release-checksum@v1
          with:
             token: ${{ secrets.METPLUS_BOT_TOKEN }}


SonarQube Scan

name: SonarQube Scan

# Run SonarQube for Pull Requests and changes to the develop and main_vX.Y branches

on:

  # Trigger analysis for pushes to develop and main_vX.Y branches
  push:
    branches:
      - develop
      - 'main_v**'
    paths-ignore:
      - 'docs/**'
      - '.github/pull_request_template.md'
      - '.github/ISSUE_TEMPLATE/**'
      - '**/README.md'
      - '**/LICENSE.md'

  # Trigger analysis for pull requests to develop and main_vX.Y branches
  pull_request:
    types: [opened, synchronize, reopened]
    branches:
      - develop
      - 'main_v**'
    paths-ignore:
      - 'docs/**'
      - '.github/pull_request_template.md'
      - '.github/ISSUE_TEMPLATE/**'
      - '**/README.md'
      - '**/LICENSE.md'

  workflow_dispatch:
    inputs:
      reference_branch:
        description: 'Reference Branch'
        default: develop
        type: string

jobs:
  sonarqube:
    name: SonarQube Scan
    runs-on: ubuntu-latest

    steps:

    - uses: actions/checkout@v4
      with:
        # Disable shallow clones for better analysis
        fetch-depth: 0

    - name: Get branch name
      id: get_branch_name
      run: echo branch_name=${GITHUB_REF#refs/heads/} >> $GITHUB_OUTPUT

    - name: Configure SonarQube
      run: .github/jobs/configure_sonarqube.sh
      env:
        SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}
        WD_REFERENCE_BRANCH: ${{ github.event.inputs.reference_branch }}
        SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
        SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

    - name: SonarQube Scan
      uses: sonarsource/sonarqube-scan-action@master
      env:
        SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
        SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

    - name: SonarQube Quality Gate check
      id: sonarqube-quality-gate-check
      uses: sonarsource/sonarqube-quality-gate-action@master
      # Force to fail step after specific time.
      timeout-minutes: 5
      env:
       SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}


Trigger METplus Workflow

name: Trigger METplus Workflow

on:
  push:
    branches:
      - develop
    paths-ignore:
      - 'docs/**'
      - '.github/pull_request_template.md'
      - '.github/ISSUE_TEMPLATE/**'
      - '**/README.md'
      - '**/LICENSE.md'

jobs:
  trigger_metplus:
    name: Trigger METplus testing workflow
    runs-on: ubuntu-latest
    steps:
      - name: Print GitHub values for reference
        env:
          GITHUB_CONTEXT: ${{ toJson(github) }}
        run: echo "$GITHUB_CONTEXT"
      - uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.METPLUS_BOT_TOKEN }}
          script: |
            await github.rest.actions.createWorkflowDispatch({
              owner: 'dtcenter',
              repo: 'METplus',
              workflow_id: 'testing.yml',
              ref: 'develop',
              inputs: {
                repository: '${{ github.repository }}',
                ref: '${{ github.ref }}',
                actor: '${{ github.actor }}',
                sha: '${{ github.sha }}',
                pusher_email: '${{ github.event.pusher.email }}'
              },
            });


Python tests

# This workflow will install Python dependencies, run tests the specified Python version
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Python tests

on:
  push:
    branches:
      - develop
      - develop-ref
      - feature_*
      - main_*
      - bugfix_*
    paths-ignore:
      - 'docs/**'
      - '.github/pull_request_template.md'
      - '.github/ISSUE_TEMPLATE/**'
      - '**/README.md'
      - '**/LICENSE.md'

  pull_request:
    types: [opened, reopened, synchronize]

jobs:
  build:

    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10"]

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Retrieve METcalcpy repository develop branch
      run: |
         /usr/bin/git clone https://github.com/dtcenter/METcalcpy
         cd METcalcpy
         /usr/bin/git checkout develop
         python -m pip install -e .

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

#   Checking the branch name, not necessary but useful when setting things up.
#     - name: Extract branch name
#       shell: bash
#       run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
#       id: extract_branch

    - name: Test with pytest
      run: |
          cd test
          cd bar
          pytest test_bar.py
          cd ../box
          pytest test_box.py
          cd ../contour
          pytest test_contour.py
          cd ../eclv
          pytest test_eclv.py
          cd ../ens_ss
          pytest test_ens_ss.py
          cd ../equivalence_testing_bounds
          pytest test_equivalence_testing_bounds.py
          cd ../line
          pytest test_line_groups_plot.py
          pytest test_line_plot.py
          cd ../mpr_plot
          pytest test_mpr_plot.py
          cd ../performance_diagram
          pytest test_performance_diagram.py
          cd ../reliability_diagram
          pytest test_reliability_diagram.py
          cd ../roc_diagram
          pytest test_roc_diagram.py
          cd ../taylor_diagram
          pytest test_taylor_diagram.py
          cd ../wind_rose
          pytest test_wind_rose.py
          cd ../histogram
          pytest test_prob_hist.py
          pytest test_rank_hist.py
          pytest test_rel_hist.py


Python vulnerability check

# This workflow will check third-party Python packages for vulnerabilities using pip-audit.
# pip-audit finds vulnerabilities from packages that are dependencies of
# the packages in the requirements.txt file. A summary is generated at the end of the action.
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Python vulnerability check

on:
  push:
    branches:
      - develop
      - develop-ref
      - feature_*
      - main_*
      - bugfix_*
      - test_*
      - issue_*
  pull_request:
    types: [opened, reopened, synchronize]

jobs:
  selftest:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: install
        run: |
          python -m venv env/
          source env/bin/activate
          python -m pip install .
      - uses: pypa/gh-action-pip-audit@v1.0.5
        with:
          inputs: requirements.txt
          virtual-environment: env/
          local: true
          summary: true


Tests

name: Tests

on:
  pull_request:
  push:
    branches:
      - og-develop
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
  cancel-in-progress: true

jobs:
  test:
    name: Run Tests
    runs-on: [self-hosted, linux, gpu, dataset-enabled]

    defaults:
      run:
        shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0}

    steps:
      - name: Fix home
        run: echo "HOME=/root" >> $GITHUB_ENV

      - name: Checkout source
        uses: actions/checkout@v2
        with:
          submodules: true
          path: omnigibson-src

      - name: Install dev requirements
        working-directory: omnigibson-src
        run: pip install -r requirements-dev.txt

      - name: Install
        working-directory: omnigibson-src
        run: pip install -e .

      - name: Run tests
        working-directory: omnigibson-src
        run: pytest --junitxml=results.xml

      - name: Test Report
        uses: dorny/test-reporter@v1
        with:
          name: Test Results
          working-directory: omnigibson-src
          path: results.xml
          reporter: java-junit
          fail-on-error: 'true'
          fail-on-empty: 'true'

      # - name: Upload coverage to Codecov
      #   uses: codecov/codecov-action@v2.1.0
      #   with:
      #     token: ${{ secrets.CODECOV_TOKEN }}


Lint

name: Lint

on: [pull_request]

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: psf/black@stable
        with:
          options: "-t py37"


Examples as Test

name: Examples as Test

on:
  schedule:
    # * is a special character in YAML so you have to quote this string
    # Format: minute hour day-of-month month day-of-week(starts on sunday)
    # Scheduled for 2 am, every sunday, tuesday, friday
    - cron: '0 10 * * 0,2,5'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
  cancel-in-progress: true

jobs:
  test:
    runs-on: [self-hosted, linux, gpu]
    if: github.repository == 'StanfordVL/OmniGibson-dev'

    steps:
      - name: Checkout source
        uses: actions/checkout@v2
        with:
          submodules: true
          path: omnigibson

      - name: Add CUDA to env
        run: echo "/usr/local/cuda/bin" >> $GITHUB_PATH

      - name: Setup python
        uses: actions/setup-python@v2
        with:
          python-version: "3.8"
          architecture: x64

      - name: Install dev requirements
        working-directory: omnigibson
        run: pip install -r requirements-dev.txt

      - name: Install additional dev requirements
        working-directory: omnigibson
        run: pip install -r tests/requirements-tests.txt

      - name: Install
        working-directory: omnigibson
        run: pip install -e .

      - name: Uninstall pip bddl
        working-directory: omnigibson
        run: pip uninstall -y bddl

      - name: Checkout BDDL
        uses: actions/checkout@v2
        with:
          repository: StanfordVL/bddl-dev
          ref: 581be50e7cfd2b3a1447aaa1b4fc2424b673339c
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}  # PAT is required since this is a different repo
          path: bddl
          submodules: recursive
          lfs: true

      - name: Install BDDL
        working-directory: bddl
        run: pip install -e .

      - name: Link Dataset
        working-directory: omnigibson
        run: ln -s /scr/ig-data omnigibson/data

      - name: Create tests of examples
        working-directory: omnigibson
        run: python tests/create_tests_of_examples.py

      - name: Run tests
        working-directory: omnigibson
        run: pytest /tmp/tests_of_examples

      - name: Remove Files
        uses: JesseTG/rm@v1.0.2
        with:
          path: /tmp/tests_of_examples


Disallow PRs to main branch

name: Disallow PRs to main branch

on: [pull_request]

jobs:
  protect-main:
    name: Disallow PRs to main branch
    runs-on: ubuntu-latest
    if: ${{ github.base_ref == 'main' }}

    steps:
    - name: Comment on PR
      uses: thollander/actions-comment-pull-request@v2
      with:
        message: |
          Hi, research friend! :wave:
          It looks like you are trying to merge a PR to the `main` branch.
          This branch is for released code only. **Please PR your code onto `og-develop` instead.**
          Thank you!

    - name: Fail
      run: exit 1