CI

name: CI

on:
  push:
    branches:
      - main
      - dev
  pull_request:
    branches:
      - main
      - dev
      - feat/*

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  Linting:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      # - name: Poetry cache
      #   uses: actions/cache@v3
      #   with:
      #     path: ~/.cache/pypoetry
      #     key: poetry-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}

      - name: Install Poetry
        uses: snok/install-poetry@v1

      - name: Install Dependencies
        # TODO: fix errors so that we can run `make dev` instead
        run: |
          make full

      - name: Lint with ruff
        run: |
          make lint

  Typing:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        pydantic-version: ["==1.10.9", ">=2.x"]
        openai-version: [">=1.x"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      # - name: Poetry cache
      #   uses: actions/cache@v3
      #   with:
      #     path: ~/.cache/pypoetry
      #     key: poetry-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ matrix.pydantic-version }}

      - name: Install Poetry
        uses: snok/install-poetry@v1

      - name: Install Dependencies
        # TODO: fix errors so that we can run `make dev` instead
        run: |
          make full
          poetry run pip install "pydantic${{ matrix.pydantic-version }}"
          poetry run pip install "openai${{ matrix.openai-version }}"

      - name: Static analysis with pyright
        run: |
          make type

  Pytests:
    runs-on: LargeBois
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        # TODO: fix errors so that we can run both `make dev` and `make full`
        # dependencies: ['dev', 'full']
        dependencies: ["full"]
        pydantic-version: ["==1.10.9", ">=2.x"]
        openai-version: [">=1.x"]
    steps:
      - uses: actions/checkout@v4

      - name: Create .guardrailsrc
        run: |
          echo 'id="SYSTEM TESTING"' > ~/.guardrailsrc
          echo 'no_metrics=false' >> ~/.guardrailsrc

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      # - name: Poetry cache
      #   uses: actions/cache@v3
      #   with:
      #     path: ~/.cache/pypoetry
      #     key: poetry-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ matrix.pydantic-version }}-${{ matrix.openai-version }}

      - name: Install Poetry
        uses: snok/install-poetry@v1

      - name: Install Dependencies
        run: |
          make ${{ matrix.dependencies }}
          poetry run pip install "pydantic${{ matrix.pydantic-version }}"
          poetry run pip install "openai${{ matrix.openai-version }}"

      - name: Run Pytests
        run: |
          make test-cov

      - name: Upload to codecov.io
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: true


Deploy static content to Pages

# Simple workflow for deploying static content to GitHub Pages
name: Deploy static content to Pages

on:
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
  contents: read
  pages: write
  id-token: write

# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
  group: "pages"
  cancel-in-progress: false

jobs:
  # Single deploy job since we're just deploying
  deploy:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Setup Pages
        uses: actions/configure-pages@v3
      - name: Poetry cache
        uses: actions/cache@v3
        with:
            path: ~/.cache/pypoetry
            key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }}
      - name: Install Poetry
        uses: snok/install-poetry@v1
      - name: Install dependencies
        run: poetry install --with docs
      - name: Build
        run: poetry run mkdocs build
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v2
        with:
          # Upload build folder
          path: 'site'
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2


Notebook Execution and Error Check

name: Notebook Execution and Error Check

on:
  schedule:
    - cron: "0 0 * * *"
  push:
    branches:
      - main
      - dev
  workflow_dispatch:  # This enables manual triggering

jobs:
  execute_notebooks:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # this line is automatically generated by the script in .github/workflows/scripts/update_notebook_matrix.sh
        notebook: ["bug_free_python_code.ipynb","check_for_pii.ipynb","competitors_check.ipynb","extracting_entities.ipynb","generate_structured_data.ipynb","generate_structured_data_cohere.ipynb","guardrails_with_chat_models.ipynb","input_validation.ipynb","llamaindex-output-parsing.ipynb","no_secrets_in_generated_text.ipynb","provenance.ipynb","recipe_generation.ipynb","regex_validation.ipynb","response_is_on_topic.ipynb","secrets_detection.ipynb","select_choice_based_on_action.ipynb","syntax_error_free_sql.ipynb","text_summarization_quality.ipynb","toxic_language.ipynb","translation_to_specific_language.ipynb","translation_with_quality_check.ipynb","valid_chess_moves.ipynb","value_within_distribution.ipynb"]
    env:
      COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      HUGGINGFACE_API_KEY: ${{ secrets.HUGGINGFACE_API_KEY }}
      NLTK_DATA: /tmp/nltk_data
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        ref: ${{ github.head_ref }}
        fetch-depth: 0
    - name: Create .guardrailsrc
      run: |
        echo 'id="SYSTEM TESTING"' > ~/.guardrailsrc
        echo 'no_metrics=false' >> ~/.guardrailsrc
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: 3.11.x
    - name: Install dependencies
      run: |

        # Setup Virtual Environment
        python3 -m venv ./.venv
        source .venv/bin/activate

        # Install the current branch
        pip install .

        # Install extra stuff for notebook runs
        pip install "huggingface_hub[cli]" jupyter nbconvert cohere==5.3.2
        pip install nltk
    - name: Huggingface Hub Login
      run: |
        source .venv/bin/activate
        huggingface-cli login --token $HUGGINGFACE_API_KEY
    - name: download nltk data
      run: |
        source .venv/bin/activate
        mkdir /tmp/nltk_data;
        python -m nltk.downloader -d /tmp/nltk_data punkt;
    - name: Execute notebooks and check for errors
      run: |
        source .venv/bin/activate
        bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }}


Broken Link Check

on:
  schedule:
    - cron: 0 12 * * * # run monthly
  workflow_dispatch:
name: Broken Link Check
jobs:
  check:
    name: Broken Link Check
    runs-on: ubuntu-latest
    steps:
      - name: Broken Link Check
        uses: technote-space/broken-link-checker-action@v2


Install from Hub

name: Install from Hub

on:
  push:
    branches:
      - main
  workflow_dispatch:  # This enables manual triggering

jobs:
  install_from_hub:
    runs-on: ubuntu-latest
    steps:
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: 3.11.x
    - name: pip install from main
      run: pip install git+https://github.com/guardrails-ai/guardrails.git@main
    - name: Install PII validator
      run: guardrails hub install hub://guardrails/detect_pii
    - name: Verify PII validator is addressable
      run: echo 'from guardrails.hub import DetectPII' | python


Release PyPi Version

name: Release PyPi Version

on:
  workflow_dispatch: # This event allows manual triggering

jobs:
  release:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v2

      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.11.x

      - name: Poetry cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pypoetry
          key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }}

      - name: Install Poetry
        uses: snok/install-poetry@v1

      - name: Install dependencies
        run: make full

      - name: Upload to PyPI
        env:
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: poetry publish --build -u __token__ -p $PYPI_PASSWORD


name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI

on: push

jobs:
  build:
    name: Build distribution 📦
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.x"
    - name: Install pypa/build
      run: >-
        python3 -m
        pip install
        build
        --user
    - name: Build a binary wheel and a source tarball
      run: python3 -m build
    - name: Store the distribution packages
      uses: actions/upload-artifact@v4
      with:
        name: python-package-distributions
        path: dist/

  publish-to-pypi:
    name: >-
      Publish Python 🐍 distribution 📦 to PyPI
    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
    needs:
    - build
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/p/polars-upgrade  # Replace <package-name> with your PyPI project name
    permissions:
      id-token: write  # IMPORTANT: mandatory for trusted publishing

    steps:
    - name: Download all the dists
      uses: actions/download-artifact@v4
      with:
        name: python-package-distributions
        path: dist/
    - name: Publish distribution 📦 to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1

  github-release:
    name: >-
      Sign the Python 🐍 distribution 📦 with Sigstore
      and upload them to GitHub Release
    needs:
    - publish-to-pypi
    runs-on: ubuntu-latest

    permissions:
      contents: write  # IMPORTANT: mandatory for making GitHub Releases
      id-token: write  # IMPORTANT: mandatory for sigstore

    steps:
    - name: Download all the dists
      uses: actions/download-artifact@v3
      with:
        name: python-package-distributions
        path: dist/
    - name: Sign the dists with Sigstore
      uses: sigstore/gh-action-sigstore-python@v2
      with:
        inputs: >-
          ./dist/*.tar.gz
          ./dist/*.whl
    - name: Create GitHub Release
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: >-
        gh release create
        '${{ github.ref_name }}'
        --repo '${{ github.repository }}'
        --notes ""
    - name: Upload artifact signatures to GitHub Release
      env:
        GITHUB_TOKEN: ${{ github.token }}
      # Upload to GitHub Release using the `gh` CLI.
      # `dist/` contains the built packages, and the
      # sigstore-produced signatures and certificates.
      run: >-
        gh release upload
        '${{ github.ref_name }}' dist/**
        --repo '${{ github.repository }}'


ci

name: ci

on:
  pull_request:
  push:
    branches: [main]

jobs:
  tox:
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
        os: [windows-latest, ubuntu-latest]

    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache multiple paths
        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            $RUNNER_TOOL_CACHE/Python/*
            ~\AppData\Local\pip\Cache
          key: ${{ runner.os }}-build-${{ matrix.python-version }}
      - name: install-reqs
        run: python -m pip install --upgrade tox virtualenv setuptools pip -r requirements-dev.txt
      - name: install
        run: pip install -e .
      - name: test
        run: pytest --cov=polars_upgrade._plugins


Deploy Builds

name: Deploy Builds

on:
  release:
    types: [ released ]
  push:
    tags:
      # v0.X.Y pre-release tags
      - 'v0.*.*a*'
      - 'v0.*.*b*'
      - 'v0.*.*rc*'

jobs:
  test:
    uses: ./.github/workflows/pytest.yml

  docker:
    name: build and deploy docker images
    needs: test
    runs-on: ubuntu-latest

    steps:
    # Fetch all history for all tags and branches
    - name: clone socs
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    # Dockerize
    - name: Build docker images
      run: |
        docker-compose build

    - name: Login to Docker Hub
      uses: docker/login-action@v3
      with:
        username: ${{ secrets.REGISTRY_USER }}
        password: ${{ secrets.REGISTRY_PASSWORD }}

    - name: Build and push official docker image
      env:
        DOCKERHUB_ORG: "simonsobs"
      run: |
        export DOCKER_TAG=`git describe --tags --always`

        # Tag all images for upload to the registry
        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:latest
        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:${DOCKER_TAG}

        # Upload to docker registry
        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:latest
        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:${DOCKER_TAG}
        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} echo ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} pushed

  wheel:
    name: build and deploy to PyPI
    needs: test
    runs-on: ubuntu-latest
    env:
      TWINE_USERNAME: __token__
      TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }}

    steps:
    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: 3.8

    - name: clone socs
      uses: actions/checkout@v4

    - name: install build dependencies
      run: |
        python3 -m pip install --upgrade build twine

    - name: build wheel
      run: |
        python3 -m build

    - name: install wheel
      run: |
        python3 -m pip install dist/socs*.whl

    - name: install requirements for testing
      run: |
        pip3 install -r requirements.txt

    - name: Run unit tests
      working-directory: ./tests
      run: |
        python3 -m pytest -m 'not integtest'

    - name: upload to PyPI
      run: |
        python3 -m twine upload dist/*


Build Test Images

name: Build Test Images

on:
  push:
    branches: [ main ]
    paths-ignore:
      - 'docs/**'
      - '**.rst'
      - '**.md'
      - '.flake8'
      - '.pre-commit-config.yaml'
      - '.readthedocs.yaml'
      - '.github/dependabot.yml'

jobs:
  test:
    uses: ./.github/workflows/pytest.yml

  build:
    name: build and deploy test images
    needs: test
    runs-on: ubuntu-latest

    steps:
    # Fetch all history for all tags and branches
    - name: clone socs
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    # Dockerize
    - name: Build docker images
      run: |
        docker-compose build

    - name: Login to Docker Hub
      uses: docker/login-action@v3
      with:
        username: ${{ secrets.REGISTRY_USER }}
        password: ${{ secrets.REGISTRY_PASSWORD }}

    - name: Tag and push test docker images
      env:
        DOCKERHUB_ORG: "simonsobs"
      run: |
        export DOCKER_TAG=`git describe --tags --always`-dev
        echo "${DOCKER_TAG}"

        # Tag all images for upload to the registry
        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:${DOCKER_TAG}

        # Upload to docker registry
        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:${DOCKER_TAG}
        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} echo ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} pushed

  # testing so we can catch any issues before release
  # if issues are found, test locally, or copy to pytest.yml for test on push
  wheel:
    name: build and test wheel
    needs: test
    runs-on: ubuntu-latest

    steps:
    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: 3.8

    - name: clone socs
      uses: actions/checkout@v4

    - name: install build dependencies
      run: |
        python3 -m pip install --upgrade build twine

    - name: build wheel
      run: |
        python3 -m build

    - name: install wheel
      run: |
        python3 -m pip install dist/socs*.whl

    - name: install requirements for testing
      run: |
        pip3 install -r requirements.txt

    - name: Run unit tests
      working-directory: ./tests
      run: |
        python3 -m pytest -m 'not integtest'


Run Tests

name: Run Tests

on:
  pull_request:
    paths-ignore:
      - 'docs/**'
      - '**.rst'
      - '**.md'
      - '.flake8'
      - '.pre-commit-config.yaml'
      - '.readthedocs.yaml'
      - '.github/dependabot.yml'
  workflow_call:

jobs:
  test:
    name: pytest with coverage
    runs-on: ubuntu-latest

    steps:
    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.12.1
      with:
        access_token: ${{ github.token }}

    - name: Install system packages
      run: |
        sudo apt-get install -y socat

    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: 3.8

    # Install
    - name: Install so3g
      run: |
        pip3 install so3g

    - name: clone socs
      uses: actions/checkout@v4

    - name: Install socs
      run: |
        pip3 install -r requirements.txt
        pip3 install -e .

    # Unit Tests
    - name: Run unit tests
      working-directory: ./tests
      run: |
        COVERAGE_FILE=.coverage.unit python3 -m pytest --cov -m 'not integtest'

    - name: Build docker test images
      run: |
        docker-compose build socs

    # Integration Tests
    - name: Run integration tests
      working-directory: ./tests
      run: |
        COVERAGE_FILE=.coverage.int python3 -m pytest --cov -m 'integtest'

    # Coverage
    - name: Report test coverage
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        mv ./tests/.coverage.* ./
        pip install -U coveralls
        coverage combine
        coverage report
        coveralls --service=github


name: Run Tests

on:
  pull_request:
    paths:
      - 'docs/**'
      - '**.rst'
      - '**.md'
      - '.flake8'
      - '.pre-commit-config.yaml'
      - '.readthedocs.yaml'
      - '.github/dependabot.yml'

jobs:
  test:
    name: pytest with coverage
    runs-on: ubuntu-latest

    steps:
      - run: 'echo "No build required" '


Test Docker Build

name: Test Docker Build

on:
  pull_request:
    paths:
      - 'docs/**'
      - '**.rst'
      - '**.md'
      - '.flake8'
      - '.pre-commit-config.yaml'
      - '.readthedocs.yaml'
      - '.github/dependabot.yml'

jobs:
  build:
    name: test image build
    runs-on: ubuntu-latest

    steps:
      - run: 'echo "No build required" '


name: Test Docker Build

on:
  pull_request:
    paths-ignore:
      - 'docs/**'
      - '**.rst'
      - '**.md'
      - '.flake8'
      - '.pre-commit-config.yaml'
      - '.readthedocs.yaml'
      - '.github/dependabot.yml'

jobs:
  build:
    name: test image build
    runs-on: ubuntu-latest

    steps:
    # Fetch all history for all tags and branches
    - name: clone socs
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    # Dockerize
    - name: Build docker images
      run: |
        docker-compose build


CI

name: CI

on:
  push:
    branches: [ main ]
    paths-ignore: ["web/**", "**.md", "README.rst"]
  pull_request:
    branches: [ main ]
    paths-ignore: ["web/**", "**.md", "README.rst"]

env:
  latest_python: "3.12"
  supported_pythons: '["3.8", "3.9", "3.10", "3.11", "3.12"]'
  miniforge_version: "23.11.0-0"
  miniforge_variant: "Mambaforge"

jobs:
  conf:
    # This job is needed to route the global environment variables into
    # a context that's available for matrix (and name, but that's unimportant)
    name: Prepare Test Plan
    runs-on: ubuntu-latest
    outputs:
      latest_python: ${{ steps.set-vars.outputs.latest_python }}
      supported_pythons: ${{ steps.set-vars.outputs.supported_pythons }}
    steps:
      - name: Report Plan
        id: set-vars
        run: |
          echo "latest_python=$latest_python" >> $GITHUB_OUTPUT
          echo "supported_pythons=$supported_pythons" >> $GITHUB_OUTPUT

  lint:
    name: Lint code (${{ needs.conf.outputs.latest_python }}, ubuntu-latest)
    needs: conf
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - uses: actions/checkout@v4
      - uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          python-version: ${{ env.latest_python }}
          miniforge-version: ${{ env.miniforge_version }}
          miniforge-variant: ${{ env.miniforge_variant }}
          environment-file: ci/conda_host_env.yml
      - name: Install dependencies
        run: |
          pip install -r ci/requirements.lint.txt
          conda list
      - name: Run Ruff
        run: |
          ruff check --output-format=github .

  doc:
    name: Build Documentation (${{ needs.conf.outputs.latest_python }}, ubuntu-latest)
    needs: ["conf", "lint"]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - uses: actions/checkout@v4
      - uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          python-version: ${{ env.latest_python }}
          miniforge-version: ${{ env.miniforge_version }}
          miniforge-variant: ${{ env.miniforge_variant }}
          environment-file: ci/conda_host_env.yml
      - name: Install dependencies
        run: |
          pip install -r ci/requirements.doc.txt
          pip install .
          conda list
      - name: Make documentation
        run: make doc
      # save built documentation (HTML) for deployment
      - name: Substitute URLs
        if: github.event_name == 'push'
        run: python doc/suburl.py
      - name: Save documentation
        if: github.event_name == 'push'
        uses: actions/upload-artifact@v4
        with:
          name: docpack
          path: doc/build/html/

  # full-scale test with latest Python
  test-latest:
    name: Test (${{ needs.conf.outputs.latest_python }}, ${{ matrix.os }})
    needs: conf
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: true
      # macos-14 and on uses Apple silicon chips (ARM-based).
      # macos-13 is the last version that uses Intel chips. See:
      # https://docs.github.com/en/actions/using-github-hosted-runners/
      # about-github-hosted-runners/about-github-hosted-runners#standard-
      # github-hosted-runners-for-public-repositories
      matrix:
        os: ["ubuntu-latest", "macos-13", "macos-14"]
    steps:
      - uses: actions/checkout@v4
      - uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          python-version: ${{ env.latest_python }}
          miniforge-version: ${{ env.miniforge_version }}
          miniforge-variant: ${{ env.miniforge_variant }}
          environment-file: ci/conda_host_env.yml
      - name: Install dependencies
        run: |
          pip install .
          conda list
      - name: Run unit tests
        env:
          WITH_COVERAGE: "TRUE"
        run: make test

        # upload coverage reports to Codecov (only under Linux)
      - name: Generate coverage reports
        if: runner.os == 'Linux'
        run: |
          cd ci && coverage lcov --rcfile ../.coveragerc
      - name: Upload coverage reports to Codecov
        if: runner.os == 'Linux'
        uses: codecov/codecov-action@v4.0.1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          slug: scikit-bio/scikit-bio

  # test under AArch64 (ARM64) architecture
  test-aarch64:
    name: Test (${{ needs.conf.outputs.latest_python }}, qemu::aarch64-centos)
    needs: conf
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        # setup-buildx-action uses the git context directly
        # but checklist wants the .git directory
      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Build and test for linux-aarch64
        id: docker_build
        uses: docker/build-push-action@v2
        with:
          context: .
          # ^ to use the local checkout, not the git context
          file: aarch64.Dockerfile
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            PYTHON_VERSION=${{ env.latest_python }}

  # test under all combinations of environments (Python versions, operating systems, PyPI vs Conda)
  test-all:
    name: Test (${{ matrix.python_version }}, ${{ matrix.os }}, ${{ fromJSON('["pypi", "conda"]')[matrix.use_conda] }})
    runs-on: ${{ matrix.os }}
    needs: ["conf", "test-latest", "lint"]
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "macos-latest"]
        python_version: ${{ fromJSON(needs.conf.outputs.supported_pythons) }}
        use_conda: [true, false]
    steps:
      - uses: actions/checkout@v4
      - uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          python-version: ${{ matrix.python_version }}
          miniforge-version: ${{ env.miniforge_version }}
          miniforge-variant: ${{ env.miniforge_variant }}
          environment-file: ci/conda_host_env.yml
      - name: Install dependencies (conda)
        if: ${{ matrix.use_conda }}
        run: |
          conda install -q --yes -c conda-forge --file ci/conda_requirements.txt
          pip install . --no-deps
          conda list
      - name: Install dependencies
        if: ${{ !matrix.use_conda }}
        run: |
          pip install .
          conda list
      - name: Run unit tests
        env:
          WITH_COVERAGE: "TRUE"
        run: make test

  # deploy the current development documentation to the website
  # only when event is push and all tests have passed
  deploy-doc:
    name: Deploy documentation
    if: github.event_name == 'push'
    needs: ["doc", "test-all", "test-aarch64"]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      # load documentation built by job "doc"
      - name: Load documentation
        uses: actions/download-artifact@v4
        with:
          name: docpack
          path: docpack
      # checkout website repo
      - name: Check out website
        uses: actions/checkout@v4
        with:
          repository: scikit-bio/scikit-bio.github.io
          path: website
          ssh-key: ${{ secrets.SSH_DEPLOY_KEY }}
      # synchronize documentation to website's docs/dev directory
      - name: Update documentation
        run: rsync -av --delete docpack/ website/docs/dev
      # push website back to repo
      - name: Push website
        run: |
          cd website
          git config user.name "${{ github.actor }}"
          git config user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com"
          git add -A
          git commit -m "Update from ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}"
          git push


Release

name: Release

on:
  push:
    tags:
      - '*'

env:
  earliest_python: "3.8"
  latest_python: "3.12"
  miniforge_version: "23.11.0-0"
  miniforge_variant: "Mambaforge"

jobs:
  pypi:
    name: Publish to PyPI
    runs-on: ubuntu-latest
    steps:
      - name: Check out repo
        uses: actions/checkout@v4
        with:
          submodules: true

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.earliest_python }}

      - name: Build distribution
        run: |
          export RELEASE_VERSION=${{ github.ref_name }}
          pip install numpy cython
          python setup.py sdist

      - name: Publish distribution
        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}

  doc:
    name: Publish documentation
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - name: Check out repo
        uses: actions/checkout@v4

      - name: Set up Python
        uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          python-version: ${{ env.latest_python }}
          miniforge-version: ${{ env.miniforge_version }}
          miniforge-variant: ${{ env.miniforge_variant }}
          environment-file: ci/conda_host_env.yml

      - name: Install dependencies
        run: |
          pip install -r ci/requirements.doc.txt
          pip install .
          conda list

      - name: Build documentation
        run: make doc

      - name: Substitute URLs
        run: python doc/suburl.py

      - name: Check out website
        uses: actions/checkout@v4
        with:
          repository: scikit-bio/scikit-bio.github.io
          path: website
          ssh-key: ${{ secrets.SSH_DEPLOY_KEY }}

      - name: Update documentation
        run: |
          rsync -av --delete doc/build/html/ website/docs/${{ github.ref_name }}
          rsync -av --delete doc/build/html/ website/docs/latest

      - name: Publish website
        run: |
          cd website
          git config user.name "${{ github.actor }}"
          git config user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com"
          git add -A
          git commit -m "Update from ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}"
          git push


Website

name: Website

on:
  push:
    branches: [ main ]
    paths: [ "web/**", "!**.md" ]
  pull_request:
    branches: [ main ]
    paths: [ "web/**", "!**.md" ]

env:
  latest_python: "3.12"
  miniforge_version: "23.11.0-0"
  miniforge_variant: "Mambaforge"

jobs:

  build:
    name: Build website
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - name: Check out repo
        uses: actions/checkout@v4

      - name: Set up conda
        uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          python-version: ${{ env.latest_python }}
          miniforge-version: ${{ env.miniforge_version }}
          miniforge-variant: ${{ env.miniforge_variant }}

      - name: Install dependencies
        run: pip install -r ci/requirements.doc.txt

      - name: Make website
        run: make web

      - name: Substitute URLs
        if: github.event_name == 'push'
        run: python web/suburl.py

      - name: Add version list
        if: github.event_name == 'push'
        run: cp web/versions.json web/_build/html/

      - name: Save website package
        if: github.event_name == 'push'
        uses: actions/upload-artifact@v4
        with:
          name: webpack
          path: web/_build/html/

  deploy:
    name: Deploy website
    if: github.event_name == 'push'
    needs: build
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - name: Check out repo
        uses: actions/checkout@v4

      - name: Load website package
        uses: actions/download-artifact@v4
        with:
          name: webpack
          path: webpack

      - name: Check out website repo
        uses: actions/checkout@v4
        with:
          repository: scikit-bio/scikit-bio.github.io
          path: website
          ssh-key: ${{ secrets.SSH_DEPLOY_KEY }}

      - name: Update website
        run: rsync -av --delete --exclude-from web/.exclude webpack/ website

      - name: Push website
        run: |
          cd website
          git config user.name "${{ github.actor }}"
          git config user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com"
          git add -A
          git commit -m "Update from ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}"
          git push


doc

name: doc
concurrency:
  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
  cancel-in-progress: true
on:  # yamllint disable-line rule:truthy
  pull_request:
  push:
    branches: [main]
  workflow_dispatch:

jobs:
  build:
    timeout-minutes: 30
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Setup Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install system dependencies
        uses: ./.github/actions/install-system-dependencies
      - name: Install optipng (for sphinx-gallery)
        run: sudo apt install optipng
      - name: Install package
        run: |
          python -m pip install --progress-bar off --upgrade pip setuptools
          python -m pip install --progress-bar off .[doc]
      - name: Display system information
        run: mne_lsl-sys_info --developer
      - name: Get dataset
        uses: ./.github/actions/get-testing-dataset
        with:
          sample: "true"
          testing: "false"
      - name: Build doc
        run: make -C doc html
      - name: Prune sphinx environment
        run: rm -R ./doc/_build/html/.doctrees
      - name: Upload documentation
        uses: actions/upload-artifact@v4
        with:
          name: doc-dev
          path: ./doc/_build/html

  deploy:
    if: github.event_name == 'push'
    needs: build
    timeout-minutes: 10
    runs-on: ubuntu-latest
    permissions:
      contents: write
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Download documentation
        uses: actions/download-artifact@v4
        with:
          name: doc-dev
          path: ./doc-dev
      - name: Deploy
        uses: JamesIves/github-pages-deploy-action@v4
        with:
          folder: ./doc-dev
          target-folder: ./dev
          git-config-name: 'github-actions[bot]'
          git-config-email: 'github-actions[bot]@users.noreply.github.com'
          single-commit: true
          force: true


pytest

name: pytest
concurrency:
  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
  cancel-in-progress: true
on:  # yamllint disable-line rule:truthy
  pull_request:
  push:
    branches: [main]
  workflow_dispatch:

jobs:
  pytest:
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]
        python-version: [3.9, "3.10", "3.11", "3.12"]
    name: ${{ matrix.os }} - py${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}-latest
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system dependencies
        uses: ./.github/actions/install-system-dependencies
      - name: Install package
        run: |
          python -m pip install --progress-bar off --upgrade pip setuptools
          python -m pip install --progress-bar off .[test]
      - name: Display system information
        run: mne_lsl-sys_info --developer
      - name: Get testing dataset
        uses: ./.github/actions/get-testing-dataset
      - name: Run pytest
        run: pytest mne_lsl --cov=mne_lsl --cov-report=xml --cov-config=pyproject.toml -s
        env:
          MNE_LSL_LOG_LEVEL: DEBUG
      - name: Upload to codecov
        uses: codecov/codecov-action@v4
        with:
          files: ./coverage.xml
          flags: unittests  # optional
          name: codecov-umbrella  # optional
          fail_ci_if_error: false  # optional (default = false)
          token: ${{ secrets.CODECOV_TOKEN }}
          verbose: true  # optional (default = false)

  pytest-pip-pre:
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
    name: pip pre-release - py${{ matrix.python-version }}
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system dependencies
        uses: ./.github/actions/install-system-dependencies
      - name: Install package
        run: |
          python -m pip install --progress-bar off --upgrade pip setuptools
          python -m pip install --progress-bar off .[test]
          python -m pip install --progress-bar off --upgrade git+https://github.com/mne-tools/mne-python
          python -m pip install matplotlib
          python -m pip install --progress-bar off --upgrade --no-deps --pre --only-binary :all: -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple --timeout=180 matplotlib
          python -m pip install --progress-bar off --upgrade --pre --only-binary :all: -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple --timeout=180 numpy scipy
      - name: Display system information
        run: mne_lsl-sys_info --developer
      - name: Get testing dataset
        uses: ./.github/actions/get-testing-dataset
      - name: Run pytest
        run: pytest mne_lsl --cov=mne_lsl --cov-report=xml --cov-config=pyproject.toml -s
        env:
          MNE_LSL_LOG_LEVEL: DEBUG
      - name: Upload to codecov
        uses: codecov/codecov-action@v4
        with:
          files: ./coverage.xml
          flags: unittests  # optional
          name: codecov-umbrella  # optional
          fail_ci_if_error: false  # optional (default = false)
          token: ${{ secrets.CODECOV_TOKEN }}
          verbose: true  # optional (default = false)

  pytest-compat:
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        mne-version: ["1.4.2", "1.5.0"]
        python-version: [3.9]
    name: mne compat ${{ matrix.mne-version }} - py${{ matrix.python-version }}
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system dependencies
        uses: ./.github/actions/install-system-dependencies
      - name: Install package
        run: |
          python -m pip install --progress-bar off --upgrade pip setuptools
          python -m pip install --progress-bar off .[test]
          python -m pip uninstall numpy mne -y
          python -m pip install --progress-bar off "numpy<2"
          python -m pip install --progress-bar off mne==${{ matrix.mne-version }}
      - name: Display system information
        run: mne_lsl-sys_info --developer
      - name: Get testing dataset
        uses: ./.github/actions/get-testing-dataset
      - name: Run pytest
        run: pytest mne_lsl --cov=mne_lsl --cov-report=xml --cov-config=pyproject.toml -s
        env:
          MNE_LSL_LOG_LEVEL: DEBUG
      - name: Upload to codecov
        uses: codecov/codecov-action@v4
        with:
          files: ./coverage.xml
          flags: unittests  # optional
          name: codecov-umbrella  # optional
          fail_ci_if_error: false  # optional (default = false)
          token: ${{ secrets.CODECOV_TOKEN }}
          verbose: true  # optional (default = false)


publish

name: publish

on:  # yamllint disable-line rule:truthy
  release:
    types: [published]

jobs:
  pypi:
    timeout-minutes: 10
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Setup Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install system dependencies
        uses: ./.github/actions/install-system-dependencies
      - name: Install package
        run: |
          python -m pip install --progress-bar off --upgrade pip setuptools
          python -m pip install --progress-bar off -e .[build,stubs]
      - name: Display system information
        run: mne_lsl-sys_info --developer
      - name: Generate stub files
        run: python tools/stubgen.py
      - name: Build and publish
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN  }}
        run: |
          python -m build
          twine check dist/*
          twine upload dist/*


pylsl

name: pylsl
concurrency:
  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
  cancel-in-progress: true
on:  # yamllint disable-line rule:truthy
  workflow_dispatch:
  schedule:
    - cron: '0 8 * * 1'

jobs:
  sha256:
    timeout-minutes: 2
    runs-on: ubuntu-latest
    env:
      SHA256_PYLSL: 0a384000acdd5f831376d142b4aea373693f1e356ab1cd3dbdef10896acf74e3
    steps:
      - name: Checkout pylsl repository
        uses: actions/checkout@v4
        with:
          repository: labstreaminglayer/pylsl
      - name: Compare checksum
        run: echo "$SHA256_PYLSL pylsl/pylsl.py" | sha256sum -c
      - name: Create GitHub issue
        if: failure()
        uses: dacbd/create-issue-action@main
        with:
          token: ${{ github.token }}
          title: pyLSL is out of sync with MNE-LSL
          body: |
            Please visit https://github.com/labstreaminglayer/pylsl and review the changes.
            [Failed Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
          assignees: mscheltienne


stubs

name: stubs
concurrency:
  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
  cancel-in-progress: true
on:  # yamllint disable-line rule:truthy
  pull_request:
  schedule:
    - cron: '0 3 * * *'
  workflow_dispatch:

jobs:
  generate:
    timeout-minutes: 10
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Setup Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install system dependencies
        uses: ./.github/actions/install-system-dependencies
      - name: Install package
        run: |
          python -m pip install --progress-bar off --upgrade pip setuptools
          python -m pip install --progress-bar off -e .[stubs]
      - name: Display system information
        run: mne_lsl-sys_info --developer
      - name: Generate stub files
        run: python tools/stubgen.py
      - name: Push stub files
        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          git config --global user.name 'github-actions[bot]'
          git config --global user.email 'github-actions[bot]@users.noreply.github.com'
          if [ -n "$(git status --porcelain)" ]; then
            branch_name="update-stub-files-$(date +'%Y%m%d-%H%M%S')"
            git checkout -b "$branch_name"
            git add mne_lsl/\*.pyi
            git commit -m "deploy stub files [ci skip]"
            git push --set-upstream origin "$branch_name"
            gh pr create --title "Update stub files" --body "Auto-generated PR for updating stub files." --base main
          fi


name: main_workflow

on:
  push:
    branches:
      - develop
      - main
  pull_request:

jobs:
  lock_check:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest, macos-latest, windows-latest ]
        python-version: [ "3.10" ]
        poetry-version: ["1.4.0"]

    defaults:
      run:
        shell: bash

    timeout-minutes: 10

    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install and configure Poetry
      uses: snok/install-poetry@v1
      with:
        version: ${{ matrix.poetry-version }}
        virtualenvs-create: true
        virtualenvs-in-project: false
        virtualenvs-path: ~/my-custom-path
        installer-parallel: true
    - name: Check if dependencies can be locked
      run: poetry lock --check
    - name: Poetry install requirements and check they can be locked
      run: poetry install --all-extras --no-interaction -vvv

  copyright_and_dependencies_check:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ "3.10" ]

    timeout-minutes: 30

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: pip install tomte[tox,cli]==0.2.14
    - name: Check copyright headers
      run: tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part mech --exclude-part mech_interact_abci  --exclude-part http_server
    - name: License compatibility check
      run: tox -e liccheck
    - name: Check dependencies
      run: tox -e check-dependencies
    - name: Check doc links
      run: tomte check-doc-links
    - name: Check doc IPFS hashes
      run: tox -e check-doc-hashes

  linter_checks:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ "3.10" ]

    timeout-minutes: 30

    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run:  |
        pip install tomte[tox,cli]==0.2.14

        # Install `mdspell` for spelling checks
        sudo npm install -g markdown-spellcheck

    - name: Security checks
      run: tomte check-security
    - name: Check packages
      run: tox -e check-packages
    - name: Check hash
      run: tox -e check-hash
    - name: Code checks
      run: tomte check-code
    - name: Check spelling
      run: tomte check-spelling
    - name: AbciApp consistency checks
      run: |
        tox -e check-abci-docstrings
        tox -e check-abciapp-specs
        tox -e check-handlers
        tox -e analyse-service

  scan:
    name: gitleaks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - run: |
          wget https://github.com/zricethezav/gitleaks/releases/download/v8.10.1/gitleaks_8.10.1_linux_x64.tar.gz && \
          tar -xzf gitleaks_8.10.1_linux_x64.tar.gz && \
          sudo install gitleaks /usr/bin && \
          gitleaks detect --report-format json --report-path leak_report -v

  test:
    continue-on-error: True
    needs:
      - lock_check
      - copyright_and_dependencies_check
      - linter_checks
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest, macos-latest, windows-latest ]
        python-version: [ "3.8", "3.9", "3.10", "3.11" ]

    timeout-minutes: 30

    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/setup-go@v3
        with:
          go-version: "1.17.7"
      - name: Create decision maker's `store_path` (data folder)
        run: mkdir data
      - if: matrix.os == 'ubuntu-latest'
        name: Install dependencies (ubuntu-latest)
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          pip install tomte[tox]==0.2.14
          pip install --user --upgrade setuptools

          # install Protobuf compiler
          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
          unzip protoc-24.3-linux-x86_64.zip -d protoc
          sudo mv protoc/bin/protoc /usr/local/bin/protoc

          # install Tendermint
          wget -O tendermint.tar.gz https://github.com/tendermint/tendermint/releases/download/v0.34.19/tendermint_0.34.19_linux_amd64.tar.gz
          tar -xf tendermint.tar.gz
          sudo mv tendermint /usr/local/bin/tendermint

      - if: matrix.os == 'ubuntu-latest'
        name: Packages unit tests ubuntu-latest
        run: |
          tox -e py${{ matrix.python-version }}-linux

      - if: matrix.os == 'macos-latest'
        name: Install dependencies (macos-latest)
        run: |
          pip install tomte[tox]==0.2.14
          brew install gcc
          # brew install protobuf
          # brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/72457f0166d5619a83f508f2345b22d0617b5021/Formula/protobuf.rb
          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-osx-x86_64.zip
          unzip protoc-24.3-osx-x86_64.zip -d protoc
          sudo mv protoc/bin/protoc /usr/local/bin/protoc
          brew tap yoheimuta/protolint
          brew install protolint

      - if: matrix.os == 'macos-latest'
        name: Packages unit tests macos-latest
        run: |
          tox -e py${{ matrix.python-version }}-darwin

      - if: matrix.os == 'windows-latest'
        name: Install dependencies (windows-latest)
        env:
          ACTIONS_ALLOW_UNSECURE_COMMANDS: true
        run: |
          python -m pip install -U pip
          echo "::add-path::C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64"
          choco install wget -y
          choco install protoc --version 24.3
          choco install mingw -y
          choco install make -y
          # to check make was installed
          make --version
          pip install tomte[tox]==0.2.14

      - if: matrix.os == 'windows-latest'
        name: Packages unit tests windows-latest
        run: |
          tox -e py${{ matrix.python-version }}-win

      - if: matrix.os == 'ubuntu-latest'
        name: Upload coverage to Codecov
        uses: codecov/codecov-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: false


Release Flow

name: Release Flow

on:
  release:
    types: [prereleased, released]

jobs:
  publish-packages:
    name: Push Packages
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-versions }}
      - uses: addnab/docker-run-action@v3
        with:
            image: valory/open-autonomy-user:latest
            options: -v ${{ github.workspace }}:/work
            run: |
              echo "Pushing Packages"
              cd /work
              export AUTHOR=$(grep 'service' packages/packages.json | awk -F/ '{print $2}' | head -1)
              autonomy init --reset --author $AUTHOR --ipfs --remote
              autonomy push-all
  publish-images:
    name: Publish Docker Images
    runs-on: ${{ matrix.os }}
    needs:
      - "publish-packages"
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10"]
    env:
        DOCKER_USER: ${{secrets.DOCKER_USER}}
        DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
    steps:
      - uses: actions/checkout@v4
        with:
            fetch-depth: 0
      - name: Set up tag and vars
        uses: addnab/docker-run-action@v3
        with:
            image: valory/open-autonomy-user:latest
            options: -v ${{ github.workspace }}:/work
            run: |
              echo "Setting Tag Images"
              cd /work
              apt-get update && apt-get install git -y || exit 1
              git config --global --add safe.directory /work
              export TAG=$(git describe --exact-match --tags $(git rev-parse HEAD)) || exit 1
              if [ $? -eq 0 ]; then
                  export TAG=`echo $TAG | sed 's/^v//'`
              else
                  echo "You are not on a tagged branch"
                  exit 1
              fi
              echo VERSION=$TAG> env.sh
              echo AUTHOR=$(grep 'service/' packages/packages.json | awk -F/ '{print $2}' | head -1) >> env.sh
              echo SERVICE=$(grep 'service/' packages/packages.json | awk -F/ '{print $3}' | head -1) >> env.sh
              echo AGENT=$(grep 'agent/' packages/packages.json | awk -F/ '{print $3}' | head -1) >> env.sh
              echo DEFAULT_IMAGE_TAG=$(cat packages/packages.json | grep agent/ | awk -F: '{print $2}' | tr -d '", ' | head -n 1) >> env.sh
              cat env.sh

      - uses: addnab/docker-run-action@v3
        name: Build Images
        with:
            image: valory/open-autonomy-user:latest
            options: -v ${{ github.workspace }}:/work
            shell: bash
            run: |
              echo "Building Docker Images"
              cd /work
              source env.sh || exit 1
              echo "Building images for $AUTHOR for service $SERVICE"
              autonomy init --reset --author $AUTHOR --ipfs --remote
              autonomy fetch $AUTHOR/$SERVICE --service --local || exit 1
              cd $SERVICE || exit 1
              autonomy build-image || exit 1
              autonomy build-image --version $VERSION || exit 1

      - name: Docker login
        run: |
            echo  $DOCKER_PASSWORD | docker login -u $DOCKER_USER --password-stdin
      - name: Docker Push
        run: |
          source env.sh
          echo "Pushing $DOCKER_USER/oar-$AGENT:$VERSION"
          echo "Pushing $DOCKER_USER/oar-$AGENT:$DEFAULT_IMAGE_TAG"
          docker push $DOCKER_USER/oar-$AGENT:$VERSION
          docker push $DOCKER_USER/oar-$AGENT:$DEFAULT_IMAGE_TAG

  deploy-service-on-propel:
    name: Deploy service on propel
    environment: ${{ github.event.action == 'released' && 'production' || 'staging' }}
    needs:
    - "publish-packages"
    - "publish-images"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10"]
    env:
      CMD: "propel -U ${{ vars.PROPEL_BASE_URL }}"
    steps:
      - uses: actions/checkout@master
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-versions }}
      - name: Install dependencies
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          python -m pip install --upgrade pip
          pip install propel-client open-autonomy

      - name: Make use proxy instead of actual nlb by dns override
        run: |
          # for staging
          export IP_ADDR=$(dig +short balancer.tcp.propel.staging.autonolas.tech)
          echo -e "\n$IP_ADDR\tapp.propel.staging.valory.xyz\n"  | sudo tee -a /etc/hosts

          # for prod
          export IP_ADDR=$(dig +short balancer.tcp.propel.autonolas.tech)
          echo -e "\n$IP_ADDR\tapp.propel.valory.xyz\n"  | sudo tee -a /etc/hosts

      - name: Login to propel
        run: |
          $CMD login -u  '${{ vars.PROPEL_USERNAME }}' -p  '${{ secrets.PROPEL_PASSWORD }}'

      - name: Do a deployment
        run: |
          # determine ipfs hash id
          export IPFS_HASH=$(jq '.dev | to_entries[] | select(.key | startswith("service"))| .value' -r  ./packages/packages.json | head -n1)
          export SERVICE_PATH=service_for_propel

          # fetch service file and check it published
          autonomy init --reset --author ci --ipfs --remote
          autonomy fetch $IPFS_HASH --service --alias $SERVICE_PATH

          # get env vars from github actions
          echo '${{ toJSON(secrets) }}' | jq -r 'to_entries|map("export \(.key)=\(.value|tojson)")|.[]' > github.vars
          echo >> github.vars
          echo '${{ toJSON(vars) }}' | jq -r 'to_entries|map("export \(.key)=\(.value|tojson)")|.[]' >> github.vars
          source github.vars
          $CMD service deploy --name  '${{ vars.SERVICE_NAME }}'  --service-dir $SERVICE_PATH --service-ipfs-hash $IPFS_HASH  --ingress-enabled true --keys '${{ vars.SERVICE_KEYS }}' --timeout 320


Publish 📦 to PyPI

name: Publish 📦 to PyPI

on:
  push:
    tags:
      - 'v*'

jobs:
  build-n-publish:
    name: Build and publish 📦 to PyPI
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.10
        uses: actions/setup-python@v5.1.0
        with:
          python-version: 3.10.8
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel build
      - name: Build a binary wheel and a source tarball
        run: >-
          python -m build .
#      - name: Publish 📦 to Test PyPI
#        uses: pypa/gh-action-pypi-publish@master
#        with:
#          password: ${{ secrets.PYPI_TEST_TOKEN }}
#          repository_url: https://test.pypi.org/legacy/
      - name: Publish 📦 to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.pypi_prod_token }}


Publish 📦 to TestPyPI

name: Publish 📦 to TestPyPI

on:
  push:
    branches:
      - development

jobs:
  build-n-publish:
    name: Build and publish 📦 to TestPyPI
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: development
          fetch-depth: 0
      - name: Set up Python 3.10
        uses: actions/setup-python@v5.1.0
        with:
          python-version: 3.10.8
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel build
      - name: Build a binary wheel and a source tarball
        run: >-
          python -m build .
      - name: Publish 📦 to Test PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_TEST_TOKEN }}
          repository_url: https://test.pypi.org/legacy/


Python package

name: Python package

on:
  push:
    branches:
      - master
      - development
  pull_request:
    branches:
      - master
      - development

jobs:
  black:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 1
      matrix:
        python-version: [3.11.4]

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5.1.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install black
      - name: Check with black
        run: |
          black --check --diff src/pyatmo/ tests/

  linter:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: [3.11.4]

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5.1.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install ruff
      - name: Lint with ruff
        run: |
          ruff check src/pyatmo

  build:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: [3.10.8, 3.11.4]

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5.1.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Run tests with tox
        run: |
          pip install tox tox-gh-actions
          tox


Create Github Release

# This is a basic workflow to help you get started with Actions

name: Create Github Release

# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the development branch
on:
  push:
    # Sequence of patterns matched against refs/tags
    tags:
      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  # This workflow contains a single job called "build"
  build:
    # The type of runner that the job will run on
    runs-on: ubuntu-latest
    permissions:
      contents: write

    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # Runs a single command using the runners shell
      - name: Create a Release
        uses: actions/create-release@v1.1.4
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          # The name of the tag. This should come from the webhook payload, `github.GITHUB_REF` when a user pushes a new tag
          tag_name: ${{ github.ref }}
          # The name of the release. For example, `Release v1.0.1`
          release_name: Release ${{ github.ref }}


Changelog updated?

name: Changelog updated?

on:
  pull_request:
    types:
      - opened
      - labeled
      - unlabeled
      - synchronize
    branches:
      - main
      - develop

jobs:
  call-changelog-check-workflow:
    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.0
    secrets:
      USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Check links

name: Check links

# TODO: switch back to the push trigger if/when this workflow gets fixed; see https://github.com/ASFHyP3/hyp3-docs/issues/340
# on: push
on:
  workflow_dispatch:

jobs:
  linkChecker:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Link Checker
        id: lychee
        uses: lycheeverse/lychee-action@v1.9.3
        env:
          GITHUB_TOKEN: ${{secrets.TOOLS_BOT_PAK}}
        with:
          args: --insecure "docs/**/*.md" "README.md" "CODE_OF_CONDUCT.md"
          fail: true


Create Jira issue

name: Create Jira issue

on:
  issues:
    types: [labeled]

jobs:
  call-create-jira-issue-workflow:
    uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.0
    secrets:
      JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
      JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
      JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }}
      JIRA_FIELDS: ${{ secrets.JIRA_FIELDS }}


Deploy to Github.io

name: Deploy to Github.io

on:
  push:
    branches:
      - main
      - develop

jobs:
  build_and_deploy:
    name: Build site and deploy
    runs-on: "ubuntu-latest"
    steps:
      - uses: actions/checkout@v4
        with:
          path: hyp3-docs

      - uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          python-version: 3.9
          activate-environment: hyp3-docs
          environment-file: hyp3-docs/environment.yml

      - name: Deploy test website
        if: github.ref == 'refs/heads/develop'
        shell: bash -l {0}
        run: |
          cd hyp3-docs
          # Remove conflicting CNAME for project site
          rm docs/CNAME
          # Set site URL for dev site
          sed -i 's|site_url: https://hyp3-docs.asf.alaska.edu/|site_url: https://hyp3-docs.asf.alaska.edu/hyp3-docs/|' mkdocs.yml
          mkdocs gh-deploy --force

      - uses: actions/checkout@v4
        if: github.ref == 'refs/heads/main'
        with:
          repository: ASFHyP3/ASFHyP3.github.io
          path: ASFHyP3.github.io
          token: ${{ secrets.TOOLS_BOT_PAK }}

      - name: deploy to github.io
        if: github.ref == 'refs/heads/main'
        shell: bash -l {0}
        run: |
          cd ASFHyP3.github.io
          mkdocs gh-deploy  --force --config-file ../hyp3-docs/mkdocs.yml --remote-branch main


Is PR labeled?

name: Is PR labeled?

on:
  pull_request:
    types:
      - opened
      - labeled
      - unlabeled
      - synchronize
    branches:
      - main

jobs:
  call-labeled-pr-check-workflow:
    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.0


on:
  pull_request:
    types:
      - opened
    branches:
      - main

jobs:
  call-release-checklist-workflow:
    uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.0
    secrets:
      USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Create Release

name: Create Release

on:
  push:
    tags:
      - 'v*'

jobs:
  call-release-workflow:
    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.0
    with:
      release_prefix: HyP3 Docs
    secrets:
      USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}


Spell Check .md

name: Spell Check .md
on: push

jobs:
  spellcheck:
    runs-on: ubuntu-latest
    name: Spellcheck
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Spellcheck
        uses: tbroadley/spellchecker-cli-action@v1
        with:
          files: "docs/**/*.md **/*.md *.md"
          quiet: true
          noSuggestions: true
          dictionaries: ".github/dictionary.txt"
          reports: "report.json"
          plugins: "spell repeated-words syntax-mentions syntax-urls frontmatter"
      - name: Report
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: report
          path: report.json


Static analysis

name: Static analysis

on: push

jobs:
  call-secrets-analysis-workflow:
    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.0


Tag New Version

name: Tag New Version

on:
  push:
    branches:
      - main

jobs:
  call-bump-version-workflow:
    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.0
    secrets:
      USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}


Test build of website

name: Test build of website

on:
  pull_request:
    branches:
      - main
      - develop

jobs:
  test_build:
    name: Test build of site
    runs-on: "ubuntu-latest"
    steps:
      - uses: actions/checkout@v4

      - uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          python-version: 3.9
          activate-environment: hyp3-docs
          environment-file: environment.yml

      - name: Test build of website
        shell: bash -l {0}
        run: |
          mkdocs build


Update ASF Tools version

name: Update ASF Tools version

on:
  workflow_dispatch:
    inputs:
      asf_tools_version:
        description: 'The new version of ASF Tools'
        required: true

jobs:
  bump_sdk_version:
    name: Bump the ASF Tools version
    runs-on: "ubuntu-latest"
    steps:
      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.TOOLS_BOT_PAK }}

      - name: create patch branch
        id: patch
        env:
          ASF_TOOLS_VERSION: ${{ github.event.inputs.asf_tools_version }}
          PATCH_BRANCH: bump-asf-tools-${{ github.event.inputs.asf_tools_version }}
        run: |
          git config user.name "tools-bot"
          git config user.email "UAF-asf-apd@alaska.edu"
          echo "::set-output name=branch::${PATCH_BRANCH}"
          git checkout -b ${PATCH_BRANCH}
          sed -i -r "s|asf-tools/v[0-9]+\.[0-9]+\.[0-9]+|asf-tools/${ASF_TOOLS_VERSION}|" docs/tools/asf_tools.md
          sed -i -r "s|\`asf_tools\` \*v[0-9]+\.[0-9]+\.[0-9]+\*|\`asf_tools\` \*${ASF_TOOLS_VERSION}\*|" docs/tools/asf_tools_api.md
          sed -i -r "s|asf_tools=[0-9]+\.[0-9]+\.[0-9]+|asf_tools=${ASF_TOOLS_VERSION#v}|" environment.yml
          git commit -am "Bump ASF Tools version to ${ASF_TOOLS_VERSION}"
          git push origin ${PATCH_BRANCH}

      - name: open PR
        uses: repo-sync/pull-request@v2
        with:
          source_branch:  ${{ steps.patch.outputs.branch }}
          destination_branch: main
          pr_title: Update ASF Tools version to ${{ github.event.inputs.asf_tools_version }}
          pr_body: |
            PR created by a `workflow_dispatch` event
          pr_assignee: ASFHyP3/tools
          pr_label: tools-bot
          pr_draft: false
          pr_allow_empty: true
          github_token: ${{ secrets.TOOLS_BOT_PAK }}


Update HyP3 SDK version

name: Update HyP3 SDK version

on:
  workflow_dispatch:
    inputs:
      sdk_version:
        description: 'The new version of the SDK'
        required: true

jobs:
  bump_sdk_version:
    name: Bump the SDK version
    runs-on: "ubuntu-latest"
    steps:
      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.TOOLS_BOT_PAK }}

      - name: create patch branch
        id: patch
        env:
          SDK_VERSION: ${{ github.event.inputs.sdk_version }}
          PATCH_BRANCH: bump-sdk-${{ github.event.inputs.sdk_version }}
        run: |
          git config user.name "tools-bot"
          git config user.email "UAF-asf-apd@alaska.edu"
          echo "::set-output name=branch::${PATCH_BRANCH}"
          git checkout -b ${PATCH_BRANCH}
          sed -i -r "s|hyp3-sdk/v[0-9]+\.[0-9]+\.[0-9]+|hyp3-sdk/${SDK_VERSION}|" docs/using/sdk.md
          sed -i -r "s|\`hyp3_sdk\` \*v[0-9]+\.[0-9]+\.[0-9]+\*|\`hyp3_sdk\` \*${SDK_VERSION}\*|" docs/using/sdk_api.md
          sed -i -r "s|hyp3_sdk=[0-9]+\.[0-9]+\.[0-9]+|hyp3_sdk=${SDK_VERSION#v}|" environment.yml
          git commit -am "Bump SDK version to ${SDK_VERSION}"
          git push origin ${PATCH_BRANCH}

      - name: open PR
        uses: repo-sync/pull-request@v2
        with:
          source_branch:  ${{ steps.patch.outputs.branch }}
          destination_branch: main
          pr_title: Update HyP3 SDK version to ${{ github.event.inputs.sdk_version }}
          pr_body: |
            PR created by a `workflow_dispatch` event
          pr_assignee: ASFHyP3/tools
          pr_label: tools-bot
          pr_draft: false
          pr_allow_empty: true
          github_token: ${{ secrets.TOOLS_BOT_PAK }}


Check Dev Container

name: Check Dev Container

on:
  push:
    branches: [ main ]
    paths:
      - ".devcontainer/**"
      - ".github/workflows/devcontainer-ci.yaml"
  pull_request:
    branches: [ main ]
    paths:
      - ".devcontainer/**"
      - ".github/workflows/devcontainer-ci.yaml"

permissions:
  contents: read

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Use Node.js 20.x
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
      - run: npm install -g @devcontainers/cli
      - run: devcontainer build --config ./.devcontainer/devcontainer.json --workspace-folder "$(pwd)"


name: Format and Lint with Ruff and MarkdownLint
on:
    pull_request:
        branches: [main]
    workflow_dispatch:
permissions:
    contents: write
    pull-requests: write
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install ruff
      # Update output format to enable automatic inline annotations.
      - name: Run Ruff
        run: |
            ruff check . --fix --output-format=github
            ruff format .

      - name: Markdown Lint base-files
        uses: DavidAnson/markdownlint-cli2-action@v16
        with:
            fix: true
            globs: |
              *.md
              .github/**/*.md
        continue-on-error: true
      - name: Markdown Lint Docs
        uses: DavidAnson/markdownlint-cli2-action@v16
        with:
            fix: true
            globs: docs/**/*.md
            config: docs/.markdownlint.json
        continue-on-error: true
      - name: Commit changes
        run: |
          if [ -n "$(git status --porcelain)" ]; then
            git config --global user.name "Ruff"
            git config --global user.email "ruff-action@users.noreply.github.com"
            git add -A
            git commit -m "Format and lint with Ruff"
            git push origin HEAD:${{ github.event.pull_request.head.ref }}
          fi


Build distribution

name: Build distribution

on:
  release:
    types: [created]

jobs:
  test:
    uses: ./.github/workflows/test.yml
  deploy:
    needs: test
    name: Deploy to PyPI
    permissions:
      id-token: write
    runs-on: ubuntu-latest
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11

      - name: Install build dependencies
        run: python -m pip install build twine

      - name: Build distributions
        run: python -m build

      - name: Publish package to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1


PyTest

name: PyTest
on:
  workflow_call:
  workflow_dispatch:
  push:
    branches:
      - main
  pull_request:
    paths:
      - "src/**"
      - "tests/**"
      - "pyproject.toml"
      - "requirements.txt"

jobs:
  test-against-matrix:
    # Only test all the supported versions when a pull request is made or the workflow is called
    if: ${{github.event_name == 'workflow_call'}} || ${{github.event_name == 'pull_request'}}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.11"]
      fail-fast: true
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install requirements
        run: |
          python -m pip install -e .[dev]
      - name: Run tests
        run: |
          python -m pytest

  test-against-latest:
    # Always run against the latest version
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install requirements
        run: |
          python -m pip install -e .[dev]
      - name: Run tests
        run: |
          python -m pytest


Defi Protocols

name: Defi Protocols

on: [push]

jobs:
  Test:

    runs-on: self-hosted

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Display Python version
        run: python -c "import sys; print(sys.version)"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install . 'defyes[all]'

      - name: Install development dependencies
        run: pip install -r requirements-dev.txt

      - name: Linter -- check black
        run: black --fast --check defyes tests

      - name: Linter -- check isort
        run: isort --check defyes tests

      - name: Linter -- check flake8
        run: flake8 defyes tests

      - name: Download cache from latest main artifact
        continue-on-error: true
        run: |
          cat << EOF | python

          import requests
          from pathlib import Path
          from zipfile import ZipFile

          s = requests.Session()
          s.headers["Accept"] = "application/vnd.github+json"
          s.headers["Authorization"] = "Bearer ${{ secrets.GITHUB_TOKEN }}"
          s.headers["X-GitHub-Api-Version"] = "2022-11-28"

          def main():
              path = Path("/tmp/kkit")
              path.mkdir(exist_ok=True)
              cache_zip = path / "cache-db.zip"
              download_last_artifact("cache-db", branch="main", to=cache_zip)
              print(f"Unzipping {cache_zip} to {path}.")
              ZipFile(cache_zip).extractall(path)
              cache_file = path / "cache.db"
              if not cache_file.exists():
                  raise FileNotFoundError(cache_file)
              file_size = len(cache_file.read_bytes())
              print(f"Cache {cache_file} ({file_size:_} bytes).")

          def download_last_artifact(name, branch, to):
              runs = get(
                  "https://api.github.com/repos/${{ github.repository }}/actions/runs",
                  params={"branch": branch, "event": "push"},
              ).json()
              runs = dict_get(runs, "workflow_runs")
              run = find_latest_succeed_run(runs)
              artifacts_url = dict_get(run, "artifacts_url")
              print(f"Using {artifacts_url=}")
              artifacts = dict_get(get(artifacts_url).json(), "artifacts")
              artifact_url = find_artifact(name, artifacts)
              print(f"Downloading {to} from {artifact_url=} ...")
              download(artifact_url, to=to)

          def get(*args, **kwargs):
              response = s.get(*args, **kwargs)
              response.raise_for_status()
              return response

          def find_latest_succeed_run(runs):
              for run in runs:
                  if dict_get(run, "conclusion") == "success":
                      return run
              else:
                  raise LookupError(f"Non succeed workflow runs found among {len(runs)} runs.")

          def find_artifact(name, artifacts):
              return first(
                  dict_get(artifact, "archive_download_url")
                  for artifact in artifacts
                  if dict_get(artifact, "name") == name
              )

          def download(url, to):
              response = get(url)
              with open(to, "wb") as f:
                  f.write(response.content)

          first = lambda seq: next(iter(seq))

          def dict_get(d, *keys):
              node = d
              try:
                  for key in keys:
                      node = node[key]
              except (KeyError, TypeError):
                  print(node)
                  raise
              else:
                  return node

          main()

          EOF

      - name: Run tests
        env:
          CI: true
          NODE_ETH: ${{ secrets.NODE_ETH }}
          NODE_XDAI: ${{ secrets.NODE_XDAI }}
          NODE_POLYGON: ${{ secrets.NODE_POLYGON }}
          NODE_OPTIMISM: ${{ secrets.NODE_OPTIMISM }}
          NODE_ARBITRUM: ${{ secrets.NODE_ARBITRUM }}
          NODE_AVALANCHE: ${{ secrets.NODE_AVALANCHE }}
          NODE_BINANCE: ${{ secrets.NODE_BINANCE }}
          NODE_BASE: ${{ secrets.NODE_BASE }}
          NODE_METIS: ${{ secrets.NODE_METIS }}
          APIKEYS_ETHERSCAN: ${{ secrets.APIKEYS_ETHERSCAN }}
          APIKEYS_POLSCAN: ${{ secrets.APIKEYS_POLSCAN }}
          APIKEYS_GNOSISSCAN: ${{ secrets.APIKEYS_GNOSISSCAN }}
          APIKEYS_OPTIMISM: ${{ secrets.APIKEYS_OPTIMISM }}
          APIKEYS_ARBITRUM: ${{ secrets.APIKEYS_ARBITRUM }}
          APIKEYS_METISEXPLORER: ${{ secrets.APIKEYS_METISEXPLORER }}
          APIKEYS_BASESCAN: ${{ secrets.APIKEYS_BASESCAN }}
          APIKEYS_SNOWTRACE: ${{ secrets.APIKEYS_SNOWTRACE }}
        run: |
          sh ci_secrets_helper.sh
          cat /tmp/tmp_config.json
          PYTHONPATH=${PYTHONPATH}:. CONFIG_PATH=/tmp/tmp_config.json pytest -vs --debug-defiproto

      - name: Archive cache
        uses: actions/upload-artifact@v3
        with:
          name: cache-db
          path: /tmp/kkit/cache.db


runs on ascend

name: runs on ascend

on:
  workflow_call:
    inputs:
      machine:
        description: If set to other value, the job need ssh
        type: string
        required: false
        default: "ASCEND"
      runner:
        description: Set up the build environment
        type: string
        required: false
        default: "dicp-ascend-ci-910b"

jobs:
  checkout_code:
    name: checkout code
    runs-on: ${{ inputs.runner }}
    steps:
      - name: Checkout Code
        uses: DeepLink-org/deeplink.framework/.github/actions/checkout-code@main

  build_test:
    runs-on: ${{ inputs.runner }}
    needs: checkout_code
    steps:
      - name: build and test on ascend
        uses: DeepLink-org/deeplink.framework/.github/actions/code-build-test@main
        with:
          build_shell: "
              source dicp/scripts/ci/ascend/dipu_env.sh && \
              rm -rf /tmp/dicp_ascend/* && \
              cd /mnt/cache/share/deeplinkci/dicp_env/transformers && \
              pip uninstall transformers -y && \
              patch -p1 < modeling_llama.diff && patch -p1 < utils.diff && \
              python setup.py clean && \
              python setup.py install --user && \
              patch -R -p1 < modeling_llama.diff && patch -R -p1 < utils.diff && \
              cd - && \
              cd /mnt/cache/share/deeplinkci/dicp_env/accelerate && \
              pip uninstall accelerate -y && \
              python setup.py clean && \
              python setup.py install --user && \
              cd - && \
              pip uninstall torch_dipu -y && \
              pip uninstall dicp -y && \
              cd dipu && python setup.py clean && python setup.py install --user && \
              cd ../dicp && python setup.py clean && python setup.py install --user && \
              source scripts/ci/ascend/test_env.sh /mnt/cache/share/deeplinkci/dicp_env/llama_models && \
              export TEST_DIR=$(pwd)/test && echo ${TEST_DIR} && \
              bash ${TEST_DIR}/ascend_scripts/ops/run_test_ops.sh false && \
              bash ${TEST_DIR}/ascend_scripts/models/run_test_models.sh false
            " #Write the script you want to execute here，If you don't know which parameters to fill in, you can refer to the actions/code-build-test
          job_name: "build_test"
          cover_job: "0"
          cleaner: "clean_all_if_error"


runs on nv step 1

name: runs on nv step 1

on:
  workflow_call:
    inputs:
      runner:
        description: Set up the runner
        type: string
        required: false
        default: "tps-sco-ci"
      deeplink_path:
        description: ci work home
        type: string
        required: false
        default: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}/'
      env_path:
        description: env file path
        type: string
        required: false
        default: '/mnt/cache/share/deeplinkci/github'

jobs:
  Build-Cuda:
    name: Build-dipu-cuda
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
    steps:
      - name: Build dipu
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            set -e
            cd ${DEEPLINK_PATH}/ && ls -al && find ${DEEPLINK_PATH}/ -maxdepth 1 -mmin +240 -type d |xargs rm -rf
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}
            srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${GITHUB_JOB}/dipu \
            && source ${ENV_PATH}/dipu_env  \
            && rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/ \
            && bash scripts/ci/nv/ci_nv_script.sh build_dipu" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
          else
            ssh SH1424 """
            set -e
            export USE_COVERAGE=ON
            cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu
            source ${ENV_PATH}/dipu_env
            rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/
            srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CUDA_PARTATION} --gres=gpu:1 --time=30 bash scripts/ci/nv/ci_nv_script.sh build_dipu \
            || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
            """
          fi

  Tidy-Cuda:
    name: Run tidy (cuda)
    needs: [Build-Cuda]
    runs-on: ${{ inputs.runner }}
    env:
      GETRUNNER: ${{ inputs.runner }}
      DEEPLINK_PATH: ${{ inputs.deeplink_path }}
      ENV_PATH: ${{ inputs.env_path }}
      CUDA_PARTATION: "pat_dev"
      CLANGD_TIDY_PATH: "/mnt/cache/share/platform/dep/clangd-tidy"
    steps:
      - name: Check SupportedDiopiFunctions.txt
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            cd $DEEPLINK_PATH/$GITHUB_RUN_NUMBER/Build-Cuda && \
            git diff -s --exit-code dipu/SupportedDiopiFunctions.txt || \
            { echo "::error file=dipu/SupportedDiopiFunctions.txt,title=File Not Match::Please commit your compiled SupportedDiopiFunctions.txt" && exit 1; }
          else
            ssh SH1424 """
            cd $DEEPLINK_PATH/$GITHUB_RUN_NUMBER/Build-Cuda && \
            git diff -s --exit-code dipu/SupportedDiopiFunctions.txt || \
            { echo "::error file=dipu/SupportedDiopiFunctions.txt,title=File Not Match::Please commit your compiled SupportedDiopiFunctions.txt" && exit 1; }
            """
          fi
      - name: Run clang-tidy
        run: |
          if [[ "${GETRUNNER}" == *sco* ]];then
            srun --job-name=$GITHUB_JOB bash -c "bash $DEEPLINK_PATH/$GITHUB_RUN_NUMBER/Build-Cuda/dipu/scripts/ci/nv/ci_nv_tidy.sh $CLANGD_TIDY_PATH"
          else
            ssh SH1424 """
            bash $DEEPLINK_PATH/$GITHUB_RUN_NUMBER/Build-Cuda/dipu/scripts/ci/nv/ci_nv_tidy.sh $CLANGD_TIDY_PATH
            """
          fi