sarplus test and package

# ---------------------------------------------------------
# Copyright (c) Recommenders contributors.
# Licensed under the MIT License.
# ---------------------------------------------------------
# This workflow will run tests and do packaging for contrib/sarplus.
#
# References:
#   * GitHub Actions workflow templates
#       + [python package](https://github.com/actions/starter-workflows/blob/main/ci/python-package.yml)
#       + [scala](https://github.com/actions/starter-workflows/blob/main/ci/scala.yml)
#   * [GitHub hosted runner - Ubuntu 20.04 LTS](https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md)
#   * [Azure Databricks runtime releases](https://docs.microsoft.com/en-us/azure/databricks/release-notes/runtime/releases)
#   * [Azure Synapse Analytics runtimes](https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-version-support)

name: sarplus test and package

on:
  push:
    paths:
      - contrib/sarplus/python/**
      - contrib/sarplus/scala/**
      - contrib/sarplus/VERSION
      - .github/workflows/sarplus.yml

  # Enable manual trigger
  workflow_dispatch:

env:
  SARPLUS_ROOT: ${{ github.workspace }}/contrib/sarplus
  PYTHON_ROOT: ${{ github.workspace }}/contrib/sarplus/python
  SCALA_ROOT: ${{ github.workspace }}/contrib/sarplus/scala

jobs:
  python:
    # Test pysarplus with different versions of Python.
    # Package pysarplus and upload as GitHub workflow artifact when merged into
    # the main branch.
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          python -m pip install -U build cibuildwheel pip twine
          python -m pip install -U flake8 pytest pytest-cov scikit-learn

      - name: Lint with flake8
        run: |
          cd "${PYTHON_ROOT}"
          # See https://flake8.pycqa.org/en/latest/user/index.html
          flake8 .

      - name: Package and check
        run: |
          # build
          cd "${PYTHON_ROOT}"
          cp "${SARPLUS_ROOT}/VERSION" ./pysarplus/VERSION
          python -m build --sdist
          PYTHON_VER='${{ matrix.python-version }}'
          MINOR_VER="${PYTHON_VER#*.}"
          CIBW_BUILD="cp3${MINOR_VER}-manylinux_x86_64" python -m cibuildwheel --platform linux --output-dir dist
          python -m twine check dist/*

          # set sarplus_version for the artifact upload step
          SARPLUS_VERSION=$(cat "${SARPLUS_ROOT}/VERSION")
          echo "sarplus_version=${SARPLUS_VERSION}" >> $GITHUB_ENV

      - name: Test
        run: |
          cd "${PYTHON_ROOT}"
          python -m pip install dist/*.whl

          cd "${SCALA_ROOT}"
          export SPARK_VERSION=$(python -m pip show pyspark | grep -i version | cut -d ' ' -f 2)
          SPARK_JAR_DIR=$(python -m pip show pyspark | grep -i location | cut -d ' ' -f2)/pyspark/jars
          SCALA_JAR=$(ls ${SPARK_JAR_DIR}/scala-library*)
          HADOOP_JAR=$(ls ${SPARK_JAR_DIR}/hadoop-client-api*)
          SCALA_VERSION=${SCALA_JAR##*-}
          export SCALA_VERSION=${SCALA_VERSION%.*}
          HADOOP_VERSION=${HADOOP_JAR##*-}
          export HADOOP_VERSION=${HADOOP_VERSION%.*}
          sbt ++"${SCALA_VERSION}"! package

          cd "${PYTHON_ROOT}"
          pytest ./tests

      - name: Upload Python wheel as GitHub artifact when merged into main
        # Upload the whl file of the specific python version
        if: github.ref == 'refs/heads/main'
        uses: actions/upload-artifact@v4
        with:
          name: pysarplus-${{ env.sarplus_version }}-cp${{ matrix.python-version }}-wheel
          path: ${{ env.PYTHON_ROOT }}/dist/*.whl

      - name: Upload Python source as GitHub artifact when merged into main
        # Only one pysarplus source tar file is needed
        if: github.ref == 'refs/heads/main' && matrix.python-version == '3.10'
        uses: actions/upload-artifact@v4
        with:
          name: pysarplus-${{ env.sarplus_version }}-source
          path: ${{ env.PYTHON_ROOT }}/dist/*.tar.gz

  scala:
    # Test sarplus with different versions of Databricks and Synapse runtime
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        include:
          # For Azure Databricks 7.3 LTS
          - scala-version: "2.12.10"
            spark-version: "3.0.1"
            hadoop-version: "2.7.4"

          # For Azure Databricks 9.1 LTS and Azure Synapse Apache Spark 3.1 runtime
          - scala-version: "2.12.10"
            spark-version: "3.1.2"
            hadoop-version: "2.7.4"

          # For Azure Databricks 10.4 LTS
          - scala-version: "2.12.14"
            spark-version: "3.2.1"
            hadoop-version: "3.3.1"

    steps:
      - uses: actions/checkout@v4

      - name: Test
        run: |
          cd "${SCALA_ROOT}"
          export SPARK_VERSION="${{ matrix.spark-version }}"
          export HADOOP_VERSION="${{ matrix.hadoop-version }}"
          sbt ++${{ matrix.scala-version }}! test

      - name: Package
        # Generate jar files for Spark below 3.2 and above 3.2
        if: github.ref == 'refs/heads/main' && matrix.spark-version != '3.0.1'
        env:
          GPG_KEY: ${{ secrets.SARPLUS_GPG_PRI_KEY_ASC }}
        run: |
          SARPLUS_VERSION=$(cat "${SARPLUS_ROOT}/VERSION")

          # generate artifacts
          cd "${SCALA_ROOT}"
          export SPARK_VERSION='${{ matrix.spark-version }}'
          export HADOOP_VERSION='${{ matrix.hadoop-version }}'
          export SCALA_VERSION='${{ matrix.scala-version }}'
          sbt ++${SCALA_VERSION}! package
          sbt ++${SCALA_VERSION}! packageDoc
          sbt ++${SCALA_VERSION}! packageSrc
          sbt ++${SCALA_VERSION}! makePom

          # sign with GPG
          cd "${SCALA_ROOT}/target/scala-2.12"
          gpg --import <(cat <<< "${GPG_KEY}")
          for file in {*.jar,*.pom}; do gpg -ab "${file}"; done

          # bundle
          [ '${{ matrix.spark-version }}' == '3.1.2' ] \
            && jar cvf sarplus-${SARPLUS_VERSION}-bundle_2.12-spark-${SPARK_VERSION}.jar \
                       sarplus_*.jar \
                       sarplus_*.pom \
                       sarplus_*.asc
          [ '${{ matrix.spark-version }}' == '3.2.1' ] \
            && jar cvf sarplus-spark-3.2-plus-bundle_2.12-${SARPLUS_VERSION}.jar \
                       sarplus-spark*.jar \
                       sarplus-spark*.pom \
                       sarplus-spark*.asc

          # set sarplus_version for the artifact upload step
          echo "sarplus_version=${SARPLUS_VERSION}" >> $GITHUB_ENV

      - name: Upload Scala bundle as GitHub artifact when merged into main
        if: github.ref == 'refs/heads/main'
        uses: actions/upload-artifact@v4
        with:
          name: sarplus-${{ env.sarplus_version }}-bundle_2.12-spark-${{ matrix.spark-version }}-jar
          path: ${{ env.SCALA_ROOT }}/target/scala-2.12/*bundle*.jar


azureml-unit-tests

# ---------------------------------------------------------
# Copyright (c) Recommenders contributors.
# Licensed under the MIT License.
# ---------------------------------------------------------

name: azureml-unit-tests

on:
  pull_request:
    branches:
      - 'staging'
      - 'main'
    paths:
      # Tests will be run only when there are changes in the code:
      - examples/**
      - '!examples/**/*.md'
      - recommenders/**
      - '!recommenders/**/*.md'
      - tests/**
      - '!tests/**/*.md'
      - setup.py
    types: [opened, synchronize, reopened, ready_for_review]

  # Enable manual trigger
  workflow_dispatch:
    input:
      tags:
        description: 'Tags to label this manual run (optional)'
        default: 'Manual trigger'

  # Make this workflow reusable
  workflow_call:

jobs:
  get-test-groups:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Get test group names
        id: get_test_groups
        uses: ./.github/actions/get-test-groups
        with:
          TEST_KIND: "unit"
      - name: Print test group names
        run: echo ${{ steps.get_test_groups.outputs.test_groups }}
        shell: bash
    outputs:
      test_groups: ${{ steps.get_test_groups.outputs.test_groups }}
    if: github.event.pull_request.draft == false

  execute-tests:
    needs: get-test-groups
    name: ${{ join(matrix.*, ', ') }}
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
      matrix:
        python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"']
        test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Execute tests
        uses: ./.github/actions/azureml-test
        id: execute_tests
        with:
          EXP_NAME: 'unit_tests'
          TEST_KIND: 'unit'
          AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }}
          AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }}
          PYTHON_VERSION: ${{ matrix.python-version }}
          TEST_GROUP: ${{ matrix.test-group }}
    if: github.event.pull_request.draft == false


azureml-gpu-nightly

# ---------------------------------------------------------
# Copyright (c) Recommenders contributors.
# Licensed under the MIT License.
# ---------------------------------------------------------

name: azureml-gpu-nightly

on:
  #          ┌───────────── minute (0 - 59)
  #          │ ┌───────────── hour (0 - 23)
  #          │ │  ┌───────────── day of the month (1 - 31)
  #          │ │  │  ┌───────────── month (1 - 12 or JAN-DEC)
  #          │ │  │  │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
  #          │ │  │  │ │
  #          │ │  │  │ │
  schedule:
    - cron: '0 0 */5 * *'  # running every 5 days at 12AM
    # cron works with default branch (main) only: # https://github.community/t/on-schedule-per-branch/17525/2

  push:
    # Because we can't schedule runs for non-main branches,
    # to ensure we are running the build on the staging branch, we can add push policy for it
    branches: [staging]
    paths:
      # Tests will be run only when there are changes in the code:
      - examples/**
      - '!examples/**/*.md'
      - recommenders/**
      - '!recommenders/**/*.md'
      - tests/**
      - '!tests/**/*.md'
      - setup.py

  # Enable manual trigger
  workflow_dispatch:
    input:
      tags:
        description: 'Tags to label this manual run (optional)'
        default: 'Manual trigger'

  # Make this workflow reusable
  workflow_call:

jobs:
  get-test-groups:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Get test group names
        id: get_test_groups
        uses: ./.github/actions/get-test-groups
        with:
          TEST_KIND: "nightly"
          TEST_ENV: "gpu"
      - name: Print test group names
        run: echo ${{ steps.get_test_groups.outputs.test_groups }}
        shell: bash
    outputs:
      test_groups: ${{ steps.get_test_groups.outputs.test_groups }}

  execute-tests:
    needs: get-test-groups
    name: ${{ join(matrix.*, ', ') }}
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
      matrix:
        python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"']
        test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Execute tests
        uses: ./.github/actions/azureml-test
        id: execute_tests
        with:
          EXP_NAME: 'nightly_tests'
          TEST_KIND: 'nightly'
          TEST_ENV: 'gpu'
          AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }}
          AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }}
          PYTHON_VERSION: ${{ matrix.python-version }}
          TEST_GROUP: ${{ matrix.test-group }}


azureml-spark-nightly

# ---------------------------------------------------------
# Copyright (c) Recommenders contributors.
# Licensed under the MIT License.
# ---------------------------------------------------------

name: azureml-spark-nightly

on:
  #          ┌───────────── minute (0 - 59)
  #          │ ┌───────────── hour (0 - 23)
  #          │ │  ┌───────────── day of the month (1 - 31)
  #          │ │  │  ┌───────────── month (1 - 12 or JAN-DEC)
  #          │ │  │  │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
  #          │ │  │  │ │
  #          │ │  │  │ │
  schedule:
    - cron: '0 0 */5 * *'  # running every 5 days at 12AM
    # cron works with default branch (main) only: # https://github.community/t/on-schedule-per-branch/17525/2

  push:
    # Because we can't schedule runs for non-main branches,
    # to ensure we are running the build on the staging branch, we can add push policy for it
    branches: [staging]
    paths:
      # Tests will be run only when there are changes in the code:
      - examples/**
      - '!examples/**/*.md'
      - recommenders/**
      - '!recommenders/**/*.md'
      - tests/**
      - '!tests/**/*.md'
      - setup.py

  # Enable manual trigger
  workflow_dispatch:
    input:
      tags:
        description: 'Tags to label this manual run (optional)'
        default: 'Manual trigger'

  # Make this workflow reusable
  workflow_call:

jobs:
  get-test-groups:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Get test group names
        id: get_test_groups
        uses: ./.github/actions/get-test-groups
        with:
          TEST_KIND: "nightly"
          TEST_ENV: "spark"
      - name: Print test group names
        run: echo ${{ steps.get_test_groups.outputs.test_groups }}
        shell: bash
    outputs:
      test_groups: ${{ steps.get_test_groups.outputs.test_groups }}

  execute-tests:
    needs: get-test-groups
    name: ${{ join(matrix.*, ', ') }}
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
      matrix:
        python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"']
        test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Execute tests
        uses: ./.github/actions/azureml-test
        id: execute_tests
        with:
          EXP_NAME: 'nightly_tests'
          TEST_KIND: 'nightly'
          TEST_ENV: 'spark'
          AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }}
          AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }}
          PYTHON_VERSION: ${{ matrix.python-version }}
          TEST_GROUP: ${{ matrix.test-group }}


azureml-cpu-nightly

# ---------------------------------------------------------
# Copyright (c) Recommenders contributors.
# Licensed under the MIT License.
# ---------------------------------------------------------

name: azureml-cpu-nightly

on:
  #          ┌───────────── minute (0 - 59)
  #          │ ┌───────────── hour (0 - 23)
  #          │ │  ┌───────────── day of the month (1 - 31)
  #          │ │  │  ┌───────────── month (1 - 12 or JAN-DEC)
  #          │ │  │  │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
  #          │ │  │  │ │
  #          │ │  │  │ │
  schedule:
    - cron: '0 0 */5 * *'  # running every 5 days at 12AM
    # cron works with default branch (main) only: # https://github.community/t/on-schedule-per-branch/17525/2

  push:
    # Because we can't schedule runs for non-main branches,
    # to ensure we are running the build on the staging branch, we can add push policy for it
    branches: [staging]
    paths:
      # Tests will be run only when there are changes in the code:
      - examples/**
      - '!examples/**/*.md'
      - recommenders/**
      - '!recommenders/**/*.md'
      - tests/**
      - '!tests/**/*.md'
      - setup.py

  # Enable manual trigger
  workflow_dispatch:
    input:
      tags:
        description: 'Tags to label this manual run (optional)'
        default: 'Manual trigger'

  # Make this workflow reusable
  workflow_call:

jobs:
  get-test-groups:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Get test group names
        id: get_test_groups
        uses: ./.github/actions/get-test-groups
        with:
          TEST_KIND: "nightly"
          TEST_ENV: "cpu"
      - name: Print test group names
        run: echo ${{ steps.get_test_groups.outputs.test_groups }}
        shell: bash
    outputs:
      test_groups: ${{ steps.get_test_groups.outputs.test_groups }}

  execute-tests:
    needs: get-test-groups
    name: ${{ join(matrix.*, ', ') }}
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 50 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
      matrix:
        python-version: ['"python=3.8"', '"python=3.9"', '"python=3.10"', '"python=3.11"']
        test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Execute tests
        uses: ./.github/actions/azureml-test
        id: execute_tests
        with:
          EXP_NAME: 'nightly_tests'
          TEST_KIND: 'nightly'
          TEST_ENV: 'cpu'
          AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }}
          AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }}
          PYTHON_VERSION: ${{ matrix.python-version }}
          TEST_GROUP: ${{ matrix.test-group }}


Create GitHub Draft Release

# Copyright (c) Recommenders contributors.
# Licensed under the MIT License.

on:
  push:
    # Sequence of patterns matched against refs/tags
    tags:
      - '0.*' # Push events to matching 0.*, 1.*
      - '1.*'

name: Create GitHub Draft Release

jobs:

  unit-test-workflow:
    uses: ./.github/workflows/azureml-unit-tests.yml
    secrets: inherit

  cpu-nightly-workflow:
    uses: ./.github/workflows/azureml-cpu-nightly.yml
    secrets: inherit

  gpu-nightly-workflow:
    uses: ./.github/workflows/azureml-gpu-nightly.yml
    secrets: inherit

  spark-nightly-workflow:
    uses: ./.github/workflows/azureml-spark-nightly.yml
    secrets: inherit

  create-release:
    runs-on: ubuntu-latest
    needs: [unit-test-workflow, cpu-nightly-workflow, gpu-nightly-workflow, spark-nightly-workflow]
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4
      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: "3.8"
      - name: Install wheel package
        run: pip install wheel
      - name: Create wheel from setup.py
        run: python setup.py bdist_wheel
      - name: Create tar gz from setup.py
        run: python setup.py sdist
      - name: Create GitHub Draft Release
        id: create_release
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: ${{ github.ref }}
          release_name: Recommenders ${{ github.ref }}
          body: |
            Changes in this Release
            - First Change
            - Second Change
          draft: true
          prerelease: false
      - name: Upload .whl and .tar.gz files as Release Assets
        id: upload-release-assets
        uses: dwenegar/upload-release-assets@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          release_id: ${{ steps.create_release.outputs.id }}
          assets_path: dist


Update Documentation

# ---------------------------------------------------------
# Copyright (c) Recommenders contributors.
# Licensed under the MIT License.
# ---------------------------------------------------------

name: Update Documentation

on:
  push:
    branches:
      - main

jobs:
  build:
    runs-on: ubuntu-22.04

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install dependencies
        run: |
          pip install -q --upgrade pip setuptools wheel
          pip install -q --no-use-pep517 lightfm
          pip install -q .[all]
          pip install -q -r docs/requirements-doc.txt

      - name: List dependencies
        run: |
          pip list

      - name: Build documentation
        run: |
          jupyter-book config sphinx docs/
          sphinx-build docs docs/_build/html -b html

      - name: Configure Git
        run: |
          git config --global user.email "actions@github.com"
          git config --global user.name "GitHub Actions"

      - name: Create and switch to gh-pages branch
        run: |
          git checkout -b gh-pages
          git pull origin gh-pages || true

      - name: Copy built documentation
        run: cp -r docs/_build/html/* .

      - name: Add and commit changes
        run: |
          git add * -f
          git commit -m "Update documentation"

      - name: Configure pull strategy (rebase)
        run: git config pull.rebase true

      - name: Pull latest changes from remote gh-pages branch
        run: git pull -Xtheirs origin gh-pages

      - name: Push changes to gh-pages
        run: git push origin gh-pages


🚀 Build & Publish

name: 🚀 Build & Publish

on:
  push:
    tags:
      - 'v\d+\.\d+\.[0-9a-z]+' # e.g. v1.0.0 or v20.15.10b5

jobs:
  build:
    name: Build Release
    runs-on: ubuntu-latest

    steps:
    - name: Check out
      uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
        cache: 'pip' # caching pip dependencies

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install build

    - name: Build artifacts
      run: |
        python -m build

    - name: Upload artifacts
      uses: actions/upload-artifact@v3
      with:
        name: wheelstorage
        path: ./dist/*
        if-no-files-found: error
        retention-days: 30

  publish:
    name: Publish Release
    needs: build
    runs-on: ubuntu-latest

    steps:
    - name: Get date & flat tag
      id: date_tag
      run: |
        export DATE=$(TZ=Europe/London date +'%Y-%m-%d')
        echo $DATE
        export FLAT_TAG=$(echo ${GITHUB_REF##*/} | sed 's/[v.]//g')
        echo $FLAT_TAG
        echo "TODAY=$DATE" >> "$GITHUB_OUTPUT"
        echo "VERSION=$FLAT_TAG" >> "$GITHUB_OUTPUT"
      shell: bash

    - name: Download artifacts
      uses: actions/download-artifact@v3
      with:
        name: wheelstorage
        path: dist

    - name: Publish to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}

    - name: Create GitHub Release
      uses: softprops/action-gh-release@v1
      with:
        body: '[Release Notes](https://github.com/fonttools/fontbakery/blob/main/CHANGELOG.md#${{ steps.date_tag.outputs.VERSION }}-${{ steps.date_tag.outputs.TODAY }})'
        token: ${{ secrets.GITHUB_TOKEN }}
        prerelease: true
        files: ./dist/*.tar.gz


⚙ Install & Run

# Workflow for testing installing, running, and uninstalling FontBakery
name: ⚙ Install & Run

on:
  push:
    branches:
      - '**'
    paths-ignore:
      - 'docs/**'
      - '**.md'
      - '**.txt'
      - '!requirements*.txt'
    tags-ignore:
      - '**'

  pull_request:
    paths-ignore:
      - 'docs/**'
      - '**.md'
      - '**.txt'
      - '!requirements*.txt'

jobs:
  install-run:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-13, windows-latest]
        profile:
          - description: Universal profile on a static font
            name: universal
            args: >-
              -x win_ascent_and_descent
              -x os2_metrics_match_hhea
              -x soft_dotted
              data/test/source-sans-pro/OTF/SourceSansPro-Regular.otf
              data/test/source-sans-pro/OTF/SourceSansPro-Italic.otf
          - description: Universal profile on a variable font
            name: universal
            args: >-
              -x win_ascent_and_descent
              -x os2_metrics_match_hhea
              -x fsselection
              -x valid_default_instance_nameids
              -x soft_dotted
              data/test/source-sans-pro/VAR/SourceSansVariable-Roman.ttf
          - description: OpenType profile on a TTC
            name: opentype
            args: >-
              -x tabular_kerning
              -x style
              -x family
              -x monospace
              data/test/ttc/NotoSerifToto.ttc
          - description: Font Bureau profile
            name: fontbureau
            args: >-
              -c ots
              -c ytlc_sanity
              data/test/fontbureau/ytlcSample.ttf
          - description: Adobe Fonts profile on a static font
            name: adobefonts
            args: >-
              data/test/source-sans-pro/OTF/SourceSansPro-Regular.otf
              data/test/source-sans-pro/OTF/SourceSansPro-Italic.otf
          - description: Adobe Fonts profile on a variable font
            name: adobefonts
            args: >-
              data/test/source-sans-pro/VAR/SourceSansVariable-Roman.ttf
          - description: UFO profile
            extras: ".[ufo]"
            name: ufo
            args: --verbose data/test/test.ufo
          - description: UFO Profile on a designspace
            extras: ".[ufo]"
            name: ufo
            args: >-
              -x designspace_has_consistent
              "data/test/stupidfont/Stupid Font.designspace"
          - description: Shaping checks
            extras: ".[shaping]"
            name: shaping
            args: data/test/mada/Mada-Regular.ttf
          - description: Google Fonts
            extras: ".[googlefonts]"
            name: googlefonts
            args: >-
              -c canonical_filename
              -c vendor_id
              -c glyph_coverage
              -c name/license
              -c hinting_impact
              -c unreachable_glyphs
              -c contour_count
              -c outline_colinear_vectors
              data/test/cabin/Cabin-*.ttf
          - description: Fontwerk
            extras: ".[fontwerk]"
            name: fontwerk
            args: >-
              -c weight_class_fvar
              -c inconsistencies_between_fvar_stat
              -c style_linking
              -c consistent_axes
              -c metadata/parses
              -c usweightclass
              data/test/source-sans-pro/VAR/SourceSansVariable-Roman.ttf
          - description: Noto fonts
            extras: ".[notofonts]"
            name: notofonts
            args: >-
              -c unicode_range_bits
              -c noto_trademark
              -c noto_vendor
              -c alien_codepoints
              -c tnum_horizontal_metrics
              -c control_chars
              -c canonical_filename
              data/test/notosanskhudawadi/NotoSansKhudawadi-Regular.ttf
          - description: Type Network
            extras: ".[typenetwork]"
            name: typenetwork
            args: >-
              -c glyph_coverage
              -c vertical_metrics
              -c font_is_centered_vertically
              -c family/tnum_horizontal_metrics
              -c family/equal_numbers_of_glyphs
              data/test/cabin/Cabin-Regular.ttf data/test/cabin/Cabin-Italic.ttf

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0  # unshallow fetch for setuptools-scm (otherwise the version is always 0.1.dev1)

    - name: Set up Python 3.10
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"
        cache: 'pip' # caching pip dependencies

    - name: Install FontBakery (no extras)
      run: |
        python -m pip install --upgrade pip
        python -m pip install .
        fontbakery -h
        fontbakery --version

    - name: Install any extras
      if: ${{ matrix.profile.extras }}
      run: |
        python -m pip install ${{ matrix.profile.extras }}

    - name: Run ${{matrix.profile.description}}
      run: >-
        fontbakery check-${{ matrix.profile.name }}
        --error-code-on ERROR
        ${{ matrix.profile.args }}


🧑🔬 Lint & Test

name: 🧑🔬 Lint & Test

on:
  push:
    branches:
      - '**'
    paths-ignore:
      - 'docs/**'
      - '**.md'
      - '**.txt'
      - '!requirements*.txt'
    tags-ignore:
      - '**'

  pull_request:
    paths-ignore:
      - 'docs/**'
      - '**.md'
      - '**.txt'
      - '!requirements*.txt'

jobs:
  lint:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0  # unshallow fetch for setuptools-scm (otherwise the version is always 0.1.dev1)

    - name: Set up Python 3.10
      uses: actions/setup-python@v4
      with:
        python-version: "3.10"
        cache: 'pip' # caching pip dependencies

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        # For pytype we need everything
        python -m pip install .[tests,docs]
        python -m pip install glyphsLib
        python -m pip install pytype # Not in requirements as it doesn't work on Windows

    - name: Run black, pylint and pytype
      run: |
        black . --check --diff --color
        pylint --recursive=true Lib tests
        pytype

  test:
    needs: lint
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-13, windows-latest]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0  # unshallow fetch for setuptools-scm (otherwise the version is always 0.1.dev1)

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip' # caching pip dependencies

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install .[tests,docs]
        python -m pip freeze --all

    - name: Install FontBakery
      run: |
        python -m pip install . --use-pep517

    - name: Test with pytest
      run: |
        python -m pytest

    - name: Run coverage (only Ubuntu/Python 3.10)
      if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'
      run: |
        python -m pytest --quiet --cov

    - name: Ensure documentation can build (only Ubuntu/Python 3.10)
      if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'
      run: |
        cd docs; make html


Lint

name: Lint

on: [pull_request]

jobs:
  lint:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        os: [ubuntu-latest]
    steps:
      - uses: actions/checkout@v3
      - uses: psf/black@stable
        with:
          options: "--check --verbose"
          src: "."
          jupyter: true


name: DeepEval Test Run with Enhanced Configuration

on:
  push:
  pull_request:

permissions:
  contents: read
  pull-requests: write

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true

      # - name: Load cached venv
      #   id: cached-poetry-dependencies
      #   uses: actions/cache@v3
      #   with:
      #     path: .venv
      #     key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction

      - name: List Installed Packages
        run: poetry run pip list

      - name: Check if 'deepeval' script is available
        run: ls -l $(poetry env info --path)/bin/deepeval || echo "deepeval script not found"

      - name: Run deepeval login
        env:
          CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
        run: poetry run deepeval login --confident-api-key "$CONFIDENT_API_KEY"

      - name: Run deepeval tests and capture output
        run: poetry run deepeval test run tests/test_deployment.py > output.txt 2>&1

      - name: Display output.txt
        if: always()
        run: cat output.txt

      - name: Remove ANSI codes from output
        run: sed -i -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]//g" output.txt

      - name: Save test results to environment variable
        run: echo "test_result<<EOF" >> $GITHUB_ENV && cat output.txt >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV


Run All Tests with Poetry

name: Run All Tests with Poetry

on:
  push:
  pull_request:

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      #----------------------------------------------
      #       check-out repo and set-up python
      #----------------------------------------------
      - name: Check out repository
        uses: actions/checkout@v3
      - name: Set up python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      #----------------------------------------------
      #  -----  install & configure poetry  -----
      #----------------------------------------------
      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      #----------------------------------------------
      #       load cached venv if cache exists
      #----------------------------------------------
      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      #----------------------------------------------
      # install dependencies if cache does not exist
      #----------------------------------------------
      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root
      #----------------------------------------------
      # install your root project, if required
      #----------------------------------------------

      - name: Install project
        run: poetry install --no-interaction

      - name: Set NLTK_DATA
        run: echo "NLTK_DATA=${{ runner.workspace }}/nltk_data" >> $GITHUB_ENV

      - name: Download NLTK resources
        run: |
          source .venv/bin/activate
          python -m nltk.downloader punkt stopwords

      #----------------------------------------------
      #              run test suite
      #----------------------------------------------
      - name: Run tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          poetry run pytest tests/ --ignore=tests/test_deployment.py


name: main_workflow

on:
  push:
    branches:
      - develop
      - main
  pull_request:

jobs:
  lock_check:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest, macos-latest, windows-latest ]
        python-version: [ "3.10" ]
        poetry-version: ["1.3.2"]
    timeout-minutes: 10
    defaults:
      run:
        shell: bash
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install and configure Poetry
      uses: snok/install-poetry@v1
      with:
        version: ${{ matrix.poetry-version }}
        virtualenvs-create: true
        virtualenvs-in-project: false
        virtualenvs-path: ~/my-custom-path
        installer-parallel: true
    - name: Check if dependencies can be locked
      run: poetry lock --check
    - name: Poetry install requirements and check they can be locked
      run: poetry install --all-extras --no-interaction -vvv

  copyright_and_dependencies_check:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ "3.10" ]

    timeout-minutes: 30

    steps:
    - uses: actions/checkout@v2
      with:
        fetch-depth: 0
    - uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run:  |
        sudo apt-get update --fix-missing
        sudo apt-get autoremove
        sudo apt-get autoclean
        pip install tomte[tox,cli]==0.2.15
        pip install --user --upgrade setuptools
    - name: Check copyright headers
      run: tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription
    - name: License compatibility check
      run: tox -e liccheck
# TODO: reactivate once false positives are fixed
#    - name: Check dependencies
#      run: tox -e check-dependencies
    - name: Check doc links
      run: tomte check-doc-links
    - name: Check doc IPFS hashes
      run: tox -e check-doc-hashes

  linter_checks:
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ "3.10" ]

    timeout-minutes: 30

    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - uses: actions/setup-go@v3
      with:
        go-version: "1.17.7"
    - name: Install dependencies
      run:  |
        sudo apt-get update --fix-missing
        sudo apt-get autoremove
        sudo apt-get autoclean
        pip install tomte[tox,cli]==0.2.15
        pip install --user --upgrade setuptools

        # install Protobuf compiler
        wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
        unzip protoc-24.3-linux-x86_64.zip -d protoc
        sudo mv protoc/bin/protoc /usr/local/bin/protoc

        # install IPFS
        sudo apt-get install -y wget
        wget -O ./go-ipfs.tar.gz https://dist.ipfs.io/go-ipfs/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz
        tar xvfz go-ipfs.tar.gz
        sudo mv go-ipfs/ipfs /usr/local/bin/ipfs
        ipfs init

        # Install `mdspell` for spelling checks
        sudo npm install -g markdown-spellcheck
        make protolint_install

    - name: Check generate protocols
      run: echo "Skipped..." # tox -e check-generate-all-protocols
    - name: Security checks
      run: tomte check-security
    - name: Check packages
      run: tox -e check-packages
    - name: Check hash
      run: tox -e check-hash
    - name: Code checks
      run: tomte check-code
    - name: Check spelling
      run: tomte check-spelling
    - name: AbciApp consistency checks
      run: |
        tox -e check-abci-docstrings
        tox -e check-abciapp-specs
        # tox -e check-handlers
        # ignore for now due to https://github.com/valory-xyz/open-autonomy/issues/1988
        # tox -e analyse-service

  scan:
    name: gitleaks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-go@v3
        with:
          go-version: "1.17.7"
      - run: |
          wget https://github.com/zricethezav/gitleaks/releases/download/v8.10.1/gitleaks_8.10.1_linux_x64.tar.gz && \
          tar -xzf gitleaks_8.10.1_linux_x64.tar.gz && \
          sudo install gitleaks /usr/bin && \
          gitleaks detect --report-format json --report-path leak_report -v

  tools_checks:
    environment: 'ci'
    continue-on-error: False
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ "3.10" ]

    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          pip install tomte[tox,cli]==0.2.15
          pip install --user --upgrade setuptools
      - name: Tool unit tests
        env:
          OPENAI_SECRET_KEY: ${{ secrets.OPENAI_API_KEY }}
          STABILITY_API_KEY: ${{ secrets.STABILITY_AI_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          GOOGLE_ENGINE_ID: ${{ vars.GOOGLE_ENGINE_ID }}
          CLAUDE_API_KEY: ${{ secrets.CLAUDE_API_KEY }}
          REPLICATE_API_KEY: ${{ secrets.REPLICATE_API_KEY }}
          NEWS_API_KEY: ${{ secrets.NEWS_API_KEY }}
          OPENROUTER_API_KEY: ${{ secrets.OPEN_ROUTER_API_KEY }}
          GNOSIS_RPC_URL: ${{ secrets.GNOSIS_RPC_URL }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
        run: |
          printenv
          tox -e check-tools


Release Flow

name: Release Flow

on:
  release:
    types: [prereleased, released]

jobs:
  publish-packages:
    name: Push Packages
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10"]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-versions }}
      - uses: addnab/docker-run-action@v3
        with:
            image: valory/open-autonomy-user:latest
            options: -v ${{ github.workspace }}:/work
            run: |
              echo "Pushing Packages"
              cd /work
              export AUTHOR=$(grep 'service' packages/packages.json | awk -F/ '{print $2}' | head -1)
              autonomy init --reset --author $AUTHOR --ipfs --remote
              autonomy push-all
  publish-images:
    name: Publish Docker Images
    runs-on: ${{ matrix.os }}
    needs:
      - "publish-packages"
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10"]
    env:
        DOCKER_USER: ${{secrets.DOCKER_USER}}
        DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
    steps:
      - uses: actions/checkout@v3
      - name: Set up tag and vars
        uses: addnab/docker-run-action@v3
        with:
            image: valory/open-autonomy-user:latest
            options: -v ${{ github.workspace }}:/work
            run: |
              echo "Setting Tag Images"
              cd /work
              apt-get update && apt-get install git -y || exit 1
              git config --global --add safe.directory /work
              export TAG=$(git describe --exact-match --tags $(git rev-parse HEAD)) || exit 1
              if [ $? -eq 0 ]; then
                  export TAG=`echo $TAG | sed 's/^v//'`
              else
                  echo "You are not on a tagged branch"
                  exit 1
              fi
              echo VERSION=$TAG> env.sh
              echo AUTHOR=$(grep 'service' packages/packages.json | awk -F/ '{print $2}' | head -1) >> env.sh
              echo SERVICE=$(grep 'service' packages/packages.json | awk -F/ '{print $3}' | head -1) >> env.sh
              echo AGENT=$(grep 'agent' packages/packages.json | awk -F/ '{print $3}' | head -1) >> env.sh
              echo DEFAULT_IMAGE_TAG=$(cat packages/packages.json | grep agent | awk -F: '{print $2}' | tr -d '", ' | head -n 1) >> env.sh
              cat env.sh

      - uses: addnab/docker-run-action@v3
        name: Build Images
        with:
            image: valory/open-autonomy-user:latest
            options: -v ${{ github.workspace }}:/work
            shell: bash
            run: |
              echo "Building Docker Images"
              cd /work
              source env.sh || exit 1
              pip install requests==2.28.1
              echo "Building images for $AUTHOR for service $SERVICE"
              autonomy init --reset --author $AUTHOR --ipfs --remote
              autonomy fetch $AUTHOR/$SERVICE --service --local || exit 1
              cd $SERVICE || exit 1
              autonomy build-image || exit 1
              autonomy build-image --version $VERSION || exit 1

      - name: Docker login
        run: |
            echo  $DOCKER_PASSWORD | docker login -u $DOCKER_USER --password-stdin
      - name: Docker Push
        run: |
          source env.sh
          echo "Pushing $DOCKER_USER/oar-$AGENT:$VERSION"
          echo "Pushing $DOCKER_USER/oar-$AGENT:$DEFAULT_IMAGE_TAG"
          docker push $DOCKER_USER/oar-$AGENT:$VERSION
          docker push $DOCKER_USER/oar-$AGENT:$DEFAULT_IMAGE_TAG
  deploy-service-on-propel:
    name: Deploy service on propel
    environment: ${{ fromJSON('[["staging", "staging"], ["production", "shorts_production"]]')[github.event.action == 'released'][endsWith(github.event.release.name, '(shorts)')] }}
    needs:
    - "publish-packages"
    - "publish-images"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10"]
    env:
      CMD: "propel -U ${{ vars.PROPEL_BASE_URL }}"
    steps:
      - uses: actions/checkout@master
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-versions }}
      - name: Install dependencies
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get autoremove
          sudo apt-get autoclean
          python -m pip install --upgrade pip
          pip install propel-client open-autonomy

      - name: Make use proxy instead of actual nlb by dns override
        run: |
          # for staging
          export IP_ADDR=$(dig +short balancer.tcp.propel.staging.autonolas.tech)
          echo -e "\n$IP_ADDR\tapp.propel.staging.valory.xyz\n"  | sudo tee -a /etc/hosts

          # for prod
          export IP_ADDR=$(dig +short balancer.tcp.propel.autonolas.tech)
          echo -e "\n$IP_ADDR\tapp.propel.valory.xyz\n"  | sudo tee -a /etc/hosts

      - name: Login to propel
        run: |
          $CMD login -u  '${{ vars.PROPEL_USERNAME }}' -p  '${{ secrets.PROPEL_PASSWORD }}'

      - name: Do a deployment
        run: |
          # determine ipfs hash id
          export IPFS_HASH=$(jq '.dev | to_entries[] | select(.key | startswith("service"))| .value' -r  ./packages/packages.json | head -n1)
          export SERVICE_PATH=service_for_propel

          # fetch service file and check it published
          autonomy init --reset --author ci --ipfs --remote
          autonomy fetch $IPFS_HASH --service --alias $SERVICE_PATH

          # get env vars from github actions
          echo '${{ toJSON(secrets) }}' | jq -r 'to_entries|map("export \(.key)=\(.value|tojson)")|.[]' > github.vars
          echo >> github.vars
          echo '${{ toJSON(vars) }}' | jq -r 'to_entries|map("export \(.key)=\(.value|tojson)")|.[]' >> github.vars
          source github.vars
          $CMD service deploy --name  '${{ vars.SERVICE_NAME }}'  --service-dir $SERVICE_PATH --service-ipfs-hash $IPFS_HASH  --ingress-enabled true --keys '${{ vars.SERVICE_KEYS }}' --timeout 320


All Bug Reproduction

name: All Bug Reproduction
on:
  schedule:
    - cron: '0 0 * * 0'  # Run every Sunday at 00:00 UTC
  workflow_dispatch:
jobs:
  bug_reproduction:
    timeout-minutes: 1440
    runs-on: [self-hosted, Linux, X64]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: '1.20.5'
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements-dev.txt
          make
      - name: Create profile dir with 777 mode
        run: |
          mkdir -m 777 -p profile/data
      - name: Run bug reproduction
        run: |
          python -m pytest test/e2e_tests -m "all_bug_reproduction" -s


counter

name: counter
on:
  # Triggers the workflow on push or pull request events but only for the "main" branch
  push:
    branches: [ "main" ]
  workflow_dispatch:

jobs:
  conditional_job_check_files:
    runs-on: 'ubuntu-20.04'
    # Declare outputs for next jobs
    outputs:
      docs_changed: ${{ steps.check_file_changed.outputs.docs_changed }}
    steps:
    - uses: actions/checkout@v2
      with:
        # Checkout as many commits as needed for the diff
        fetch-depth: 2
    - shell: pwsh
      id: check_file_changed
      run: |
        # Diff HEAD with the previous commit
        $diff = git diff --name-only HEAD^ HEAD

        # Check if a file under docs/ or with the .md extension has changed (added, modified, deleted)
        $SourceDiff = $diff | Where-Object { $_ -match 'bugs.md$' }
        $HasDiff = $SourceDiff.Length -gt 0

        # Set the output named "docs_changed"
        Write-Host "::set-output name=docs_changed::$HasDiff"

  # Run the job only with "docs_changed" equals "True"
  conditional_job:
    runs-on: 'ubuntu-20.04'
    needs: [ conditional_job_check_files ]
    if: needs.conditional_job_check_files.outputs.docs_changed == 'True'
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: 'Set up Python'
      uses: actions/setup-python@v4
      with:
          python-version: '3.10'
    - name: Change bugs.md
      shell: bash
      run: python3 .github/workflows/bug_counter.py

    - name: Check if bugs.md is changed by the script
      id: verify_diff
      run: |
        git diff --quiet bugs.md || echo "::set-output name=are_bugs_changed::true"

    - name: Commit files
      if: needs.conditional_job_check_files.outputs.docs_changed == 'True' || github.event_name == 'workflow_dispatch'
      run: |
        git config --local user.name github-actions[bot]
        git config --local user.email github-actions[bot]@users.noreply.github.com
        git status
        git add bugs.md
        git commit -m "update: bug number"
        git log
        git push


End-to-End Test

name: End-to-End Test
on:
  pull_request:
  workflow_dispatch:
jobs:
  bug_reproduction:
    timeout-minutes: 60
    runs-on: [self-hosted, Linux, X64]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: '1.20.5'
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements-dev.txt
          make
      - name: Create profile dir with 777 mode
        run: |
          mkdir -m 777 -p profile/data
      - name: Run bug reproduction
        run: |
          python -m pytest test/e2e_tests -m "single_bug_reproduction" -s


Kubernetes Engine Test

name: Kubernetes Engine Test
on:
  push:
    paths:
      - acto/kubernetes_engine/**
  workflow_dispatch:
jobs:
  KubernetesEngineTest:
    runs-on:  ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v4
        with:
          go-version: '1.20.5'
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements-dev.txt
          make
      - name: Install Kind
        run: |
          go install sigs.k8s.io/kind@v0.20.0
      - name: Install minikube
        run: |
          curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
          sudo install minikube-linux-amd64 /usr/local/bin/minikube
      - name: Run KubernetesEngineTest
        run: |
          pytest -m "kubernetes_engine"


Regression Test

name: Regression Test
on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  workflow_dispatch:
jobs:
  unittest:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: "3.10"
      - name: Install Packages
        run: pip install codespell flake8
      - name: Check Spelling
        run: codespell --skip="./data"|| true
      - name: Check Syntax Error
        run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
      - uses: actions/setup-go@v4
        with:
          go-version: '1.20.5'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements-dev.txt
          make
      - name: Run unittest
        run: >
          COVERAGE_FILE=.coverage.${{ github.sha }}.unittest
          pytest acto
          --junitxml=pytest.xml
          --cov-report=
          --cov=acto
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: .coverage.${{ github.sha }}.unittest
          path: .coverage.${{ github.sha }}.unittest
          retention-days: 1
  integration-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: "3.10"
      - name: Install Packages
        run: pip install codespell flake8
      - name: Check Spelling
        run: codespell --skip="./data"|| true
      - name: Check Syntax Error
        run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
      - uses: actions/setup-go@v4
        with:
          go-version: '1.20.5'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements-dev.txt
          make
      - name: Run integration test
        run: >
          COVERAGE_FILE=.coverage.${{ github.sha }}.integration-test
          pytest test/integration_tests
          --junitxml=pytest.xml
          --cov-report=
          --cov=acto
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: .coverage.${{ github.sha }}.integration-test
          path: .coverage.${{ github.sha }}.integration-test
          retention-days: 1
  coverage-report:
    runs-on: ubuntu-latest
    needs: [unittest, integration-test]
    permissions:
      pull-requests: write
      contents: write
    steps:
      - uses: actions/checkout@v3
      - name: Download unittest coverage
        uses: actions/download-artifact@v4
        with:
          name: .coverage.${{ github.sha }}.unittest
      - name: Download integration test coverage
        uses: actions/download-artifact@v4
        with:
          name: .coverage.${{ github.sha }}.integration-test
      - name: Merge coverage files
        run: |
          pip install coverage
          coverage combine
      - name: list files
        run: ls -al
      - name: Coverage comment
        uses: py-cov-action/python-coverage-comment-action@v3
        with:
          GITHUB_TOKEN: ${{ github.token }}
          ANNOTATE_MISSING_LINES: true


ci

# Workflow file for continuous integration
# Conda environment is caches to avoid long wait times for CI
# Based on code from: https://github.com/epassaro/cache-conda-envs/blob/main/.github/workflows/ci.yml
# Based on code from: https://github.com/marketplace/actions/setup-miniconda

name: ci

on: [push, pull_request]

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: psf/black@stable
  build:
    name: linux-64
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
    - name: Checkout project
      uses: actions/checkout@v2
    - name: Setup miniconda
      uses: conda-incubator/setup-miniconda@v2
      with:
          auto-activate-base: false
          activate-environment: climakitae-tests
          environment-file: conda-linux-64.lock
          use-only-tar-bz2: true
    - name: Output conda info
      run: conda info
    - name: Install pip packages
      run: pip install dask-gateway==2023.1.1 dask-geopandas==0.3.1 xclim==0.42.0
    - name: Install climakitae
      run: pip install .
    - name: Output conda list
      run: conda list
    - name: Test with pytest
      run: pytest --no-header -vv


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ master ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ master ]
  schedule:
    - cron: '28 6 * * 5'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
    - name: Checkout repository
      uses: actions/checkout@v3

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v3

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3


Package

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Package

on:
  push:
    branches: [ "master" ]
  pull_request: {}

env:
  PYTHON_VERSION: "3.9.14"

jobs:
  bandit:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v3
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Install dependencies
        uses: ./.github/actions/install-dependencies
        with:
          test-requirements: "true"

      - name: Run bandit
        run: bandit --ini .bandit -r instagrapi

  flake8:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v3
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Install dependencies
        uses: ./.github/actions/install-dependencies
        with:
          test-requirements: "true"

      - name: Run flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --statistics
  isort:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v3
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Install dependencies
        uses: ./.github/actions/install-dependencies
        with:
          requirements: "true"
          test-requirements: "true"

      - name: Run isort
        run: isort --check-only instagrapi

  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ "3.10" ]
    env:
      TEST_ACCOUNTS_URL: ${{ secrets.TEST_ACCOUNTS_URL }}
    steps:
      - name: Check out code
        uses: actions/checkout@v3
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        uses: ./.github/actions/install-dependencies
        with:
          requirements: "true"
          test-requirements: "true"
      - name: Run media test
        run: pytest -sv tests.py::ClientMediaTestCase
      - name: Run user test
        run: pytest -sv tests.py::ClientUserTestCase
      # - name: Run comment test
      #   run: pytest -sv tests.py::ClientCommentTestCase
      # - name: Run location test
      #   run: pytest -sv tests.py::ClientLocationTestCase
      # - name: Run hashtag test
      #   run: pytest -sv tests.py::ClientHashtagTestCase
      # - name: Run share test
      #   run: pytest -sv tests.py::ClientShareTestCase
      - name: Run highlight test
        run: pytest -sv tests.py::ClientHighlightTestCase

  build-docs:
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    steps:
      - name: Check out code
        uses: actions/checkout@v3

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies
        uses: ./.github/actions/install-dependencies
        with:
          requirements: "true"
          test-requirements: "true"

      - name: Build Docs
        run: mkdocs build --strict

      # upload artifact for dev build
      - name: Upload coverage results artifact
        if: github.ref != 'refs/heads/main'
        uses: actions/upload-artifact@v4
        with:
          name: docs-site
          path: site/

  update-dev-docs:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    steps:
      - name: Check out code
        uses: actions/checkout@v3

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies
        uses: ./.github/actions/install-dependencies
        with:
          requirements: "true"
          test-requirements: "true"

      - name: Push documentaiton changes
        uses: ./.github/actions/publish-docs-with-mike
        with:
          version_name: dev


CI

name: CI

on:
  push:
    # Sequence of patterns matched against refs/tags
    tags:
      - "v*"

env:
  DOCKERHUB_REPO: dataelement/

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build_bisheng_langchain:
    runs-on: ubuntu-latest
    #if: startsWith(github.event.ref, 'refs/tags')
    steps:
      - name: checkout
        uses: actions/checkout@v2

      - name: Get version
        id: get_version
        run: |
          echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}

      - name: Set Environment Variable
        run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV

      # 构建 bisheng_langchain
      - name: Set python version 3.8
        uses: actions/setup-python@v1
        with:
          python-version: 3.8

      - name: Build PyPi bisheng-langchain and push
        id: pypi_build_bisheng_langchain
        run: |
          pip install Cython
          pip install wheel
          pip install twine
          cd ./src/bisheng-langchain
          python setup.py bdist_wheel
          set +e
          twine upload dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }} --repository pypi
          set -e

  build_bisheng:
    needs: build_bisheng_langchain
    runs-on: ubuntu-latest
    # if: startsWith(github.event.ref, 'refs/tags')
    steps:
      - name: checkout
        uses: actions/checkout@v2

      - name: Get version
        id: get_version
        run: |
          echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}

      - name: Set Environment Variable
        run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV

      # 登录 docker hub
      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          # GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
          # DOCKERHUB_USERNAME 是 docker hub 账号名.
          # DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      # - name: Login to DockerHub
      #   uses: docker/login-action@v1
      #   with:
      #     registry: https://cr.dataelem.com/
      #     username: ${{ secrets.CR_DOCKERHUB_USERNAME }}
      #     password: ${{ secrets.CR_DOCKERHUB_TOKEN }}

      # 构建 backend 并推送到 Docker hub
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1

      - name: set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: install poetry
        uses: snok/install-poetry@v1
        with:
          installer-parallel: true
      - name: build lock
        run: |
          cd ./src/backend
          pip install bisheng_langchain==$RELEASE_VERSION
          sed -i 's/^bisheng_langchain.*/bisheng_langchain = "'$RELEASE_VERSION'"/g' pyproject.toml
          poetry lock
          cd ../../

      - name: Build backend and push
        id: docker_build_backend
        uses: docker/build-push-action@v2
        with:
          # backend 的context目录
          context: "./src/backend/"
          # 是否 docker push
          push: true
          # docker build arg, 注入 APP_NAME/APP_VERSION
          platforms: linux/amd64,linux/arm64
          build-args: |
            APP_NAME="bisheng-backend"
            APP_VERSION=${{ steps.get_version.outputs.VERSION }}
          # 生成两个 docker tag: ${APP_VERSION} 和 latest
          tags: |
            ${{ env.DOCKERHUB_REPO }}bisheng-backend:latest
            ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
      # 构建 Docker frontend 并推送到 Docker hub
      - name: Build frontend and push
        id: docker_build_frontend
        uses: docker/build-push-action@v2
        with:
          # frontend 的context目录
          context: "./src/frontend/"
          # 是否 docker push
          push: true
          # docker build arg, 注入 APP_NAME/APP_VERSION
          platforms: linux/amd64,linux/arm64
          build-args: |
            APP_NAME="bisheng-frontend"
            APP_VERSION=${{ steps.get_version.outputs.VERSION }}
          # 生成两个 docker tag: ${APP_VERSION} 和 latest
          tags: |
            ${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest
            ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}

      - name: Process git message
        id: process_message
        run: |
          value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g')
          value=$(echo "${value}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\r/%0A/g')
          echo "message=${value}" >> $GITHUB_ENV
        shell: bash

      - name: notify feishu
        uses: fjogeleit/http-request-action@v1
        with:
          url: ${{ secrets.FEISHU_WEBHOOK }}
          method: 'POST'
          data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{ steps.get_version.outputs.VERSION }}发布成功",  "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}'


name: release

on:
  push:
    # Sequence of patterns matched against refs/tags
    branches:
      - "release"

env:
  DOCKERHUB_REPO: dataelement/
  PY_NEXUS: 110.16.193.170:50083
  DOCKER_NEXUS: 110.16.193.170:50080

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    runs-on: ubuntu-latest
    #if: startsWith(github.event.ref, 'refs/tags')
    steps:
      # deploy
      - name: checkout
        uses: actions/checkout@v2

      - name: Process git message
        id: process_message
        run: |
          value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g' )
          echo "message=${value}" >> $GITHUB_ENV
        shell: bash

      - name: notify feishu
        uses: fjogeleit/http-request-action@v1
        with:
          url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b'
          method: 'POST'
          data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{github.event.pusher.name}}提交代码，开始编译",  "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}'

      - name: Set Environment Variable
        run: echo "RELEASE_VERSION=99.99.99" >> $GITHUB_ENV

      - name: Set python version 3.8
        uses: actions/setup-python@v1
        with:
          python-version: 3.8

      - name: Build PyPi bisheng-langchain and push
        id: pypi_build_bisheng_langchain
        run: |
          pip install Cython
          pip install wheel
          pip install twine
          cd ./src/bisheng-langchain
          python setup.py bdist_wheel
          repo="http://${{ env.PY_NEXUS }}/repository/pypi-hosted/"
          twine upload --verbose -u ${{ secrets.NEXUS_USER }}  -p ${{ secrets.NEXUS_PASSWORD }} --repository-url $repo  dist/*.whl
          cd ../../

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1

      - name: set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      # 发布到 私有仓库
      - name: set insecure registry
        run: |
          echo "{ \"insecure-registries\": [\"http://${{ env.DOCKER_NEXUS }}\"] }" | sudo tee /etc/docker/daemon.json
          sudo service docker restart

      # - name: Set up QEMU
      #   uses: docker/setup-qemu-action@v1
      - name: Login Nexus Container Registry
        uses: docker/login-action@v2
        with:
          registry: http://${{ env.DOCKER_NEXUS }}/
          username: ${{ secrets.NEXUS_USER }}
          password: ${{ secrets.NEXUS_PASSWORD }}

      # 替换poetry编译为私有服务
      - name: replace self-host repo
        uses: snok/install-poetry@v1
        with:
          installer-parallel: true

      - name: build lock
        run: |
          cd ./src/backend
          sed -i 's/^bisheng_langchain.*/bisheng_langchain = "'$RELEASE_VERSION'"/g' pyproject.toml
          poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple
          poetry lock
          cd ../../

      # 构建 backend 并推送到 Docker hub
      - name: Build backend and push
        id: docker_build_backend
        uses: docker/build-push-action@v2
        with:
          # backend 的context目录
          context: "./src/backend/"
          # 是否 docker push
          push: true
          platforms: linux/amd64,linux/arm64
          # docker build arg, 注入 APP_NAME/APP_VERSION
          build-args: |
            APP_NAME="bisheng-backend"
            APP_VERSION="release"
          # 生成两个 docker tag: ${APP_VERSION} 和 latest
          tags: |
            ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-backend:release
      # 构建 Docker frontend 并推送到 Docker hub
      - name: Build frontend and push
        id: docker_build_frontend
        uses: docker/build-push-action@v2
        with:
          # frontend 的context目录
          context: "./src/frontend/"
          # 是否 docker push
          push: true
          platforms: linux/amd64,linux/arm64
          # docker build arg, 注入 APP_NAME/APP_VERSION
          build-args: |
            APP_NAME="bisheng-frontend"
            APP_VERSION="release"
          # 生成两个 docker tag: ${APP_VERSION} 和 latest
          tags: |
            ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-frontend:release
      # deploy
      - name: notify feishu
        uses: fjogeleit/http-request-action@v1
        with:
          url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b'
          method: 'POST'
          data: '{"msg_type":"text","content":{"text":"release 编译成功, 准备部署"}}'

      - name: Deploy Stage
        uses: fjogeleit/http-request-action@v1
        with:
          timeout: 200000
          url: 'https://bisheng.dataelem.com/deploy/cgi-bin/deploy_script.py'
          method: 'GET'

      - name: notify feishu
        uses: fjogeleit/http-request-action@v1
        with:
          url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b'
          method: 'POST'
          data: '{"msg_type":"text","content":{"text":"release 部署成功"}}'


main

name: main

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main, develop ]

jobs:
  tii-deb-build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        ros2_distro: [ humble ]
    steps:

      - name: Checkout mesh_com
        uses: actions/checkout@v2
        with:
          path: mesh_com

      - name: Run mesh_com docker build
        env:
          ROS: 1
          ROS_DISTRO: ${{ matrix.ros2_distro }}
          PACKAGE_NAME: mesh_com
        run: |
          set -eux
          mkdir bin
          pushd mesh_com
          ./build.sh ../bin/
          popd

      - uses: jfrog/setup-jfrog-cli@v2
        env:
          JF_ARTIFACTORY_1: ${{ secrets.ARTIFACTORY_CLOUD_TOKEN }}

      - name: Upload to Artifactory
        env:
          ARTIFACTORY_REPO: ssrc-deb-public-local
          DISTRIBUTION: focal
          COMPONENT: fog-sw
          ARCHITECTURE: amd64
          BUILD_NAME: mesh-com
          CI: true
        if: github.event_name == 'push'
        run: |
          set -exu
          jfrog rt ping
          pkg=$(find bin -name 'ros-${{ matrix.ros2_distro }}-mesh-com*.deb')
          pkg_name=$(basename $pkg)
          jfrog rt u --deb "$DISTRIBUTION/$COMPONENT/$ARCHITECTURE" \
                     --target-props COMMIT="$GITHUB_SHA" \
                     --build-name "$BUILD_NAME" \
                     --build-number "$GITHUB_SHA" \
                     "$pkg" \
                     "$ARTIFACTORY_REPO/$pkg_name"
          jfrog rt build-publish "$BUILD_NAME" "$GITHUB_SHA"
          jfrog rt bpr "$BUILD_NAME" "$GITHUB_SHA" "$ARTIFACTORY_REPO" \
                       --status dev \
                       --comment "development build"


tii-mesh-com

name: tii-mesh-com

permissions:
  contents: read
  packages: write

on:
  push:
    branches: [ main, develop ]
    tags:
      - 'v*'
      - v[0-9]+.[0-9]+.[0-9]+
      - v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+
  pull_request:

jobs:
  tii-mesh-com:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false

      - uses: actions/checkout@v4

      - uses: docker/setup-buildx-action@v3

      - name: Setup QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: amd64,arm64,riscv64

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ghcr.io/tiiuae/tii-mesh-com
          tags: |
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=sha
            type=raw,value=latest

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build tii-mesh-com image and push
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64,linux/riscv64
          file: ./modules/mesh_com/Dockerfile
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}


tests

name: tests

on:
  push:
    branches: [ master ]
  pull_request:
  workflow_call:

jobs:
  pytest:
    # Containers must run in Linux based operating systems
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: 3.8

      - name: Cache environment
        uses: actions/cache@master
        id: cache
        with:
          path: ${{ env.pythonLocation }}/lib/python3.8/site-packages/*
          key: ${{ runner.os }}-packages_only-${{ hashFiles('pyproject.toml') }}
      - name: Install dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          pip install -e .[dev]

      - name: Run pytest
        run: python -m pytest -vv

  mypy:
    # Containers must run in Linux based operating systems
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: 3.8

      - name: Cache environment
        uses: actions/cache@master
        id: cache
        with:
          path: ${{ env.pythonLocation }}/lib/python3.8/site-packages/*
          key: ${{ runner.os }}-packages_only-${{ hashFiles('pyproject.toml') }}
      - name: Install dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          pip install -e .[dev]

      - name: Run mypy
        run: python -m mypy .


lint

name: lint

on:
  push:
    branches: [ master ]
  pull_request:
  workflow_call:

jobs:
  black:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: psf/black@stable
        with:
          options: "--check"
          version: "23.1.0"

  isort:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: isort/isort-action@master


Documentation

name: Documentation

on:
  push:
    branches:
      - master
    paths:
      # For the current generated documentation, these trigger paths are sufficient.
      # Long-term, job-level triggers are more appropriate.
      # https://how.wtf/run-workflow-step-or-job-based-on-file-changes-github-actions.html
      - src/fundus/publishers/**
      - scripts/generate_tables.py

  # https://github.com/stefanzweifel/git-auto-commit-action/issues/211#issuecomment-1100105924
  pull_request_target:
    paths:
      - src/fundus/publishers/**
      - scripts/generate_tables.py

  workflow_dispatch:

env:
  PYTHONPATH: .
  CI_COMMIT_MESSAGE: >-
    ${{
      github.event_name == 'pull_request'
      && format('Update documentation from @ {0} (#{1})', github.event.pull_request.head.sha, github.event.number)
      || format('Update documentation from @ {0}', github.sha)
    }}

jobs:
  supported_publishers:
    runs-on: ubuntu-latest

    permissions:
      contents: write

    steps:
      - name: Set up Git repository
        uses: actions/checkout@v4
    # https://github.com/stefanzweifel/git-auto-commit-action?tab=readme-ov-file#use-in-forks-from-public-repositories
        with:
          # Checkout the fork/head-repository and push changes to the fork.
          # If you skip this, the base repository will be checked out and changes
          # will be committed to the base repository!
          repository: ${{ github.event.pull_request.head.repo.full_name }}

          # Checkout the branch made in the fork. Will automatically push changes
          # back to this branch.
          ref: ${{ github.head_ref }}

      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: Install Fundus
        run: pip install -e .[dev]

      - name: Generate 'supported_publishers.md'
        run: python scripts/generate_tables.py

      - name: Commit changes
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          commit_message: ${{ env.CI_COMMIT_MESSAGE }}
          file_pattern: docs


Publisher Coverage

name: Publisher Coverage

on:
  schedule:
    - cron: '0 1 * * *'  # Runs at 01:00

  workflow_dispatch:

jobs:
  validate_crawlers:
    runs-on: ubuntu-latest

    steps:
      - name: Set up Git repository
        uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}

      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: Install Fundus
        run: pip install -e .

      - name: Validate Crawlers
        env:
          PYTHONPATH: .
        run: |
          set -o pipefail
          exec python scripts/publisher_coverage.py | tee publisher_coverage.txt

      - name: Upload Coverage Report
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: Publisher Coverage
          path: publisher_coverage.txt

  create_badge:
    runs-on: ubuntu-latest
    needs: validate_crawlers
    if: success() || failure()

    steps:
      - name: Set up Git repository
        uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}

      - name: Download Coverage Report
        uses: actions/download-artifact@v4
        with:
          name: Publisher Coverage

      - name: Get Success Rate
        run: echo "SUCCESS_RATE=$(tail -n 1 publisher_coverage.txt | grep -P -o '\d+\/\d+')" >> $GITHUB_ENV

      - name: Get Coverage Bounds
        run: |
          echo "TOTAL_PUBLISHERS=$(echo ${{ env.SUCCESS_RATE }} | grep -P -o '\d+' | tail -1)" >> $GITHUB_ENV
          echo "PASSED_PUBLISHERS=$(echo ${{ env.SUCCESS_RATE }} | grep -P -o '\d+' | head -1)" >> $GITHUB_ENV

      - name: Get Red Threshold
        # We set the badge colour to red when at least one publisher failed the tests.
        run: echo "RED_THRESHOLD=$(( ${{ env.TOTAL_PUBLISHERS }} - 1 ))" >> $GITHUB_ENV

      - name: Create Badge
        uses: schneegans/dynamic-badges-action@v1.6.0
        with:
          auth: ${{ secrets.DOBBERSC_GIST_SECRET }}
          gistID: ca0ae056b05cbfeaf30fa42f84ddf458
          filename: fundus_publisher_coverage.json
          label: Publisher Coverage
          message: ${{ env.SUCCESS_RATE }}
          valColorRange: ${{ env.PASSED_PUBLISHERS }}
          maxColorRange: ${{ env.TOTAL_PUBLISHERS }}
          minColorRange: ${{ env.RED_THRESHOLD }}


# This release workflow was created using the following guide
# https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/

name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI

on:
  release:
    types:
      - released

jobs:

  test:
    name: Test the latest release commit
    uses: ./.github/workflows/tests.yml

  lint:
    name: Lint the latest release commit
    uses: ./.github/workflows/lint.yml

  build:
    name: Build distribution 📦
    needs:
      - test
      - lint
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'

      - name: Install pypa/build
        run: >-
          python3 -m
          pip install
          build
          --user

      - name: Build a binary wheel and a source tarball
        run: python3 -m build

      - name: Store the distribution packages
        uses: actions/upload-artifact@v3
        with:
          name: python-package-distributions
          path: dist/

  publish-to-testpypi:
    name: Publish Python 🐍 distribution 📦 to TestPyPI
    needs:
      - build
    runs-on: ubuntu-latest

    environment:
      name: testpypi
      url: https://test.pypi.org/p/fundus

    permissions:
      id-token: write

    steps:
      - name: Download all the dists
        uses: actions/download-artifact@v3
        with:
          name: python-package-distributions
          path: dist/

      - name: Publish distribution 📦 to TestPyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: https://test.pypi.org/legacy/

      - name: Sleep for 2 minutes
        run: sleep 2m
        shell: bash

  test-distribution:
    name: Install and test TestPyPi distribution
    needs:
      - publish-to-testpypi
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'

      - name: Install package
        run: >-
          python3 -m
          pip install
          --index-url https://test.pypi.org/simple/
          --extra-index-url https://pypi.org/simple/
          fundus==${{ github.event.release.tag_name }}

  publish-to-pypi:
    name: Publish Python 🐍 distribution 📦 to PyPI
    needs:
      - test-distribution
    runs-on: ubuntu-latest

    environment:
      name: pypi
      url: https://pypi.org/p/fundus

    permissions:
      id-token: write

    steps:
      - name: Download all the dists
        uses: actions/download-artifact@v3
        with:
          name: python-package-distributions
          path: dist/

      - name: Publish distribution 📦 to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1


GH Pages Deploy

name: GH Pages Deploy

on:
  push:
    branches: [main, development]
    paths-ignore:
      - README.md

jobs:
  build-deploy:
    name: Build and deploy

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
        with:
          ref: main
          path: source

      - uses: actions/checkout@v2
        with:
          ref: development
          path: source-dev

      - uses: actions/checkout@v2
        with:
          ref: gh-pages
          path: build

      - name: remove all files on the build folder
        working-directory: build
        run: rm -vfr *

      - uses: nicholasphair/sphinx-action@7.0.0
        with:
          docs-folder: "source/docs/"
          # required by #1291
          pre-build-command: "apt-get update -y && apt-get install -y git"
          build-command: "sphinx-build . ../../build/en/latest -W -D version=latest"

      - uses: nicholasphair/sphinx-action@7.0.0
        with:
          docs-folder: "source-dev/docs/"
          # required by #1291
          pre-build-command: "apt-get update -y && apt-get install -y git"
          build-command: "sphinx-build . ../../build/en/development -W -D version=development"

      - name: disable jekyll
        working-directory: build
        run: touch .nojekyll

      - name: create subdomain CNAME file
        working-directory: build
        run: echo docs.inductiva.ai > CNAME

      - name: configure client side redirection
        working-directory: source/docs
        run: cp redirect.html ../../build/index.html

      - name: copy robots.txt file to root
        working-directory: source/docs
        run: cp robots.txt ../../build/

      - name: Commit and push changes to gh-pages
        working-directory: build
        run: |
          git config user.name "${GITHUB_ACTOR}"
          git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
          git add .
          git commit -m "Update GitHub pages from Sphinx build for commit ${GITHUB_SHA}"
          git push origin gh-pages


Deploy Docs to Staging Server

name: Deploy Docs to Staging Server

on:
  workflow_dispatch:

jobs:
  build-deploy:
    name: Build and deploy

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
        with:
          path: source

      - uses: nicholasphair/sphinx-action@7.0.0
        with:
          docs-folder: "source/docs/"
          # required by #1291
          pre-build-command: "apt-get update -y && apt-get install -y git"
          build-command: "sphinx-build . ../../build/en/latest -W -D version=latest"

      - name: Deploy to staging webserver via rsync
        uses: burnett01/rsync-deployments@7.0.0
        with:
          switches: -avzr --delete --no-perms --no-owner --no-group --omit-dir-times
          path: build/en/latest/
          remote_path: /var/www/${{ secrets.STAGING_SERVER }}/docs/
          remote_host: ${{secrets.STAGING_SERVER}}
          remote_user: inductiva-bot
          remote_key: ${{secrets.STAGING_WEB_SSH_KEY}}


name: Deploy Tutorials to Production Web Server

on:
  push:
    branches: [main]

jobs:
  build-deploy:
    name: Build and deploy

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
        with:
          ref: main
          path: source

      - uses: nicholasphair/sphinx-action@7.0.0
        with:
          docs-folder: "source/tutorials/"
          # required by #1291
          pre-build-command: "apt-get update -y && apt-get install -y git"
          build-command: "sphinx-build . ../../build/en/latest -W -D version=latest"

      - name: Deploy to staging webserver via rsync
        uses: burnett01/rsync-deployments@7.0.0
        with:
          switches: -avzr --delete --no-perms --no-owner --no-group --omit-dir-times
          path: build/en/latest/
          remote_path: /var/www/tutorials.inductiva.ai/
          remote_host: ${{secrets.STAGING_SERVER}}
          remote_user: inductiva-bot
          remote_key: ${{secrets.STAGING_WEB_SSH_KEY}}


Deploy Tutorials to Staging Server

name: Deploy Tutorials to Staging Server

on:
  workflow_dispatch:

jobs:
  build-deploy:
    name: Build and deploy

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
        with:
          path: source

      - uses: nicholasphair/sphinx-action@7.0.0
        with:
          docs-folder: "source/tutorials/"
          # required by #1291
          pre-build-command: "apt-get update -y && apt-get install -y git"
          build-command: "sphinx-build . ../../build/en/latest -W -D version=latest"

      - name: Deploy to staging webserver via rsync
        uses: burnett01/rsync-deployments@7.0.0
        with:
          switches: -avzr --delete --no-perms --no-owner --no-group --omit-dir-times
          path: build/en/latest/
          remote_path: /var/www/${{ secrets.STAGING_SERVER }}/tutorials/
          remote_host: ${{secrets.STAGING_SERVER}}
          remote_user: inductiva-bot
          remote_key: ${{secrets.STAGING_WEB_SSH_KEY}}


Publish to PyPI

# Workflow that publishes package to PyPI.
# The workflow is triggered on push of a tag that starts with "v", e.g. v0.3.0.
# The workflow will build the package and then publish it to PyPI. Note that the
# version specified in the tag is not the used for the package version on PyPI.
# The version used is the one specified in the package's setup.cfg file. Ensure
# that the version specified in the setup.cfg file in the repository is the
# same used in the tag that triggers the workflow for consistency between the
# repository and the published package.
name: Publish to PyPI

on:
  push:
    tags:
      - v*
  workflow_dispatch:
    inputs:
      branch:
        description: "Branch from where to deploy"
        required: true
        default: "main"

jobs:
  build-and-publish:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11

      - name: Install build dependencies
        run: |
          python -m pip install build

      - name: Generate Autocompletion File (zsh)
        run: |
          pip install shtab
          pip install .
          mkdir -p inductiva/assets/completions/zsh
          shtab --shell=zsh -u inductiva._cli.main.get_main_parser > inductiva/assets/completions/zsh/_inductiva

      - name: Build package
        run: |
          python -m build

      - name: Save artifact
        uses: actions/upload-artifact@v4
        with:
          name: build-artifact
          path: dist/*

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}


Python package

# This workflow will install Python dependencies, check formatting, lint and run
# tests with a variety of Python versions. For more information see:
# https://docs.github.com/en/actions/guides/building-and-testing-python

name: Python package

on:
  push:
    branches: [main]
  pull_request:

jobs:
  build:

    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.8]

    steps:
    - uses: actions/checkout@v3
    - uses: actions/checkout@v3
      with:
        ref: Inductiva-badges
        path: badges-folder
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      env:
        BOT_REPO_ACCESS: ${{ secrets.BOT_REPO_ACCESS }}
      run: |
        sudo apt-get install -y libglu1-mesa ca-certificates
        python -m pip install --upgrade pip setuptools wheel
        python -m pip install mypy yapf coverage coverage-badge pylint pytest pytest-custom_exit_code toml pytest-markdown-docs
        if [[ -f requirements.txt ]]; then python -m pip install -r requirements.txt; fi
    - name: Check formatting with yapf
      if: ${{ always() }}
      run: |
        yapf . --diff --recursive --parallel --exclude=third_party
    - name: Lint with pylint
      if: ${{ always() }}
      run: |
        find . -type f -not -path "./third_party/*" -name '*.py' | xargs pylint
    - name: Test with pytest
      if: ${{ always() }}
      run: |
        pytest --suppress-no-test-exit-code

    - name: Test docs with pytest pytest-markdown-docs
      if: ${{ always() }}
      run: |
        if [ "${{ github.ref }}" = "refs/heads/main" ]; then
          export INDUCTIVA_API_KEY=${{ secrets.API_TEST_KEY }}
        else
          export INDUCTIVA_API_KEY=${{ secrets.API_DEV_TEST_KEY }}
          export INDUCTIVA_API_URL=https://api-dev.inductiva.ai
        fi
        pytest --confcutdir=docs/ --markdown-docs docs/markdown_sample.md
    - name: Link Checker
      uses: lycheeverse/lychee-action@v1.9.0
      with:
        fail: true
        args: --base . --no-progress --exclude ccrm.vims.edu './**/*.md'
    - name: Coverage report
      if: ${{ always() }}
      run: | # conftest crashes with coverage due to markdown-docs
        coverage run --source=./inductiva -m pytest --noconftest
        coverage report -m
    - name: Generate badge
      if: ${{ always() }}
      run: |
        coverage-badge -f -o ./badges-folder/badges/cov.svg
    - name: Push badge
      if: ${{ always() }}
      working-directory: badges-folder
      run: |
        git config user.name "${GITHUB_ACTOR}"
        git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
        git diff --quiet || ( \
          git add ./badges/cov.svg && \
          git commit -m "Update coverage badge for commit ${GITHUB_SHA}")
        git push origin Inductiva-badges
    - name: Mypy type checking
      continue-on-error: true
      if: ${{ always() }}
      run: |
        mypy . | nl > mypy_output.txt

        cat mypy_output.txt
        echo "###############"

        : # Argument passed to a function has the wrong type
        echo "--- Errors arg-type ---"
        grep arg-type mypy_output.txt
        grep arg-type mypy_output.txt | wc -l
        echo "###############"

        : # function defenitions without types or missing types
        echo "--- Errors no-untyped-def ---"
        grep no-untyped-def mypy_output.txt
        grep no-untyped-def mypy_output.txt | wc -l
        echo "###############"


Repository setup

name: Repository setup

on:
  push:
    branches: [main]

jobs:
  setup:

    if: ${{ github.event.created }}
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - name: Replace variables
        uses: jacobtomlinson/gha-find-replace@master
        with:
          find: "\\{\\{GITHUB_REPOSITORY\\}\\}"
          replace: ${{ github.repository }}
      - name: Commit changes
        uses: stefanzweifel/git-auto-commit-action@v4
        with:
          commit_message: Automated repository setup
          commit_options: "--no-verify --signoff"