Release

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Release

run-name: Releasing ${{ inputs.version }} (${{ github.ref_name }})

permissions:

  # Create release and upload artifact to releases
  contents: write

  # Allow the action to upload images to ghcr
  packages: write

on:
  workflow_dispatch:
    inputs:
      version:
        description: 'The version to release, without prefix v (e.g. 1.1.0-rc10). if not provided, will be calculated from the current version and bump_version_mode'
        default: ''
        type: string
      bump_version_mode:
        description: 'The version bump mode. Whether to bump rc version or set stable version'
        default: 'bump-rc'
        type: choice
        options: [ 'bump-rc', 'stable' ]
      skip_images:
        description: 'Comma separated list of images to skip building, example with all possible images: mlrun,mlrun-gpu,ui,api,base,jupyter,test'
        required: false
        default: ''
      skip_publish_pypi:
        description: 'Whether to skip publishing the python package to Pypi (Auto skip for feature branch)'
        required: false
        default: 'false'
        type: choice
        options: ['true', 'false']
      skip_create_tag_release:
        description: 'Whether to skip creating tag & release in Github (Auto skip for feature branch)'
        required: false
        default: 'false'
        type: choice
        options:
          - 'true'
          - 'false'
      ui_ref:
        description: 'The UI reference (branch / tag name) to use for the UI image, (development, 1.3.3, etc)'
        required: false
        default: ''

jobs:
  prepare-inputs:
    name: Prepare inputs
    runs-on: ubuntu-latest
    outputs:
      is_stable_version: ${{ steps.resolve.outputs.is_stable_version }}
      is_feature_branch: ${{ steps.resolve.outputs.is_feature_branch }}
      version: ${{ steps.resolve.outputs.version }}
      previous_version: ${{ steps.resolve.outputs.previous_version }}
      ui_ref: ${{ steps.resolve.outputs.ui_ref }}
    steps:
      - uses: actions/checkout@v4
        with:

          # Fetch all history for all tags and branches
          fetch-depth: 0
      - name: Resolve inputs
        id: resolve
        run: |

          # map the input to the actual mode
          declare -A bump_version_mode=(["bump-rc"]="rc" ["stable"]="rc-grad")
          NEXT_VERSION_MODE=${bump_version_mode[$BUMP_VERSION_MODE_INPUT]}
          version=$(python ./automation/version/version_file.py next-version --mode $NEXT_VERSION_MODE)
          echo "Calculated version: $version"
          if [[ -n "$VERSION_INPUT" ]]; then \
            version=$VERSION_INPUT; \
            echo "Using version from input: $version"; \
          fi

          echo "is_stable_version=$(python ./automation/version/version_file.py is-stable $version)" >> $GITHUB_OUTPUT
          echo "is_feature_branch=$(python ./automation/version/version_file.py is-feature-branch)" >> $GITHUB_OUTPUT
          echo "version=$version" >> $GITHUB_OUTPUT
          echo "previous_version=$(python ./automation/version/version_file.py current-version)" >> $GITHUB_OUTPUT
          echo "ui_ref=${UI_REF_INPUT:-`echo ${{ github.ref_name }}`}" >> $GITHUB_OUTPUT
          cat $GITHUB_OUTPUT
        env:
          UI_REF_INPUT: ${{ github.event.inputs.ui_ref }}
          BUMP_VERSION_MODE_INPUT: ${{ github.event.inputs.bump_version_mode }}
          VERSION_INPUT: ${{ github.event.inputs.version }}

  trigger-and-wait-for-mlrun-image-building:
    name: Trigger build workflow in mlrun/mlrun and wait to finish
    needs: prepare-inputs
    uses: ./.github/workflows/build-internal.yaml
    with:
      docker_registries: "ghcr.io/,quay.io/,registry.hub.docker.com/"
      version: ${{ needs.prepare-inputs.outputs.version }}
      skip_images: ${{ github.event.inputs.skip_images }}
    secrets: inherit

  # TODO: Move to reuseable-workflow too.
  # Requires cross-repo validation for passing github token as secret
  trigger-and-wait-for-ui-image-building:
    name: Trigger build workflow in mlrun/ui and wait to finish
    runs-on: ubuntu-latest
    needs: prepare-inputs
    steps:
      - uses: convictional/trigger-workflow-and-wait@v1.6.5

        # since some steps relay on the ui image, we need to wait for it to finish building
        # the condition is here and not on job because some other jobs "needs" this job to be done (and not skipped)
        if: ${{ !contains(github.event.inputs.skip_images, 'ui') }}
        with:
          owner: ${{ github.repository_owner }}
          repo: ui
          github_token: ${{ secrets.RELEASE_GITHUB_ACCESS_TOKEN }}
          workflow_file_name: build.yaml
          ref: ${{ needs.prepare-inputs.outputs.ui_ref }}
          wait_interval: 60
          client_payload: '{"docker_registries": "ghcr.io/,quay.io/,registry.hub.docker.com/", "version": "${{ needs.prepare-inputs.outputs.version }}"}'

  publish-to-pypi:
    name: Publish package to pypi
    runs-on: ubuntu-latest

    # skip releasing to pypi for feature branches. installing mlrun can be done by:
    # pip install 'mlrun[complete] @ git+https://github.com/mlrun/mlrun@feature/branch'
    # pypi do not allow local version identifier (https://peps.python.org/pep-0440/#local-version-identifiers)
    if: needs.prepare-inputs.outputs.is_feature_branch == 'false'

    # publishing to pypi is (kind of) irreversible, therefore do it only if both previous steps finished successfully
    needs: [ prepare-inputs, trigger-and-wait-for-ui-image-building, trigger-and-wait-for-mlrun-image-building ]
    steps:
      - uses: actions/checkout@v4
      - name: Set up python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
          cache: pip
      - name: Install dependencies
        run: |
          # needed for updating the version file prior to publishing
          python -m pip install -r automation/requirements.txt

          # TODO: move to a separated requirements file
          python -m pip install \
            build~=1.0 \
            wheel~=0.42 \
            twine~=4.0
      - name: Build & push to pypi
        if: github.event.inputs.skip_publish_pypi != 'true'
        run: |
          export TWINE_USERNAME=${{ secrets.PYPI_USERNAME }}
          export TWINE_PASSWORD=${{ secrets.PYPI_PASSWORD }}
          MLRUN_VERSION="$INPUT_VERSION" make publish-package
        env:
          INPUT_VERSION: ${{ needs.prepare-inputs.outputs.version }}

  create-releases:
    name: Create release & tag v${{ needs.prepare-inputs.outputs.version }}
    runs-on: ubuntu-latest
    if: needs.prepare-inputs.outputs.is_feature_branch == 'false'
    needs: [ prepare-inputs, publish-to-pypi ]
    steps:
      - uses: ncipollo/release-action@v1
        if: github.event.inputs.skip_create_tag_release != 'true'
        with:
          tag: v${{ needs.prepare-inputs.outputs.version }}
          commit: ${{ github.ref_name }}
          token: ${{ secrets.GITHUB_TOKEN }}
          prerelease: ${{ needs.prepare-inputs.outputs.is_stable_version == 'false' }}
      - uses: ncipollo/release-action@v1
        if: github.event.inputs.skip_create_tag_release != 'true'
        with:
          repo: ui
          tag: v${{ needs.prepare-inputs.outputs.version }}
          commit: ${{ github.ref_name }}
          token: ${{ secrets.RELEASE_GITHUB_ACCESS_TOKEN }}
          # experienced 500 errors when trying to create release notes for ui repo with `prerelease flag`
          # prerelease: ${{ needs.prepare-inputs.outputs.prerelease }}

  create-tag:
    name: Create tag v${{ needs.prepare-inputs.outputs.version }}
    runs-on: ubuntu-latest

    # run this specific job for feature branch only as we wont have a github release for it
    # the tag would be useful to track feature branch released versions
    if: needs.prepare-inputs.outputs.is_feature_branch == 'true'
    needs: [ prepare-inputs, trigger-and-wait-for-mlrun-image-building ]
    steps:
      - name: Create feature branch tag
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.git.createRef({
              owner: context.repo.owner,
              repo: context.repo.repo,
              ref: 'refs/tags/v${{ needs.prepare-inputs.outputs.version }}',
              sha: context.sha
            })

  update-release-notes:
    name: Update release notes
    runs-on: ubuntu-latest
    if: github.event.inputs.skip_create_tag_release != 'true'
    needs: [ prepare-inputs, create-releases ]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
          cache: pip
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r automation/requirements.txt
      - name: Generate release notes
        id: release-notes
        run: |
          make release-notes
        env:
          MLRUN_SKIP_CLONE: true
          MLRUN_RELEASE_BRANCH: ${{ github.ref_name }}
          MLRUN_RELEASE_NOTES_OUTPUT_FILE: release_notes.md
          MLRUN_RAISE_ON_ERROR: false
          MLRUN_OLD_VERSION: "v${{ needs.prepare-inputs.outputs.previous_version }}"
          MLRUN_VERSION: "v${{ needs.prepare-inputs.outputs.version }}"

      - name: resolve release notes
        id: resolve-release-notes
        run: |
          echo "body<<EOF" >> $GITHUB_OUTPUT
          cat release_notes.md >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - uses: ncipollo/release-action@v1
        with:
          tag: v${{ needs.prepare-inputs.outputs.version }}
          body: ${{ steps.resolve-release-notes.outputs.body }}
          token: ${{ secrets.GITHUB_TOKEN }}
          allowUpdates: true
          prerelease: ${{ needs.prepare-inputs.outputs.is_stable_version == 'false' }}

  update-tutorials:
    name: Bundle tutorials
    needs: [ prepare-inputs, create-releases ]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Create tutorials tar
        run: |
          wget -c https://github.com/v3io/tutorials/blob/mlrun-release-stable/welcome.ipynb -P docs
          wget -c https://github.com/v3io/tutorials/blob/mlrun-release-stable/README.md -P docs
          tar -cvf mlrun-tutorials.tar docs/tutorials docs/README.md docs/welcome.ipynb
          rm -rf docs/welcome.ipynb docs/README.md
      - name: Add tutorials tar to release
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: true
          tag: v${{ needs.prepare-inputs.outputs.version }}
          token: ${{ secrets.GITHUB_TOKEN }}
          artifacts: mlrun-tutorials.tar
          prerelease: ${{ needs.prepare-inputs.outputs.is_stable_version == 'false' }}

  update-demos:
    name: Bundle demos
    needs: [ prepare-inputs, create-releases ]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: install prerequisits
        run: |
          pip install gitpython
      - name: Create mlrun demos tar
        run: |
          python automation/scripts/bundle_demos.py
      - name: Add mlrun-demos tar to release
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: true
          tag: v${{ needs.prepare-inputs.outputs.version }}
          token: ${{ secrets.GITHUB_TOKEN }}
          artifacts: mlrun-demos.tar
          prerelease: ${{ needs.prepare-inputs.outputs.is_stable_version == 'false' }}


System Tests Enterprise

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: System Tests Enterprise

on:
  push:
    branches:
      - '.+-system-tests'

  schedule:

    # * is a special character in YAML so you have to quote this string
    # Run the system tests every 8 hours (cron hours should divide 24 equally, otherwise there will be an overlap at the end of the day)
    - cron: '0 */8 * * *'

  workflow_dispatch:
    inputs:
      docker_registry:
        description: 'Docker registry to pull images from (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)'
        required: true
        default: 'ghcr.io/'
      clean_resources_in_teardown:
        description: 'Clean resources created by test (like project) in each test teardown (default: true - perform clean)'
        required: true
        default: 'true'
        type: choice
        options:
          - 'true'
          - 'false'
      override_iguazio_version:
        description: 'Override the configured target system iguazio version (leave empty to resolve automatically)'
        required: false

concurrency: one-at-a-time

jobs:
  system-test-cleanup:
    name: System Test Cleanup
    runs-on: [ self-hosted, Linux ]
    container:
      image: ubuntu:latest
    timeout-minutes: 10
    # let's not run this on every fork, change to your fork when developing
    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'

    steps:
    - uses: actions/checkout@v4
    - name: Install dependencies
      run: |
        apt-get update -qqy && apt-get install -y sshpass
    - name: cleanup docker images from registries
      # SSH to datanode and delete all docker images created by the system tests by restarting the docker-registry
      # deployment and the datanode docker-registry
      run: |
        sshpass \
          -p "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }}" \
          ssh \
          -o StrictHostKeyChecking=no \
          -o ServerAliveInterval=180 \
          -o ServerAliveCountMax=3 \
          ${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_USERNAME }}@${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_IP }} \
          kubectl -n default-tenant rollout restart deployment docker-registry

        sshpass \
          -p "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }}" \
          scp \
          automation/system_test/cleanup.py \
          ${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_USERNAME }}@${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_IP }}:/home/iguazio/cleanup.py

        sshpass \
          -p "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }}" \
          scp \
          automation/system_test/dev_utilities.py \
          ${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_USERNAME }}@${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_IP }}:/home/iguazio/dev_utilities.py

  prepare-system-tests-enterprise-ci:
    # When increasing the timeout make sure it's not larger than the schedule cron interval
    timeout-minutes: 55
    name: Prepare System Tests Enterprise
    runs-on: [ self-hosted, Linux ]
    container:
      image: python:3.9
    needs: [system-test-cleanup]
    # let's not run this on every fork, change to your fork when developing
    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'

    steps:
    - uses: actions/checkout@v4
    - name: Install dependencies
      run: |
        apt-get update -qqy && apt-get install -y sshpass jq curl gnupg nodejs
    - uses: actions/setup-node@v4
      with:
        node-version: 18
    - name: Copy state branch file from remote
      run: |
        sshpass \
          -p "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }}" \
          scp \
          -o StrictHostKeyChecking=no \
          ${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_USERNAME }}@${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_IP }}:/tmp/system-tests-branches-list.txt \
          system-tests-branches-list.txt

    - name: Resolve Branch To Run System Tests
      id: current-branch
      shell: bash
      # we store a file named /tmp/system-tests-branches-list.txt which contains a list of branches to run system tests
      # on the branches are separated with commas, so each run we pop the first branch in the list and append it to the
      # end of the list.
      # This mechanism allows us to run on multiple branches without the need to modify the file or secrets each time
      # a new branch is added or removed
      run: |
        # Read branches from local file
        branches=$(cat system-tests-branches-list.txt)
        echo "branches found in system-tests-branches-list.txt: $branches"

        # Split branches into an array
        IFS=',' read -ra branches_array <<< "$branches"

        # Get the first branch in the list to work on
        first_branch="${branches_array[0]}"
        echo "working on $first_branch"

        # Remove the first branch from the list
        branches_array=("${branches_array[@]:1}")

        # Add the first branch at the end of the list
        branches_array+=("$first_branch")

        # Join branches back into a string
        branches=$(printf ",%s" "${branches_array[@]}")
        branches=${branches:1}

        # Output the new list of branches
        echo "$branches"

        # Write new branches order to a local file
        echo "$branches" | cat > system-tests-branches-list.txt

        # Set output
        echo "name=$(echo $first_branch)" >> $GITHUB_OUTPUT

    - name: Override remote file from local resolved branch list
      run: |
        # Override the remote file with the new list of branches
        sshpass \
          -p "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }}" \
          scp \
          -o StrictHostKeyChecking=no system-tests-branches-list.txt \
          ${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_USERNAME }}@${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_IP }}:/tmp/

      # checking out to base branch and not the target(resolved) branch, to be able to run the changed preparation code
      # before merging the changes to upstream.
    - name: Checkout base branch
      uses: actions/checkout@v4

    - name: Install automation scripts dependencies
      run: |
        python -m pip install -r automation/requirements.txt

    - name: Extract git hashes from upstream and latest version
      id: git_upstream_info
      run: |

        # Get the latest commit of mlrun/mlrun (that is older than 1 hour)
        echo "mlrun_hash=$( \
          cd /tmp && \
          git clone --single-branch --branch ${{ steps.current-branch.outputs.name }} https://github.com/mlrun/mlrun.git mlrun-upstream 2> /dev/null && \
          cd mlrun-upstream && \
          git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \
          cd .. && \
          rm -rf mlrun-upstream)" >> $GITHUB_OUTPUT

        # Get the latest commit of mlrun/ui (that is older than 1 hour)
        echo "ui_hash=$( \
          cd /tmp && \
          git clone --single-branch --branch ${{ steps.current-branch.outputs.name }} https://github.com/mlrun/ui.git mlrun-ui 2> /dev/null && \
          cd mlrun-ui && \
          git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \
          cd .. && \
          rm -rf mlrun-ui)" >> $GITHUB_OUTPUT

        # Get the tested mlrun version
        echo "unstable_version_prefix=$( \
          curl https://raw.githubusercontent.com/mlrun/mlrun/${{ steps.current-branch.outputs.name }}/automation/version/unstable_version_prefix \
        )" >> $GITHUB_OUTPUT

    - name: Set computed versions params
      id: computed_params
      run: |
        action_mlrun_hash=${{ steps.git_action_info.outputs.mlrun_hash }} && \
        upstream_mlrun_hash=${{ steps.git_upstream_info.outputs.mlrun_hash }} && \
        export mlrun_hash=${upstream_mlrun_hash:-`echo $action_mlrun_hash`}
        echo "mlrun_hash=$(echo $mlrun_hash)" >> $GITHUB_OUTPUT
        action_mlrun_ui_hash=${{ steps.git_action_ui_info.outputs.ui_hash }} && \
        upstream_mlrun_ui_hash=${{ steps.git_upstream_info.outputs.ui_hash }} && \
        export ui_hash=${upstream_mlrun_ui_hash:-`echo $action_mlrun_ui_hash`}
        echo "ui_hash=$(echo $ui_hash)" >> $GITHUB_OUTPUT
        echo "mlrun_version=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}+$mlrun_hash)" >> $GITHUB_OUTPUT
        echo "mlrun_docker_tag=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$mlrun_hash)" >> $GITHUB_OUTPUT
        echo "mlrun_ui_version=${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$ui_hash" >> $GITHUB_OUTPUT
        echo "mlrun_docker_registry=$( \
          input_docker_registry=$INPUT_DOCKER_REGISTRY && \
          echo ${input_docker_registry:-ghcr.io/})" >> $GITHUB_OUTPUT
        echo "mlrun_system_tests_clean_resources=$( \
          input_system_tests_clean_resources=$INPUT_CLEAN_RESOURCES_IN_TEARDOWN && \
          echo ${input_system_tests_clean_resources:-true})" >> $GITHUB_OUTPUT
        echo "override_iguazio_version=$INPUT_OVERRIDE_IGUAZIO_VERSION" >> $GITHUB_OUTPUT
      env:
        INPUT_DOCKER_REGISTRY: ${{ github.event.inputs.docker_registry }}
        INPUT_OVERRIDE_IGUAZIO_VERSION: ${{ github.event.inputs.override_iguazio_version }}
        INPUT_CLEAN_RESOURCES_IN_TEARDOWN: ${{ github.event.inputs.clean_resources_in_teardown }}

    - name: Prepare System Test Environment and Install MLRun
      env:
        IP_ADDR_PREFIX: ${{ secrets.IP_ADDR_PREFIX }}
      timeout-minutes: 50
      run: |
        python automation/system_test/prepare.py run \
          --data-cluster-ip "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_IP }}" \
          --data-cluster-ssh-username "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_USERNAME }}" \
          --data-cluster-ssh-password "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }}" \
          --provctl-download-url "${{ secrets.LATEST_SYSTEM_TEST_PROVCTL_DOWNLOAD_PATH }}" \
          --provctl-download-s3-access-key "${{ secrets.LATEST_SYSTEM_TEST_PROVCTL_DOWNLOAD_URL_S3_ACCESS_KEY }}" \
          --provctl-download-s3-key-id "${{ secrets.LATEST_SYSTEM_TEST_PROVCTL_DOWNLOAD_URL_S3_KEY_ID }}" \
          --username "${{ secrets.LATEST_SYSTEM_TEST_USERNAME }}" \
          --access-key "${{ secrets.LATEST_SYSTEM_TEST_ACCESS_KEY }}" \
          --iguazio-version "${{ steps.computed_params.outputs.iguazio_version }}" \
          --mlrun-version "${{ steps.computed_params.outputs.mlrun_version }}" \
          --mlrun-ui-version "${{ steps.computed_params.outputs.mlrun_ui_version }}" \
          --mlrun-commit "${{ steps.computed_params.outputs.mlrun_hash }}" \
          --override-image-registry "${{ steps.computed_params.outputs.mlrun_docker_registry }}"

    - name: Prepare System Test env.yml and MLRun installation from current branch
      timeout-minutes: 5
      run: |
        python automation/system_test/prepare.py env \
          --data-cluster-ip "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_IP }}" \
          --data-cluster-ssh-username "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_USERNAME }}" \
          --data-cluster-ssh-password "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }}" \
          --username "${{ secrets.LATEST_SYSTEM_TEST_USERNAME }}" \
          --access-key "${{ secrets.LATEST_SYSTEM_TEST_ACCESS_KEY }}" \
          --slack-webhook-url "${{ secrets.LATEST_SYSTEM_TEST_SLACK_WEBHOOK_URL }}" \
          --branch "${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunBranch }}" \
          --github-access-token "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}" \
          --save-to-path "$GITHUB_WORKSPACE/env.yml"

    - name: Encrypt file
      id: encrypt_file
      run: |
        gpg \
          --batch \
          --passphrase "${{ env.GPG_PASSPHRASE }}" \
          --output "$GITHUB_WORKSPACE/env.yml.gpg" \
          --symmetric "$GITHUB_WORKSPACE/env.yml"

        echo "env_file_path=$(echo $GITHUB_WORKSPACE/env.yml.gpg)" >> $GITHUB_OUTPUT
      env:
        GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}

    - name: Upload env file
      uses: actions/upload-artifact@v4
      with:
        name: env
        path: "${{ steps.encrypt_file.outputs.env_file_path }}"
        if-no-files-found: error

    outputs:
      mlrunVersion: ${{ steps.computed_params.outputs.mlrun_version }}
      mlrunBranch: ${{ steps.current-branch.outputs.name }}
      mlrunSystemTestsCleanResources: ${{ steps.computed_params.outputs.mlrun_system_tests_clean_resources }}

  run-system-tests-enterprise-ci:
    # When increasing the timeout make sure it's not larger than the schedule cron interval
    timeout-minutes: 360
    name: Test ${{ matrix.test_component }} [${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunBranch }}]
    # requires prepare to finish before starting
    needs: [prepare-system-tests-enterprise-ci]
    runs-on: [ self-hosted, Linux ]
    container:
      image: python:3.9
    # let's not run this on every fork, change to your fork when developing
    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        test_component:
        - api
        - runtimes
        - projects
        - model_monitoring
        - examples
        - backwards_compatibility
        - datastore
        - logs
        - feature_store
    steps:
    - name: Install Dependencies
      run: |
        # gnupg         for decrypting env.yml.gpg
        # nodejs        for installing github action dependencies
        # graphviz      for generating graphs (feature store tests)
        # git-core      for cloning repos and etc, required by many suites
        # gcc, make     for installing python packages
        apt-get update -qqy && \
          apt-get install -y \
            gnupg \
            nodejs \
            graphviz \
            git-core \
            gcc \
            make

    - uses: actions/setup-node@v4
      with:
        node-version: 18

    - uses: actions/checkout@v4
      # checking out to the resolved branch to run system tests on, as now we run the actual tests, we don't want to run
      # the system tests of the branch that triggered the system tests as it might be in a different version
      # than the mlrun version we deployed on the previous job (can have features that the resolved branch doesn't have)
      with:
        ref: ${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunBranch }}

    - uses: actions/download-artifact@v4
      with:
        name: env
        path: /tmp

    - name: Decrypt file
      run: |
        gpg \
          --batch \
          --passphrase "${{ env.GPG_PASSPHRASE }}" \
          --output "$GITHUB_WORKSPACE/tests/system/env.yml" \
          --decrypt "/tmp/env.yml.gpg"

        # ensure file is created
        test -f "$GITHUB_WORKSPACE/tests/system/env.yml"
      env:
        GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}

    - name: Pre Tests Requirements
      run: |

        # due to
        # https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9
        # ensure mlrun git is safe to use
        git config --global --add safe.directory "$GITHUB_WORKSPACE"
        MLRUN_VERSION="${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunVersion }}" \
          make install-requirements install-complete-requirements update-version-file

    - name: Run System Tests
      run: |
        MLRUN_SYSTEM_TESTS_GITHUB_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
        MLRUN_SYSTEM_TESTS_CLEAN_RESOURCES="${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunSystemTestsCleanResources }}" \
        MLRUN_VERSION="${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunVersion }}" \
        MLRUN_SYSTEM_TESTS_COMPONENT="${{ matrix.test_component }}" \
        MLRUN_SYSTEM_TESTS_BRANCH="${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunBranch }}" \
          make test-system


Security Scan

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Currently supported running against prebuilt images
name: Security Scan
run-name: Scanning ${{ inputs.tag }}

on:
  workflow_dispatch:
    inputs:
      tag:
        description: 'MLRun image tag to scan (unstable-cache, unstable-cache-13x, 1.3.0-wwwwwwww, 1.3.0 or any other tag)'
        required: false
        default: 'unstable'
      registry:
        description: 'MLRun image registry'
        required: false
        default: 'ghcr.io/'
      skip_images:
        description: 'Comma separated list of images to skip scanning'
        required: false
        # test images are not scanned as they are not used in production
        default: 'test'
      publish_results:
        description: 'Whether to publish results to Github or not (default empty - no publish)'
        required: false
        default: ''
      severity_threshold:
        description: 'The minimum severity of vulnerabilities to report ("negligible", "low", "medium", "high" and "critical".)'
        required: false
        default: 'medium'
        type: choice
        options:
          - 'negligible'
          - 'low'
          - 'medium'
          - 'high'
          - 'critical'
      only_fixed:
        description: 'Whether to scan only fixed vulnerabilities ("true" or "false")'
        required: false
        default: 'true'
        type: choice
        options:
          - 'true'
          - 'false'

jobs:
  matrix_prep:
    name: Prepare image list
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - id: set-matrix
        uses: ./.github/actions/image-matrix-prep
        with:
          skip_images: ${{ github.event.inputs.skip_images }}

  build_and_scan_docker_images:
    name: Scan ${{ matrix.image-name }} (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    needs: matrix_prep
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v4

      - name: Cleanup disk
        run: |
          "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"

      - name: Resolving image name
        id: resolve_image_name
        run: |
          echo "image_name=$(make pull-${{ matrix.image-name }} | tail -1)" >> $GITHUB_OUTPUT
        env:
          MLRUN_DOCKER_REGISTRY: ${{ github.event.inputs.registry }}
          MLRUN_VERSION: ${{ github.event.inputs.tag }}
          MLRUN_PYTHON_VERSION: ${{ matrix.python-version }}

      - name: Define output format
        id: output-format

        # this section is duplicated in the other jobs.
        # make sure to update all when changed.
        run: |
          if [[ -n "${{ github.event.inputs.publish_results }}" ]]; then \
            echo "format=sarif" >> $GITHUB_OUTPUT; \
            echo "fail_build=false" >> $GITHUB_OUTPUT; \
          else \
            echo "format=table" >> $GITHUB_OUTPUT; \
            echo "fail_build=true" >> $GITHUB_OUTPUT; \
          fi

      - name: Scan image
        uses: anchore/scan-action@v3
        id: scan
        with:
          image: ${{ steps.resolve_image_name.outputs.image_name }}
          only-fixed: ${{ github.event.inputs.only_fixed }}
          output-format: ${{ steps.output-format.outputs.format }}
          fail-build: ${{ steps.output-format.outputs.fail_build }}
          severity-cutoff: ${{ github.event.inputs.severity_threshold }}

      - name: Upload scan results
        if: github.event.inputs.publish_results != ''
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: ${{ steps.scan.outputs.sarif }}
          category: ${{ matrix.image-name }}-${{ matrix.python-version }})

  scan_fs:
    name: Scan file system
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Define output format
        id: output-format

        # this section is duplicated in the other jobs.
        # make sure to update all when changed.
        run: |
          if [[ -n "${{ github.event.inputs.publish_results }}" ]]; then \
            echo "format=sarif" >> $GITHUB_OUTPUT; \
            echo "fail_build=false" >> $GITHUB_OUTPUT; \
          else \
            echo "format=table" >> $GITHUB_OUTPUT; \
            echo "fail_build=true" >> $GITHUB_OUTPUT; \
          fi

      - name: Scan fs
        uses: anchore/scan-action@v3
        id: scan
        with:
          path: "."
          only-fixed: ${{ github.event.inputs.only_fixed }}
          output-format: ${{ steps.output-format.outputs.format }}
          fail-build: ${{ steps.output-format.outputs.fail_build }}
          severity-cutoff: ${{ github.event.inputs.severity_threshold }}

      - name: Upload scan results
        if: github.event.inputs.publish_results != ''
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: ${{ steps.scan.outputs.sarif }}
          category: "repository"


Scheduled Re-Build Images

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Scheduled Re-Build Images

on:
  schedule:
  # every night at 2am
  - cron: "0 2 * * *"

jobs:
  resolve-branch-to-rebuild:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-branch.outputs.matrix }}
    steps:
      - name: Resolve Branch to Re-Build
        id: set-branch
        run: |

          # take mlrun branches, find branches that are in format of 9.9.x (where 9 could by any number)
          # sort them - so latest version would be last
          # take last 2 elements (aka last 2 versions)
          # wrap each name with "" and join with ,
          # join with development
          branches=$(curl --retry 3 --retry-all-errors --silent https://api.github.com/repos/mlrun/mlrun/branches \
            | jq -r 'map(select(.name | test("^\\d+.\\d+.x$"))) | sort_by(.name) | .[-2:] | map(.name | tostring | "\"" + . + "\"") | join(",")' \
          )
          # output would be like
          # 1.3.x,1.4.x,development
          branches="$branches,\"development\""

          matrix="{\"repo\": [\"mlrun\",\"ui\"], \"branch\": [$branches]}"
          echo "matrix=$(echo $matrix)" >> $GITHUB_OUTPUT

  re-build-images:
    if: github.repository == 'mlrun/mlrun'
    needs: resolve-branch-to-rebuild
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.resolve-branch-to-rebuild.outputs.matrix)}}
    runs-on: ubuntu-latest
    steps:

      # TODO: move to reuseable workflow once all branches have backported with the new workflow
      - name: Re-Build MLRun Image
        if: matrix.repo == 'mlrun'
        uses: convictional/trigger-workflow-and-wait@v1.6.5
        with:
            owner: mlrun
            repo: mlrun
            github_token: ${{ secrets.RELEASE_GITHUB_ACCESS_TOKEN }}
            workflow_file_name: build.yaml
            ref: ${{ matrix.branch }}
            wait_interval: 60

            # models is removed on 1.5.x but still exists on 1.4.x and 1.3.x
            client_payload: '{"skip_images": "models,tests", "build_from_cache": "false"}'
      - name: Re-Build UI Image
        if: matrix.repo == 'ui'
        uses: convictional/trigger-workflow-and-wait@v1.6.5
        with:
          owner: mlrun
          repo: ui
          github_token: ${{ secrets.RELEASE_GITHUB_ACCESS_TOKEN }}
          workflow_file_name: build.yaml
          ref: ${{ matrix.branch }}
          wait_interval: 60


Build (internal)

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Build (internal)

run-name: Building ${{ inputs.version }} ${{ github.ref_name }}

permissions:

  # Allow the action to upload images to ghcr
  packages: write

on:

  # FYI
  # The event payload in the called workflow is the same event payload from the calling workflow
  # The inputs are the inputs defined in the called workflow (by the "with")
  workflow_call:
    inputs:
      docker_registries:
        description: 'Comma separated list of docker registries to push images to (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)'
        default: 'ghcr.io/'
        type: string
      docker_repo:
        description: 'Docker repo to push images to (default: lowercase github repository owner name)'
        default: ''
        type: string
      version:
        description: 'The version to build, without prefix v (e.g. 1.1.0), if not provided version will be <unstable-version-prefix>-<commit-hash>, where <unstable-version-prefix> is taken from automation/version/unstable_version_prefix'
        default: ''
        type: string
      skip_images:
        description: 'Comma separated list of images to skip building, example with all possible images: mlrun,api,base,models,jupyter,test'
        default: ''
        type: string
      build_from_cache:
        description: 'Whether to build images from cache or not. Default: true, set to false only if required because that will cause a significant increase in build time'
        default: 'true'
        type: string

jobs:
  matrix_prep:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4
      - id: set-matrix
        uses: ./.github/actions/image-matrix-prep
        with:
          skip_images: ${{ inputs.skip_images }}

  build-images:
    name: Build and push image - ${{ matrix.image-name }} (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    needs: matrix_prep
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
    steps:
    - uses: actions/checkout@v4

    # since github-actions gives us 14G only, and fills it up with some garbage
    - name: Freeing up disk space
      run: |
        "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"

    - name: Install curl and jq
      run: sudo apt-get install curl jq

    - name: Extract git hash, ref and latest version
      id: git_info
      run: |
        echo "mlrun_commit_hash=$(git rev-parse --short=8 $GITHUB_SHA)"  >> $GITHUB_OUTPUT
        echo "unstable_version_prefix=$(cat automation/version/unstable_version_prefix)"  >> $GITHUB_OUTPUT

    - name: Resolve docker cache tag
      id: docker_cache
      run: |
        export version_suffix=$(echo "$GITHUB_REF_NAME" | grep -E "^[0-9]+\.[0-9]+\.x$" | tr -d '.');
        export unstable_tag=$(if [ -z "$version_suffix" ]; then echo "unstable-cache"; else echo "unstable-cache-$version_suffix";fi);
        export build_from_cache=$(if [ -z "$INPUT_BUILD_FROM_CACHE" ]; then echo "true" ; else echo "$INPUT_BUILD_FROM_CACHE";fi);
        export no_cache=$(if [ "$build_from_cache" = "false" ]; then echo "true" ; else echo "";fi);
        echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
        echo "no_cache=$(echo $no_cache)" >> $GITHUB_OUTPUT
      env:
        INPUT_BUILD_FROM_CACHE: ${{ inputs.build_from_cache }}

    - name: Set computed versions params
      id: computed_params
      run: |
        echo "mlrun_version=$( \
          input_mlrun_version=$INPUT_VERSION && \
          default_mlrun_version=$(echo ${{ steps.git_info.outputs.unstable_version_prefix }}+${{ steps.git_info.outputs.mlrun_commit_hash }}) && \
          echo ${input_mlrun_version:-`echo $default_mlrun_version`})" >> $GITHUB_OUTPUT
        echo "mlrun_docker_repo=$( \
          input_docker_repo=$INPUT_DOCKER_VERSION && \
          default_docker_repo=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') && \
          echo ${input_docker_repo:-`echo $default_docker_repo`})" >> $GITHUB_OUTPUT
        echo "mlrun_docker_registries=$( \
          input_docker_registries=$INPUT_DOCKER_REGISTRIES && \
          echo ${input_docker_registries:-ghcr.io/})" >> $GITHUB_OUTPUT
        echo "mlrun_cache_date=$(date +%s)" >> $GITHUB_OUTPUT
      env:
        INPUT_VERSION: ${{ inputs.version }}
        INPUT_DOCKER_VERSION: ${{ inputs.docker_repo }}
        INPUT_DOCKER_REGISTRIES: ${{ inputs.docker_registries }}

    - name: Docker login (ghcr)
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ github.repository_owner }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Docker login (quay.io)
      continue-on-error: true
      uses: docker/login-action@v3
      with:
        registry: quay.io
        username: ${{ secrets.QUAY_IO_DOCKER_REGISTRY_USERNAME }}
        password: ${{ secrets.QUAY_IO_DOCKER_REGISTRY_PASSWORD }}

    - name: Docker login (docker.com)
      continue-on-error: true
      uses: docker/login-action@v3
      with:
        registry: registry.hub.docker.com
        username: ${{ secrets.DOCKER_HUB_DOCKER_REGISTRY_USERNAME }}
        password: ${{ secrets.DOCKER_HUB_DOCKER_REGISTRY_PASSWORD }}

    - name: Pull cache, build and push image

      # we don't really want per-commit test image we just want to build and push the cache image so CI will be able
      # to use it and run much faster
      if: ${{ matrix.image-name != 'test' }}
      run: |
        for registry in $(echo ${{ steps.computed_params.outputs.mlrun_docker_registries }} | sed "s/,/ /g"); \
          do \
            MLRUN_CACHE_DATE=${{ steps.computed_params.outputs.mlrun_cache_date }} \
            MLRUN_DOCKER_REGISTRY=$registry \
            MLRUN_DOCKER_CACHE_FROM_REGISTRY=ghcr.io/ \
            MLRUN_DOCKER_REPO=${{ steps.computed_params.outputs.mlrun_docker_repo }} \
            MLRUN_VERSION=${{ steps.computed_params.outputs.mlrun_version }} \
            MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} \
            MLRUN_NO_CACHE=${{ steps.docker_cache.outputs.no_cache }} \
            MLRUN_PUSH_DOCKER_CACHE_IMAGE="true" \
            MLRUN_PYTHON_VERSION=${{ matrix.python-version }} \
            INCLUDE_PYTHON_VERSION_SUFFIX=${{ matrix.include-suffix }} \
            make push-${{ matrix.image-name }}; \
          done;

    - name: Build and push unstable tag

      # we don't need to have unstable tag for the test image
      # And we don't need to run this when triggered manually (workflow dispatch)
      if: matrix.image-name != 'test' && github.event_name != 'workflow_dispatch' && github.ref_name == 'development'
      run: |
        for registry in "ghcr.io/" "quay.io/" "registry.hub.docker.com/"; \
          do \
            MLRUN_CACHE_DATE=${{ steps.computed_params.outputs.mlrun_cache_date }} \
            MLRUN_DOCKER_REGISTRY=$registry \
            MLRUN_DOCKER_CACHE_FROM_REGISTRY=ghcr.io/ \
            MLRUN_DOCKER_REPO=${{ steps.computed_params.outputs.mlrun_docker_repo }} \
            MLRUN_VERSION=unstable \
            MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} \
            MLRUN_PYTHON_VERSION=${{ matrix.python-version }} \
            INCLUDE_PYTHON_VERSION_SUFFIX=${{ matrix.include-suffix }} \
            make push-${{ matrix.image-name }}; \
          done;
    - name: Pull cache, build and push test image
      # When version is given we're probably in a release process, we don't need the test image in that case
      if: matrix.image-name == 'test' && inputs.version == ''
      run: |
        MLRUN_CACHE_DATE=${{ steps.computed_params.outputs.mlrun_cache_date }} \
        MLRUN_DOCKER_REGISTRY=ghcr.io/ \
        MLRUN_DOCKER_CACHE_FROM_REGISTRY=ghcr.io/ \
        MLRUN_DOCKER_REPO=${{ steps.computed_params.outputs.mlrun_docker_repo }} \
        MLRUN_VERSION=${{ steps.docker_cache.outputs.tag }} \
        MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} \
        MLRUN_PUSH_DOCKER_CACHE_IMAGE=true \
        MLRUN_PYTHON_VERSION=${{ matrix.python-version }} \
        INCLUDE_PYTHON_VERSION_SUFFIX=${{ matrix.include-suffix }} \
        make push-${{ matrix.image-name }}


CI (Go)

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: CI (Go)

on:
  pull_request:
    branches:
    - development
    - '[0-9]+.[0-9]+.x'
    - 'feature/**'
    paths:
      - server/log-collector/**

  # Run CI also on push to backport release branches - we sometimes push code there by cherry-picking, meaning it
  # doesn't go through CI (no PR)
  # For master all code is coming from development where it's coming from PRs (meaning it's passing CI) but there's the
  # rare scenario where the combination of merged PRs won't pass CI
  push:
    branches:
    - master
    - '[0-9]+.[0-9]+.x'
    paths:
      - server/log-collector/**

env:
  NAMESPACE: mlrun-integ-test

jobs:
  go-lint:
    name: Lint code
    runs-on: ubuntu-latest
    steps:
      - name: Dump runner context
        run: echo "$RUNNER_CONTEXT"
        env:
          RUNNER_CONTEXT: ${{ toJson(runner) }}
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version-file: "server/log-collector/go.mod"
          cache-dependency-path: "server/log-collector/go.sum"
          cache: true
      - name: Install protobuf compiler
        run: |
          sudo apt-get update && sudo apt install -y protobuf-compiler

          # version is coupled to `go/cmd/schemas_compiler/docker/Dockerfile`
          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28
          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2
      - name: Lint
        run: make lint-go

  go-unit-tests:
    name: Run Unit Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run GO tests
        run: |
          make test-go-unit-dockerized

  go-integration-tests:
    name: Run Integration Tests in Kubernetes Environment
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      # since github-actions gives us 14G only, and fills it up with some garbage
      - name: Freeing up disk space
        run: |
          "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"
      - uses: manusa/actions-setup-minikube@v2.11.0
        with:
          minikube version: "v1.28.0"
          kubernetes version: "v1.25.3"
          driver: docker
          github token: ${{ github.token }}
      - name: Prepare minikube
        run: |
          minikube kubectl -- config view --flatten > kubeconfig_flatten
          echo "MLRUN_TEST_KUBECONFIG=$(pwd)/kubeconfig_flatten" >> $GITHUB_ENV
      - name: Run GO tests
        run: |
          make test-go-integration-dockerized
      - name: Output some logs in case of failure
        if: ${{ failure() }}
        # add set -x to print commands before executing to make logs reading easier
        run: |
          set -x
          minikube ip
          minikube logs
          minikube kubectl -- --namespace ${NAMESPACE} get events
          minikube kubectl -- --namespace ${NAMESPACE} logs -l app.kubernetes.io/component=api,app.kubernetes.io/name=mlrun --tail=-1
          minikube kubectl -- --namespace ${NAMESPACE} get pods
          minikube kubectl -- --namespace ${NAMESPACE} get pods -o yaml
          minikube kubectl -- --namespace ${NAMESPACE} describe pods
          set +x


PR Title Checker

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "PR Title Checker"
on:

  # when opening PRs from forks
  pull_request_target:
    types:
      - opened
      - edited
      - synchronize
      - labeled
      - unlabeled

jobs:
  check:
    runs-on: ubuntu-latest
    steps:
      - uses: thehanimo/pr-title-checker@v1.4.2
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          configuration_path: .github/pr-title-validation-config.json


Release Pipeline Adapters

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Release Pipeline Adapters
on:
  workflow_dispatch:
    inputs:
      packages:
        description: 'Pipeline adapters package to release. Use "*" to release all packages.'
        default: '*'
        options:
          - '*'
          - 'mlrun-pipelines-kfp-common'
          - 'mlrun-pipelines-kfp-v1-8'
          - 'mlrun-pipelines-kfp-v2'

jobs:
  build-packages-matrix:
    name: From packages input to matrix
    runs-on: ubuntu-latest
    outputs:
      packages: ${{ steps.packages.outputs.packages }}
    steps:
      - name: Install jq
        run: sudo apt-get install jq
      - name: resolve packages to release
        id: packages
        shell: bash
        run: |
          if [[ "$PACKAGES" = "*" ]]; then \
            export packages=("mlrun-pipelines-kfp-common" "mlrun-pipelines-kfp-v1-8" "mlrun-pipelines-kfp-v2") ;\
          else \
            export packages=("${PACKAGES}") ;\
          fi
          # "(X Y Z)" -> "{\"include\":[{\"name\":\"X\"},{\"name\":\"Y\"},{\"name\":\"Z\"}]}"
          export packages=$(jq --null-input --argjson arr "$(printf '%s\n' "${packages[@]}" | jq -R . | jq -s .)" '($arr | map({name: .}))')
          echo "packages={\"include\":$(echo $packages)}" >> $GITHUB_OUTPUT
        env:
          PACKAGES: ${{ github.event.inputs.packages }}

  pypi-publish:
    needs:
      - build-packages-matrix
    name: Release ${{ matrix.name }} to PyPI
    runs-on: ubuntu-latest
    environment: release
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.build-packages-matrix.outputs.packages) }}
    permissions:
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip build
      - name: Build package distributions
        run: |
          cd pipeline-adapters
          make build
        env:
          PACKAGES: ${{ matrix.name }}

      - name: Publish package distributions to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          skip-existing: true
          print-hash: true
          packages-dir: ${{ github.workspace }}/pipeline-adapters/${{ matrix.name }}/dist


System Tests Open Source

# Copyright 2023 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: System Tests Open Source

on:
  push:
    branches:
      - '.+-system-tests'

  schedule:

    # * is a special character in YAML so you have to quote this string
    # Run the system tests every 3 hours
    - cron:  '0 */3 * * *'

  workflow_dispatch:
    inputs:
      docker_registry:
        description: 'Docker registry to pull images from (default: ghcr.io/, use registry.hub.docker.com/ for docker hub)'
        required: true
        default: 'ghcr.io/'
      docker_repo:
        description: 'Docker repo to pull images from (default: mlrun)'
        required: true
        default: 'mlrun'
      clean_resources_in_teardown:
        description: 'Clean resources created by test (like project) in each test teardown (default: true - perform clean)'
        required: true
        default: 'true'
        type: choice
        options:
          - 'true'
          - 'false'
      debug_enabled:
        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: 'false'
        type: choice
        options:
          - 'true'
          - 'false'

env:
  NAMESPACE: mlrun
  MLRUN_API_NODE_PORT: 30070

jobs:
  run-system-tests-opensource-ci:
    name: Run System Tests Open Source
    runs-on: ubuntu-latest

    # let's not run this on every fork, change to your fork when developing
    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'

    steps:
    - uses: actions/checkout@v4
    # since github-actions gives us 14G only, and fills it up with some garbage
    - name: Freeing up disk space
      run: |
        "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"
    - name: Set up python
      uses: actions/setup-python@v5
      with:
        python-version: 3.9
        cache: pip
    - name: Install automation scripts dependencies and add mlrun to dev packages
      run: |
        pip install \
          -r automation/requirements.txt \
          -r dockerfiles/mlrun-api/requirements.txt \
          -r dev-requirements.txt \
          -r extras-requirements.txt \
        && pip install -e .
        sudo apt-get install curl jq

      # TODO: How can we avoid these duplicate lines from the enterprise system tests
    - name: Extract git branch
      id: git_info
      run: |
        echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
    - name: Extract git hashes from upstream and latest version
      id: git_upstream_info
      run: |
        echo "mlrun_hash=$( \
          cd /tmp && \
          git clone --single-branch --branch development https://github.com/mlrun/mlrun.git mlrun-upstream 2> /dev/null && \
          cd mlrun-upstream && \
          git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \
          cd .. && \
          rm -rf mlrun-upstream)" >> $GITHUB_OUTPUT
        echo "ui_hash=$( \
          cd /tmp && \
          git clone --single-branch --branch development https://github.com/mlrun/ui.git mlrun-ui 2> /dev/null && \
          cd mlrun-ui && \
          git rev-list --until="1 hour ago" --max-count 1 --abbrev-commit HEAD && \
          cd .. && \
          rm -rf mlrun-ui)" >> $GITHUB_OUTPUT
        echo "unstable_version_prefix=$(cat automation/version/unstable_version_prefix)" >> $GITHUB_OUTPUT
    - name: Set computed versions params
      id: computed_params
      run: |
        action_mlrun_hash=${{ steps.git_action_info.outputs.mlrun_hash }} && \
        upstream_mlrun_hash=${{ steps.git_upstream_info.outputs.mlrun_hash }} && \
        export mlrun_hash=${action_mlrun_hash:-`echo $upstream_mlrun_hash`}
        echo "mlrun_hash=$(echo $mlrun_hash)" >> $GITHUB_OUTPUT
        action_mlrun_ui_hash=${{ steps.git_action_ui_info.outputs.ui_hash }} && \
        upstream_mlrun_ui_hash=${{ steps.git_upstream_info.outputs.ui_hash }} && \
        export ui_hash=${action_mlrun_ui_hash:-`echo $upstream_mlrun_ui_hash`}
        echo "ui_hash=$(echo $ui_hash)" >> $GITHUB_OUTPUT
        echo "mlrun_version=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}+$mlrun_hash)" >> $GITHUB_OUTPUT
        echo "mlrun_docker_tag=$(echo ${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$mlrun_hash)" >> $GITHUB_OUTPUT
        echo "mlrun_ui_version=${{ steps.git_upstream_info.outputs.unstable_version_prefix }}-$ui_hash" >> $GITHUB_OUTPUT
        echo "mlrun_docker_repo=$( \
          input_docker_repo=$INPUT_DOCKER_REPO && \
          echo ${input_docker_repo:-mlrun})" >> $GITHUB_OUTPUT
        echo "mlrun_docker_registry=$( \
          input_docker_registry=$INPUT_DOCKER_REGISTRY && \
          echo ${input_docker_registry:-ghcr.io/})" >> $GITHUB_OUTPUT
        echo "mlrun_system_tests_clean_resources=$( \
          input_system_tests_clean_resources=$INPUT_CLEAN_RESOURCES_IN_TEARDOWN && \
          echo ${input_system_tests_clean_resources:-true})" >> $GITHUB_OUTPUT
      env:
        INPUT_DOCKER_REPO: ${{ github.event.inputs.docker_repo }}
        INPUT_DOCKER_REGISTRY: ${{ github.event.inputs.docker_registry }}
        INPUT_CLEAN_RESOURCES_IN_TEARDOWN: ${{ github.event.inputs.clean_resources_in_teardown }}

    - uses: azure/setup-helm@v4
      with:
        version: "v3.9.1"

    - uses: manusa/actions-setup-minikube@v2.11.0
      with:
        minikube version: "v1.32.0"
        kubernetes version: "v1.28.5"
        driver: docker
        github token: ${{ github.token }}
        # I couldn't find a way to configure the IP (https://github.com/kubernetes/minikube/issues/951)
        # but this seems to work
        start args: '--addons=registry --insecure-registry="192.168.49.2:5000"'

    - name: Install MLRun CE helm chart
      run: |
        # TODO: There is a modification to the helm chart that we are doing right now:
        #          The mlrun DB is set as the old SQLite db. There is a bug in github workers when trying to run a mysql
        #          server pod in minikube installed on the worker, the mysql pod crashes. There isn't much information
        #          about this issue online as this isn't how github expect you to use mysql in workflows - the worker
        #          has a mysql server installed directly on it and should be enabled and used as the DB. So we might
        #          want in the future to use that instead, unless the mysql will be able to come up without crashing.
        #
        # TODO: Align the mlrun config env vars with the ones in the prepare.py script to avoid further inconsistencies.
        python automation/deployment/ce.py deploy \
            --verbose \
            --minikube \
            --namespace=${NAMESPACE} \
            --registry-secret-name="" \
            --sqlite /mlrun/db/mlrun.db \
            --override-mlrun-api-image="${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/mlrun-api:${{ steps.computed_params.outputs.mlrun_docker_tag }}" \
            --override-mlrun-log-collector-image="${{ steps.computed_params.outputs.mlrun_docker_registry }}${{ steps.computed_params.outputs.mlrun_docker_repo }}/log-collector:${{ steps.computed_params.outputs.mlrun_docker_tag }}" \
            --override-mlrun-ui-image="ghcr.io/mlrun/mlrun-ui:${{ steps.computed_params.outputs.mlrun_ui_version }}" \
            --set 'mlrun.api.extraEnvKeyValue.MLRUN_HTTPDB__BUILDER__MLRUN_VERSION_SPECIFIER="mlrun[complete] @ git+https://github.com/mlrun/mlrun@${{ steps.computed_params.outputs.mlrun_hash }}"' \
            --set mlrun.api.extraEnvKeyValue.MLRUN_IMAGES_REGISTRY="${{ steps.computed_params.outputs.mlrun_docker_registry }}" \
            --set mlrun.api.extraEnvKeyValue.MLRUN_LOG_LEVEL="DEBUG" \
            --set 'mlrun.api.extraEnvKeyValue.MLRUN_HTTPDB__SCHEDULING__MIN_ALLOWED_INTERVAL="0 seconds"' \
            --set mlrun.api.extraEnvKeyValue.MLRUN_MODEL_ENDPOINT_MONITORING__PARQUET_BATCHING_MAX_EVENTS="100"

    - name: Prepare system tests env
      run: |
        python automation/system_test/prepare.py env \
          --mlrun-dbpath "http://$(minikube ip):${MLRUN_API_NODE_PORT}" \
          --github-access-token "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}"

    # Enable tmate debugging of manually-triggered workflows if the input option was provided
    - name: Setup tmate session
      uses: mxschmitt/action-tmate@v3
      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
      with:

        # run in detach mode to allow the workflow to continue running while session is active
        # this will wait up to 10 minutes AFTER the entire job is done. Once user connects to the session,
        # it will wait until the user disconnects before finishing up the job.
        detached: true

    - name: Run system tests
      timeout-minutes: 180
      run: |
        MLRUN_SYSTEM_TESTS_CLEAN_RESOURCES="${{ steps.computed_params.outputs.mlrun_system_tests_clean_resources }}" \
        MLRUN_VERSION="${{ steps.computed_params.outputs.mlrun_version }}" \
          make test-system-open-source

    - name: Output some logs in case of failure
      if: ${{ failure() }}
      # add set -x to print commands before executing to make logs reading easier
      run: |
        set -x
        minikube ip
        minikube logs
        minikube kubectl -- --namespace ${NAMESPACE} logs -l app.kubernetes.io/component=api,app.kubernetes.io/name=mlrun --tail=-1
        minikube kubectl -- --namespace ${NAMESPACE} get all
        minikube kubectl -- --namespace ${NAMESPACE} get all -o yaml
        minikube kubectl -- --namespace ${NAMESPACE} describe pods
        minikube kubectl -- --namespace ${NAMESPACE} get cm
        minikube kubectl -- --namespace ${NAMESPACE} get cm -o yaml
        minikube kubectl -- --namespace ${NAMESPACE} get secrets
        minikube kubectl -- --namespace ${NAMESPACE} get secrets -o yaml
        minikube kubectl -- --namespace ${NAMESPACE} get pvc
        minikube kubectl -- --namespace ${NAMESPACE} get pv
        set +x


deploy

name: deploy
on:
  push:
    branches:
      - stable
jobs:
  # This job upload the Python library to PyPI
  deploy_pip:
    if: ${{ startsWith(github.event.head_commit.message, 'Bump version:') }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v1
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.8'
      - name: Install dependencies
        run: |
         python3 -m pip install virtualenv
      - name: Build Python package
        run: |
         make build_library
      - name: Upload package to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}
  # This job creates a new GitHub release
  github_release:
    if: ${{ startsWith(github.event.head_commit.message, 'Bump version:') }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}
      - name: Create a release
        run: |
         # Find the last two commits
         export LATEST_TAG="$(git describe --tags --abbrev=0)"
         export PREVIOUS_TAG="$(git describe --tags --abbrev=0 $(git rev-list --tags --skip=1 --max-count=1))"
         # Write release notes to a file
         touch release_notes.txt
         printf "## Install\nPython package is available at [PyPI](https://pypi.org/project/openram/).\n" >> release_notes.txt
         printf "## Documentation\nDocumentation is available [here](https://github.com/VLSIDA/OpenRAM/blob/stable/docs/source/index.md).\n" >> release_notes.txt
         printf "## Changes\n" >> release_notes.txt
         printf "Full changelog: https://github.com/VLSIDA/OpenRAM/compare/${PREVIOUS_TAG}...${LATEST_TAG}\n" >> release_notes.txt
         printf "## Contributors\n" >> release_notes.txt
         printf "$(git log --pretty='format:+ %an' ${LATEST_TAG}...${PREVIOUS_TAG} | sort -u)\n" >> release_notes.txt
         # Create the release via GitHub CLI
         gh release create ${LATEST_TAG} --verify-tag -F release_notes.txt
        env:
          GITHUB_TOKEN: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}


regress

name: regress
on:
  push:
    branches-ignore:
      - stable
jobs:
  # All tests should be run from this job
  regression_test:
    # This job runs on pull requests or any push that doesn't have a new version
    if: ${{ startsWith(github.event.head_commit.message, 'Bump version:') == false }}
    runs-on: self-hosted
    steps:
      - name: Checkout code
        uses: actions/checkout@v1
      - name: Install dependencies
        run: |
         pip3 install -r requirements.txt
      - name: Library build
        run: |
         rm -rf ~/.local/lib/python3.8/site-packages/openram*
         make library
      - name: Build conda
        run: |
         ./install_conda.sh
      - name: PDK Install
        run: |
         export OPENRAM_HOME="${{ github.workspace }}/compiler"
         export OPENRAM_TECH="${{ github.workspace }}/technology"
         export PDK_ROOT="${{ github.workspace }}/pdk"
         # Add make targets to install PDKs of all technologies that need it
         make sky130-pdk
         make sky130-install
      - name: Regress
        run: |
         export OPENRAM_HOME="${{ github.workspace }}/compiler"
         export OPENRAM_TECH="${{ github.workspace }}/technology"
         export PDK_ROOT="${{ github.workspace }}/pdk"
         export FREEPDK45="~/FreePDK45"
         # KLAYOUT_PATH breaks klayout installation. Unset it for now...
         unset KLAYOUT_PATH
         #cd $OPENRAM_HOME/.. && make pdk && make install
         #export OPENRAM_TMP="${{ github.workspace }}/scn4me_subm_temp"
         #python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 12 -t scn4m_subm
         #$OPENRAM_HOME/tests/regress.py -j 24 -t scn4m_subm
         cd $OPENRAM_HOME/tests
         make clean
         make -k -j 48
      - name: Archive
        if: ${{ failure() }}
        uses: actions/upload-artifact@v2
        with:
          name: Regress Archives
          path: ${{ github.workspace }}/compiler/tests/results/*
  # This job triggers sync.yml workflow
  sync_trigger:
    if: ${{ always() && github.ref_name == 'dev' && github.repository == 'VLSIDA/PrivateRAM' && needs.regression_test.result == 'failure' }}
    needs: regression_test
    uses: ./.github/workflows/sync.yml
    secrets:
      WORKFLOW_ACCESS_TOKEN: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}
  # This job triggers version.yml workflow
  version_trigger:
    if: ${{ github.ref_name == 'dev' }}
    needs: regression_test
    uses: ./.github/workflows/version.yml
    secrets:
      WORKFLOW_ACCESS_TOKEN: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}


sync

name: sync
on:
  workflow_call:
    secrets:
      WORKFLOW_ACCESS_TOKEN:
        required: true
jobs:
  # This job synchronizes the 'dev' branch of OpenRAM repo with the current branch
  sync_dev_no_version:
    if: ${{ github.repository == 'VLSIDA/PrivateRAM' }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}
      - name: Synchronize OpenRAM repo
        run: |
         # Configure pusher account
         git config --global user.name "vlsida-bot"
         git config --global user.email "mrg+vlsidabot@ucsc.edu"
         # Add remote repo
         git remote add public-repo https://${{ secrets.WORKFLOW_ACCESS_TOKEN }}@github.com/VLSIDA/OpenRAM.git
         git pull public-repo dev
         # Push the latest changes
         git push -u public-repo HEAD:dev


name: sync_tag
on:
  push:
    branches:
      - dev
jobs:
  # This job makes sure that a version commit has a tag (manually bumped versions might have tags missing)
  sync_dev_tag_check:
    if: ${{ github.repository == 'VLSIDA/PrivateRAM' && startsWith(github.event.head_commit.message, 'Bump version:') }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}
      - name: Compare version and tag
        run: |
         # Configure pusher account
         git config --global user.name "vlsida-bot"
         git config --global user.email "mrg+vlsidabot@ucsc.edu"
         # Add both repos
         git remote add private-repo https://${{ secrets.WORKFLOW_ACCESS_TOKEN }}@github.com/VLSIDA/PrivateRAM.git
         git remote add public-repo https://${{ secrets.WORKFLOW_ACCESS_TOKEN }}@github.com/VLSIDA/OpenRAM.git
         # Read the version file
         echo "LATEST_VERSION=v$(cat VERSION)" >> $GITHUB_ENV
         # Read the tag name of the last commit
         echo "HEAD_TAG=$(git describe --tags HEAD)" >> $GITHUB_ENV
      - name: Make a new tag and push
        if: ${{ env.LATEST_VERSION != env.HEAD_TAG }}
        run: |
         # Tag the commit
         git tag ${{ env.LATEST_VERSION }} HEAD
         # Push to private/dev
         git pull private-repo dev
         git push private-repo HEAD:dev ${{ env.LATEST_VERSION }}
         # Push to public-repo/dev
         git pull public-repo dev
         git push public-repo HEAD:dev ${{ env.LATEST_VERSION }}
         # Push to public/stable
         git pull public-repo stable
         git push public-repo HEAD:stable ${{ env.LATEST_VERSION }}


version

name: version
on:
  workflow_call:
    secrets:
      WORKFLOW_ACCESS_TOKEN:
        required: true
jobs:
  make_version:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}
      - name: Configure git
        run: |
         # Configure the committer
         git config --global user.name "vlsida-bot"
         git config --global user.email "mrg+vlsidabot@ucsc.edu"
         # Set remote repos
         git remote add private-repo https://${{ secrets.WORKFLOW_ACCESS_TOKEN }}@github.com/VLSIDA/PrivateRAM.git
         git remote add public-repo https://${{ secrets.WORKFLOW_ACCESS_TOKEN }}@github.com/VLSIDA/OpenRAM.git
      - name: Make new version number
        run: |
         # Read the current version number
         export CURRENT_VERSION="$(cat VERSION)"
         # Increment the version number
         export NEXT_VERSION="$(echo ${CURRENT_VERSION} | awk -F. -v OFS=. '{$NF += 1 ; print}')"
         echo "${NEXT_VERSION}" > VERSION
         # Commit the change and tag the commit
         git commit -a -m "Bump version: ${CURRENT_VERSION} -> ${NEXT_VERSION}"
         git tag "v${NEXT_VERSION}" HEAD
      - name: Push changes
        run: |
         # Read next tag
         export NEXT_TAG="v$(cat VERSION)"
         # Push to private/dev
         git pull private-repo dev
         git push private-repo HEAD:dev ${NEXT_TAG}
         # Push to public/dev
         git pull public-repo dev
         git push public-repo HEAD:dev ${NEXT_TAG}
         # Push to public/stable
         git pull public-repo stable
         git push public-repo HEAD:stable ${NEXT_TAG}


Tests

---
name: Tests

on: [push, pull_request]

jobs:
  static:
    name: static checks / linters
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.8, 3.9, '3.10', '3.11']

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache pip
      uses: actions/cache@v4
      with:
        # This path is specific to Ubuntu
        path: ${{ env.pythonLocation }}
        # Look to see if there is a cache hit for the setup file
        key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-new3
          ${{ runner.os }}-new3
    - name: Install numpy
      run: pip install -U numpy
    - name: Install dependencies
      run: pip install --upgrade --upgrade-strategy eager .[full,test,dev]
    - name: pip list packages
      run: pip list
    - name: show pip dependencies
      run: |
        pip install pipdeptree
        pipdeptree -fl
    - name: pre-commit checks
      run: pre-commit run -a
  docs:
    name: Documentation build
    runs-on: ubuntu-latest
    needs: [static]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: 3.8
    - name: Cache pip
      uses: actions/cache@v4
      with:
        # This path is specific to Ubuntu
        path: ${{ env.pythonLocation }}
        # Look to see if there is a cache hit for the setup file
        key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-new3
          ${{ runner.os }}-new3
    - name: Install doc dependencies
      run: pip install -r docs/requirements.txt
    - name: Install lale
      run: pip install --no-dependencies -e .
    - name: pip list packages
      run: pip list
    - name: Install latex (used for math notation in sphinx)
      run: >
        sudo apt-get update -y &&
        sudo apt-get install -y
        latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended
        dvipng librsvg2-bin
    - name: Run sphinx
      run: sphinx-build "." "_build" -W --keep-going
      working-directory: ./docs

  # test_matrix_pr and test_matrix_master have a lot of redundancy
  # If GitHub actions adds support for conditional matrix expressions
  # these should be combined, but current workarounds seem overly complex
  test_matrix_pr:
    name: Test (PR)
    runs-on: ubuntu-latest
    if: ${{github.event_name == 'pull_request' }}
    needs: [static]
    strategy:
      fail-fast: false
      matrix:
        test-case:
        - test/test_core_classifiers.py
        - test/test_core_regressors.py
        - test/test_core_transformers.py
        - test/test_core_pipeline.py
        - test/test_custom_schemas.py
        - test/test_json_pretty_viz.py
        - test/test_grammar.py
        - test/test_type_checking.py
        - test/test_interoperability.py
        - test/test_optimizers.py
        - test/test_pipeline.py
        - test/test_autogen_lib.py
        - test/test_snapml.py
        - test/test_relational.py
        - test/test_category_encoders.py
        python-version: [3.9]
        setup-target: ['.[full,test]']
        include:
        - test-case: test/test_core_misc.py
          python-version: 3.8
          setup-target: '.[full,test]'
        - test-case: test/test_core_misc.py
          python-version: 3.9
          setup-target: '.[full,test]'
        - test-case: test/test_core_misc.py
          python-version: '3.10'
          setup-target: '.[full,test]'
        - test-case: test/test_core_misc.py
          python-version: '3.11'
          setup-target: '.[full,test]'
        - test-case: test/test_autogen_lib.py
          python-version: 3.8
          setup-target: '.[test]'
        - test-case: test/test_halving_gridsearchcv.py
          python-version: 3.8
          setup-target: '.[test]'
        - test-case: test/test_aif360.py
          python-version: 3.8
          setup-target: '.[full,test]'
        - test-case: test/test_relational_sklearn.py
          python-version: 3.8
          setup-target: '.[full,test]'

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install system packages
      run: sudo apt-get install graphviz swig
    - name: Cache pip
      uses: actions/cache@v4
      with:
        # This path is specific to Ubuntu
        path: ${{ env.pythonLocation }}
        # Look to see if there is a cache hit for the setup file
        key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-new3
          ${{ runner.os }}-new3
    - name: Install numpy
      run: pip install -U numpy
    - name: Install dependencies
      run: pip install --upgrade --upgrade-strategy eager ${{matrix.setup-target}}
    - name: Install deps for test_relational_sklearn
      if: ${{ matrix.test-case == 'test/test_relational_sklearn.py' }}
      run: pip install 'scikit-learn>=1.0'
    - name: Install deps for test_snapml
      if: ${{ matrix.test-case == 'test/test_snapml.py' }}
      run: pip install 'numpy>=1.20.1,<1.24'
    - name: Imbalanced learn version for Python 3.8
      if: ${{ matrix.test-case == 'test/test_interoperability.py' &&  matrix.python-version == 3.8}}
      run: pip install 'imbalanced-learn==0.9.0' 'scikit-learn==1.0.2'
    - name: pip list packages
      run: pip list
    - name: show pip dependencies
      run: |
        pip install pipdeptree
        pipdeptree -fl
    - name: Run test
      run: py.test -v --capture=tee-sys ${{matrix.test-case}}

  test_matrix_master:
    name: Test (master)
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'push' }}
    needs: [static]
    strategy:
      fail-fast: false
      matrix:
        test-case:
        - test/test_core_classifiers.py
        - test/test_core_regressors.py
        - test/test_core_transformers.py
        - test/test_core_misc.py
        - test/test_core_pipeline.py
        - test/test_custom_schemas.py
        - test/test_json_pretty_viz.py
        - test/test_grammar.py
        - test/test_type_checking.py
        - test/test_interoperability.py
        - test/test_optimizers.py
        - test/test_pipeline.py
        - test/test_autogen_lib.py
        - test/test_relational.py
        - test/test_category_encoders.py
        python-version: [3.8, 3.9, '3.10', '3.11']
        setup-target: ['.[full,test]']
        include:
        - test-case: test/test_autogen_lib.py
          python-version: 3.8
          setup-target: '.[test]'
        - test-case: test/test_halving_gridsearchcv.py
          python-version: 3.8
          setup-target: '.[test]'
        - test-case: test/test_aif360.py
          python-version: 3.8
          setup-target: '.[full,test]'
        - test-case: test/test_relational_sklearn.py
          python-version: 3.8
          setup-target: '.[test]'
        - test-case: test/test_snapml.py
          python-version: 3.8
          setup-target: '.[full,test]'
        - test-case: test/test_snapml.py
          python-version: 3.9
          setup-target: '.[full,test]'

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install system packages
      run: sudo apt-get install graphviz swig
    - name: Cache pip
      uses: actions/cache@v4
      with:
        # This path is specific to Ubuntu
        path: ${{ env.pythonLocation }}
        # Look to see if there is a cache hit for the setup file
        key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-new3
          ${{ runner.os }}-new3
    - name: Install numpy
      run: pip install -U numpy
    - name: Install dependencies
      run: pip install --upgrade --upgrade-strategy eager ${{matrix.setup-target}}
    - name: Install deps for test_relational_sklearn
      if: ${{ matrix.test-case == 'test/test_relational_sklearn.py' }}
      run: pip install 'scikit-learn>=1.0'
    - name: Install deps for test_snapml
      if: ${{ matrix.test-case == 'test/test_snapml.py' }}
      run: pip install 'numpy>=1.20.1,<1.24'
    - name: Imbalanced learn version for Python 3.8
      if: ${{ matrix.test-case == 'test/test_interoperability.py' &&  matrix.python-version == 3.8}}
      run: pip install 'imbalanced-learn==0.9.0' 'scikit-learn==1.0.2'
    - name: pip list packages
      run: pip list
    - name: show pip dependencies
      run: |
        pip install pipdeptree
        pipdeptree -fl
    - name: Run test
      run: py.test -v --capture=tee-sys ${{matrix.test-case}}

  test_newer:
    name: Test with newest sklearn
    runs-on: ubuntu-latest
    needs: [static]
    strategy:
      fail-fast: false
      matrix:
        test-case:
        - test/test_core_classifiers.py
        - test/test_core_regressors.py
        - test/test_core_transformers.py
        python-version: ['3.10', '3.11']
        setup-target: ['.']
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install system packages
      run: sudo apt-get install graphviz swig
    - name: Cache pip
      uses: actions/cache@v4
      with:
        # This path is specific to Ubuntu
        path: ${{ env.pythonLocation }}
        # Look to see if there is a cache hit for the setup file
        key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-new3
          ${{ runner.os }}-new3
    - name: Install numpy
      run: pip install -U numpy
    - name: Install dependencies
      run: pip install --upgrade --upgrade-strategy eager ${{matrix.setup-target}}
    - name: Install pytest
      run: pip install pytest
    - name: Install latest scikit-learn, xgboost, and lightgbm
      run: pip install --upgrade --upgrade-strategy eager scikit-learn xgboost lightgbm
    - name: pip list packages
      run: pip list
    - name: show pip dependencies
      run: |
        pip install pipdeptree
        pipdeptree -fl
    - name: Run test
      run: py.test -v --capture=tee-sys ${{matrix.test-case}}
      continue-on-error: true  # we may not yet support the latest scikit-learn version

  test_notebooks_pr:
    name: Test Notebooks (PR)
    needs: [static]
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'pull_request' }}
    env:
      ALL_NOTEBOOK_CATEGORIES: demo_ docs_ talk_
    strategy:
      fail-fast: false
      matrix:
        category:
        - demo_
        - docs_
        - talk_
        python-version: [3.9]
        setup-target: ['.[full,test]']
        test-case: ['test/test_notebooks.py']
        dir: ['examples']
        nbexcludes: ['']
        include:
        - dir: 'examples/kdd22'
          python-version: 3.9
          setup-target: '.[tutorial,test]'
          test-case: 'test/test_notebooks.py'
          # nbexcludes: '06_multobj.ipynb'

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache pip
      uses: actions/cache@v4
      with:
        # This path is specific to Ubuntu
        path: ${{ env.pythonLocation }}
        # Look to see if there is a cache hit for the setup file
        key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-new3
          ${{ runner.os }}-new3
    - name: Install packages
      run: sudo apt-get install graphviz swig
    - name: Install numpy
      run: pip install -U numpy
    - name: Install dependencies
      run: pip install --upgrade --upgrade-strategy eager ${{matrix.setup-target}}
    - name: Fix tutorial dependencies
      if: ${{ matrix.dir == 'examples/kdd22' }}
      run: |
        pip install "pandas<2.0.0" "xgboost<=1.5.1"
    - name: pip list packages
      run: pip list
    - name: show pip dependencies
      run: |
        pip install pipdeptree
        pipdeptree -fl
    - name: Run test
      run: >
        NOTEBOOK_EXCLUDES="${{matrix.nbexcludes}}"
        NOTEBOOK_DIR=${{matrix.dir}}
        NOTEBOOK_CATEGORY=${{matrix.category}}
        py.test -v --capture=tee-sys ${{matrix.test-case}}

  test_notebooks_master:
    name: Test Notebooks (master)
    needs: [static]
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'push' }}
    env:
      ALL_NOTEBOOK_CATEGORIES: demo_ docs_ talk_
    strategy:
      fail-fast: false
      matrix:
        category:
        - demo_
        - docs_
        - talk_
        python-version: [3.8, 3.9, '3.10', '3.11']
        setup-target: ['.[full,test]']
        test-case: ['test/test_notebooks.py']
        dir: ['examples']
        nbexcludes: ['']
        include:
        - dir: 'examples'
          python-version: 3.8
          setup-target: '.[full,test]'
          test-case: 'test/test_notebooks.py'
          nbexcludes: 'demo_auto_pipeline.ipynb'
        - dir: 'examples/kdd22'
          python-version: 3.8
          setup-target: '.[tutorial,test]'
          test-case: 'test/test_notebooks.py'
          nbexcludes: '02_total.ipynb'
        - dir: 'examples/kdd22'
          python-version: 3.9
          setup-target: '.[tutorial,test]'
          test-case: 'test/test_notebooks.py'
          # nbexcludes: '06_multobj.ipynb'
        - dir: 'examples/kdd22'
          python-version: '3.10'
          setup-target: '.[tutorial,test]'
          test-case: 'test/test_notebooks.py'
          # nbexcludes: '06_multobj.ipynb'
        - dir: 'examples/kdd22'
          python-version: '3.11'
          setup-target: '.[tutorial,test]'
          test-case: 'test/test_notebooks.py'
          # nbexcludes: '06_multobj.ipynb'

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache pip
      uses: actions/cache@v4
      with:
        # This path is specific to Ubuntu
        path: ${{ env.pythonLocation }}
        # Look to see if there is a cache hit for the setup file
        key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
        restore-keys: |
          ${{ runner.os }}-pip-new3
          ${{ runner.os }}-new3
    - name: Install packages
      run: sudo apt-get install graphviz swig
    - name: Install numpy
      run: pip install -U numpy
    - name: Install dependencies
      run: pip install --upgrade --upgrade-strategy eager ${{matrix.setup-target}}
    - name: Fix tutorial dependencies
      if: ${{ matrix.dir == 'examples/kdd22' }}
      run: |
        pip install "pandas<2.0.0" "xgboost<=1.5.1"
    - name: pip list packages
      run: pip list
    - name: show pip dependencies
      run: |
        pip install pipdeptree
        pipdeptree -fl
    - name: Run test
      run: >
        NOTEBOOK_EXCLUDES="${{matrix.nbexcludes}}"
        NOTEBOOK_DIR=${{matrix.dir}}
        NOTEBOOK_CATEGORY=${{matrix.category}}
        py.test -v --capture=tee-sys ${{matrix.test-case}}

  test_fairness:
    name: Test fairness install
    runs-on: ubuntu-latest
    needs: [static]
    strategy:
      fail-fast: false
      matrix:
        test-case:
        - test/test_aif360.py
        - test/test_aif360_ensembles.py
        python-version: [3.8]
        setup-target: ['.[fairness]']
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install system packages
      run: sudo apt-get install graphviz swig
    - name: Install pytest
      run: pip install pytest
    - name: Install dependencies
      run: pip install --upgrade --upgrade-strategy eager ${{matrix.setup-target}}
    - name: pip list packages
      run: pip list
    - name: show pip dependencies
      run: |
        pip install pipdeptree
        pipdeptree -fl
    - name: Run test
      run: py.test -v --capture=tee-sys ${{matrix.test-case}}

  deploy:
    name: Deploy to TestPyPI
    runs-on: ubuntu-latest
    # This should run only on the master branch of the main repo
    if: github.repository == 'IBM/lale' && github.ref == 'refs/heads/master' && success()
    needs: [static, test_matrix_master, test_newer, test_fairness, test_notebooks_master, docs]
    strategy:
      matrix:
        python-version: [3.8]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: pip install wheel twine
    - name: Build distribution
      env:
        # integrate with the current system for timestamping versions
        TRAVIS: " "
      run: python setup.py sdist bdist_wheel
    - name: Publish package to TestPyPI
      run: twine upload -u "__token__" -p "${{ secrets.test_pypi_password }}" -r testpypi dist/*


Deploy Release

---
name: Deploy Release

on:
  release:
    types: [released]

jobs:
  deploy:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.8]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: pip install wheel twine
    - name: Build distribution
      run: python setup.py sdist bdist_wheel
    - name: Publish package to (not-test) PyPi
      run: twine upload -u "__token__" -p "${{ secrets.pypi_password }}" dist/*
    - name: Publish package to TestPyPI
      run: twine upload -u "__token__" -p "${{ secrets.test_pypi_password }}" -r testpypi dist/*


Develop Push

name: Develop Push

on:
  push:
    branches:
      - develop

jobs:
  docker-publish-unstable:
    name: Docker Publish
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}

      - name: Build and Publish Unstable Images
        run: make publish-docker-unstable

  CodeCoverage:
    runs-on: ${{ matrix.os }}

    defaults:
      run:
        working-directory: ./src/app

    strategy:
      matrix:
        python-version: ["3.11"]
        os: ["ubuntu-latest"]

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python-version }}

    name: Code Coverage OS ${{ matrix.os }} - Python ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v3

      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      # Need to update if we support other OS's
      - name: Cache PIP Install
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-${{ matrix.python-version }}-pip-coverage-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.python-version }}-pip-coverage

      - name: Make Deps
        run: |
          make deps
          pip install pytest
          pip install pytest-cov

      - name: Generate coverage report
        continue-on-error: true
        run: |
          coverage run --source beer_garden -m pytest test --tb=no
          coverage report -m
          coverage xml

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v1