CI

name: CI
# NOTE: this CI workflow is running a number of unit/integration/system tests jobs
# and also builds a set of "test" docker images (required for integration/system tests)
# if all tests are passing, and the CI runs on the master branch, then it will also deploy
# the "test" docker images to dockerhub registry
#
# in PRs: only the jobs that needs to run will be run by detecting file changes
#
# NOTE2: to keep the github branch protection simpler, the required jobs are the ones named:
# unit-tests, integration-tests, and system-tests

on:
  push:
    branches:
      - "*"
    tags-ignore:
      - "*"
  pull_request:
    branches:
      - "*"
  # https://github.blog/changelog/2023-02-08-pull-request-merge-queue-public-beta/
  merge_group:
    branches:
      - "master"

  workflow_dispatch:
    inputs:
      tests:
        description: tests to run
        required: true
        default: "all"
        type: choice
        options:
          - unit-tests
          - integration-tests
          - system-tests
          - all

env:
  DEFAULT_MAX_NANO_CPUS: 10000000
  DEFAULT_MAX_MEMORY: 268435456

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    name: detect file changes since last push
    runs-on: ubuntu-latest
    # Set job outputs to values from filter step
    outputs:
      aws-library: ${{ steps.filter.outputs.aws-library }}
      dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }}
      models-library: ${{ steps.filter.outputs.models-library }}
      notifications-library: ${{ steps.filter.outputs.notifications-library }}
      postgres-database: ${{ steps.filter.outputs.postgres-database }}
      service-integration: ${{ steps.filter.outputs.service-integration }}
      service-library: ${{ steps.filter.outputs.service-library }}
      settings-library: ${{ steps.filter.outputs.settings-library }}
      simcore-sdk: ${{ steps.filter.outputs.simcore-sdk }}
      agent: ${{ steps.filter.outputs.agent }}
      api: ${{ steps.filter.outputs.api }}
      api-server: ${{ steps.filter.outputs.api-server }}
      autoscaling: ${{ steps.filter.outputs.autoscaling }}
      catalog: ${{ steps.filter.outputs.catalog }}
      clusters-keeper: ${{ steps.filter.outputs.clusters-keeper }}
      dask-sidecar: ${{ steps.filter.outputs.dask-sidecar }}
      datcore-adapter: ${{ steps.filter.outputs.datcore-adapter }}
      director: ${{ steps.filter.outputs.director }}
      director-v2: ${{ steps.filter.outputs.director-v2 }}
      dynamic-sidecar: ${{ steps.filter.outputs.dynamic-sidecar }}
      efs-guardian: ${{ steps.filter.outputs.efs-guardian }}
      invitations: ${{ steps.filter.outputs.invitations }}
      migration: ${{ steps.filter.outputs.migration }}
      osparc-gateway-server: ${{ steps.filter.outputs.osparc-gateway-server }}
      payments: ${{ steps.filter.outputs.payments }}
      dynamic-scheduler: ${{ steps.filter.outputs.dynamic-scheduler }}
      resource-usage-tracker: ${{ steps.filter.outputs.resource-usage-tracker }}
      static-webserver: ${{ steps.filter.outputs.static-webserver }}
      storage: ${{ steps.filter.outputs.storage }}
      webserver: ${{ steps.filter.outputs.webserver }}
      anything: ${{ steps.filter.outputs.anything }}
      anything-py: ${{ steps.filter.outputs.anything-py }}
      anything-js: ${{ steps.filter.outputs.anything-js }}
    steps:
      - uses: actions/checkout@v4
      # For pull requests it's not necessary to checkout the code
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            aws-library:
              - 'packages/aws-library/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            dask-task-models-library:
              - 'packages/dask-task-models-library/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            models-library:
              - 'packages/models-library/**'
              - 'packages/postgres-database/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            notifications-library:
              - 'packages/notifications-library/**'
              - 'packages/postgres-database/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            postgres-database:
              - 'packages/postgres-database/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            service-integration:
              - 'packages/models-library/**'
              - 'packages/pytest-simcore/**'
              - 'packages/service-integration/**'
              - 'services/docker-compose*'
            service-library:
              - 'packages/pytest-simcore/**'
              - 'packages/service-library/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            settings-library:
              - 'packages/pytest-simcore/**'
              - 'packages/settings-library/**'
              - 'services/docker-compose*'
            simcore-sdk:
              - 'packages/**'
              - 'services/docker-compose*'
            agent:
              - 'packages/**'
              - 'services/agent/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            api:
              - 'api/**'
            api-server:
              - 'packages/**'
              - 'services/api-server/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            autoscaling:
              - 'packages/**'
              - 'services/autoscaling/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            catalog:
              - 'packages/**'
              - 'services/catalog/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            clusters-keeper:
              - 'packages/**'
              - 'services/clusters-keeper/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            dask-sidecar:
              - 'packages/**'
              - 'services/dask-sidecar/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            datcore-adapter:
              - 'packages/**'
              - 'services/datcore-adapter/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            director:
              - 'packages/**'
              - 'services/director/**'
              - 'services/docker-compose*'
            director-v2:
              - 'packages/**'
              - 'services/director-v2/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            dynamic-sidecar:
              - 'packages/**'
              - 'services/dynamic-sidecar/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            efs-guardian:
              - 'packages/**'
              - 'services/efs-guardian/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            invitations:
              - 'packages/**'
              - 'services/invitations/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            migration:
              - 'packages/**'
              - 'services/migration/**'
              - 'services/docker-compose*'
            osparc-gateway-server:
              - 'packages/**'
              - 'services/osparc-gateway-server/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            payments:
                - 'packages/**'
                - 'services/payments/**'
                - 'services/docker-compose*'
                - 'scripts/mypy/*'
                - 'mypy.ini'
            dynamic-scheduler:
                - 'packages/**'
                - 'services/dynamic-scheduler/**'
                - 'services/docker-compose*'
                - 'scripts/mypy/*'
                - 'mypy.ini'
            resource-usage-tracker:
              - 'packages/**'
              - 'services/resource-usage-tracker/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            static-webserver:
              - 'services/static-webserver/**'
              - 'services/docker-compose*'
            storage:
              - 'packages/**'
              - 'services/storage/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            webserver:
              - 'packages/**'
              - 'services/web/**'
              - 'services/docker-compose*'
            anything:
              - 'packages/**'
              - 'services/**'
              - 'tests/**'
            anything-py:
              - '**/*.py'
            anything-js:
              - '**/*.js'
  build-test-images:
    # this step comes first, so that it is executed as first job in push calls
    # in PR calls this step is anyway skipped
    needs: changes
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    name: "[build] docker images"
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      - name: show system environs
        run: ./ci/helpers/show_system_versions.bash
      - name: build images
        run: |
          export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
          mkdir --parents /${{ runner.temp }}/build
          make build local-dest=/${{ runner.temp }}/build
      - name: upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
          path: /${{ runner.temp }}/build

  unit-test-webserver-01:
    needs: changes
    if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/webserver.bash typecheck
      - name: test isolated
        if: always()
        run: ./ci/github/unit-testing/webserver.bash test_isolated
      - name: test
        if: always()
        run: ./ci/github/unit-testing/webserver.bash test_with_db 01
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-webserver-02:
    needs: changes
    if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: test
        run: ./ci/github/unit-testing/webserver.bash test_with_db 02
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-webserver-03:
    needs: changes
    if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 03"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: test
        run: ./ci/github/unit-testing/webserver.bash test_with_db 03
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-storage:
    needs: changes
    if: ${{ needs.changes.outputs.storage == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] storage"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/storage.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/storage.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/storage.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-agent:
    needs: changes
    if: ${{ needs.changes.outputs.agent == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] agent"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/agent.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/agent.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/agent.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-api:
    needs: changes
    if: ${{ needs.changes.outputs.api == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] api"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install api
        run: ./ci/github/unit-testing/api.bash install
      - name: test
        run: ./ci/github/unit-testing/api.bash test

  unit-test-api-server:
    needs: changes
    if: ${{ needs.changes.outputs.api-server == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] api-server"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/api-server.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/api-server.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/api-server.bash test
      - name: OAS backwards compatibility check
        if: always()
        run: ./ci/github/unit-testing/api-server.bash openapi-diff
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-autoscaling:
    needs: changes
    if: ${{ needs.changes.outputs.autoscaling == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] autoscaling"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/autoscaling.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/autoscaling.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/autoscaling.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-catalog:
    needs: changes
    if: ${{ needs.changes.outputs.catalog == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] catalog"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/catalog.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/catalog.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/catalog.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/catalog/test_failures
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-clusters-keeper:
    needs: changes
    if: ${{ needs.changes.outputs.clusters-keeper == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] clusters-keeper"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/clusters-keeper && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          pushd services/clusters-keeper && \
          make mypy
      - name: test
        if: always()
        run: |
          source .venv/bin/activate && \
          pushd services/clusters-keeper && \
          make test-ci-unit
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-datcore-adapter:
    needs: changes
    if: ${{ needs.changes.outputs.datcore-adapter == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] datcore-adapter"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/datcore-adapter.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/datcore-adapter.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/datcore-adapter.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/datcore-adapter/test_failures
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-director:
    needs: changes
    if: ${{ needs.changes.outputs.director == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] director"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        # KEEP 3.6 Development of this service is frozen
        # KEEP ubuntu 20.04, else no python 3.6
        python: [3.6]
        os: [ubuntu-20.04]
        docker_buildx: [v0.10.4]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/director.bash install
      - name: test
        run: ./ci/github/unit-testing/director.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-director-v2:
    needs: changes
    if: ${{ needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] director-v2"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/director-v2.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/director-v2.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/director-v2.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-aws-library:
    needs: changes
    if: ${{ needs.changes.outputs.aws-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] aws-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/aws-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/aws-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/aws-library.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-dask-task-models-library:
    needs: changes
    if: ${{ needs.changes.outputs.dask-task-models-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dask-task-models-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dask-task-models-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dask-task-models-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/dask-task-models-library.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-dask-sidecar:
    needs: changes
    if: ${{ needs.changes.outputs.dask-sidecar == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dask-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dask-sidecar.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dask-sidecar.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/dask-sidecar.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-osparc-gateway-server:
    needs: changes
    if: ${{ needs.changes.outputs.osparc-gateway-server == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] osparc-gateway-server"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make mypy
      - name: test
        if: always()
        run: |
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make test-ci-unit
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-payments:
    needs: changes
    if: ${{ needs.changes.outputs.payments == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] payments"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/payments.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/payments.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/payments.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-dynamic-scheduler:
    needs: changes
    if: ${{ needs.changes.outputs.dynamic-scheduler == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dynamic-scheduler"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dynamic-scheduler.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dynamic-scheduler.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/dynamic-scheduler.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-resource-usage-tracker:
    needs: changes
    if: ${{ needs.changes.outputs.resource-usage-tracker == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] resource-usage-tracker"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/resource-usage-tracker && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          pushd services/resource-usage-tracker && \
          make mypy
      - name: test
        if: always()
        run: |
          source .venv/bin/activate && \
          pushd services/resource-usage-tracker && \
          make test-ci-unit
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-dynamic-sidecar:
    needs: changes
    if: ${{ needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dynamic-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dynamic-sidecar.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dynamic-sidecar.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/dynamic-sidecar.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-efs-guardian:
    needs: changes
    if: ${{ needs.changes.outputs.efs-guardian == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] efs-guardian"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/efs-guardian && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          pushd services/efs-guardian && \
          make mypy
      - name: test
        if: always()
        run: |
          source .venv/bin/activate && \
          pushd services/efs-guardian && \
          make test-ci-unit
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-frontend:
    needs: changes
    if: ${{ needs.changes.outputs.static-webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] frontend"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        node: [14]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - uses: actions/setup-node@v4.0.2
        with:
          node-version: ${{ matrix.node }}
          cache: "npm"
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/frontend.bash install
      - name: test
        run: ./ci/github/unit-testing/frontend.bash test

  unit-test-python-linting:
    needs: changes
    if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] python-linting"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10", "3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/python-linting.bash install
      - name: test
        run: ./ci/github/unit-testing/python-linting.bash test

  unit-test-postgres-database:
    needs: changes
    if: ${{ needs.changes.outputs.postgres-database == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] postgres-database"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/postgres-database.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/postgres-database.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/postgres-database.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-invitations:
    needs: changes
    if: ${{ needs.changes.outputs.invitations == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] invitations"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/invitations.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/invitations.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/invitations.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-service-integration:
    needs: changes
    if: ${{ needs.changes.outputs.service-integration == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] service-integration"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/service-integration.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/service-integration.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/service-integration.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-service-library:
    needs: changes
    if: ${{ needs.changes.outputs.service-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] service-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/service-library.bash install_all
      - name: typecheck
        run: ./ci/github/unit-testing/service-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/service-library.bash test_all
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-settings-library:
    needs: changes
    if: ${{ needs.changes.outputs.settings-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] settings-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/settings-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/settings-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/settings-library.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-models-library:
    needs: changes
    if: ${{ needs.changes.outputs.models-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] models-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/models-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/models-library.bash typecheck
      - name: test
        run: ./ci/github/unit-testing/models-library.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-notifications-library:
    needs: changes
    if: ${{ needs.changes.outputs.notifications-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit]notifications-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
        docker_buildx: [v0.10.4]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/notifications-library/requirements/ci.txt"
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/notifications-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/notifications-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/notifications-library.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-test-simcore-sdk:
    needs: changes
    if: ${{ needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] simcore-sdk"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/simcore-sdk.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/simcore-sdk.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/simcore-sdk.bash test
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: unittests #optional

  unit-tests:
    # NOTE: this is a github required status check!
    if: ${{ always() }}
    needs:
      [
        unit-test-agent,
        unit-test-api-server,
        unit-test-api,
        unit-test-autoscaling,
        unit-test-catalog,
        unit-test-clusters-keeper,
        unit-test-dask-sidecar,
        unit-test-aws-library,
        unit-test-dask-task-models-library,
        unit-test-datcore-adapter,
        unit-test-director-v2,
        unit-test-director,
        unit-test-dynamic-sidecar,
        unit-test-efs-guardian,
        unit-test-frontend,
        unit-test-models-library,
        unit-test-notifications-library,
        unit-test-osparc-gateway-server,
        unit-test-payments,
        unit-test-dynamic-scheduler,
        unit-test-postgres-database,
        unit-test-python-linting,
        unit-test-resource-usage-tracker,
        unit-test-service-integration,
        unit-test-service-library,
        unit-test-settings-library,
        unit-test-simcore-sdk,
        unit-test-storage,
        unit-test-webserver-01,
        unit-test-webserver-02,
        unit-test-webserver-03,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: a previous unit-test job failed
        if: ${{ contains(join(needs.*.result, ','), 'failure') }}
        run: |
          echo "::error title=ERROR::one of the unit-tests failed!"
          echo "${{ join(needs.*.result, ',') }}"
          exit 1
      - name: all the previous unit-tests were run successfully or skipped
        if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
        run: echo "::notice All good!"

  integration-test-webserver-01:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] webserver 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/webserver.bash install
      - name: test
        run: ./ci/github/integration-testing/webserver.bash test 01
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/web/server/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/webserver.bash clean_up
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: integrationtests #optional

  integration-test-webserver-02:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] webserver 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/webserver.bash install
      - name: test
        run: ./ci/github/integration-testing/webserver.bash test 02
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/web/server/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/webserver.bash clean_up
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: integrationtests #optional

  integration-test-director-v2-01:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push'}}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] director-v2 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/director-v2.bash install
      - name: test
        run: ./ci/github/integration-testing/director-v2.bash test 01
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/director-v2.bash clean_up
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: integrationtests #optional

  integration-test-director-v2-02:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] director-v2 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup rclone docker volume plugin
        run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/director-v2.bash install
      - name: test
        run: ./ci/github/integration-testing/director-v2.bash test 02
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/director-v2.bash clean_up
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: integrationtests #optional

  integration-test-dynamic-sidecar:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push'}}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] dynamic-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/dynamic-sidecar.bash install
      - name: test
        run: ./ci/github/integration-testing/dynamic-sidecar.bash test 01
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/dynamic-sidecar/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/dynamic-sidecar.bash clean_up
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: integrationtests #optional

  integration-test-osparc-gateway-server:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.osparc-gateway-server == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] osparc-gateway-server"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container

      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv && \
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make install-ci
      - name: integration-test
        run: |
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make test-ci-integration
      - name: system-test
        run: |
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make test-system
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: always()
        run: |
          pushd services/osparc-gateway-server && \
          make down
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: integrationtests #optional

  integration-test-simcore-sdk:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] simcore-sdk"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/simcore-sdk.bash install
      - name: test
        run: ./ci/github/integration-testing/simcore-sdk.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./packages/simcore-sdk/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/simcore-sdk.bash clean_up
      - uses: codecov/codecov-action@v4.5.0
        with:
          flags: integrationtests

  integration-tests:
    # NOTE: this is a github required status check!
    if: always()
    needs:
      [
        integration-test-director-v2-01,
        integration-test-director-v2-02,
        integration-test-dynamic-sidecar,
        integration-test-osparc-gateway-server,
        integration-test-simcore-sdk,
        integration-test-webserver-01,
        integration-test-webserver-02,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: a previous integration-test job failed
        if: ${{ contains(join(needs.*.result, ','), 'failure') }}
        run: |
          echo "::error title=ERROR::one of the integration-tests failed!"
          exit 1
      - name: all the previous integration-tests were run successfully or skipped
        if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
        run: echo "::notice All good!"

  system-test-public-api:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[sys] public api"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: ./ci/github/system-testing/public-api.bash install
      - name: test
        run: ./ci/github/system-testing/public-api.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/public-api.bash clean_up

  system-test-swarm-deploy:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] deploy simcore"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/system-testing/swarm-deploy.bash install
      - name: test
        run: ./ci/github/system-testing/swarm-deploy.bash test
      - name: dump services setting schemas
        run: export DOCKER_REGISTRY=local; export DOCKER_IMAGE_TAG=production; make settings-schema.json
      - name: upload services settings schemas
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_services_settings_schemas
          path: ./services/**/settings-schema.json
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/swarm-deploy.bash clean_up

  system-test-e2e:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] e2e"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        node: [14]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - uses: actions/setup-node@v4.0.2
        with:
          node-version: ${{ matrix.node }}
          cache: "npm"
          cache-dependency-path: "tests/e2e/package-lock.json"
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: setup
        run: ./ci/github/system-testing/e2e.bash install
      - name: test
        run: ./ci/github/system-testing/e2e.bash test
      - name: dump docker logs
        id: docker_logs_dump
        if: failure()
        run: ./ci/github/system-testing/e2e.bash dump_docker_logs
      - name: upload docker logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./tests/e2e/test_failures
      - name: upload screenshots
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_screenshots
          path: tests/e2e/screenshots
      - name: upload e2e logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_logs
          path: tests/e2e/logs
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/e2e.bash clean_up

  system-test-e2e-playwright:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] e2e-playwright"
    runs-on: ${{ matrix.os }}
    # NOTE: this is an interesting way, but generate a load of issues like not having docker installed, etc etc.
    # container:
    #   image: mcr.microsoft.com/playwright/python:v1.39.0-jammy
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      - name: download docker images
        uses: actions/download-artifact@v4
        with:
          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
          path: /${{ runner.temp }}/build
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: prepare devenv
        run: make devenv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: setup
        run: |
          ./ci/github/system-testing/e2e-playwright.bash install
      - name: test
        run: |
          ./ci/github/system-testing/e2e-playwright.bash test
      - name: dump docker logs
        id: docker_logs_dump
        if: failure()
        run: ./ci/github/system-testing/e2e-playwright.bash dump_docker_logs
      - name: upload docker logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./tests/e2e-playwright/test_failures
      - name: upload tracing if failed
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_tracing
          path: tests/e2e-playwright/test-results

  system-test-environment-setup:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] environment setup"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/system-testing/environment-setup.bash install
      - name: test
        run: ./ci/github/system-testing/environment-setup.bash test
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/environment-setup.bash clean_up

  system-tests:
    # NOTE: this is a github required status check!
    if: always()
    needs:
      [
        system-test-e2e,
        system-test-e2e-playwright,
        system-test-environment-setup,
        system-test-public-api,
        system-test-swarm-deploy,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: a previous system-test job failed
        if: ${{ contains(join(needs.*.result, ','), 'failure') }}
        run: |
          echo "::error title=ERROR::one of the system-tests failed!"
          exit 1
      - name: all the previous system-tests were run successfully or skipped
        if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
        run: echo "::notice All good!"

  deploy:
    name: deploy to dockerhub
    if: github.event_name == 'push'
    needs: [unit-tests, integration-tests, system-tests]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    env:
      # secrets can be set in settings/secrets on github
      DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: |
          make load-images local-src=/${{ runner.temp }}/build
      - name: set owner variable
        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
      - if: github.ref == 'refs/heads/master'
        name: deploy master image
        env:
          TAG_PREFIX: master-github
        run: ./ci/deploy/dockerhub-deploy.bash -n
      - if: contains(github.ref, 'refs/heads/hotfix_v')
        name: deploy release hotfix image
        env:
          TAG_PREFIX: hotfix-github
        run: ./ci/deploy/dockerhub-deploy.bash -n
      - if: contains(github.ref, 'refs/heads/hotfix_staging_')
        name: deploy staging hotfix image
        env:
          TAG_PREFIX: hotfix-staging-github
        run: ./ci/deploy/dockerhub-deploy.bash -n


CodeQL

#
# codeQL       : https://codeql.github.com/docs/codeql-overview/
# GH-Action    : https://github.com/github/codeql-action
# LGTM         : https://lgtm.com/#explore
# codeql-config: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-a-custom-configuration-file
#
name: "CodeQL"

on:
  schedule:
    - cron: "26 5 * * 1"

jobs:
  analyze:
    name: Analyze Code
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: ["javascript", "python"]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Initialize CodeQL tools for scanning
        uses: github/codeql-action/init@v3
        with:
          languages: ${{ matrix.language }}
          config-file: ./.github/codeql/codeql-config.yml

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3


Tests

name: Tests

on:
  push:
    branches: [ master ]
  pull_request:
  release:
    types: [created]

permissions:
  contents: read

jobs:
  build:
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.9]
        backend: [tensorflow, jax, torch, numpy]
    name: Run tests
    runs-on: ubuntu-latest
    env:
      PYTHON: ${{ matrix.python-version }}
      KERAS_HOME: .github/workflows/config/${{ matrix.backend }}
    steps:
      - uses: actions/checkout@v4
      - name: Check for changes in keras/src/applications
        uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            applications:
              - 'keras/src/applications/**'
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Get pip cache dir
        id: pip-cache
        run: |
          python -m pip install --upgrade pip setuptools
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
      - name: pip cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}
      - name: Install dependencies
        run: |
          pip install -r requirements.txt --progress-bar off --upgrade
          pip uninstall -y keras keras-nightly
          pip install tf_keras==2.16.0 --progress-bar off --upgrade
          pip install -e "." --progress-bar off --upgrade
      - name: Test applications with pytest
        if: ${{ steps.filter.outputs.applications == 'true' }}
        run: |
          pytest keras/src/applications --cov=keras/src/applications
          coverage xml --include='keras/src/applications/*' -o apps-coverage.xml
      - name: Codecov keras.applications
        if: ${{ steps.filter.outputs.applications == 'true' }}
        uses: codecov/codecov-action@v4
        with:
          env_vars: PYTHON,KERAS_HOME
          flags: keras.applications,keras.applications-${{ matrix.backend }}
          files: apps-coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false
      - name: Test integrations
        if: ${{ matrix.backend != 'numpy'}}
        run: |
          python integration_tests/import_test.py
          python integration_tests/numerical_test.py
      - name: Test TF-specific integrations
        if: ${{ matrix.backend == 'tensorflow'}}
        run: |
          python integration_tests/tf_distribute_training_test.py
      - name: Test Torch-specific integrations
        if: ${{ matrix.backend == 'torch'}}
        run: |
          pytest integration_tests/torch_workflow_test.py
      - name: Test with pytest
        run: |
          pytest keras --ignore keras/src/applications --cov=keras
          coverage xml --omit='keras/src/applications/*,keras/api' -o core-coverage.xml
      - name: Codecov keras
        uses: codecov/codecov-action@v4
        with:
          env_vars: PYTHON,KERAS_HOME
          flags: keras,keras-${{ matrix.backend }}
          files: core-coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false

  format:
    name: Check the code format
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - name: Get pip cache dir
        id: pip-cache
        run: |
          python -m pip install --upgrade pip setuptools
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
      - name: pip cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}
      - name: Install dependencies
        run: |
          pip install -r requirements.txt --progress-bar off --upgrade
          pip uninstall -y keras keras-nightly
          pip install -e "." --progress-bar off --upgrade
      - name: Lint
        run: bash shell/lint.sh
      - name: Check for API changes
        run: |
          bash shell/api_gen.sh
          git status
          clean=$(git status | grep "nothing to commit")
          if [ -z "$clean" ]; then
            echo "Please run shell/api_gen.sh to generate API."
            exit 1
          fi


auto-assignment

name: auto-assignment
on:
  issues:
    types:
      - opened

permissions:
  contents: read
  issues: write
  pull-requests: write

jobs:
  welcome:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/github-script@v7
        with:
          script: |
            const script = require('./\.github/workflows/scripts/auto-assignment.js')
            script({github, context})


Labeler

# Copyright 2024 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This workflow automatically identifies issues and pull requests (PRs) and add the
# appropriate label as per defined rules.
# First Labeler workflow: It searches for the keyword "Gemma" (case-insensitive) in both the title
# and description of the issue/PR. If a match is found, the workflow adds the label 'Gemma' to the issue/PR.

name: 'Labeler'
on:
  issues:
    types: [edited,opened]
  pull_request_target:
    types: [opened, edited]

permissions:
  contents: read
  issues: write
  pull-requests: write

jobs:
  welcome:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/github-script@v7
        with:
          script: |
            const script = require('./\.github/workflows/scripts/labeler.js')
            script({github, context})