integration-tests-and-build

name: integration-tests-and-build

on:
  push:
    branches:
      - master

jobs:
  integration-test-python:
    if: github.repository == 'feast-dev/feast'
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.10", "3.11"]
        os: [ ubuntu-latest ]
    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python-version }}
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - name: Set up AWS SDK
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Use AWS CLI
        run: aws sts get-caller-identity
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Get uv cache dir
        id: uv-cache
        run: |
          echo "::set-output name=dir::$(uv cache dir)"
      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.uv-cache.outputs.dir }}
          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
      - name: Install dependencies
        run: make install-python-ci-dependencies-uv
      - name: Setup Redis Cluster
        run: |
          docker pull vishnunair/docker-redis-cluster:latest
          docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
      - name: Test python and go
        env:
          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
        run: make test-python-integration
      - name: Benchmark python
        env:
          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
        run: pytest --verbose --color=yes sdk/python/tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5
      - name: Upload Benchmark Artifact to S3
        run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmark

  build-all-docker-images:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        component: [ feature-server-java, feature-transformation-server ]
    env:
      MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
      REGISTRY: gcr.io/kf-feast
    steps:
      - uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
        with:
          install: true
      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - run: gcloud auth configure-docker --quiet
      - name: Build image
        run: |
          make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
      - name: Push image
        run: |
          make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}

          docker tag ${REGISTRY}/${{ matrix.component }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.component }}:develop
          docker push ${REGISTRY}/${{ matrix.component }}:develop


release

name: release

on:
  workflow_dispatch:
    inputs:
      dry_run:
        description: 'Dry Run'
        required: true
        default: true
        type: boolean
      token:
        description: 'Personal Access Token'
        required: true
        default: ""
        type: string
      publish_ui:
        description: 'Publish to NPM?'
        required: true
        default: true
        type: boolean

jobs:

  get_dry_release_versions:
    runs-on: ubuntu-latest
    env:
      GITHUB_TOKEN: ${{ github.event.inputs.token }}
    outputs:
      current_version: ${{ steps.get_versions.outputs.current_version }}
      next_version: ${{ steps.get_versions.outputs.next_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: "lts/*"
      - name: Release (Dry Run)
        id: get_versions
        run: |
          CURRENT_VERSION=$(npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run | grep "associated with version " | sed -E 's/.* version//' | sed -E 's/ on.*//')
          NEXT_VERSION=$(npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run | grep 'The next release version is' | sed -E 's/.* ([[:digit:].]+)$/\1/')
          echo ::set-output name=current_version::$CURRENT_VERSION
          echo ::set-output name=next_version::$NEXT_VERSION
          echo "Current version is ${CURRENT_VERSION}"
          echo "Next version is ${NEXT_VERSION}"

  validate_version_bumps:
    if: github.repository == 'feast-dev/feast'
    needs: get_dry_release_versions
    runs-on: ubuntu-latest
    env:
      # This publish is working using an NPM automation token to bypass 2FA
      NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
      HELM_VERSION: v3.8.0
      CURRENT_VERSION: ${{ needs.get_dry_release_versions.outputs.current_version }}
      NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: "lts/*"
      - name: Bump file versions
        run: python ./infra/scripts/release/bump_file_versions.py ${CURRENT_VERSION} ${NEXT_VERSION}
      - name: Install yarn dependencies
        working-directory: ./ui
        run: yarn install
      - name: Build yarn rollup
        working-directory: ./ui
        run: yarn build:lib
      - name: Bundle UI in SDK
        run: make build-ui
      - name: Remove previous Helm
        run: sudo rm -rf $(which helm)
      - name: Set up Homebrew
        uses: Homebrew/actions/setup-homebrew@master
      - name: Setup Helm-docs
        run: |
          brew install norwoodj/tap/helm-docs
      - name: Generate helm chart READMEs
        run: make build-helm-docs
      - name: Install Helm
        run: ./infra/scripts/helm/install-helm.sh
      - name: Validate Helm chart prior to publishing
        run: ./infra/scripts/helm/validate-helm-chart-publish.sh
      - name: Validate all version consistency
        run: ./infra/scripts/helm/validate-helm-chart-versions.sh $NEXT_VERSION

  publish-web-ui-npm:
    needs: [validate_version_bumps, get_dry_release_versions]
    runs-on: ubuntu-latest
    env:
      # This publish is working using an NPM automation token to bypass 2FA
      NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
      CURRENT_VERSION: ${{ needs.get_dry_release_versions.outputs.current_version }}
      NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: "lts/*"
      - name: Bump file versions (temporarily for Web UI publish)
        run: python ./infra/scripts/release/bump_file_versions.py ${CURRENT_VERSION} ${NEXT_VERSION}
      - name: Install yarn dependencies
        working-directory: ./ui
        run: yarn install
      - name: Build yarn rollup
        working-directory: ./ui
        run: yarn build:lib
      - name: Publish UI package
        working-directory: ./ui
        if: github.event.inputs.dry_run == 'false' && github.event.inputs.publish_ui == 'true'
        run: npm publish
        env:
          # This publish is working using an NPM automation token to bypass 2FA
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

  release:
    name: release
    runs-on: ubuntu-latest
    needs: publish-web-ui-npm
    env:
      GITHUB_TOKEN: ${{ github.event.inputs.token }}
      GIT_AUTHOR_NAME: feast-ci-bot
      GIT_AUTHOR_EMAIL: feast-ci-bot@willem.co
      GIT_COMMITTER_NAME: feast-ci-bot
      GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co
    steps:
    - name: Checkout
      uses: actions/checkout@v4
      with:
        persist-credentials: false
    - name: Setup Node.js
      uses: actions/setup-node@v3
      with:
        node-version: "lts/*"
    - name: Set up Homebrew
      id: set-up-homebrew
      uses: Homebrew/actions/setup-homebrew@master
    - name: Setup Helm-docs
      run: |
        brew install norwoodj/tap/helm-docs
    - name: Release (Dry Run)
      if: github.event.inputs.dry_run == 'true'
      run: |
        npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run
    - name: Release
      if: github.event.inputs.dry_run == 'false'
      run: |
        npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release


linter

name: linter

on: [push, pull_request]

jobs:
  lint-python:
    runs-on: [ubuntu-latest]
    env:
      PYTHON: 3.11
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          architecture: x64
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Install dependencies
        run: |
          make install-python-ci-dependencies-uv
      - name: Lint python
        run: make lint-python


pr-integration-tests

name: pr-integration-tests

on:
  pull_request_target:
    types:
      - opened
      - synchronize
      - labeled

# concurrency is currently broken, see details https://github.com/actions/runner/issues/1532
#concurrency:
#  group: pr-integration-tests-${{ github.event.pull_request.number }}
#  cancel-in-progress: true

jobs:
  integration-test-python:
    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
    if:
      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
      github.repository == 'feast-dev/feast'
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: [ "3.11" ]
        os: [ ubuntu-latest ]
    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python-version }}
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4
        with:
          # pull_request_target runs the workflow in the context of the base repo
          # as such actions/checkout needs to be explicit configured to retrieve
          # code from the PR.
          ref: refs/pull/${{ github.event.pull_request.number }}/merge
          submodules: recursive
      - name: Setup Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - name: Set up AWS SDK
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Use AWS CLI
        run: aws sts get-caller-identity
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Get uv cache dir
        id: uv-cache
        run: |
          echo "::set-output name=dir::$(uv cache dir)"
      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.uv-cache.outputs.dir }}
          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
      - name: Install dependencies
        run: make install-python-ci-dependencies-uv
      - name: Setup Redis Cluster
        run: |
          docker pull vishnunair/docker-redis-cluster:latest
          docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
      - name: Test python
        if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
        env:
          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
        run: make test-python-integration


java-integration-tests-and-build

name: java-integration-tests-and-build

on:
  push:
    branches:
      - master
    tags:
      - 'v*.*.*'

jobs:
  build-docker-images:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        component: [feature-server-java]
    env:
      MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
      REGISTRY: gcr.io/kf-feast
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'true'
      - name: Setup Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: "3.11"
          architecture: x64
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - run: gcloud auth configure-docker --quiet
      - name: Get version
        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
      - name: Build image
        run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
      - name: Push image
        run: make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
      - name: Push development Docker image
        run: |
          if [ ${GITHUB_REF#refs/*/} == "master" ]; then
            docker tag ${REGISTRY}/${{ matrix.component }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.component }}:develop
            docker push ${REGISTRY}/${{ matrix.component }}:develop
          fi

  lint-java:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'true'
      - name: Lint java
        run: make lint-java

  unit-test-java:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'true'
      - name: Set up JDK 11
        uses: actions/setup-java@v1
        with:
          java-version: '11'
          java-package: jdk
          architecture: x64
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-it-maven-
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-ut-maven-
      - name: Test java
        run: make test-java-with-coverage
      - uses: actions/upload-artifact@v3
        with:
          name: java-coverage-report
          path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/

  integration-test:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    env:
      PYTHON: 3.11
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'true'
      - name: Set up JDK 11
        uses: actions/setup-java@v1
        with:
          java-version: '11'
          java-package: jdk
          architecture: x64
      - name: Setup Python (to call feast apply)
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: 3.11
          architecture: x64
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Get uv cache dir
        id: uv-cache
        run: |
          echo "::set-output name=dir::$(uv cache dir)"
      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.uv-cache.outputs.dir }}
          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}

      - name: Install Python dependencies
        run: make install-python-ci-dependencies-uv
      - uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-it-maven-
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - name: Set up AWS SDK
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Use AWS CLI
        run: aws sts get-caller-identity
      - name: Run integration tests
        run:  make test-java-integration


name: java_pr

on:
  pull_request_target:
    types:
      - opened
      - synchronize
      - labeled

jobs:
  lint-java:
    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
    if:
      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
      github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          # pull_request_target runs the workflow in the context of the base repo
          # as such actions/checkout needs to be explicit configured to retrieve
          # code from the PR.
          ref: refs/pull/${{ github.event.pull_request.number }}/merge
          submodules: recursive
      - name: Lint java
        run: make lint-java

  unit-test-java:
    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
    if:
      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
      github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    needs: lint-java
    steps:
      - uses: actions/checkout@v4
        with:
          # pull_request_target runs the workflow in the context of the base repo
          # as such actions/checkout needs to be explicit configured to retrieve
          # code from the PR.
          ref: refs/pull/${{ github.event.pull_request.number }}/merge
          submodules: recursive
      - name: Set up JDK 11
        uses: actions/setup-java@v1
        with:
          java-version: '11'
          java-package: jdk
          architecture: x64
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-it-maven-
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-ut-maven-
      - name: Test java
        run: make test-java-with-coverage
      - uses: actions/upload-artifact@v3
        with:
          name: java-coverage-report
          path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/

  build-docker-image-java:
    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
    if:
      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
      github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        component: [ feature-server-java ]
    env:
      MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
      REGISTRY: gcr.io/kf-feast
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'true'
      - name: Setup Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: "3.11"
          architecture: x64
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - run: gcloud auth configure-docker --quiet
      - name: Build image
        run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}

  integration-test-java-pr:
    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
    if:
      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
      github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    needs: unit-test-java
    env:
      PYTHON: 3.11
    steps:
      - uses: actions/checkout@v4
        with:
          # pull_request_target runs the workflow in the context of the base repo
          # as such actions/checkout needs to be explicit configured to retrieve
          # code from the PR.
          ref: refs/pull/${{ github.event.pull_request.number }}/merge
          submodules: recursive
      - name: Set up JDK 11
        uses: actions/setup-java@v1
        with:
          java-version: '11'
          java-package: jdk
          architecture: x64
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          architecture: 'x64'
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-it-maven-
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - name: Set up AWS SDK
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Use AWS CLI
        run: aws sts get-caller-identity
      - name: Setup Python (to call feast apply)
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: 3.11
          architecture: x64
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Get uv cache dir
        id: uv-cache
        run: |
          echo "::set-output name=dir::$(uv cache dir)"
      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.uv-cache.outputs.dir }}
          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
      - name: Install dependencies
        run: make install-python-ci-dependencies-uv
      - name: Run integration tests
        run:  make test-java-integration
      - name: Save report
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: it-report
          path: spark/ingestion/target/test-reports/TestSuite.txt
          retention-days: 5


lint-pr

name: lint-pr

on:
  pull_request_target:
    types:
      - opened
      - edited
      - synchronize

permissions:
  # read-only perms specified due to use of pull_request_target in lieu of security label check
  pull-requests: read

jobs:
  validate-title:
    if:
      github.repository == 'feast-dev/feast'
    name: Validate PR title
    runs-on: ubuntu-latest
    steps:
      - uses: amannn/action-semantic-pull-request@v4
        with:
          # Must use uppercase
          subjectPattern: ^(?=[A-Z]).+$
          subjectPatternError: |
            The subject "{subject}" found in the pull request title "{title}"
            didn't match the configured pattern. Please ensure that the subject
            starts with an uppercase character.
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


publish

name: publish

on:
  push:
    tags:
      - 'v*.*.*'

jobs:
  get-version:
    if: github.repository == 'feast-dev/feast'
    runs-on:  ubuntu-latest
    outputs:
      release_version: ${{ steps.get_release_version.outputs.release_version }}
      version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
      highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
    steps:
      - uses: actions/checkout@v4
      - name: Get release version
        id: get_release_version
        run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
      - name: Get release version without prefix
        id: get_release_version_without_prefix
        env:
          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
        run: |
          echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
      - name: Get highest semver
        id: get_highest_semver
        env:
          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
        run: |
          source infra/scripts/setup-common-functions.sh
          SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
          if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
            echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
          fi
      - name: Check output
        env:
          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
          VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
          HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
        run: |
          echo $RELEASE_VERSION
          echo $VERSION_WITHOUT_PREFIX
          echo $HIGHEST_SEMVER_TAG

  build-publish-docker-images:
    runs-on: ubuntu-latest
    needs: [get-version, publish-python-sdk]
    strategy:
      matrix:
        component: [feature-server, feature-server-java, feature-transformation-server, feast-operator]
    env:
      MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
      REGISTRY: feastdev
    steps:
      - uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - run: gcloud auth configure-docker --quiet
      - name: Build image
        run: |
          make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX}
        env:
          RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
          VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
          HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
      - name: Push versioned images
        env:
          RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
          VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
          HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
        run: |
          make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX}

          echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"
          if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ]
          then
            docker tag feastdev/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/${{ matrix.component }}:latest
            docker push feastdev/${{ matrix.component }}:latest
          fi

  publish-helm-charts:
    if: github.repository == 'feast-dev/feast'
    runs-on:  ubuntu-latest
    needs: get-version
    env:
      HELM_VERSION: v3.8.0
      VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
    steps:
      - uses: actions/checkout@v4
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - run: gcloud auth configure-docker --quiet
      - name: Remove previous Helm
        run: sudo rm -rf $(which helm)
      - name: Install Helm
        run: ./infra/scripts/helm/install-helm.sh
      - name: Validate Helm chart prior to publishing
        run: ./infra/scripts/helm/validate-helm-chart-publish.sh
      - name: Validate all version consistency
        run: ./infra/scripts/helm/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX
      - name: Publish Helm charts
        run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX

  build_wheels:
    uses: ./.github/workflows/build_wheels.yml

  publish-python-sdk:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    needs: [build_wheels]
    steps:
      - uses: actions/download-artifact@v2
        with:
          name: wheels
          path: dist
      - uses: pypa/gh-action-pypi-publish@v1.4.2
        with:
          user: __token__
          password: ${{ secrets.PYPI_PASSWORD }}

  publish-java-sdk:
    if: github.repository == 'feast-dev/feast'
    container: maven:3.6-jdk-11
    runs-on: ubuntu-latest
    needs: get-version
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'true'
      - name: Set up JDK 11
        uses: actions/setup-java@v1
        with:
          java-version: '11'
          java-package: jdk
          architecture: x64
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          architecture: 'x64'
      - uses: actions/cache@v2
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-it-maven-
      - name: Publish java sdk
        env:
          VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
          GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }}
          GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
          MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }}
        run: |
          echo -n "$GPG_PUBLIC_KEY" > /root/public-key
          echo -n "$GPG_PRIVATE_KEY" > /root/private-key
          mkdir -p /root/.m2/
          echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml
          infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root


name: build_wheels

# Call this workflow from other workflows in the repository by specifying "uses: ./.github/workflows/build_wheels.yml"
# Developers who are starting a new release should use this workflow to ensure wheels will be built correctly.
# Devs should check out their fork, add a tag to the last master commit on their fork, and run the release off of their fork on the added tag to ensure wheels will be built correctly.
on:
  workflow_dispatch:
      tags:
        - 'v*.*.*'
  workflow_call:

jobs:
  get-version:
    runs-on:  ubuntu-latest
    outputs:
      release_version: ${{ steps.get_release_version.outputs.release_version }}
      version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
      highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: Get release version
        id: get_release_version
        run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
      - name: Get release version without prefix
        id: get_release_version_without_prefix
        env:
          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
        run: |
          echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
      - name: Get highest semver
        id: get_highest_semver
        env:
          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
        run: |
          source infra/scripts/setup-common-functions.sh
          SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
          if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
            echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
          fi
      - name: Check output
        id: check_output
        env:
          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
          VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
          HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
        run: |
          echo $RELEASE_VERSION
          echo $VERSION_WITHOUT_PREFIX
          echo $HIGHEST_SEMVER_TAG

  build-python-wheel:
    name: Build wheels
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          architecture: x64
      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: '17.x'
          registry-url: 'https://registry.npmjs.org'
      - name: Build UI
        run: make build-ui
      - name: Build wheels
        run: |
          python -m pip install build
          python -m build --wheel --outdir wheelhouse/
      - uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: ./wheelhouse/*.whl

  build-source-distribution:
    name: Build source distribution
    runs-on: macos-13
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          architecture: x64
      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: '17.x'
          registry-url: 'https://registry.npmjs.org'
      - name: Build and install dependencies
        # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum.
        run: |
          pip install -U pip setuptools wheel twine
          make install-protoc-dependencies
          make build-ui
          git status
          git restore go.mod go.sum
          git restore sdk/python/feast/ui/yarn.lock
      - name: Build
        run: |
          python3 setup.py sdist
      - uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: dist/*

  # We add this step so the docker images can be built as part of the pre-release verification steps.
  build-docker-images:
    runs-on: ubuntu-latest
    needs: get-version
    strategy:
      matrix:
        component: [feature-server, feature-server-java, feature-transformation-server]
    env:
      REGISTRY: feastdev
    steps:
      - uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Build image
        run: |
          make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX}
        env:
          VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}

  verify-python-wheels:
    runs-on: ${{ matrix.os }}
    needs: [build-python-wheel, build-source-distribution, get-version]
    strategy:
      matrix:
        os: [ubuntu-latest,  macos-13 ]
        python-version: ["3.9", "3.10", "3.11"]
        from-source: [ True, False ]
    env:
      # this script is for testing servers
      # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself
      TEST_SCRIPT: |
        timeout 10s $@ & pid=$!
        wait $pid
        ret=$?
        if [[ $ret -ne 124 ]]
        then
           exit $ret
        else
           echo "Succeeded!"
        fi
      VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
    steps:
      - name: Setup Python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64
      - uses: actions/download-artifact@v2
        with:
          name: wheels
          path: dist
      - name: Install OS X dependencies
        if: matrix.os == 'macos-13'
        run: brew install coreutils
      - name: Install wheel
        if: ${{ !matrix.from-source }}
        # try to install all wheels; only the current platform wheel should be actually installed
        run: |
          cd dist/
          pip install wheel
          for f in *.whl; do pip install $f || true; done
      - name: Install sdist
        # try to install the sdist
        if: ${{ matrix.from-source }}
        run: pip install dist/*tar.gz
      # Validate that the feast version installed is not development and is the correct version of the tag we ran it off of.
      - name: Validate Feast Version
        run: |
          VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+'
          OUTPUT_REGEX='^Feast SDK Version: "$VERSION_REGEX"$'
          VERSION_OUTPUT=$(feast version)
          VERSION=$(echo $VERSION_OUTPUT | grep -oE "$VERSION_REGEX")
          OUTPUT=$(echo $VERSION_OUTPUT | grep -E "$REGEX")
          if  [ -n "$OUTPUT" ] && [ "$VERSION" = "$VERSION_WITHOUT_PREFIX" ]; then
            echo "Correct Feast Version Installed"
          else
            echo "$VERSION_OUTPUT from installed wheel is not in the correct format or doesn't have the right version $VERSION."
            exit 1
          fi
      # This is temporarily disabled.
      # - name: Smoke test
      #   run: |
      #     feast init test_repo
      #     cd test_repo/feature_repo
      #     feast apply
      #     echo "$TEST_SCRIPT" > run-and-wait.sh
      #     bash run-and-wait.sh feast serve
      #     bash run-and-wait.sh feast ui


pr-local-integration-tests

name: pr-local-integration-tests
# This runs local tests with containerized stubs of online stores. This is the main dev workflow

on:
  pull_request_target:
    types:
      - opened
      - synchronize
      - labeled

jobs:
  integration-test-python-local:
    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
    if:
      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
      github.repository == 'feast-dev/feast'
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: [ "3.11" ]
        os: [ ubuntu-latest ]
    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python-version }}
    steps:
      - uses: actions/checkout@v4
        with:
          # pull_request_target runs the workflow in the context of the base repo
          # as such actions/checkout needs to be explicit configured to retrieve
          # code from the PR.
          ref: refs/pull/${{ github.event.pull_request.number }}/merge
          submodules: recursive
      - name: Setup Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Get uv cache dir
        id: uv-cache
        run: |
          echo "::set-output name=dir::$(uv cache dir)"
      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.uv-cache.outputs.dir }}
          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
      - name: Install dependencies
        run: make install-python-ci-dependencies-uv
      - name: Test local integration tests
        if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
        run: make test-python-integration-local


nightly-ci

name: nightly-ci

on:
  schedule:
    - cron: '00 08 * * *'  # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST.

# concurrency is currently broken, see details https://github.com/actions/runner/issues/1532
#concurrency:
#  group: pr-integration-tests-${{ github.event.pull_request.number }}
#  cancel-in-progress: true

jobs:
  check_date:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    name: Check latest commit
    outputs:
      WAS_EDITED: ${{ steps.check_date.outputs.WAS_EDITED }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: master
      - id: check_date
        name: Check if there were commits in the last day
        if: ${{ github.event_name == 'schedule' }}
        run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false')
  cleanup_dynamo_tables:
    if: github.repository == 'feast-dev/feast'
    runs-on: ubuntu-latest
    name: Cleanup Bigtable / Dynamo tables which can fail to cleanup
    steps:
      - uses: actions/checkout@v4
        with:
          ref: master
      - name: Setup Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: "3.11"
          architecture: x64
      - name: Set up AWS SDK
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Install Python dependencies
        run: |
          pip install boto3
          pip install google-cloud-bigtable
          pip install tqdm
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - name: Run DynamoDB / Bigtable cleanup script
        run: python infra/scripts/cleanup_ci.py
  integration-test-python:
    if: github.repository == 'feast-dev/feast'
    needs: [check_date, cleanup_dynamo_tables]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: [ "3.11" ]
        os: [ ubuntu-latest ]
    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python-version }}
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4
        with:
          ref: master
          submodules: recursive
      - name: Setup Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64
      - name: Setup Go
        id: setup-go
        uses: actions/setup-go@v2
        with:
          go-version: 1.18.0
      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'
      - name: Set up gcloud SDK
        uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
      - name: Use gcloud CLI
        run: gcloud info
      - name: Set up AWS SDK
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Use AWS CLI
        run: aws sts get-caller-identity
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Get uv cache dir
        id: uv-cache
        run: |
          echo "::set-output name=dir::$(uv cache dir)"
      - name: uv cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.uv-cache.outputs.dir }}
          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
      - name: Install apache-arrow on ubuntu
        if: matrix.os == 'ubuntu-latest'
        run: |
            sudo apt update
            sudo apt install -y -V ca-certificates lsb-release wget
            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
            sudo apt update
            sudo apt install -y -V libarrow-dev
      - name: Install apache-arrow on macos
        if: matrix.os == 'macos-13'
        run: brew install apache-arrow
      - name: Install dependencies
        run: make install-python-ci-dependencies-uv
      - name: Setup Redis Cluster
        run: |
          docker pull vishnunair/docker-redis-cluster:latest
          docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
      - name: Test python
        if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
        env:
          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
        run: make test-python-integration


Helm Chart Release

name: Helm Chart Release

on:
  release:
    types: [published]
  workflow_dispatch:

jobs:
  chart-release:
    name: Create chart release
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Configure Git
        run: |
          git config user.name "$GITHUB_ACTOR"
          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
      - name: Install Helm
        uses: azure/setup-helm@v4
      - name: Add bitnami repository
        run: helm repo add bitnami https://charts.bitnami.com/bitnami
      - name: Run chart-releaser
        uses: ./.github/actions/chart_releaser
        with:
          config: .github/actions/chart_releaser/cr.yaml
        env:
          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"


Controller Unit Tests

name: Controller Unit Tests

on:
  push:
    branches:
      - "main"
  pull_request:
    branches:
      - "main"

jobs:
  build:
    name: Build, Lint, Unit Test
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        working-directory: ./oidc-controller
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements-dev.txt
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
      - name: Lint with flake8
        working-directory: ./oidc-controller
        run: |
          flake8 . --count --show-source --statistics
      - name: Test with pytest
        working-directory: ./oidc-controller
        run: |
          pytest --log-cli-level=INFO --cov --cov-report lcov
      - name: Coveralls Parallel
        uses: coverallsapp/github-action@v2
        with:
          flag-name: python-${{ matrix.python-version }}
          parallel: true
  code-coverage:
    name: Generate Code Coverage
    needs: build
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - name: Coveralls Finished
        uses: coverallsapp/github-action@v2
        with:
          parallel-finished: true
          carryforward: "python-3.11"


Build & Deploy Development

name: Build & Deploy Development
on:
  workflow_dispatch:
  push:
    branches:
      - main
jobs:
  build:
    name: "Build VC-AuthN"
    if: github.repository_owner == 'bcgov'
    uses: ./.github/workflows/publish.yml
    with:
      tag: "dev"
      ref: "main"
      platforms: "linux/amd64"

  # Build vc-authn
  deploy:
    name: Deploy VC-AuthN to Dev
    environment: dev
    runs-on: ubuntu-latest
    needs:
      - build
    if: ${{ always() && !(contains(needs.*.result, 'failed')) && (github.repository_owner == 'bcgov') }}

    steps:
      - uses: actions/checkout@v4

      - name: Authenticate and set context
        uses: redhat-actions/oc-login@v1
        with:
          openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }}
          openshift_token: ${{ secrets.OPENSHIFT_TOKEN }}
          namespace: ${{ secrets.OPENSHIFT_NAMESPACE }}

      - name: VC-Authn Dev Helm
        run: |
          helm upgrade --install vc-authn-oidc \
          -f https://raw.githubusercontent.com/bcgov/trust-over-ip-configurations/main/helm-values/vc-authn-oidc/dev.yaml \
          --set image.tag=${{ needs.build.outputs.image_version }} \
          ./charts/vc-authn-oidc --wait
      - name: Restart Deployments
        run: |
          oc rollout restart deployment/vc-authn-oidc-agent
          oc rollout restart deployment/vc-authn-oidc


Publish VC-AuthN Image

name: Publish VC-AuthN Image
run-name: Publish VC-AuthN ${{ inputs.tag || github.event.release.tag_name }} Image
on:
  release:
    types: [published]
  workflow_call:
    inputs:
      tag:
        description: "Image tag"
        required: true
        type: string
      platforms:
        description: "Platforms - Comma separated list of the platforms to support."
        required: true
        default: linux/amd64
        type: string
      ref:
        description: "Optional - The branch, tag or SHA to checkout."
        required: false
        type: string
    outputs:
      image_tag:
        description: "The tag used for this image"
        value: ${{ jobs.publish_image.outputs.image_tag }}
      image_version:
        description: "The tag used for this image"
        value: ${{ jobs.publish_image.outputs.image_version }}

  workflow_dispatch:
    inputs:
      tag:
        description: "Image tag"
        required: true
        type: string
      platforms:
        description: "Platforms - Comma separated list of the platforms to support."
        required: true
        default: linux/amd64
        type: string
      ref:
        description: "Optional - The branch, tag or SHA to checkout."
        required: false
        type: string

env:
  PLATFORMS: ${{ inputs.platforms || 'linux/amd64,linux/arm64' }}

jobs:
  publish_image:
    if: github.repository_owner == 'bcgov'
    strategy:
      fail-fast: false

    name: Publish VC-AuthN Image
    runs-on: ubuntu-latest
    outputs:
      image_tag: ${{ steps.values.outputs.image_tag }}
      image_version: ${{ steps.values.outputs.image_version }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || '' }}

      - name: Gather image info
        id: info
        run: |
          echo "repo-owner=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_OUTPUT

      - name: Cache Docker layers
        uses: actions/cache@v4
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Log in to the GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Setup Image Metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            ghcr.io/${{ steps.info.outputs.repo-owner }}/vc-authn-oidc
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=ref,event=pr
            type=sha

      - name: Build and Push Image to ghcr.io
        uses: docker/build-push-action@v6
        with:
          push: true
          context: .
          file: docker/oidc-controller/Dockerfile
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          target: main
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
          platforms: ${{ env.PLATFORMS }}

      - name: Get Build Info
        id: values
        shell: bash
        run: |
          echo "image_tag=${{ fromJSON(steps.meta.outputs.json).tags[0] }}" >> $GITHUB_OUTPUT
          echo "image_version=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}" >> $GITHUB_OUTPUT

      # Temp fix
      # https://github.com/docker/build-push-action/issues/252
      # https://github.com/moby/buildkit/issues/1896
      - name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache


Github Actions

name: Github Actions

on: ["push", "pull_request"]

jobs:
  test:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
        numpy-version: [""]
        architecture: [x86, x64]
        os:
          [
            ubuntu-latest,
            ubuntu-20.04,
            macos-13,
            macos-12,
            windows-latest,
            windows-2019,
          ]
        exclude:
          - os: ubuntu-latest
            architecture: x86
          - os: ubuntu-20.04
            architecture: x86
          - os: macos-13
            architecture: x86
          - os: macos-12
            architecture: x86
        include:
          - python-version: "3.12"
            numpy-version: ">=2.0.0rc1"
            architecture: x64
            os: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          architecture: ${{ matrix.architecture }}

      - name: Install
        run: |
          pip install .

      - name: Test with pytest
        run: |
          pip install pytest "numpy${{ matrix.numpy-version }}"
          pytest --pyargs bottleneck

  check:
    needs: test
    runs-on: ubuntu-latest
    steps:
      - name: Placeholder for CI checks in PRs
        run: echo "Done"

  build_wheels:
    needs: test
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [macos-latest, windows-latest, ubuntu-latest]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Build wheels
        uses: pypa/cibuildwheel@v2.16.5
        env:
          CIBW_SKIP: pp* cp36-*

      - name: Store wheel artifacts
        uses: actions/upload-artifact@v4
        with:
          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
          path: ./wheelhouse/*.whl

  build_sdist:
    needs: test
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Build sdist
        run: pipx run build --sdist

      - uses: actions/upload-artifact@v4
        with:
          path: dist/*.tar.gz

  release:
    needs: [build_wheels, build_sdist]
    # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
    if: github.event_name == 'push'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/download-artifact@v4
        with:
          path: dist
          merge-multiple: true

      - uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN}}


CI

name: CI

on: [push, pull_request]

permissions:
  contents: read

jobs:

  sdist:
    name: Create sdist
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: true
      - uses: actions/setup-python@v4
        name: Install Python
        with:
          python-version: 3.x
      - name: Ubuntu library headers
        run:  |
          sudo apt update
          # Keep in sync with "Prerequisites" in User's Guide.
          sudo apt install libblosc-dev libbz2-dev libhdf5-dev liblz4-dev liblzo2-dev libsnappy-dev libzstd-dev zlib1g-dev
      - name: Source distribution
        run: |
          pip install build
          python -m build --sdist
      - name: Save source distribution
        uses: actions/upload-artifact@v3
        with:
          name: tables-sdist
          path: dist/*.tar.gz

  build:
    name: ${{ matrix.os }} ${{ matrix.python }} ${{ matrix.numpy }} ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    needs: sdist
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        # Keep in sync with ``project.classifiers`` in ``pyproject.toml``.
        python: ['3.9', '3.10', '3.11', '3.12']
        numpy: ['numpy']
        include:
          - os: ubuntu-latest
            python: '3.12'
            numpy: 'numpy>=2.dev0'

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        submodules: true
    - name: Set up Python ${{ matrix.python }}
      uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: ${{ matrix.python }}
        auto-update-conda: true
        channels: conda-forge
        channel-priority: strict
    - name: Install dependencies
      run: |
        # Keep in sync with
        # "Prerequisites" in User's Guide,
        # ``build-system.requires`` in ``pyproject.toml``.
        conda install -q setuptools pip wheel build packaging numpy cython bzip2 hdf5 lzo
        python -m pip install --require-hashes -r requirements.txt
        python -m pip install --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple '${{ matrix.numpy }}'
        # Keep in sync with ``project.optional-dependencies.doc`` in ``pyproject.toml``.
        # conda install sphinx sphinx_rtd_theme numpydoc ipython
    - name: Get source distribution
      uses: actions/download-artifact@v3
      with:
        name: tables-sdist
    - name: Installation
      run: |
        python -m pip install -v tables-*.tar.gz
    - name: 'Run test'
      run: |
        cd .. && python -m tables.tests.test_all -v
        pt2to3 -h
        ptrepack -h
        ptdump -h
        pttree -h

  build_cblosc:
    name: Sdist with cblosc2
    runs-on: ubuntu-latest
    needs: sdist
    defaults:
      run:
        shell: bash -l {0}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        submodules: true
    - name: Set up Python
      uses: conda-incubator/setup-miniconda@v3
      with:
        auto-update-conda: true
        channels: conda-forge
        channel-priority: strict
    - name: Install dependencies
      run: |
        # Keep in sync with "Prerequisites" in User's Guide,
        # ``build-system.requires`` in ``pyproject.toml``.
        conda install setuptools pip wheel build packaging py-cpuinfo numpy cython numexpr bzip2 hdf5 lzo 'typing_extensions>=4.4.0' c-blosc2
    - name: Get source distrubtion
      uses: actions/download-artifact@v3
      with:
        name: tables-sdist
    - name: Installation
      run: |
        python -m pip install -v tables-*.tar.gz --no-deps
    - name: Python blosc2 wheel or conda pkg is not installed
      run: |
        ! conda list | grep -E '^(python-)?blosc2'
    - name: 'Run test'
      run: |
        cd .. && python -m tables.tests.test_all -v
        pt2to3 -h
        ptrepack -h
        ptdump -h
        pttree -h


ubuntu

name: ubuntu

on: [push, pull_request]

permissions:
  contents: read

jobs:
  build:
    # Please try to use as many packages as possible from Ubuntu APT repos,
    # and as few as possible from PyPI,
    # as the purpose here is to test against system packages.
    runs-on: ubuntu-22.04
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        submodules: true
    - name: Install APT packages
      run: |
        sudo apt-get update
        # Keep in sync with "Prerequisites" in User's Guide.
        sudo apt install libblosc-dev libbz2-dev libhdf5-dev liblz4-dev liblzo2-dev libsnappy-dev libzstd-dev zlib1g-dev
        # ``python3-build`` (which uses ``python3-venv``) is needed for ``make (s)dist``.
        # However, Ubuntu 22.04's ``python3-build`` is broken,
        # so try with ``python3-virtualenv`` instead as suggested in
        # <https://github.com/pypa/build/issues/215#issuecomment-1335841041>.
        # At any rate, the toolchain is going to come from PyPI.
        sudo apt install python3-all-dev python3-pip  # python3-virtualenv python3-build
        # Keep in sync with ``build-system.requires`` in ``pyproject.toml``.
        # Packages commented out are either missing or too old, use PyPI packages below.
        sudo apt install python3-numpy python3-packaging python3-cpuinfo  # python3-setuptools python3-wheel # cython3 blosc2
        # Keep in sync with ``project.dependencies`` in ``pyproject.toml``.
        # Packages commented out are either missing or too old, use PyPI packages below (if not already installed).
        sudo apt install python3-numexpr  # python3-numpy python3-packaging python3-cpuinfo # blosc2
        sudo apt install python3-pytest python3-pytest-doctestplus
        # Keep in sync with ``project.optional-dependencies`` in  ``pyproject.toml``.
        sudo apt install python3-sphinx python3-sphinx-rtd-theme python3-numpydoc python3-ipython
        sudo apt install latexmk texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-plain-generic
    - name: Install dependencies
      run: |
        # A recent build toolchain is necessary for different tasks in the makefile,
        # though it would not be used to create an Ubuntu package anyway.
        # Ubuntu's toolchain is outdated for this purpose,
        # so make sure that we use PyPI packages by using ``--upgrade``.
        # All the tools below are upgraded together as mixing with system packages
        # breaks "Source distribution" (can't remove system packages)
        # or "Test" ("build" does nothing and PyTest finds no doctests).
        python3 -m pip install --user --upgrade setuptools pip wheel build
        # Keep in sync with ``build-system.requires`` in ``pyproject.toml``.
        # Install build requirements which would also be used to create an Ubuntu package
        # but have old or missing Ubuntu APT packages.
        # Do not use ``--upgrade`` here: we want to be sure whether we use system or PyPI packages.
        python3 -m pip install --user cython
        python3 -m pip install --require-hashes --user -r requirements.txt
        python3 -c "import numpy as np; print('numpy:', np.__version__)"
        python3 -c "import blosc2; print('blosc2:', blosc2.__version__)"
    - name: Build PyTables
      run: make build
      env:
        PYTABLES_NO_EMBEDDED_LIBS: TRUE
    - name: Build HTML documentation
      run: make html
    - name: Build LaTeX documentation
      run: make latex
    - name: Source distribution
      run: make dist
    - name: Test
      run: make check


Wheels

name: Wheels

# Publish when a (published) GitHub Release is created.
on:
  push:
    branches:
      - master
      - 'releases/**'
      - 'ci/**'
    tags:
      - v*
    pull_request:
  release:
    types:
      - published

permissions:
  contents: read

jobs:
  build_sdist:
    name: Build source distribution
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          submodules: true

      - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
        name: Install Python
        with:
          python-version: 3.x

      - name: Install APT packages
        if: contains(${{ matrix.os }}, 'ubuntu')
        run: |
          sudo apt update
          # Keep in sync with "Prerequisites" in User's Guide.
          sudo apt install libbz2-dev libhdf5-serial-dev liblzo2-dev
          sudo apt install latexmk texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-plain-generic

      - name: Install dependencies
        run: |
          # Keep in sync with ``build-system.requires`` in ``pyproject.toml``.
          python -m pip install --require-hashes -r ./.github/workflows/requirements/build-requirements.txt
          python -m pip install --require-hashes -r requirements.txt
          # Keep in sync with ``project.optional-dependencies.doc`` in ``pyproject.toml``.
          python -m pip install --require-hashes -r ./.github/workflows/requirements/optional-requirements.txt

      - name: Build sdist
        run: make PYTHON=python dist

      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
        with:
          path: dist/*

  build_wheels:
    name: Build ${{ matrix.os }} wheels for ${{ matrix.arch }}
    runs-on: ${{ matrix.os }}
    env:
      # Keep in sync with "Prerequisites" in User's Guide.
      HDF5_VERSION: 1.14.3  # H5Dchunk_iter needs at least 1.14.1
      MACOSX_DEPLOYMENT_TARGET: "10.9"
      # Keep Python in sync with ``project.classifiers`` in ``pyproject.toml``.
      # Skip 3.6, 3.7 and 3.8 wheels
      CIBW_SKIP: "*-musllinux_* cp36-* cp37-* cp38-* "
      CIBW_BUILD: "cp*"
    strategy:
      matrix:
        # Please note that some architectures are not tested,
        # see "Build wheels" and ``CIBW_TEST_SKIP`` below.
        os: [ 'ubuntu-latest' ]
        arch: [ 'x86_64',  'aarch64' ]
        include:
          - os: macos-13
            arch: x86_64
          - os: macos-14
            arch: arm64

    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          submodules: true

      - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
        name: Install Python
        with:
          python-version: 3.x

      - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
        if: runner.os == 'Linux'
        name: Set up QEMU

      - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
        id: deps-cache
        with:
          path: hdf5_build
          key: ${{ runner.os }}-${{ matrix.arch }}-deps-cache-${{ hashFiles('**/get_hdf5.sh') }}-${{ env.HDF5_VERSION }}-${{ hashFiles('**/wheels.yml') }}

      - name: Build dependencies
        env:
          CFLAGS: -g0
          CIBW_ARCHS: ${{ matrix.arch }}
        if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }}
        run: |
          mkdir hdf5_build
          if [[ ${{ runner.os }} = 'Linux' ]]; then
            docker run --rm -eMACOSX_DEPLOYMENT_TARGET=${{ env.MACOSX_DEPLOYMENT_TARGET }} -e HDF5_DIR=/io/hdf5_build -e CFLAGS=-g0 -e HDF5_VERSION=${{ env.HDF5_VERSION }} -v `pwd`:/io:rw quay.io/pypa/manylinux2014_${{ matrix.arch }} /io/ci/github/get_hdf5.sh
          else
            HDF5_DIR=`pwd`/hdf5_build ci/github/get_hdf5.sh
          fi

      - name: Install cibuildwheel
        run: |
          python -m pip install --require-hashes -r ./.github/workflows/requirements/wheel-requirements.txt

      - name: Build wheels
        run: |
          python -m cibuildwheel --output-dir wheelhouse
        env:
          CFLAGS: -g0
          CIBW_BUILD_VERBOSITY_MACOS: 3
          CIBW_ARCHS: ${{ matrix.arch }}
          # Keep in sync with ``build-system.requires`` in ``pyproject.toml``.
          CIBW_BEFORE_BUILD: >
            python -m pip install -U --require-hashes -r ./.github/workflows/requirements/build-requirements.txt &&
            python -m pip install --require-hashes -r requirements.txt
          CIBW_ENVIRONMENT: DISABLE_AVX2='TRUE' CFLAGS=-g0 HDF5_DIR=/tmp/hdf5 LD_LIBRARY_PATH="/tmp/hdf5/lib:${LD_LIBRARY_PATH}" PKG_CONFIG_PATH="/tmp/hdf5/lib/pkgconfig:${PKG_CONFIG_PATH}"
          CIBW_ENVIRONMENT_MACOS: CC=/usr/bin/clang CXX=/usr/bin/clang HDF5_DIR=/tmp/hdf5 LZO_DIR=/tmp/hdf5 BZIP2_DIR=/tmp/hdf5 LD_LIBRARY_PATH="/tmp/hdf5/lib:${LD_LIBRARY_PATH}" PKG_CONFIG_PATH="/tmp/hdf5/lib/pkgconfig:${PKG_CONFIG_PATH}"
          CIBW_BEFORE_ALL_MACOS: cp -r `pwd`/hdf5_build /tmp/hdf5
          # Keep in sync with "Prerequisites" in User's Guide.
          CIBW_BEFORE_ALL_LINUX: >
            cp -r `pwd`/hdf5_build /tmp/hdf5 &&
            yum -y update &&
            yum install -y zlib-devel bzip2-devel lzo-devel
          CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
            DYLD_FALLBACK_LIBRARY_PATH=/tmp/hdf5/lib delocate-listdeps {wheel} &&
            DYLD_FALLBACK_LIBRARY_PATH=/tmp/hdf5/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}
          CIBW_TEST_COMMAND: "python -m tables.tests.test_all"
          CIBW_TEST_SKIP: "*_aarch64 *_arm64"

      - name: Copy requirements.txt
        run: |
          cp requirements.txt ./wheelhouse/

      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
        with:
          path: |
            ./wheelhouse/*.whl
            ./wheelhouse/*.txt

  build_wheels_windows:
    name: Build wheels for ${{matrix.arch}} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest]
        arch: [win_amd64]

    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          submodules: true

      - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
        name: Install Python
        with:
          python-version: 3.x

      - name: Install Miniconda
        uses: conda-incubator/setup-miniconda@3b0f2504dd76ef23b6d31f291f4913fb60ab5ff3 # v2.2.0
        with:
          channels: defaults,conda-forge
          use-only-tar-bz2: true

      - name: Install cibuildwheel
        run: |
          python -m pip install --require-hashes -r ./.github/workflows/requirements/wheel-requirements.txt

      - name: Build wheels for Windows (${{ matrix.arch }})
        run: cibuildwheel --output-dir wheelhouse
        env:
          # Keep Python in sync with ``project.classifiers`` in ``pyproject.toml``.
          CIBW_BUILD: "cp39-${{ matrix.arch }} cp310-${{ matrix.arch }} cp311-${{ matrix.arch }} cp312-${{ matrix.arch }}"
          # Keep in sync with "Prerequisites" in User's Guide.
          CIBW_BEFORE_ALL_WINDOWS: >
            conda create --yes --name=build &&
            conda activate build &&
            conda config --env --set subdir ${{ matrix.arch == 'win32' && 'win-32' || 'win-64' }} &&
            conda install --yes blosc bzip2 cython hdf5 lz4 lzo snappy zstd zlib
          CIBW_ENVIRONMENT_WINDOWS: 'CONDA_PREFIX="C:\\Miniconda\\envs\\build" PATH="$PATH;C:\\Miniconda\\envs\\build\\Library\\bin"'
          CIBW_ENVIRONMENT: "PYTABLES_NO_EMBEDDED_LIBS=true DISABLE_AVX2=true"
          CIBW_BEFORE_BUILD: >
            python -m pip install --require-hashes -r requirements.txt &&
            python -m pip install delvewheel
          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
          CIBW_TEST_COMMAND: "python -m tables.tests.test_all"

      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
        with:
          path: ./wheelhouse/*.whl

  twine_check:
    needs: [ build_sdist, build_wheels, build_wheels_windows ]
    name: Twine check
    runs-on: 'ubuntu-latest'

    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          submodules: true

      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          path: ./wheelhouse/

      - name: Install Python
        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
        with:
          python-version: '3.x'

      - name: Install twine
        run: |
          python -m pip install --require-hashes -r ./.github/workflows/requirements/wheel-requirements.txt

      - name: Check sdist and wheels
        run: |
          python -m twine check wheelhouse/artifact/tables*.tar.gz wheelhouse/artifact/*.whl


build deb packages

name: build deb packages
on: pull_request

jobs:
  build_deb_packages:
    runs-on: "ubuntu-20.04"
    container: debian:11
    steps:
      - name: Change Owner of Container Working Directory
        run: chown root:root .

      - name: Install git
        run: |
          apt-get update -q
          apt-get install -y git

      - name: Check out repository code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Install depedencies
        run: |
          echo "deb http://deb.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/backports.list
          apt-get update -q
          apt-get install -y --no-install-recommends git python3 python3-requests python3-gnupg s3cmd wget

      - name: Fetch debops-ci
        run: |
          wget https://raw.githubusercontent.com/ooni/sysadmin/master/tools/debops-ci
          chmod +x debops-ci

      - name: Build the package
        run: ./debops-ci --show-commands ci --bucket-name ooni-internal-deb
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          DEB_GPG_KEY: ${{ secrets.DEB_GPG_KEY }}


test ooniapi/oonirun

name: test ooniapi/oonirun
on: push
jobs:
  run_tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python 3.11
        uses: actions/setup-python@v4
        with:
          python-version: 3.11

      - name: Install hatch
        run: pip install hatch

      - name: Run all tests
        run: make test-cov
        working-directory: ./ooniapi/services/oonirun/

      - name: Upload coverage to codecov
        uses: codecov/codecov-action@v3
        with:
          flags: oonirun
          working-directory: ./ooniapi/services/oonirun/


test legacy/ooniapi

name: test legacy/ooniapi
on:
  push:
    paths:
      - "api/**"
      - "analysis/**"

  workflow_dispatch:
    inputs:
      debug_enabled:
        description: "Run the build with tmate debugging enabled"
        required: false
        default: false

jobs:
  mypy:
    runs-on: ubuntu-latest
    container: debian:11
    steps:
      - name: Check out repository code
        uses: actions/checkout@v2

      - name: Setup APT
        run: |
          apt-get update
          apt-get install --no-install-recommends -y ca-certificates gnupg
          echo "deb http://deb-ci.ooni.org unstable main" >> /etc/apt/sources.list
          apt-key adv --verbose --keyserver hkp://keyserver.ubuntu.com --recv-keys "B5A08F01796E7F521861B449372D1FF271F2DD50"

      - name: Install dependencies
        run: |
          apt-get update
          apt-get install --no-install-recommends -qy mypy

      - name: Run tests
        # see the mypy.ini file
        run: cd api && mypy **/*.py

  integration_test:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v2

      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
        with:
          limit-access-to-actor: true

      - name: Build docker image
        run: cd api && make build

      - name: Check apispec.json
        run: cd api && tools/check_apispec_changes

      - name: Setup database fixtures and run fastpath to populate the DB
        run: cd api && make initdb

      - name: Run all tests
        run: cd api && T="--show-capture=no -s -vv" make tests
        #run: T="--show-capture=no -s -vv --junitxml=pytest.xml" make tests

      #- name: debug docker
      #  if: always()
      #  run: docker ps -a

      # - run: find / -name pytest.xml 2> /dev/null
      #   if: success() || failure() # run even if previous step failed

      # - name: Test report
      #   uses: dorny/test-reporter@v1
      #   if: success() || failure() # run even if previous step failed
      #   with:
      #     name: Test report
      #     path: '/home/runner/work/api/api/newapi/pytest.xml'
      #     reporter: java-junit     # compatible with pytest --junitxml


test fastpath

name: test fastpath
on:
  push:
    paths:
      - 'fastpath/**'

jobs:
  test:
    runs-on: ubuntu-latest
    container: debian:11
    steps:
      - uses: actions/checkout@v2
      - name: Install deps
        run: |
          apt-get update
          apt-get -y --no-install-recommends install python3-boto3 python3-lz4 python3-psycopg2 python3-setuptools \
          python3-statsd python3-systemd python3-ujson nginx python3-pytest python3-yaml python3-gunicorn \
          python3-pytest-cov mypy python3-clickhouse-driver

      - name: Run mypy
        run: mypy --ignore-missing-imports af/fastpath/fastpath || true

      - name: Run tests
        run: |
          cd fastpath
          PYTHONPATH=. pytest-3 -v --cov=fastpath --cov-report=term \
          fastpath/tests/test_functional.py \
          fastpath/tests/test_functional_nodb.py \
          fastpath/tests/test_unit.py

      - name: Archive code coverage HTML pages
        uses: actions/upload-artifact@v2
        with:
          name: coverage
          path: fastpath/htmlcov


build docs

name: build docs
on: push

jobs:
  build_docs:
    runs-on: "ubuntu-20.04"
    steps:
      - name: Check out repository code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Build docs
        run: make docs

      - name: Get current git ref
        id: rev_parse
        run: echo "COMMIT_HASH=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - name: Checkout ooni/docs
        uses: actions/checkout@v2
        with:
          repository: "ooni/docs"
          ssh-key: ${{ secrets.OONI_DOCS_DEPLOYKEY }}
          path: "ooni-docs"

      - name: Update docs
        run: cp -R dist/docs/* ooni-docs/src/content/docs/backend/

      - name: Check for conflicting slugs
        run: |
          cat ooni-docs/src/content/docs/backend/*.md \
            | grep "^slug:" | awk -F':' '{gsub(/^ +/, "", $2); print $2}' | sort | uniq -c \
            | awk '{if ($1 > 1) { print "duplicate slug for: " $2; exit 1}}'

      - name: Print the lines of the generated docs
        run: wc -l ooni-docs/src/content/docs/backend/*

      - name: Commit changes
        # Only push the docs update when we are in master
        if: github.ref == 'refs/heads/master'
        run: |
          cd ooni-docs
          git config --global user.email "github+backend@ooni.org"
          git config --global user.name "OONI Github Actions Bot"
          git add .
          git commit -m "auto: update backend docs to ${{ steps.rev_parse.outputs.COMMIT_HASH }}" || echo "No changes to commit"
          git push origin


test ooniapi/ooniauth

name: test ooniapi/ooniauth
on: push
jobs:
  run_tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python 3.11
        uses: actions/setup-python@v4
        with:
          python-version: 3.11

      - name: Install hatch
        run: pip install hatch

      - name: Run all tests
        run: make test-cov
        working-directory: ./ooniapi/services/ooniauth/

      - name: Upload coverage to codecov
        uses: codecov/codecov-action@v3
        with:
          flags: ooniauth
          working-directory: ./ooniapi/services/ooniauth/


test ooniapi/ooniprobe

name: test ooniapi/ooniprobe
on: push
jobs:
  run_tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python 3.11
        uses: actions/setup-python@v4
        with:
          python-version: 3.11

      - name: Install hatch
        run: pip install hatch

      - name: Run all tests
        run: make test-cov
        working-directory: ./ooniapi/services/ooniprobe/

      - name: Upload coverage to codecov
        uses: codecov/codecov-action@v3
        with:
          flags: ooniprobe
          working-directory: ./ooniapi/services/ooniprobe/


test ooniapi/oonifindings

name: test ooniapi/oonifindings
on: push
jobs:
  run_tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python 3.11
        uses: actions/setup-python@v4
        with:
          python-version: 3.11

      - name: Install hatch
        run: pip install hatch

      - name: Run all tests
        run: make test-cov
        working-directory: ./ooniapi/services/oonifindings/

      - name: Upload coverage to codecov
        uses: codecov/codecov-action@v3
        with:
          flags: oonifindings
          working-directory: ./ooniapi/services/oonifindings/


Lint

name: Lint

on: [pull_request]

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
      - uses: psf/black@20.8b1


Upload Python Package

# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Upload Python Package

on:
  release:
    types: [created]

jobs:
  deploy:
    name: upload release to PyPI
    runs-on: ubuntu-latest
    permissions:
      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install setuptools wheel twine
    - name: Build and publish
      run: |
        python setup.py sdist bdist_wheel
    - name: Publish package distributions to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1


Run pytests

name: Run pytests

on:
  pull_request:
    branches: [master, dev]
  workflow_dispatch:
    inputs: null

jobs:
  pytest:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: ["3.8", "3.10"]
        os: [ubuntu-latest]

    steps:
      - uses: actions/checkout@v2

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dev dependencies
        run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi

      - name: Install test dependencies
        run: if [ -f requirements/requirements-test.txt ]; then pip install -r requirements/requirements-test.txt; fi

      - name: Install package
        run: python -m pip install .

      - name: Run pytest tests
        run: pytest tests -x -vv --remote-data


Build

name: Build

on: [push, pull_request]

jobs:
  build:
    name: Build
    runs-on: ubuntu-latest
    # Specifying a GitHub environment is optional, but strongly encouraged
    environment: release
    permissions:
      # IMPORTANT: this permission is mandatory for trusted publishing
      id-token: write
    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v4
      with:
        python-version: '3.8'
        cache: 'pip'
    - run: pip install -r requirements.txt
    - run: pip install -e .
    - run: flake8 kuyruk/
    - run: mypy kuyruk/
    - run: pytest -v --cov=kuyruk --cov-report xml tests/
    - uses: coverallsapp/github-action@v2
    - run: echo ${{github.ref_name}} > VERSION
    - name: Install pypa/build
      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
      run: >-
        python3 -m
        pip install
        build
        --user
    - name: Build a binary wheel and a source tarball
      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
      run: >-
        python3 -m
        build
        --sdist
        --wheel
        --outdir dist/
        .
    - name: Publish package
      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
      uses: pypa/gh-action-pypi-publish@release/v1


name: Build

on:
  push:
  pull_request:
  schedule:
  - cron: '0 15 * * 5'

jobs:
  ubuntu:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
    env:
      PYTHON: ${{ matrix.python-version }}
      OS: ubuntu

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        allow-prereleases: true
    - name: Install OS dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y libldap2-dev libsasl2-dev ldap-utils sasl2-bin
        sudo apt-get install -y krb5-user libsasl2-modules-gssapi-mit libkrb5-dev
        apt list --installed
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install poetry wheel
        poetry config virtualenvs.create false
        poetry install --no-root -E gevent -E tornado -E trio
        python -m pip list
    - name: Build Docker image
      run: docker build -t bonsai -f ./.ci/docker/Dockerfile .
    - name: Configure Docker container
      run: |
        mkdir /tmp/bonsai
        sudo chown 2001:2000 /tmp/bonsai
        docker run --cap-add=NET_ADMIN -v /tmp/bonsai/:/home/ldap/run/:z -d -h bonsai.test --name server bonsai
        export BONSAI_DOCKER_IP=`docker inspect --format '{{ .NetworkSettings.IPAddress }}' server`
        sudo bash -c 'echo -e "\n'$BONSAI_DOCKER_IP'        bonsai.test" >> /etc/hosts'
        cat /etc/hosts
        ping -c 2 bonsai.test
        sleep 4
    - name: Check container and LDAP tools
      run: |
        docker exec server ps aux
        ldapwhoami -Y DIGEST-MD5 -H ldap://bonsai.test -U admin -w p@ssword
        ldapsearch -H ldap://bonsai.test -b "" -s base 'objectclass=*' -x -LLL +
        ldapsearch -VV
        saslpluginviewer
    - name: Check Python sysconfig
      run: python -m sysconfig
    - name: Install package
      run: |
        printf "\n\n[options]\nzip_safe = False" >> setup.cfg
        export CFLAGS="-coverage"
        python -m pip install -v .
    - name: Run tests
      run: |
        export BONSAI_DOCKER_IP=`docker inspect --format '{{ .NetworkSettings.IPAddress }}' server`
        sed -i.bak "s/127.0.0.1/$BONSAI_DOCKER_IP/g" ./tests/test.ini
        export KRB5_CONFIG="`pwd`/.ci/krb5/krb5.conf"
        export BONSAI_INSTALL_PATH=`python -c "import bonsai; print(bonsai.__file__[:-12])"`
        echo $BONSAI_INSTALL_PATH
        python -m pytest -v --cov-config .coveragerc --cov-report= --cov=$BONSAI_INSTALL_PATH
    - name: Upload coverage
      uses: codecov/codecov-action@v4
      with:
        directory: "."
        env_vars: OS,PYTHON
        fail_ci_if_error: true
        token: ${{ secrets.CODECOV_TOKEN }}

  macos:
    runs-on: macos-13
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
    env:
      PYTHON: ${{ matrix.python-version }}
      OS: macos

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        allow-prereleases: true
    - name: Install Python dependencies
      run: |
        echo "python_final_release=$(python -c 'import sys;print(str(sys.version_info.releaselevel == "final").lower())')" >> $GITHUB_ENV
        python -m pip install --upgrade pip
        python -m pip install poetry wheel
        poetry config virtualenvs.create false
        poetry install --no-root -E gevent -E tornado -E trio
        python -m pip install --force-reinstall --no-binary gevent delocate
        python -m pip list
    - name: Install Docker
      run: bash ./.github/workflows/scripts/install_docker_mac.sh
    - name: Wait on Docker
      run: |
        bash ./.github/workflows/scripts/wait_docker_mac.sh
        sudo launchctl unload -w /System/Library/LaunchDaemons/com.apple.Kerberos.kdc.plist
    - name: Build Docker image
      run: |
        docker build -t bonsai -f ./.ci/docker/Dockerfile .
    - name: Configure Docker container
      run: |
        mkdir /tmp/bonsai
        sudo chown 2001:2000 /tmp/bonsai
        docker run --cap-add=NET_ADMIN -p 389:389 -p 636:636 -p 88:88/udp -p 749:749 -p 8000:8000 -d -h bonsai.test --name server bonsai
        sudo bash -c 'echo -e "\n127.0.0.1        bonsai.test" >> /etc/hosts'
        sudo scutil --set HostName bonsai.test
        cat /etc/hosts
        ping -c 2 bonsai.test
        sleep 5
    - name: Check container and LDAP tools
      run: |
        docker exec server ps aux
        /usr/local/opt/openldap/bin/ldapwhoami -Y DIGEST-MD5 -H ldap://bonsai.test -U admin -w p@ssword
        /usr/local/opt/openldap/bin/ldapsearch -x -H ldap://bonsai.test -b "" -s base 'objectclass=*' -LLL +
        /usr/local/opt/openldap/bin/ldapsearch -VV
    - name: Check Python sysconfig
      run: python -m sysconfig
    - name: Install package
      run: |
        printf "[build_ext]\ninclude_dirs=/usr/local/opt/openldap/include\nlibrary_dirs=/usr/local/opt/openldap/lib" > ./setup.cfg
        printf "\n\n[options]\nzip_safe = False" >> setup.cfg
        export CFLAGS="-coverage"
        python -m pip install -v .
    - name: Check linking
      run: |
        otool -L ./build/*/bonsai/*.so
    - name: Run tests
      run: |
        export KRB5_CONFIG="`pwd`/.ci/krb5/krb5.conf"
        export BONSAI_INSTALL_PATH=`python -c "import bonsai; print(bonsai.__file__[:-12])"`
        echo $BONSAI_INSTALL_PATH
        py.test -v --cov-config .coveragerc --cov-report= --cov=$BONSAI_INSTALL_PATH
    - name: Upload coverage
      uses: codecov/codecov-action@v4
      with:
        directory: "."
        env_vars: OS,PYTHON
        fail_ci_if_error: true
        token: ${{ secrets.CODECOV_TOKEN }}
    - name: Create wheel
      run: |
        rm -rf ./build
        ARCHFLAGS='-arch x86_64' _PYTHON_HOST_PLATFORM=$(python -c "import sysconfig;print(f\"{'-'.join(sysconfig.get_platform().split('-')[:-1])}-x86_64\")") python -m build -w
        delocate-wheel -v ./dist/bonsai-*.whl
      if: ${{ env.python_final_release == 'true' }}
    - name: Upload wheel
      uses: actions/upload-artifact@v4
      with:
        name: wheel-amd64-${{ matrix.python-version }}
        path: ./dist/bonsai-*.whl
        retention-days: 5
      if: ${{ env.python_final_release == 'true' }}

  macos-arm64-wheels:
    needs: macos
    runs-on: macos-14
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.10', '3.11', '3.12']
    env:
      PYTHON: ${{ matrix.python-version }}
      OS: macos
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install Python build dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install delocate build wheel
        python -m pip list
    - name: Check Python sysconfig
      run: python -m sysconfig
    - name: Create wheel
      run: |
        printf "[build_ext]\ninclude_dirs=/usr/include/sasl:/opt/homebrew/opt/openldap/include\nlibrary_dirs=/opt/homebrew/opt/openldap/lib" > ./setup.cfg
        printf "\n\n[options]\nzip_safe = False" >> setup.cfg
        ARCHFLAGS='-arch arm64' _PYTHON_HOST_PLATFORM=$(python -c "import sysconfig;print(f\"{'-'.join(sysconfig.get_platform().split('-')[:-1])}-arm64\")") python -m build -w
        delocate-wheel --require-archs arm64 -v ./dist/bonsai-*.whl
    - name: Upload wheel
      uses: actions/upload-artifact@v4
      with:
        name: wheel-arm64-${{ matrix.python-version }}
        path: ./dist/bonsai-*.whl
        retention-days: 5