CICD

name: CICD

on:
  workflow_dispatch:
    inputs:
      run-reset-deployments:
        description: "Reset deployment: Clean start"
        required: false
        default: false
        type: boolean
      run-tests:
        description: "Run tests step"
        required: false
        default: true
        type: boolean
      run-regression-tests:
        description: "Run regression tests step"
        required: false
        default: true
        type: boolean

  push:
    branches:
      - "development"
    tags:
      - "v*"
  pull_request:
    branches:
      - "development"
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review

jobs:
  build:
    if: ${{ !github.event.pull_request.draft }}
    name: Build GHCR
    permissions:
      id-token: write # This is required for requesting the JWT
      packages: write # To push to GHCR.io
    runs-on: ubuntu-latest

    concurrency:
      group: cicd-build-${{ matrix.image }}-${{ github.ref }}
      cancel-in-progress: false

    outputs:
      image_version: ${{ steps.meta.outputs.version }}

    strategy:
      matrix:
        image:
          [
            ledger-nodes,
            ledger-browser,
            tails-server,
            governance-ga-agent,
            governance-trust-registry,
            governance-multitenant-web,
            governance-ga-web,
            governance-tenant-web,
            governance-public-web,
            governance-webhooks-web,
            governance-multitenant-agent,
            governance-endorser,
            pytest,
          ]
        include:
          - image: governance-ga-agent
            context: .
            file: dockerfiles/agents/Dockerfile.agent
          - image: governance-trust-registry
            context: .
            file: dockerfiles/trustregistry/Dockerfile
          - image: governance-multitenant-web
            context: .
            file: dockerfiles/fastapi/Dockerfile
          - image: governance-ga-web
            context: .
            file: dockerfiles/fastapi/Dockerfile
          - image: governance-tenant-web
            context: .
            file: dockerfiles/fastapi/Dockerfile
          - image: governance-public-web
            context: .
            file: dockerfiles/fastapi/Dockerfile
          - image: governance-webhooks-web
            context: .
            file: dockerfiles/webhooks/Dockerfile
          - image: governance-multitenant-agent
            context: .
            file: dockerfiles/agents/Dockerfile.author.agent
          - image: ledger-browser
            context: https://github.com/bcgov/von-network.git#v1.8.0
            file: Dockerfile
          - image: ledger-nodes
            context: https://github.com/bcgov/von-network.git#v1.8.0
            file: Dockerfile
          - image: governance-endorser
            context: .
            file: dockerfiles/endorser/Dockerfile
          - image: tails-server
            context: https://github.com/bcgov/indy-tails-server.git#v1.1.0
            file: docker/Dockerfile.tails-server
          - image: pytest
            context: .
            file: dockerfiles/tests/Dockerfile

    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ github.token }}

      - name: Docker Metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
          tags: |
            type=raw,value=latest,enable=${{ github.event.repository.default_branch == github.ref_name }}
            type=sha,prefix=pr-${{ github.event.pull_request.number }}-,priority=601,enable=${{ github.event_name == 'pull_request' }}
            type=sha,prefix={{branch}}-,priority=601,enable=${{ github.event_name == 'push' && github.ref_type == 'branch' }}
            type=ref,event=branch,priority=600
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Build and push Docker images
        uses: docker/build-push-action@v5
        with:
          context: ${{ matrix.context }}
          file: ${{ matrix.file }}
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: |
            type=gha,scope=build-${{ matrix.image }}
            type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:latest
          cache-to: type=gha,mode=max,scope=build-${{ matrix.image }}

  lint:
    name: Lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.12.3
        uses: actions/setup-python@v5
        with:
          python-version: 3.12.3
          cache: pip
      - name: Install test dependencies.
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt --upgrade
      - name: Run Pylint
        run: |
          pylint app/ endorser/ shared/ trustregistry/ webhooks/ --rcfile=.pylintrc -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" --exit-zero > pylintreport.txt

  test:
    if: github.event.pull_request.draft == false
    name: Test
    needs:
      - build
    runs-on: ubuntu-latest
    outputs:
      test_success: ${{ steps.test.outputs.test_success }}
    strategy:
      matrix:
        module:
          - { test: "app/tests/ endorser trustregistry webhooks --ignore=app/tests/e2e/", id: "1" } # fast tests, ignore e2e
          - { test: "app/tests/e2e/issuer/", id: "2" } # then some individual, slower e2e tests
          - { test: "app/tests/e2e/verifier/", id: "3" }
          - { test: "app/tests/e2e/test_definitions.py app/tests/e2e/test_revocation.py", id: "4" }
          - { test: "app/tests/e2e/ \
                --ignore=app/tests/e2e/issuer/ \
                --ignore=app/tests/e2e/verifier/ \
                --ignore=app/tests/e2e/test_definitions.py \
                --ignore=app/tests/e2e/test_revocation.py", id: "5" } # all other e2e tests
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.12.3
        uses: actions/setup-python@v5
        with:
          python-version: 3.12.3
          cache: pip
      - name: Install test dependencies.
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt --upgrade
      - uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ github.token }}
      - name: Start Test Harness
        run: ./manage up-as-daemon-no-build
        shell: bash
        env:
          REGISTRY: ghcr.io/${{ github.repository_owner }}
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}
      - name: Test with pytest
        id: test
        run: |
          set +e
          pytest --durations=0 ${{ matrix.module.test }} --cov | tee test_output.txt
          EXIT_CODE=${PIPESTATUS[0]}
          set -e
          echo "Exit code: $EXIT_CODE"

          mkdir -p coverage-files
          mv .coverage coverage-files/.coverage.${{ matrix.module.id }}

          # very hacky way to get around the fact that teardown fails even if tests pass
          TEARDOWN_ERROR=false
          SINGLE_ERROR=false
          TEST_FAILURES=0

          if grep -q "ERROR at teardown" test_output.txt; then
              echo "ERROR at teardown"
              TEARDOWN_ERROR=true
          fi

          if grep -q ", 1 error in" test_output.txt; then
              echo "Only 1 error total"
              SINGLE_ERROR=true
          fi

          # Count the number of test failures
          TEST_FAILURES=$(grep -c "^FAILED" test_output.txt || true)
          echo "Number of test failures: $TEST_FAILURES"

          if [ "$TEARDOWN_ERROR" = true ] && [ "$SINGLE_ERROR" = true ] && [ "$TEST_FAILURES" -eq 0 ]; then
              echo "Tests passed with teardown error"
              exit 0
          else
              if [ "$EXIT_CODE" -ne 0 ]; then
                echo "test_success=false" >> $GITHUB_OUTPUT
              else
                echo "test_success=true" >> $GITHUB_OUTPUT
              fi
              exit $EXIT_CODE
          fi
      - name: Upload .coverage files as artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-files-${{ matrix.module.id }}
          path: coverage-files/.coverage.${{ matrix.module.id }}

  status-check:
    name: Status Check
    runs-on: ubuntu-latest
    needs: test
    if: always()
    steps:
      - name: Check if any test failed
        run: exit 1
        if: needs.test.outputs.test_success == 'false'

  combine-coverage:
    if: github.event.pull_request.draft == false
    name: Coverage
    runs-on: ubuntu-latest
    needs: status-check
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python 3.12.3
        uses: actions/setup-python@v5
        with:
          python-version: 3.12.3
      - name: Install dependencies
        run: pip install coverage
      - name: Download all .coverage artifacts
        uses: actions/download-artifact@v4
        with:
          path: coverage-files
      - name: Move coverage files to top-level directory
        run: |
          for dir in coverage-files/coverage-files-*; do
            mv "$dir"/.coverage.* .
          done
      - name: Combine coverage files
        run: |
          coverage combine
          coverage report
      - name: Generate XML coverage report
        run: coverage xml
      - name: Upload coverage to Codacy
        run: bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r coverage.xml
        env:
          CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}

  deploy:
    if: github.actor != 'dependabot[bot]' && github.event.pull_request.draft == false
    name: Deploy to EKS
    environment:
      name: dev
    needs:
      - build
    permissions:
      id-token: write # Required to authenticate with AWS
      contents: read # Required to clone this repository
      checks: write # Required for action-junit-report
      pull-requests: write # Required to comment on PRs for Pytest coverage comment
    runs-on: ubuntu-latest

    timeout-minutes: 35

    env:
      TAILSCALE_VERSION: 1.62.1
      HELMFILE_VERSION: v0.163.1
      HELM_VERSION: v3.14.3
      OUTPUT_FILE: test_output.xml
      COVERAGE_FILE: test_coverage.txt
      PYTEST_COMPLETIONS: 1

    concurrency:
      group: cicd-deploy
      cancel-in-progress: false

    outputs:
      output: ${{ steps.updated_deployments.outputs.success }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Authenticate GitHub CLI
        run: echo "${{ secrets.PAT }}" | gh auth login --with-token

      - name: Set branch name
        id: set_branch_name
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
          else
            echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | cut -d/ -f2-)" >> $GITHUB_ENV
          fi

      - name: Check if branch exists
        id: check_branch
        run: |
          DEFAULT_BRANCH=master

          if gh api repos/didx-xyz/charts/git/ref/heads/$BRANCH_NAME; then
            echo "branch_exists=true" >> $GITHUB_ENV
            echo "branch_name=$BRANCH_NAME" >> $GITHUB_ENV
          else
            echo "branch_exists=false" >> $GITHUB_ENV
            echo "branch_name=$DEFAULT_BRANCH" >> $GITHUB_ENV
          fi

      - name: Checkout Charts
        uses: actions/checkout@v4
        with:
          repository: didx-xyz/charts
          token: ${{ secrets.PAT }}
          path: charts
          ref: ${{ env.branch_name }}

      - name: Install dependencies
        run: sudo apt-get install -y postgresql-client redis-tools

      - name: Sops Binary Installer
        uses: mdgreenwald/mozilla-sops-action@v1.6.0

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: af-south-1
          role-to-assume: arn:aws:iam::402177810328:role/cicd
          role-session-name: github-cicd

      - name: Update Kubeconfig
        run: aws eks update-kubeconfig --name cloudapi-dev

      - uses: tailscale/github-action@main
        with:
          authkey: ${{ secrets.TAILSCALE_AUTHKEY }}
          version: ${{ env.TAILSCALE_VERSION }}

      # - name: Helmfile Diff
      #   if: github.event.inputs.run-reset-deployments == 'true'
      #   uses: helmfile/helmfile-action@v1.9.1
      #   with:
      #     helmfile-args: |
      #       diff \
      #         --environment ${{ vars.ENVIRONMENT }} \
      #         -f ./charts/helmfiles/aries-cloudapi-python.yaml \
      #         --set image.tag=${{ env.IMAGE_TAG }} \
      #         --set image.registry=ghcr.io/${{ github.repository_owner }}
      #     helm-plugins: |
      #       https://github.com/databus23/helm-diff,
      #       https://github.com/jkroepke/helm-secrets
      #     helmfile-version: ${{ env.HELMFILE_VERSION }}
      #     helm-version: ${{ env.HELM_VERSION }}
      #   env:
      #     IMAGE_TAG: ${{ needs.build.outputs.image_version }}

      - name: Helmfile Destroy
        id: destroy_deployments
        if: github.event.inputs.run-reset-deployments == 'true'
        uses: helmfile/helmfile-action@v1.9.1
        with:
          helmfile-args: |
            destroy \
              --environment ${{ vars.ENVIRONMENT }} \
              -f ./charts/helmfiles/aries-cloudapi-python.yaml
          helm-plugins: |
            https://github.com/databus23/helm-diff,
            https://github.com/jkroepke/helm-secrets
          helmfile-version: ${{ env.HELMFILE_VERSION }}
          helm-version: ${{ env.HELM_VERSION }}
        env:
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}

      - name: Drop DBs
        id: drop_dbs
        if: github.event.inputs.run-reset-deployments == 'true'
        env:
          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
          DB_HOST: ${{ secrets.DB_HOST }}
          DB_USER: ${{ secrets.DB_USER}}
          DB_PORT: ${{ secrets.DB_PORT }}
          DB_EXCLUDE: ${{ secrets.DB_EXCLUDE }}
          GA_ACAPY_WALLET_NAME: ${{ secrets.GA_ACAPY_WALLET_NAME }}
          MT_ACAPY_WALLET_NAME: ${{ secrets.MT_ACAPY_WALLET_NAME }}
          TRUST_REGISTRY_DB_OWNER: ${{ secrets.TRUST_REGISTRY_DB_OWNER }}
        run: |
          bash ./scripts/aurora-delete.sh -o $GA_ACAPY_WALLET_NAME -d
          bash ./scripts/aurora-delete.sh -o $MT_ACAPY_WALLET_NAME -d
          bash ./scripts/aurora-delete.sh -o $TRUST_REGISTRY_DB_OWNER -d
          bash ./scripts/aurora-delete.sh -o $TRUST_REGISTRY_DB_OWNER -c

      - name: List Elasticache Redis
        if: github.event.inputs.run-reset-deployments == 'true'
        env:
          REDIS_HOST: ${{ secrets.REDIS_HOST }}
          REDIS_PORT: ${{ secrets.REDIS_PORT }}
          REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
        run: |
          redis-cli \
            -h $REDIS_HOST \
            -p $REDIS_PORT \
            --tls --pass $REDIS_PASSWORD \
            --scan --pattern '*'

      - name: Clean Elasticache Redis
        if: github.event.inputs.run-reset-deployments == 'true'
        env:
          REDIS_HOST: ${{ secrets.REDIS_HOST }}
          REDIS_PORT: ${{ secrets.REDIS_PORT }}
          REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
        run: |
          redis-cli \
            --tls --pass $REDIS_PASSWORD \
            --cluster call --cluster-only-masters \
            $REDIS_HOST:$REDIS_PORT \
            FLUSHALL

      - name: Helmfile Apply # Apply default helmfile (without RDS proxy) when resetting deployments.
        if: github.event.inputs.run-reset-deployments == 'true'
        uses: helmfile/helmfile-action@v1.9.1
        with:
          helmfile-args: |
            apply \
              --environment ${{ vars.ENVIRONMENT }} \
              -f ./charts/helmfiles/aries-cloudapi-python.yaml \
              --state-values-set image.tag=${{ env.IMAGE_TAG }} \
              --state-values-set image.registry=ghcr.io/${{ github.repository_owner }}
          helm-plugins: |
            https://github.com/databus23/helm-diff,
            https://github.com/jkroepke/helm-secrets
          helmfile-version: ${{ env.HELMFILE_VERSION }}
          helm-version: ${{ env.HELM_VERSION }}
        env:
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}

      - name: Helmfile Apply (Exclude Agents) # Always exclude agents and apply (zero diff if resetting deployments)
        if: github.event.inputs.run-reset-deployments != 'true'
        uses: helmfile/helmfile-action@v1.9.1
        with:
          helmfile-args: |
            apply \
              --environment ${{ vars.ENVIRONMENT }} \
              -f ./charts/helmfiles/aries-cloudapi-python.yaml \
              --state-values-set image.tag=${{ env.IMAGE_TAG }} \
              --state-values-set image.registry=ghcr.io/${{ github.repository_owner }} \
              --selector app!=governance-ga-agent,app!=governance-multitenant-agent
          helm-plugins: |
            https://github.com/databus23/helm-diff,
            https://github.com/jkroepke/helm-secrets
          helmfile-version: ${{ env.HELMFILE_VERSION }}
          helm-version: ${{ env.HELM_VERSION }}
        env:
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}

      - name: Create values file
        run: |
          echo 'env:' > ./charts/helmfiles/acapy-wallet-storage-config.yaml
          echo '  ACAPY_WALLET_STORAGE_CONFIG: |-' >> ./charts/helmfiles/acapy-wallet-storage-config.yaml
          echo '    ${{ secrets.ACAPY_WALLET_STORAGE_CONFIG }}' >> ./charts/helmfiles/acapy-wallet-storage-config.yaml
          pwd
          ls -la

      - name: Helmfile Patch Proxy # Always patch agents with proxy.
        id: patch_proxy
        uses: helmfile/helmfile-action@v1.9.1
        with:
          helmfile-args: |
            apply \
              --environment ${{ vars.ENVIRONMENT }} \
              -f ./charts/helmfiles/aries-cloudapi-python.yaml \
              --state-values-set image.tag=${{ env.IMAGE_TAG }} \
              --state-values-set image.registry=ghcr.io/${{ github.repository_owner }} \
              --selector app=governance-ga-agent \
              --selector app=governance-multitenant-agent \
              --set env.WALLET_DB_HOST=${{ secrets.DB_PROXY_HOST }} \
              --values acapy-wallet-storage-config.yaml
          helm-plugins: |
            https://github.com/databus23/helm-diff,
            https://github.com/jkroepke/helm-secrets
          helmfile-version: ${{ env.HELMFILE_VERSION }}
          helm-version: ${{ env.HELM_VERSION }}
        env:
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}

      - name: Helmfile run pytest
        if: ${{ github.event_name != 'workflow_dispatch' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run-tests != 'false') }}
        id: pytest
        uses: helmfile/helmfile-action@v1.9.1
        with:
          helmfile-args: |
            apply \
              --environment ${{ vars.ENVIRONMENT }} \
              -f ./charts/helmfiles/aries-capi-test.yaml \
              --set image.tag=${{ env.IMAGE_TAG }} \
              --set image.registry=ghcr.io/${{ github.repository_owner }} \
              --set completions=${{ env.PYTEST_COMPLETIONS }} \
              --state-values-set release=cloudapi-pytest \
              --set fullnameOverride=cloudapi-pytest
          helm-plugins: |
            https://github.com/databus23/helm-diff,
            https://github.com/jkroepke/helm-secrets
          helmfile-version: ${{ env.HELMFILE_VERSION }}
          helm-version: ${{ env.HELM_VERSION }}
        env:
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}

      - name: Helmfile init regression pytest
        if: github.event.inputs.run-reset-deployments == 'true' && github.event.inputs.run-regression-tests == 'true'
        id: pytest-init-regression
        uses: helmfile/helmfile-action@v1.9.1
        with:
          helmfile-args: |
            apply \
              --environment ${{ vars.ENVIRONMENT }} \
              -f ./charts/helmfiles/aries-capi-test.yaml \
              --set image.tag=${{ env.IMAGE_TAG }} \
              --set image.registry=ghcr.io/${{ github.repository_owner }} \
              --set completions=${{ env.PYTEST_COMPLETIONS }} \
              --state-values-set release=cloudapi-pytest-regression \
              --set fullnameOverride=cloudapi-pytest-regression \
              --set env.RUN_REGRESSION_TESTS="true"
          helm-plugins: |
            https://github.com/databus23/helm-diff,
            https://github.com/jkroepke/helm-secrets
          helmfile-version: ${{ env.HELMFILE_VERSION }}
          helm-version: ${{ env.HELM_VERSION }}
        env:
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}

      - name: Helmfile run regression pytest
        if: ${{ github.event_name != 'workflow_dispatch' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run-reset-deployments == 'false' && github.event.inputs.run-regression-tests == 'true') }}
        id: pytest-run-regression
        uses: helmfile/helmfile-action@v1.9.1
        with:
          helmfile-args: |
            apply \
              --environment ${{ vars.ENVIRONMENT }} \
              -f ./charts/helmfiles/aries-capi-test.yaml \
              --set image.tag=${{ env.IMAGE_TAG }} \
              --set image.registry=ghcr.io/${{ github.repository_owner }} \
              --set completions=${{ env.PYTEST_COMPLETIONS }} \
              --state-values-set release=cloudapi-pytest-regression \
              --set fullnameOverride=cloudapi-pytest-regression \
              --set env.RUN_REGRESSION_TESTS="true" \
              --set env.FAIL_ON_RECREATING_FIXTURES="true"
          helm-plugins: |
            https://github.com/databus23/helm-diff,
            https://github.com/jkroepke/helm-secrets
          helmfile-version: ${{ env.HELMFILE_VERSION }}
          helm-version: ${{ env.HELM_VERSION }}
        env:
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}

      - name: Wait for pytest and print logs
        if: steps.pytest.outcome == 'success'
        run: |
          while true; do
            # Check if the job is complete or failed
            COMPLETION_STATUS=$(kubectl get job $JOB_NAME -n $NAMESPACE -o jsonpath='{.status.succeeded}')
            FAILURE_STATUS=$(kubectl get job $JOB_NAME -n $NAMESPACE -o jsonpath='{.status.failed}')

            if [ "$COMPLETION_STATUS" == "${{ env.PYTEST_COMPLETIONS }}" ] || [ "$FAILURE_STATUS" == "1" ]; then
                echo "Job $JOB_NAME has completed."
                break
            else
                echo "Waiting for job to complete..."
                sleep 10
            fi
          done

          # Get all pods for the job
          pods=$(kubectl get pods -n $NAMESPACE --selector=job-name=$JOB_NAME -o jsonpath='{.items[*].metadata.name}')

          # Loop through the pods and get logs
          for pod in $pods
          do
              echo "Logs for Pod: $pod"
              kubectl logs -n $NAMESPACE $pod
          done

        env:
          JOB_NAME: cloudapi-pytest
          NAMESPACE: dev-cloudapi

      - name: Wait for pytest regression and print logs
        if: steps.pytest-init-regression.outcome == 'success' || steps.pytest-run-regression.outcome == 'success'
        run: |
          while true; do
            # Check if the job is complete or failed
            COMPLETION_STATUS=$(kubectl get job $JOB_NAME -n $NAMESPACE -o jsonpath='{.status.succeeded}')
            FAILURE_STATUS=$(kubectl get job $JOB_NAME -n $NAMESPACE -o jsonpath='{.status.failed}')

            if [ "$COMPLETION_STATUS" == "${{ env.PYTEST_COMPLETIONS }}" ] || [ "$FAILURE_STATUS" == "1" ]; then
                echo "Job $JOB_NAME has completed."
                break
            else
                echo "Waiting for job to complete..."
                sleep 10
            fi
          done

          # Get all pods for the job
          pods=$(kubectl get pods -n $NAMESPACE --selector=job-name=$JOB_NAME -o jsonpath='{.items[*].metadata.name}')

          # Loop through the pods and get logs
          for pod in $pods
          do
              echo "Logs for Pod: $pod"
              kubectl logs -n $NAMESPACE $pod
          done

        env:
          JOB_NAME: cloudapi-pytest-regression
          NAMESPACE: dev-cloudapi

      - name: Copy k8s pytest results
        if: steps.pytest.outcome == 'success'
        run: |
          echo "apiVersion: v1
          kind: Pod
          metadata:
            name: $POD_NAME
            namespace: $NAMESPACE
          spec:
            containers:
            - name: $POD_NAME
              image: $CONTAINER_IMAGE
              command: [\"sleep\", \"3600\"]
              volumeMounts:
              - name: my-volume
                mountPath: $MOUNT_PATH
            volumes:
            - name: my-volume
              persistentVolumeClaim:
                claimName: $PVC_NAME
            restartPolicy: Never" > busybox-pod.yaml

          kubectl apply -f busybox-pod.yaml

          # Wait for the pod to be ready
          echo "Waiting for pod to be ready..."
          kubectl -n $NAMESPACE wait --for=condition=ready pod/$POD_NAME --timeout=60s

          # Copy the file from the pod to your local system
          echo "Copying file from pod..."
          mkdir -p $LOCAL_PATH
          kubectl -n $NAMESPACE cp $POD_NAME:$MOUNT_PATH/$OUTPUT_FILE $LOCAL_PATH/$OUTPUT_FILE
          kubectl -n $NAMESPACE cp $POD_NAME:$MOUNT_PATH/$COVERAGE_FILE $LOCAL_PATH/$COVERAGE_FILE

          # Clean up: delete the temporary pod
          echo "Cleaning up..."
          kubectl -n $NAMESPACE delete pod $POD_NAME

          echo "Done!"
        env:
          PVC_NAME: cloudapi-pytest
          POD_NAME: pytest-results
          CONTAINER_IMAGE: busybox
          MOUNT_PATH: /mnt
          LOCAL_PATH: ./pytest
          NAMESPACE: dev-cloudapi

      - name: Copy k8s pytest regression results
        if: steps.pytest-init-regression.outcome == 'success' || steps.pytest-run-regression.outcome == 'success'
        run: |
          echo "apiVersion: v1
          kind: Pod
          metadata:
            name: $POD_NAME
            namespace: $NAMESPACE
          spec:
            containers:
            - name: $POD_NAME
              image: $CONTAINER_IMAGE
              command: [\"sleep\", \"3600\"]
              volumeMounts:
              - name: my-volume
                mountPath: $MOUNT_PATH
            volumes:
            - name: my-volume
              persistentVolumeClaim:
                claimName: $PVC_NAME
            restartPolicy: Never" > busybox-pod.yaml

          kubectl apply -f busybox-pod.yaml

          # Wait for the pod to be ready
          echo "Waiting for pod to be ready..."
          kubectl -n $NAMESPACE wait --for=condition=ready pod/$POD_NAME --timeout=60s

          # Copy the file from the pod to your local system
          echo "Copying file from pod..."
          mkdir -p $LOCAL_PATH
          kubectl -n $NAMESPACE cp $POD_NAME:$MOUNT_PATH/$OUTPUT_FILE $LOCAL_PATH/$OUTPUT_FILE
          kubectl -n $NAMESPACE cp $POD_NAME:$MOUNT_PATH/$COVERAGE_FILE $LOCAL_PATH/$COVERAGE_FILE

          # Clean up: delete the temporary pod
          echo "Cleaning up..."
          kubectl -n $NAMESPACE delete pod $POD_NAME

          echo "Done!"
        env:
          PVC_NAME: cloudapi-pytest-regression
          POD_NAME: pytest-results-regression
          CONTAINER_IMAGE: busybox
          MOUNT_PATH: /mnt
          LOCAL_PATH: ./pytest-regression
          NAMESPACE: dev-cloudapi

      - name: Publish Pytest Report
        uses: mikepenz/action-junit-report@v4
        if: steps.pytest.outcome == 'success'
        with:
          check_name: JUnit Test Report
          report_paths: "./pytest/test_output.xml"
          fail_on_failure: true
          detailed_summary: true
          require_passed_tests: true

      - name: Publish Pytest Regression Report
        uses: mikepenz/action-junit-report@v4
        if: steps.pytest-init-regression.outcome == 'success' || steps.pytest-run-regression.outcome == 'success'
        with:
          check_name: JUnit Test Report Regression
          report_paths: "./pytest-regression/test_output.xml"
          fail_on_failure: true
          detailed_summary: true
          require_passed_tests: true

      - name: Pytest coverage comment
        if: steps.pytest.outcome == 'success'
        uses: MishaKav/pytest-coverage-comment@v1.1.51
        with:
          pytest-coverage-path: ./pytest/test_coverage.txt
          junitxml-path: ./pytest/test_output.xml
          create-new-comment: true
          title: "K8s Test Coverage"
          # Resolves `Warning: Your comment is too long (maximum is 65536 characters), coverage report will not be added.`
          hide-report: ${{ github.event_name != 'pull_request' }}
          hide-comment: ${{ github.event_name != 'pull_request' }}

      - name: Pytest coverage comment
        if: steps.pytest-init-regression.outcome == 'success' || steps.pytest-run-regression.outcome == 'success'
        uses: MishaKav/pytest-coverage-comment@v1.1.51
        with:
          pytest-coverage-path: ./pytest-regression/test_coverage.txt
          junitxml-path: ./pytest-regression/test_output.xml
          create-new-comment: true
          title: "K8s Regression Test Coverage"
          # Resolves `Warning: Your comment is too long (maximum is 65536 characters), coverage report will not be added.`
          hide-report: ${{ github.event_name != 'pull_request' }}
          hide-comment: ${{ github.event_name != 'pull_request' }}

      - name: Helmfile destroy pytest
        uses: helmfile/helmfile-action@v1.9.1
        if: always()
        with:
          helmfile-args: |
            destroy \
              --environment ${{ vars.ENVIRONMENT }} \
              -f ./charts/helmfiles/aries-capi-test.yaml \
              --state-values-set release=cloudapi-pytest
          helm-plugins: |
            https://github.com/databus23/helm-diff
          helmfile-version: ${{ env.HELMFILE_VERSION }}
          helm-version: ${{ env.HELM_VERSION }}
        env:
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}

      - name: Helmfile destroy pytest regression
        uses: helmfile/helmfile-action@v1.9.1
        if: always()
        with:
          helmfile-args: |
            destroy \
              --environment ${{ vars.ENVIRONMENT }} \
              -f ./charts/helmfiles/aries-capi-test.yaml \
              --state-values-set release=cloudapi-pytest-regression
          helm-plugins: |
            https://github.com/databus23/helm-diff
          helmfile-version: ${{ env.HELMFILE_VERSION }}
          helm-version: ${{ env.HELM_VERSION }}
        env:
          IMAGE_TAG: ${{ needs.build.outputs.image_version }}


Python code style check

name: Python code style check

on:
  pull_request:
    branches:
      - main
      - development

jobs:
  style:
    name: style
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.12.3
        uses: actions/setup-python@v5
        with:
          python-version: 3.12.3

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install \
            isort~=5.13.2 \
            black~=24.4.2

      - name: Check import style with isort
        run: isort . --check --profile black --diff

      - name: Check code style with Black
        run: black . --check --diff


Run Backend tests

name: Run Backend tests

on:
  pull_request:
    paths:
      - 'agenta-backend/**'
      - 'docker-compose.yml'
      - 'docker-compose.*.yml'
  workflow_dispatch:
    inputs:
      debug:
        description: 'Run Docker logs step (true/false)'
        required: true
        default: 'false'

jobs:
  pytest:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set Environment Variables
        run: |
          echo "OPENAI_API_KEY=${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}" >> $GITHUB_ENV

      - name: Install Curl
        run: sudo apt install curl -y

      - name: Start Docker Compose
        run: OPENAI_API_KEY=${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }} ENVIRONMENT=github docker-compose -f "docker-compose.test.yml" up -d --build

      - name: Restart Backend Service To Fetch Template Images
        run: docker container restart agenta-backend-test

      - name: Check Templates Exists
        run: |
          sleep 10 && curl -i http://localhost/api/containers/templates/

      - name: Wait for Backend Service
        run: |
          sleep 10 && curl -i http://localhost/api/openapi.json

      - name: Run tests
        run: sleep 10 && docker exec agenta-backend-test pytest

      - name: Docker logs
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true'
        run: docker ps -q | xargs -I {} docker logs {}


Run Frontend tests

name: Run Frontend tests

on:
  pull_request:
    paths:
      - 'agenta-web/**'
      - 'docker-compose.yml'
      - 'docker-compose.*.yml'
  workflow_dispatch:
    inputs:
      debug:
        description: 'Run Docker logs step (true/false)'
        required: true
        default: 'false'

env:
  OPENAI_API_KEY: ${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}

jobs:
  cypress:
    runs-on: ubuntu-latest

    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Set Environment Variables
        run: |
          echo "NEXT_PUBLIC_OPENAI_API_KEY=${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}" >> $GITHUB_ENV

      - name: Install Curl & Start Docker Compose
        env:
          NEXT_PUBLIC_OPENAI_API_KEY: ${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}
        run: |
          sudo apt install curl -y
          docker-compose -f "docker-compose.test.yml" up -d --build

      - name: Restart Backend Service To Fetch Template(s)
        run: docker container restart agenta-backend-test

      - name: Wait for Backend Service
        run: curl http://localhost/api/health/

      - name: Know the Response of Organization & Container Templates
        run: |
          curl http://localhost/api/organizations/own/
          curl http://localhost/api/containers/templates/

      - name: Set Node.js 18
        uses: actions/setup-node@v4
        with:
          node-version: 18

      - name: Install Frontend Dependencies
        run: |
          cd agenta-web/ && npm install

      - name: Run Cypress
        env:
          NEXT_PUBLIC_OPENAI_API_KEY: ${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}
        run: cd agenta-web/ && npm run test

      - name: Docker logs
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true'
        run: docker ps -q | xargs -I {} docker logs {}


Check Python Code - Black

name: Check Python Code - Black
on:
  pull_request:
    paths:
      - "agenta-backend/**"
      - "agenta-cli/**"
      - "examples/**"

jobs:
  linter_name:
    name: runner / black check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: psf/black@23.12.0


Check Web Code - Lint and Build

name: Check Web Code - Lint and Build

on:
  pull_request:
    paths:
      - "agenta-web/**"

jobs:
  lint_and_build:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Install dependencies
        run: npm ci --omit=dev
        working-directory: agenta-web

      - name: Run Lint
        run: npm run lint
        working-directory: agenta-web

      - name: Run Build
        run: npm run build
        working-directory: agenta-web


Check web Code - Prettier

name: Check web Code - Prettier

on:
  pull_request:
    paths:
      - "agenta-web/**"

jobs:
  prettier:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Run Prettier
        run: npx prettier --check .
        working-directory: agenta-web


Publish to PyPI

name: Publish to PyPI

on:
  push:
    branches:
      - main

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
    - name: Check out the repo
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Set working directory
      run: cd ${{ github.workspace }}

    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: 3.9

    - name: Install poetry
      run: |
        curl -sSL https://install.python-poetry.org | python3 -

    - name: Check for agenta version changes
      id: version_check
      run: |

        # # Check if there are any tags in the repository
        # if [ -z "$(git tag -l)" ]; then
        #   echo "No tags found. Skipping version check."
        #   exit 78
        # fi

        # Navigate to the cli directory
        cd ${{ github.workspace }}/agenta-cli

        # Verify that the pyproject.toml file exists
        if [ ! -f pyproject.toml ]; then
          echo "pyproject.toml file not found. Exiting..."
          exit 1
        fi

        previous_version=$(git describe --abbrev=0 --tags)
        current_version=$(poetry version --no-interaction | awk '{print $2}')
        if [ "$previous_version" == "$current_version" ]; then
          echo "No version change detected. Skipping package publication."
          exit 78
        fi
        echo "::set-output name=version_bumped::true"
      continue-on-error: true

    - name: Build and publish agenta to PyPI
      if: steps.version_check.outputs.version_bumped == 'true'
      run: |
        # Navigate to the cli directory
        cd ${{ github.workspace }}

        # Build and publish agenta-cli
        cd agenta-cli
        poetry build
        poetry publish --username __token__ --password ${{ secrets.PYPI_API_TOKEN }}
      continue-on-error: true


Push to GitHub Container Registry

name: Push to GitHub Container Registry

on:
  workflow_dispatch:
  push:
    branches:
      - main

env:
  BACKEND_IMAGE_NAME: agenta-backend
  WEB_IMAGE_NAME: agenta-web
  IMAGE_OWNER_NAME: agenta-ai
  REPOSITORY_USERNAME: ${{ github.actor }}

jobs:
  build-and-publish:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Clear Docker build cache
        run: docker builder prune --all --force

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ env.REPOSITORY_USERNAME }}
          password: ${{ secrets.DOCKER_GITHUB_SECRETS }}

      - name: Build, tag and push Backend image to Github Container Registry
        id: build-backend-image
        run: |
          cd agenta-backend
          docker build -t ghcr.io/${{ env.IMAGE_OWNER_NAME }}/${{ env.BACKEND_IMAGE_NAME }} -f Dockerfile.gh .
          docker push ghcr.io/${{ env.IMAGE_OWNER_NAME }}/${{ env.BACKEND_IMAGE_NAME }}

      - name: Build, tag and push Web image to Github Container Registry
        id: build-web-image
        run: |
          cd agenta-web
          docker build --no-cache -t ghcr.io/${{ env.IMAGE_OWNER_NAME }}/${{ env.WEB_IMAGE_NAME }} -f prod.gh.Dockerfile .
          docker push ghcr.io/${{ env.IMAGE_OWNER_NAME }}/${{ env.WEB_IMAGE_NAME }}


🆙 Bump versions

name: "🆙 Bump versions"

on:
  workflow_dispatch:
    inputs:
      bump-type:
        description: "Bump Type"
        required: true
        default: "patch"
        type: choice
        options:
          - patch
          - minor
          - major

jobs:
  migrate-db-schema-stage:
    name: Bump version
    continue-on-error: false
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ github.ref }}

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: Install Poetry
        run: |
          curl -sSL https://install.python-poetry.org | python3 -

      - name: Bump versions
        run: |
          BUMP_TYPE=${{ inputs.bump-type }}
          echo "Bumping versions to $BUMP_TYPE"
          cd agenta-web
          npm version $BUMP_TYPE
          cd ..
          cd agenta-cli
          poetry version $BUMP_TYPE
          cd ..
          cd agenta-backend
          poetry version $BUMP_TYPE

      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v6
        with:
          commit-message: Bump versions
          author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com>
          branch: bump-versions
          delete-branch: true
          title: 'Bump versions'
          body: |
            New version in
            - agenta-web
            - agenta-backend
            - agenta-cli


Publish a PRE-RELEASE PyPI

name: Publish a PRE-RELEASE PyPI

on:
  workflow_dispatch:
    inputs:
      branch:
        description: 'Branch to publish'
        required: true
        default: 'main'

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
    - name: Check out the repo
      uses: actions/checkout@v4
      with:
        ref: ${{ github.event.inputs.branch }}
        fetch-depth: 0

    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: 3.9

    - name: Install poetry
      run: |
        curl -sSL https://install.python-poetry.org | python3 -

    - name: Check for agenta version changes
      id: version_check
      run: |
        cd ${{ github.workspace }}/agenta-cli

        # Verify that the pyproject.toml file exists
        if [ ! -f pyproject.toml ]; then
          echo "pyproject.toml file not found. Exiting..."
          exit 1
        fi

        previous_version=$(git describe --abbrev=0 --tags)
        current_version=$(poetry version --no-interaction | awk '{print $2}')
        if [ "$previous_version" == "$current_version" ]; then
          echo "No version change detected. Skipping package publication."
          exit 78
        fi
        if [[ "$current_version" != *a* ]] && [[ "$current_version" != *b* ]]; then
          echo "Current version is not an alpha or beta release. Skipping package publication."
          exit 78
        fi
        echo "::set-output name=version_bumped::true"
      continue-on-error: true

    - name: Build and publish agenta to  PyPI
      if: steps.version_check.outputs.version_bumped == 'true'
      run: |
        # Navigate to the cli directory
        cd ${{ github.workspace }}

        # Build and publish agenta-cli
        cd agenta-cli
        poetry build
        poetry publish --username __token__ --password ${{ secrets.PYPI_API_TOKEN }}
      continue-on-error: true


Run SDK tests

name: Run SDK tests

on:
  pull_request:
    paths:
      - 'agenta-cli/**'
      - 'agenta-cli/pyproject.toml'
      - 'agenta-cli/poetry.lock'
  workflow_dispatch:

jobs:
  pytest:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'

      - name: Install Poetry
        run: pip install poetry

      - name: Install dependencies and agenta SDK
        run: |
          cd agenta-cli
          poetry install

      - name: Run pytest
        run: |
          cd agenta-cli
          poetry run pytest tests


cli commands tests

name: cli commands tests

on:
    pull_request:
        paths:
            - "agenta-backend/**"
            - "agenta-cli/**"

jobs:
    serve-to-oss:
        runs-on: ubuntu-latest
        environment: oss
        steps:
            - name: Check out repository
              uses: actions/checkout@v4

            - name: Set up python
              id: setup-python
              uses: actions/setup-python@v5
              with:
                  python-version: "3.12"

            - name: Install dependencies
              run: |
                  python -m pip install --upgrade pip setuptools
                  pip install poetry
                  cd agenta-cli
                  pip install -e .

            - name: Create .env file
              run: |
                  echo "OPENAI_API_KEY=${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}" > .env
              working-directory: examples/baby_name_generator

            - name: Run agenta init
              run: |
                  APP_NAME="gh-cli-$(date +'%d-%m-%y_%H-%M-%S')"
                  cd examples/baby_name_generator
                  agenta init --app-name $APP_NAME --backend-host ${{ secrets.BACKEND_HOST }}
              shell: bash
              continue-on-error: false

            - name: Run agenta variant serve
              run: |
                  cd examples/baby_name_generator
                  agenta variant serve --file_name app.py 2>&1 | tee ../../serve_output.log
              shell: bash
              continue-on-error: false

            - name: Check if app is running
              uses: ./.github/actions/check-app-accessibility
              with:
                log-file: serve_output.log
              continue-on-error: false

            - name: Run agenta variant serve with overwrite
              run: |
                  cd examples/baby_name_generator
                  agenta variant serve --file_name app.py --overwrite
              shell: bash
              continue-on-error: false

    serve-to-cloud:
        runs-on: ubuntu-latest
        environment: oss
        steps:
            - name: Check out repository
              uses: actions/checkout@v4

            - name: Set up python
              id: setup-python
              uses: actions/setup-python@v5
              with:
                  python-version: "3.12"

            - name: Install dependencies
              run: |
                  python -m pip install --upgrade pip setuptools
                  pip install poetry
                  cd agenta-cli
                  pip install -e .

            - name: Create .env file
              run: |
                  echo "OPENAI_API_KEY=${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}" > .env
              working-directory: examples/baby_name_generator

            - name: Run agenta init
              run: |
                  APP_NAME="gh-cli-$(date +'%d-%m-%y_%H-%M-%S')"
                  cd examples/baby_name_generator
                  AGENTA_API_KEY=${{ secrets.AGENTA_API_KEY }} agenta init --app-name $APP_NAME --backend-host https://cloud.agenta.ai --organisation-name team
              shell: bash
              continue-on-error: false

            - name: Run agenta variant serve
              run: |
                  cd examples/baby_name_generator
                  agenta variant serve --file_name app.py 2>&1 | tee ../../serve_output.log
              shell: bash
              continue-on-error: false

            - name: Check if app is running
              uses: ./.github/actions/check-app-accessibility
              with:
                log-file: serve_output.log
              continue-on-error: false

            - name: Run agenta variant serve with overwrite
              run: |
                  cd examples/baby_name_generator
                  agenta variant serve --file_name app.py --overwrite
              shell: bash
              continue-on-error: false


Tests

name: Tests
concurrency:
  group: ${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

on:
  push:
    branches:
      - master
    paths:
      - .github/workflows/tests.yml
      - httpie/**/*.py
      - setup.*
      - tests/**/*.py
  pull_request:
    paths:
      - .github/workflows/tests.yml
      - httpie/**/*.py
      - setup.*
      - tests/**/*.py

jobs:
  test:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version:
          - '3.12'
          - '3.11'
          - '3.10'
          - '3.9'
          - '3.8'
          - '3.7'
        pyopenssl: [0, 1]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Windows setup
        if: matrix.os == 'windows-latest'
        run: |
          python -m pip install --upgrade pip wheel
          python -m pip install --upgrade '.[dev]'
          python -m pytest --verbose ./httpie ./tests
        env:
          HTTPIE_TEST_WITH_PYOPENSSL: ${{ matrix.pyopenssl }}
      - name: Linux & Mac setup
        if: matrix.os != 'windows-latest'
        run: |
          make install
          make test
        env:
          HTTPIE_TEST_WITH_PYOPENSSL: ${{ matrix.pyopenssl }}


Coverage

name: Coverage

on:
  pull_request:
    paths:
      - .github/workflows/coverage.yml
      - httpie/**/*.py
      - setup.*
      - tests/**/*.py

jobs:
  coverage:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"
      - run: make install
      - run: make test-cover
      - run: make codecov-upload
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_REPO_TOKEN }}
      - run: make test-dist


Code Style Check

name: Code Style Check

on:
  pull_request:
    paths:
      - .github/workflows/code-style.yml
      - extras/*.py
      - httpie/**/*.py
      - setup.py
      - tests/**/*.py

jobs:
  code-style:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - run: make venv
      - run: make codestyle


Check Markdown Style

name: Check Markdown Style

on:
  pull_request:
    paths:
      - "*.md"
      - "**/*.md"

jobs:
  doc:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: 2.7
      - name: Install the linter
        run: sudo gem install mdl
      - name: Check files
        run: make doc-check


Test Brew Package (MacOS)

name: Test Brew Package (MacOS)

on:
  pull_request:
    paths:
      - .github/workflows/test-package-mac-brew.yml
      - docs/packaging/brew/httpie.rb
  workflow_dispatch:

jobs:
  brew:
    runs-on: macos-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup brew
        run: |
          brew developer on
          brew update
      - name: Build and test the formula
        run: make brew-test


Test Snap Package (Linux)

name: Test Snap Package (Linux)

on:
  pull_request:
    paths:
      - .github/workflows/test-package-linux-snap.yml
      - snapcraft.yaml
  workflow_dispatch:

jobs:
  snap:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Build
        uses: snapcore/action-build@v1
        id: snapcraft
      - name: Install
        run: sudo snap install --dangerous ${{ steps.snapcraft.outputs.snap }}
      - name: Test
        run: |
          httpie.http --version
          httpie.https --version
          httpie --version
          # Auto-aliases cannot be tested when installing a snap outside the store.
          # http --version
          # https --version


Deploy Documentation

name: Deploy Documentation

on:
  push:
    branches:
      - master
    paths:
      - docs/README.md
      - docs/config.json
  release:
    types:
      - published
      - unpublished
      - deleted
jobs:
  trigger-doc-build:
    runs-on: ubuntu-latest
    steps:
      - name: Install HTTPie
        run: sudo pip install httpie
      - name: Trigger new documentation build
        run: http --ignore-stdin POST ${{ secrets.DOCS_UPDATE_VERCEL_HOOK }}


Release on Snap

name: Release on Snap

on:
  workflow_dispatch:
    inputs:
      branch:
        description: "The branch, tag or SHA to release from"
        required: true
        default: "master"

jobs:
  snap-build-and-release:
    name: Build & Release the Snap Package
    runs-on: ubuntu-latest

    strategy:
      # If any of the stages fail, then we'll stop the action
      # to give release manager time to investigate the underlying
      # issue.
      fail-fast: true
      matrix:
        level: [edge, beta, candidate, stable]

    # Set the concurrency level for this version, so
    # that we'll release one by one.
    concurrency: ${{ github.event.inputs.branch }}

    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.inputs.branch }}

      - uses: snapcore/action-build@v1
        id: build

      - uses: snapcore/action-publish@v1
        env:
          SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAP_STORE_LOGIN }}
        with:
          snap: ${{ steps.build.outputs.snap }}
          release: ${{ matrix.level }}


Benchmark

name: Benchmark

on:
  pull_request:
    types: [ labeled ]

permissions:
  issues: write
  pull-requests: write

jobs:
  test:
    if: github.event.label.name == 'benchmark'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - id: benchmarks
        name: Run Benchmarks
        run: |
          python -m pip install pyperf>=2.3.0
          python extras/profiling/run.py --fresh --complex  --min-speed=6 --file output.txt
          body=$(cat output.txt)
          body="${body//'%'/'%25'}"
          body="${body//$'\n'/'%0A'}"
          body="${body//$'\r'/'%0D'}"
          echo "::set-output name=body::$body"

      - name: Find Comment
        uses: peter-evans/find-comment@v2
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: '# Benchmarks'

      - name: Create or update comment
        uses: peter-evans/create-or-update-comment@v2
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            # Benchmarks
            ${{ steps.benchmarks.outputs.body }}
          edit-mode: replace

      - uses: actions-ecosystem/action-remove-labels@v1
        with:
          labels: benchmark


Mark stale pull requests

name: Mark stale pull requests

on: workflow_dispatch

permissions:
  pull-requests: write

jobs:
  stale:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/stale@v8
      with:
        close-pr-message: 'Thanks for the pull request, but since it was stale for more than a 30 days we are closing it. If you want to work back on it, feel free to re-open it or create a new one.'
        stale-pr-label: 'stale'

        days-before-stale: -1
        days-before-issue-stale: -1
        days-before-pr-stale: 30

        days-before-close: -1
        days-before-issue-close: -1
        days-before-pr-close: 0

        operations-per-run: 300


Release on Homebrew

name: Release on Homebrew

on:
  workflow_dispatch:
    inputs:
      branch:
        description: "The branch, tag or SHA to release from"
        required: true
        default: "master"

jobs:
  brew-release:
    name: Release the Homebrew Package
    runs-on: macos-latest

    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.inputs.branch }}

      - uses: mislav/bump-homebrew-formula-action@v2
        with:
          formula-name: httpie
          tag-name: ${{ github.events.inputs.branch }}
        env:
          COMMITTER_TOKEN: ${{ secrets.BREW_UPDATE_TOKEN }}


Release on Chocolatey

name: Release on Chocolatey

on:
  workflow_dispatch:
    inputs:
      branch:
        description: "The branch, tag or SHA to release from"
        required: true
        default: "master"

jobs:
  brew-release:
    name: Release the Chocolatey
    runs-on: windows-2019
    env:
        package-dir: docs\packaging\windows-chocolatey

    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.inputs.branch }}

      # Chocolatey comes already installed on the Windows GHA image
      - name: Build the Choco package
        shell: cmd
        run: choco pack -v
        working-directory: ${{ env.package-dir }}

      - name: Check the Choco package
        run: choco info httpie -s .
        working-directory: ${{ env.package-dir }}

      - name: Local installation
        run: |
          choco install httpie -y -dv -s "'.;https://community.chocolatey.org/api/v2/'"
        working-directory: ${{ env.package-dir }}

      - name: Test the locally installed binaries
        run: |
          # Source: https://stackoverflow.com/a/46760714/15330941

          # Make `refreshenv` available right away, by defining the $env:ChocolateyInstall
          # variable and importing the Chocolatey profile module.
          $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."
          Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
          refreshenv

          http --version
          https --version
          httpie --version
          choco uninstall -y httpie
        working-directory: ${{ env.package-dir }}

      - name: Publish on Chocolatey
        shell: bash
        env:
          CHOCO_API_KEY: ${{ secrets.CHOCO_API_KEY }}
        run: |
          choco apikey --key $CHOCO_API_KEY --source https://push.chocolatey.org/
          choco push httpie*.nupkg --source https://push.chocolatey.org/
        working-directory: ${{ env.package-dir }}


Release on PyPI

name: Release on PyPI

on:
  workflow_dispatch:
    inputs:
      branch:
        description: "The branch, tag or SHA to release from"
        required: true
        default: "master"

jobs:
  pypi-build-and-release:
    name: Build and Release
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.inputs.branch }}

      - uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Build a binary wheel and a source tarball
        run: make install && make build

      - name: Release on PyPI
        uses: pypa/gh-action-pypi-publish@master
        with:
          password: ${{ secrets.PYPI_TOKEN }}


Release as Standalone Linux Package

name: Release as Standalone Linux Package

on:
  workflow_dispatch:
    inputs:
      branch:
        description: "The branch, tag or SHA to release from"
        required: true
        default: "master"
      tag_name:
        description: "Which release to upload the artifacts to (e.g., 3.0)"
        required: true

  release:
    types: [released, prereleased]

jobs:
  binary-build-and-release:
    name: Build and Release
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.inputs.branch }}

      - uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Build Artifacts
        run: |
          cd extras/packaging/linux
          ./get_release_artifacts.sh

      - uses: actions/upload-artifact@v3
        with:
          name: http
          path: extras/packaging/linux/artifacts/dist/http

      - uses: actions/upload-artifact@v3
        with:
          name: httpie.deb
          path: extras/packaging/linux/artifacts/dist/*.deb

      - uses: actions/upload-artifact@v3
        with:
          name: httpie.rpm
          path: extras/packaging/linux/artifacts/dist/*.rpm

      - name: Determine the release upload upload_url
        id: release_id
        run: |
          pip install httpie
          export API_URL="api.github.com/repos/httpie/cli/releases/tags/${{ github.event.inputs.tag_name }}"
          export UPLOAD_URL=`https --ignore-stdin GET $API_URL | jq -r ".upload_url"`
          echo "::set-output name=UPLOAD_URL::$UPLOAD_URL"

      - name: Publish Debian Package
        uses: actions/upload-release-asset@v1.0.2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.release_id.outputs.UPLOAD_URL }}
          asset_path: extras/packaging/linux/artifacts/dist/httpie_${{ github.event.inputs.tag_name }}_amd64.deb
          asset_name: httpie-${{ github.event.inputs.tag_name }}.deb
          asset_content_type: binary/octet-stream

      - name: Publish Single Executable
        uses: actions/upload-release-asset@v1.0.2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.release_id.outputs.UPLOAD_URL }}
          asset_path: extras/packaging/linux/artifacts/dist/http
          asset_name: http
          asset_content_type: binary/octet-stream


Update Generated Content

name: Update Generated Content
on:
  push:
    branches:
      - master
jobs:
  update-content:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - run: make content
      - name: Create Pull Request
        id: cpr
        uses: peter-evans/create-pull-request@v4
        with:
          commit-message: "[automated] Update generated content"
          title: "[automated] Update generated content"
          delete-branch: true
          token: ${{ secrets.GITHUB_TOKEN }}


Unit tests

---
name: Unit tests

#
# Documentation:
# https://help.github.com/en/articles/workflow-syntax-for-github-actions
#

#############################
# Start the job on all push #
#############################
on: [push, pull_request]

###############
# Set the Job #
###############
jobs:
  build:
    # Name the Job
    name: Unit Test Code Base
    # Set the agent to run on
    runs-on: ubuntu-latest

    ##################
    # Load all steps #
    ##################
    steps:
      ##########################
      # Checkout the code base #
      ##########################
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          # Full git history is needed to get a proper list of changed files within `super-linter`
          fetch-depth: 0

      - name: Setup helm
        uses: azure/setup-helm@v3
        with:
          version: 'v3.13.2'
        id: install

      ################################
      # Run Linter against code base #
      ################################
      # - name: Lint Code Base
      #   uses: github/super-linter@v4
      #   env:
      #     VALIDATE_ALL_CODEBASE: false
      #     DEFAULT_BRANCH: main
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Run make test
        run: |
          make test

      - name: Run make helmlint
        run: |
          make helmlint

      # - name: Run make helm kubeconform
      #   run: |
      #     curl -L -O https://github.com/yannh/kubeconform/releases/download/v0.4.13/kubeconform-linux-amd64.tar.gz
      #     tar xf kubeconform-linux-amd64.tar.gz
      #     sudo mv -v kubeconform /usr/local/bin
      #     make kubeconform


Super linter

---
name: Super linter

on: [push, pull_request]

jobs:
  build:
    # Name the Job
    name: Super linter
    # Set the agent to run on
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          # Full git history is needed to get a proper list of changed files within `super-linter`
          fetch-depth: 0

      ################################
      # Run Linter against code base #
      ################################
      - name: Lint Code Base
        uses: github/super-linter/slim@v5
        env:
          VALIDATE_ALL_CODEBASE: true
          DEFAULT_BRANCH: main
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          # These are the validation we disable atm
          VALIDATE_BASH: false
          VALIDATE_JSCPD: false
          VALIDATE_KUBERNETES_KUBECONFORM: false
          VALIDATE_YAML: false
          VALIDATE_ANSIBLE: false
          VALIDATE_DOCKERFILE_HADOLINT: false
          # VALIDATE_MARKDOWN: false
          # VALIDATE_NATURAL_LANGUAGE: false
          VALIDATE_TEKTON: false


Ansible Lint

name: Ansible Lint # feel free to pick your own name

on: [push, pull_request]

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      # Important: This sets up your GITHUB_WORKSPACE environment variable
      - uses: actions/checkout@v4

      - name: Lint Ansible Playbook
        uses: ansible/ansible-lint-action@v6
        # Let's point it to the path
        with:
          path: "ansible/"


Verify json schema

---
name: Verify json schema

#
# Documentation:
# https://help.github.com/en/articles/workflow-syntax-for-github-actions
#

#############################
# Start the job on all push #
#############################
on: [push, pull_request]

###############
# Set the Job #
###############
jobs:
  jsonschema_tests:
    # Name the Job
    name: Json Schema tests
    strategy:
      matrix:
        python-version: [3.11]
    # Set the agent to run on
    runs-on: ubuntu-latest

    ##################
    # Load all steps #
    ##################
    steps:
      ##########################
      # Checkout the code base #
      ##########################
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          # Full git history is needed to get a proper list of changed files within `super-linter`
          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install check-jsonschema

      - name: Install yq
        uses: chrisdickinson/setup-yq@latest
        with:
          yq-version: v4.30.7

      - name: Verify secrets json schema against templates
        run: |
          # check-jsonschema needs .yaml as an extension
          cp ./values-secret.yaml.template ./values-secret.yaml
          check-jsonschema --schemafile ./common/ansible/roles/vault_utils/values-secrets.v2.schema.json values-secret.yaml
          rm -f ./values-secret.yaml

      - name: Verify ClusterGroup values.schema.json against values-*yaml files
        run: |
          set -e; for i in values-hub.yaml values-group-one.yaml; do
            echo "$i"
            # disable shellcheck of single quotes in yq
            # shellcheck disable=2016
            yq eval-all '. as $item ireduce ({}; . * $item )' values-global.yaml "$i" > tmp.yaml
            check-jsonschema --schemafile ./common/clustergroup/values.schema.json tmp.yaml
            rm -f tmp.yaml
          done


Unit tests

---
name: Unit tests

#
# Documentation:
# https://help.github.com/en/articles/workflow-syntax-for-github-actions
#

#############################
# Start the job on all push #
#############################
on: [push, pull_request]

###############
# Set the Job #
###############
jobs:
  build:
    # Name the Job
    name: Unit Test Code Base
    # Set the agent to run on
    runs-on: ubuntu-latest

    ##################
    # Load all steps #
    ##################
    steps:
      ##########################
      # Checkout the code base #
      ##########################
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          # Full git history is needed to get a proper list of changed files within `super-linter`
          fetch-depth: 0
      - name: Setup helm
        uses: azure/setup-helm@v4
        with:
          version: 'v3.14.0'
        id: install

      ################################
      # Run Linter against code base #
      ################################
      # - name: Lint Code Base
      #   uses: github/super-linter@v4
      #   env:
      #     VALIDATE_ALL_CODEBASE: false
      #     DEFAULT_BRANCH: main
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Run make test
        run: |
          make test

      - name: Run make helmlint
        run: |
          make helmlint

      # Disable kubeconform for the time being
      # - name: Run make helm kubeconform
      #   run: |
      #     curl -L -O https://github.com/yannh/kubeconform/releases/download/v0.4.13/kubeconform-linux-amd64.tar.gz
      #     tar xf kubeconform-linux-amd64.tar.gz
      #     sudo mv -v kubeconform /usr/local/bin
      #     make kubeconform


Super linter

---
name: Super linter

on: [push, pull_request]

jobs:
  build:
    # Name the Job
    name: Super linter
    # Set the agent to run on
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          # Full git history is needed to get a proper list of changed files within `super-linter`
          fetch-depth: 0

      ################################
      # Run Linter against code base #
      ################################
      - name: Lint Code Base
        uses: github/super-linter/slim@v6
        env:
          VALIDATE_ALL_CODEBASE: true
          DEFAULT_BRANCH: main
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          # These are the validation we disable atm
          VALIDATE_ANSIBLE: false
          VALIDATE_BASH: false
          VALIDATE_CHECKOV: false
          VALIDATE_JSCPD: false
          VALIDATE_KUBERNETES_KUBECONFORM: false
          VALIDATE_PYTHON_PYLINT: false
          VALIDATE_SHELL_SHFMT: false
          VALIDATE_YAML: false
          # VALIDATE_DOCKERFILE_HADOLINT: false
          # VALIDATE_MARKDOWN: false
          # VALIDATE_NATURAL_LANGUAGE: false
          # VALIDATE_TEKTON: false


ack

# See https://github.com/ansible/team-devtools/blob/main/.github/workflows/ack.yml
name: ack

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

on:
  pull_request_target:
    types: [opened, labeled, unlabeled, synchronize]

jobs:
  ack:
    uses: ansible/team-devtools/.github/workflows/ack.yml@token_revised
    secrets: inherit


push

---
# See https://github.com/ansible/team-devtools/blob/main/.github/workflows/push.yml
name: push
on:
  push:
    branches:
      - main
      - "releases/**"
      - "stable/**"
  workflow_dispatch:

jobs:
  ack:
    uses: ansible/team-devtools/.github/workflows/push.yml@main


redirects

---
# Sync RTD redirects
name: redirects

on:
  push:
    branches:
      - main
    paths:
      - docs/redirects.yml
      - .github/workflows/redirects.yml

  # Manually triggered using GitHub's UI
  workflow_dispatch:

jobs:
  docs:
    environment: release
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5

      - name: Upgrade Python toolchain
        run: python3 -m pip install --upgrade pip setuptools wheel

      - name: Install readthedocs-cli
        run: python3 -m pip install readthedocs-cli

      - name: Sync redirects
        run: rtd projects ansible-lint redirects sync -f docs/redirects.yml --wet-run
        env:
          RTD_TOKEN: ${{ secrets.RTD_TOKEN }}


release

name: release

on:
  release:
    types: [published]

jobs:
  pypi:
    name: release ${{ github.event.ref }}
    environment: release
    runs-on: ubuntu-22.04
    permissions:
      id-token: write

    env:
      FORCE_COLOR: 1
      PY_COLORS: 1
      TOX_PARALLEL_NO_SPINNER: 1

    steps:
      - name: Switch to using Python 3.12 by default
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install tox
        run: python3 -m pip install --user "tox>=4.0.0"
      - name: Check out src from Git
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # needed by setuptools-scm
      - name: Build dists
        run: python3 -m tox -e pkg
      - name: Publish to pypi.org
        uses: pypa/gh-action-pypi-publish@release/v1


tox

---
name: tox

on:
  push:
    branches:
      - "main"
  pull_request:
    branches:
      - "main"
  workflow_call:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  tox:
    uses: ansible/team-devtools/.github/workflows/tox.yml@main
    with:
      other_names_also: "eco"


Docker Daily

name: Docker Daily

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.
permissions: read-all

on:
  push:
    branches: "*"
    # Publish semver tags as releases.
    tags: ["*.*.*"]

jobs:
  release_ghcr:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: get tag
        run: echo release=$(git describe --tags `git rev-list --tags --max-count=1`) >> $GITHUB_ENV
      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v2
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: login to ghcr
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN}}
          # Extract metadata (tags, labels) for Docker
      # https://github.com/docker/metadata-action
      - name: Extract Docker metadata ghrc.io
        id: meta2
        uses: docker/metadata-action@v4
        with:
          images: |
            ghcr.io/datawhores/of-scraper
          tags: |
            type=raw,value={{branch}}
            type=raw,value={{branch}}-{{sha}}

      - name: push to image ghrc.io
        id: build-and-push2
        uses: docker/build-push-action@v4
        with:
          context: .
          tags: ${{ steps.meta2.outputs.tags }}
          labels: ${{ steps.meta2.outputs.labels }}
          cache-to: type=gha,mode=max
          push: true

  release_github:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: get tag
        run: echo release=$(git describe --tags `git rev-list --tags --max-count=1`) >> $GITHUB_ENV
      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v2
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      # Extract metadata (tags, labels) for Docker
      # https://github.com/docker/metadata-action
      - name: Extract Docker metadata for Docker Hub
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: |
            datawhores/of-scraper
          tags: |
            type=raw,value={{branch}}
            type=raw,value={{branch}}-{{sha}}

      - name: push to image Docker Hub
        id: build-and-push
        uses: docker/build-push-action@v4
        with:
          context: .
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-to: type=gha,mode=max
          push: true

      - name: cosign-installer Docker Hub
        uses: sigstore/cosign-installer@v3.0.5

      - name: Write signing key to disk Docker Hub
        run: 'echo "$KEY" > cosign.key'
        shell: bash
        env:
          KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}

      - name: Sign the published Docker image Docker Hub
        env:
          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
          COSIGN_EXPERIMENTAL: "true"
        run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign --yes --key cosign.key {}@${{ steps.build-and-push.outputs.digest }}

      #       # Sign the resulting Docker image digest except on PRs.
      #       # This will only write to the public Rekor transparency log when the Docker
      #       # repository is public to avoid leaking data.  If you would like to publish
      #       # transparency data even for private images, pass --force to cosign below.
      #       # https://github.com/sigstore/cosign
      #       - name: Sign the published Docker image
      #         env:
      #           COSIGN_EXPERIMENTAL: "true"
      #         # This step uses the identity token to provision an ephemeral certificate
      #         # against the sigstore community Fulcio instance.
      #         run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
      #Login against a Docker registry except on PR
      #https://github.com/docker/login-action


codecov

name: codecov
permissions: read-all
on:
  push:
    branches: main
    # Publish semver tags as releases.
    tags: [v*.*.*]
  pull_request:
    branches: main

jobs:
  run:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v4
        with:
          python-version: 3.11.5
      - name: install poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: 1.4.1
      - name: Install dependencies
        run: poetry install --with test
      - name: create required folder
        run: mkdir -p /home/runner/.config/ofscraper/main_profile
      - name: Run tests and collect coverage
        run: poetry run pytest --cov-report term --cov-report xml:coverage.xml --cov=src
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3


Delete Package

name: Delete Package
permissions: read-all
on:
  workflow_dispatch:
jobs:
  cleanup:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write
    steps:
      - uses: snok/container-retention-policy@v2
        with:
          image-names: ofscraper
          cut-off: two hours ago UTC+2
          timestamp-to-use: updated_at
          account-type: personal
          keep-at-least: 10
          skip-tags: latest,1.*
          token: ${{ secrets.DOCKER_DELETE}}


Docker Purge

name: Docker Purge
on:
  workflow_dispatch:
  schedule:
    - cron: "0 0 * * *" # every day at midnight
permissions: read-all

jobs:
  cleanup:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write

    steps:
      - name: Main purger
        uses: snok/container-retention-policy@v2
        with:
          image-names: of-scraper
          timestamp-to-use: created_at
          account-type: personal
          token: ${{ secrets.DOCKER_DELETE}}
          filter-tags: sha*
          cut-off: 1 second ago UTC
      - name: Delete untagged
        uses: snok/container-retention-policy@v2
        with:
          image-names: of-scraper
          cut-off: 1 second ago UTC
          account-type: personal
          token: ${{ secrets.DOCKER_DELETE}}
          untagged-only: true