Unittests

name: Unittests

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master
  schedule:
    - cron: '0 4 * * *'

jobs:
  # Special job which automatically cancels old runs for the same branch, prevents runs for the
  # same file set which has already passed, etc.
  pre_job:
    if: github.ref_name != '_release_build' && vars.UNIT_TESTS_DISABLED != 'true'
    name: Skip Duplicate Jobs Pre Job
    runs-on: ubuntu-latest
    permissions:
      actions: write  # Needed for skip-duplicate-jobs job
      contents: read
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
      - id: skip_check
        uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v4.0.0
        with:
          cancel_others: 'true'
          github_token: ${{ github.token }}

  coverage:
    runs-on: ubuntu-latest
    if: (success() || failure()) && (${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }})
    needs:
      - pre_job
      - unittests
      - monitor-smoketests
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Setup Python
        uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: 3.8
      - name: Install pycoverage
        run: pip install coverage==4.5.4
      - name: Download coverage reports
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          path: reports

      - name: Prepare Coverage Data for codecov.io
        run: |
          coverage combine reports/**/.coverage
          coverage xml -i -o coverage.xml

      - name: Upload Coverage to Codecov.io
        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4
        with:
          files: ./coverage.xml
          fail_ci_if_error: true

  unittests:
    name: Unittests - Python ${{ matrix.python-version }} - ${{ matrix.platform }}
    runs-on: ${{ matrix.os }}

    needs: pre_job
    # NOTE: We always want to run job on master branch
    if: (success() || failure()) && (${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }})

    timeout-minutes: 10
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
        os:
          - ubuntu-20.04
        platform:
          - ubuntu
        include:
          - os: windows-2022
            platform: windows
            python-version: "3.10"
          - os: macos-11
            platform: macos
            python-version: "3.10"
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Setup Python
        uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip' # caching pip dependencies
          cache-dependency-path: |
            dev-requirements-new.txt
            benchmarks/micro/requirements-compression-algorithms.txt
      - name: Install tox
        env:
          TOX_VERSION: 3.20.1
          TOX_GH_ACTIONS_VERSION: 2.9.1
        run: |
          python -m pip install --upgrade pip
          pip install "tox==$TOX_VERSION" "tox-gh-actions==$TOX_GH_ACTIONS_VERSION"

      - name: Run Unit Tests
        run: tox
        env:
          PLATFORM: ${{ matrix.platform }}
          KIND: unittests

      - name: Upload pytest test results
        uses: actions/upload-artifact@v3
        with:
          name: pytest-results-${{ matrix.python-version }}-${{ matrix.platform }}
          path: |
            test-results/junit*.xml
            .coverage
        if: ${{ success() || failure() }}

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'

  monitor-smoketests:
    name: Monitor Smoketests - Python ${{ matrix.python-version }}
    runs-on: ubuntu-20.04
    timeout-minutes: 10

    needs: pre_job
    # NOTE: We always want to run job on master branch
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}

    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.8"
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Setup Python
        uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip' # caching pip dependencies
      - name: Install tox
        env:
          TOX_VERSION: 3.20.1
          TOX_GH_ACTIONS_VERSION: 2.9.1
        run: |
          python -m pip install --upgrade pip
          pip install "tox==$TOX_VERSION" "tox-gh-actions==$TOX_GH_ACTIONS_VERSION"

      - name: Run Monitor Smoketests
        run: |
          rm -rf artifacts test-results .coverage
          mkdir -p artifacts test-results
          tox -e agent_monitors_ubuntu -- --artifacts-path artifacts -n 2 --junitxml=test-results/junit-1.xml
        env:
          SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
          READ_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
          SCALYR_SERVER: https://agent.scalyr.com
          AGENT_HOST_NAME: monitors-test-${{ github.run_number }}

      - name: Upload pytest test results
        uses: actions/upload-artifact@v3
        with:
          name: pytest-results-${{ matrix.python-version }}-monitor-smoketests
          path: |
            .coverage
            test-results/junit*.xml
            artifacts
        if: ${{ success() || failure() }}

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'

  # Runs agent from it source and performs some basic sanity checks.
  agent-source-tests:
    name: Python ${{ matrix.python-version  }} ${{ matrix.platform }} Agent Source End to End Test
    runs-on: ${{ matrix.os }}
    needs: pre_job
    # NOTE: We always want to run job on master branch
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}
    container: ${{ matrix.container }}
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
        os:
          - ubuntu-20.04
        platform:
          - ubuntu
        include:
          - os: macos-11
            platform: macos
            python-version: "3.10"

    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Setup Python
        uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip' # caching pip dependencies
          cache-dependency-path: |
            dev-requirements-new.txt
            benchmarks/micro/requirements-compression-algorithms.txt

      - name: Install tox
        env:
          TOX_VERSION: 3.20.1
          TOX_GH_ACTIONS_VERSION: 2.9.1
        run: |
          python -m pip install --upgrade pip
          pip install "tox==$TOX_VERSION" "tox-gh-actions==$TOX_GH_ACTIONS_VERSION"

      - name: Run End to End Tests
        run: tox
        env:
          SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
          READ_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
          TEST_SESSION_SUFFIX: ${{ matrix.python-version  }}-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }}
          KIND: end2end

      - name: Upload pytest test results
        uses: actions/upload-artifact@v3
        with:
          name: pytest-results-${{ matrix.python-version }}-${{ matrix.variant }}
          path: |
            test-results/junit*.xml
        if: ${{ success() || failure() }}

  publish-test-results:
    runs-on: ubuntu-latest
    permissions:
      checks: write   # Needed for publish test results job
      pull-requests: write   # Needed for publish test results job
    if: success() || failure()
    needs:
      - pre_job
      - unittests
      - monitor-smoketests
    steps:
      - name: Download test results
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          path: reports
      - name: Publish Test Results
        uses: EnricoMi/publish-unit-test-result-action@283dea176069279a9076e77b548668a8e4f0c31b # pin@v2
        with:
          junit_files: "reports/**/junit*.xml"


Lint

name: Lint

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master
  schedule:
    - cron: '0 4 * * *'

permissions:
  actions: write  # Needed for skip-duplicate-jobs job
  contents: read

jobs:
  # Special job which automatically cancels old runs for the same branch, prevents runs for the
  # same file set which has already passed, etc.
  pre_job:
    name: Skip Duplicate Jobs Pre Job
    runs-on: ubuntu-latest
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
      - id: skip_check
        uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v4.0.0
        with:
          cancel_others: 'true'
          github_token: ${{ github.token }}

  tox-lint:
    runs-on: ubuntu-20.04
    needs: pre_job
    # NOTE: We always want to run job on master branch
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Install Latest Stable Shellcheck
        run: |
          wget https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz
          tar xvf shellcheck-stable.linux.x86_64.tar.xz
          sudo cp shellcheck-stable/shellcheck /usr/local/bin
          rm -rf shellcheck-stable*
      - name: Setup Python
        uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: 3.6.15

      - name: Install tox
        env:
          TOX_VERSION: 3.20.1
          TOX_GH_ACTIONS_VERSION: 2.9.1
        run: |
          python -m pip install --upgrade pip
          pip install "tox==$TOX_VERSION" "tox-gh-actions==$TOX_GH_ACTIONS_VERSION" requests

      - name: Run tox lint target
        run: tox -e lint

      - name: Verify Codecov config
        # Small safety check to make sure the build fails if codecov.yml file is invalid.
        # By default codecov doesn't fail on invalid config and simply falls back to
        # system wide default config in case repo local config is invalid. This usually results
        # in confused and undesired behavior.
        run: ./scripts/circleci/verify-codecov-config.sh

      - name: Verify add events api response headers
        # Runs script which verifies /addEvents API returns correct response headers. In ideal
        # world, this script would live somewhere else (perhaps as part of monitoring tooling,
        # but for now it will live here.
        env:
          PYTHONPATH: .
          SCALYR_TOKEN_PROD_US: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
          SCALYR_TOKEN_PROD_EU: ${{ secrets.CT_SCALYR_TOKEN_PROD_EU_MONITOR_WRITE }}
          SCALYR_TOKEN_STAGING_EU: ${{ secrets.CT_SCALYR_TOKEN_STAGING_EU_PROD_WRITE }}
        run: ./scripts/verify-add-events-api-response-headers.py

      - name: Run Shell Scripts Lint Checks
        env:
          IGNORE_BINARY_DOESNT_EXIST: "0"
        run: ./scripts/shell-scripts-lint.sh

      - name: Generate Monitor Docs
        run: |
          tox -e generate-monitor-docs
          # Verify there are no changes after generating the docs. If there
          # are, this indicates developer didn't run this target locally and
          # that some files are not up to date
          git status
          git status -- *docs/monitors/*.md */docs/monitorsDmd | (grep -q "nothing to commit" || (echo "Auto-generate monitor doc files are not up to date. Make sure you run tox -e generate-monitor-docs and commit any changed files." && exit 1))

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'


Agent Build Refactored

name: Agent Build Refactored

on:
  push:
    branches:
      - master
    tags:
      - v*.*.*
  pull_request:
    branches:
      - master
  workflow_dispatch:

  schedule:
    - cron: '0 4 * * *'

permissions:
  contents: read

jobs:
  pre_job:
    name: Skip Duplicate Jobs Pre Job
    runs-on: ubuntu-22.04
    permissions:
      actions: write  # Needed for skip-duplicate-jobs job
      contents: read
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
      python_version: ${{ steps.define_constants.outputs.python_version }}
      cache_version: ${{ steps.define_constants.outputs.cache_version }}
      aws_region: ${{ steps.define_constants.outputs.aws_region }}
      aws_private_key_name: ${{ steps.define_constants.outputs.aws_private_key_name }}
      cicd_workflow: ${{ steps.define_constants.outputs.cicd_workflow }}
    steps:
      - id: skip_check
        uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v4.0.0
        with:
          cancel_others: 'true'
          github_token: ${{ github.token }}

      - name: Checkout repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - id: define_constants
        run: |
          echo "python_version=3.8" >> "${GITHUB_OUTPUT}"
          echo "cache_version=$GITHUB_REF_NAME" >> "${GITHUB_OUTPUT}"
          echo "aws_region=us-east-1" >> "${GITHUB_OUTPUT}"
          echo "cicd_workflow=${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" >> "${GITHUB_OUTPUT}"

  build_linux_packages:
    name: "Build Linux packages"
    needs:
      - pre_job
    uses: ./.github/workflows/reusable-agent-build-linux-packages-new.yml
    with:
      python_version: ${{ needs.pre_job.outputs.python_version }}
      cache_version: ${{ needs.pre_job.outputs.cache_version }}
      aws_region: ${{ needs.pre_job.outputs.aws_region }}
      cicd_workflow: ${{ needs.pre_job.outputs.cicd_workflow }}
    secrets:
      CT_AWS_DEV_EC2_PRIVATE_KEY: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY }}
      CT_AWS_DEV_EC2_PRIVATE_KEY_NAME: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY_NAME }}
      CT_AWS_DEV_EC2_ACCESS_KEY: ${{ secrets.CT_AWS_DEV_EC2_ACCESS_KEY }}
      CT_AWS_DEV_EC2_SECRET_KEY: ${{ secrets.CT_AWS_DEV_EC2_SECRET_KEY }}
      CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
      CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}

  # Invoke workflow that builds all agent container mages.
  build_container_images:
    name: "Build Container Images ${{matrix.builder.builder-name}}"
    needs:
      - pre_job
    uses: ./.github/workflows/reusable-agent-build-container-images.yml
    strategy:
      matrix:
        builder: ${{ fromJson(vars.ENABLED_BUILDERS) }}
    with:
      python_version: ${{ needs.pre_job.outputs.python_version }}
      cache_version: ${{ needs.pre_job.outputs.cache_version }}
      aws_region: ${{ needs.pre_job.outputs.aws_region }}
      cicd_workflow: ${{ needs.pre_job.outputs.cicd_workflow }}
      builder_name: ${{ matrix.builder.builder_name }}
      base_image: ${{ matrix.builder.base_image }}
      architectures: ${{ toJson(matrix.builder.architectures) }}
    secrets:
      CT_AWS_DEV_EC2_PRIVATE_KEY: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY }}
      CT_AWS_DEV_EC2_PRIVATE_KEY_NAME: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY_NAME }}
      CT_AWS_DEV_EC2_ACCESS_KEY: ${{ secrets.CT_AWS_DEV_EC2_ACCESS_KEY }}
      CT_AWS_DEV_EC2_SECRET_KEY: ${{ secrets.CT_AWS_DEV_EC2_SECRET_KEY }}
      CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
      CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
      SCALYR_API_KEY_READ_2: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_2_READ }}
      SCALYR_API_KEY_READ_3: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_3_READ }}
      SCALYR_API_KEY_READ_4: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_4_READ }}
      SCALYR_API_KEY_READ_5: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_5_READ }}
      SCALYR_API_KEY_READ_6: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_6_READ }}
      SCALYR_API_KEY_READ_7: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_7_READ }}
      SCALYR_API_KEY_WRITE_2: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_2_WRITE }}
      SCALYR_API_KEY_WRITE_3: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_3_WRITE }}
      SCALYR_API_KEY_WRITE_4: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_4_WRITE }}
      SCALYR_API_KEY_WRITE_5: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_5_WRITE }}
      SCALYR_API_KEY_WRITE_6: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_6_WRITE }}
      SCALYR_API_KEY_WRITE_7: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_7_WRITE }}
      PULL_REGISTRY_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME_PROD_ACCOUNT }}
      PULL_REGISTRY_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD_PROD_ACCOUNT }}
      PUBLISH_REGISTRY_USERNAME: ${{ (github.ref_type == 'tag' || github.ref_name == 'master') && secrets.DOCKER_HUB_USERNAME_PROD_ACCOUNT || secrets.DOCKER_HUB_USERNAME_TEST_ACCOUNT }}
      PUBLISH_REGISTRY_PASSWORD: ${{ (github.ref_type == 'tag' || github.ref_name == 'master') && secrets.DOCKER_HUB_PASSWORD_PROD_ACCOUNT || secrets.DOCKER_HUB_PASSWORD_TEST_ACCOUNT }}
      INTERNAL_BUILD_REGISTRY_USERNAME: ${{ secrets.INTERNAL_BUILD_REGISTRY_USERNAME }}
      INTERNAL_BUILD_REGISTRY_PASSWORD: ${{ secrets.INTERNAL_BUILD_REGISTRY_PASSWORD }}

  build-windows-package:
    name: "Build Windows package"
    uses: ./.github/workflows/reusable-agent-build-windows.yml
    secrets:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

  build_tarball:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Install python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8.13"

      - name: Build tarball
        run: |
          mkdir -p build/tarball/noarch
          pushd build/tarball/noarch
          python3 ../../../build_package.py tarball
          popd

      - name: Save tarball as artifact
        uses: actions/upload-artifact@v3
        with:
          name: tarball-noarch
          path: build
          retention-days: 1

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'

  # Aggregate all agent build artifacts that we build by this workflow and put in a single artifact,
  # so it's just need to download only one artifact, for example during a release process.
  prepare-artifacts:
    runs-on: ubuntu-20.04
    needs:
      - build_linux_packages
      - build-windows-package
      - build_tarball

    steps:
      - name: Checkout repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Download artifacts
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          path: /tmp/all_artifacts

      - name: Prepare artifacts
        run: |
          mkdir -p /tmp/result_artifacts
          cp -a /tmp/all_artifacts/linux-packages-*/. /tmp/result_artifacts
          cp -a /tmp/all_artifacts/tarball-noarch/. /tmp/result_artifacts
          cp -a /tmp/all_artifacts/windows-msi/. /tmp/result_artifacts

      - name: Save result artifact
        uses: actions/upload-artifact@v3
        with:
          name: result-artifacts
          path: /tmp/result_artifacts
          retention-days: 1

  clean-ec2-tests-objects:
    name: Remove ec2 object that were created by this workflow
    if: ${{ always() }}
    needs:
      - pre_job
      - build_container_images
      - build_linux_packages
      - build-windows-package
      - build_tarball
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ needs.pre_job.outputs.python_version }}

      - name: Cleanup old prefix lists for ec2 test security group.
        env:
          AWS_ACCESS_KEY: ${{ secrets.CT_AWS_DEV_EC2_ACCESS_KEY }}
          AWS_SECRET_KEY: ${{ secrets.CT_AWS_DEV_EC2_SECRET_KEY }}
          AWS_PRIVATE_KEY_NAME: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY_NAME }}
          AWS_PRIVATE_KEY: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY }}
          AWS_REGION: ${{ needs.pre_job.outputs.aws_region }}
          CICD_WORKFLOW: ${{ needs.pre_job.outputs.cicd_workflow }}
        run: |
          python3 agent_build_refactored/utils/scripts/cicd/cleanup_ec2_objects.py


Agent Linux Packages Build Refactored

name: Agent Linux Packages Build Refactored

on:
  workflow_call:
    inputs:
      python_version:
        description: "Version of Python to use."
        type: string

      cache_version:
        description: "Version suffix for the cache key. May be useful when it is needed to invalidate the cache."
        type: string

      aws_region:
        description: "AWS region for remote builds."
        type: string

      cicd_workflow:
        description: "Additional string that has to differentiate AWS object from another workflows."
        type: string

    outputs:
      cache_miss:
        description: "Flag that indicates that packages are can not be build by using cache."
        value: ${{ jobs.build_packages.outputs.cache_miss }}

    secrets:
      CT_AWS_DEV_EC2_PRIVATE_KEY:
        required: true
      CT_AWS_DEV_EC2_PRIVATE_KEY_NAME:
        required: true
      CT_AWS_DEV_EC2_ACCESS_KEY:
        required: true
      CT_AWS_DEV_EC2_SECRET_KEY:
        required: true
      CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE:
        required: true
      CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ:
        required: true

env:
  DOCKER_BUILDKIT: 1
  # Set this variable to tell the agent build code that it runs in CI/CD and it needs to use caching.
  AGENT_BUILD_IN_CICD: "1"

# This job skips another workflows with identical content and also generates strategy matrices for all further jobs.
# Since we can do a "master" run (on push and PR to a master branch) and "non-master" run, it generates matrices with different
#  size according to that information.
jobs:
  build_dependencies:
    name: Build dependencies ${{ matrix.builder.name }}
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        builder:
          - { "name": "aio-aarch64" }
          - { "name": "aio-x86_64" }

    steps:
      - name: Checkout repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ inputs.python_version }}

      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
          platforms: all

      - name: Set up Docker Context for Buildx
        id: buildx-context
        run: |
          docker context create builders || echo Already exists

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3
        with:
          driver-opts: network=host
          endpoint: builders

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Build Dependencies
        shell: bash
        id: build
        env:
          AWS_ACCESS_KEY: ${{ secrets.CT_AWS_DEV_EC2_ACCESS_KEY }}
          AWS_SECRET_KEY: ${{ secrets.CT_AWS_DEV_EC2_SECRET_KEY }}
          AWS_PRIVATE_KEY_NAME: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY_NAME }}
          AWS_PRIVATE_KEY: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY }}
          AWS_REGION: ${{ inputs.aws_region }}
          USE_GHA_CACHE: "1"
          ALLOW_FALLBACK_TO_REMOTE_BUILDER: "1"
          CACHE_VERSION: ${{ inputs.cache_version }}
          CICD_WORKFLOW: ${{ inputs.cicd_workflow }}
          CICD_JOB: ${{ matrix.builder.name }}
        run: |
          python3 build_package_new.py package ${{ matrix.builder.name }} build-dependencies

  build_packages:
    name: Build package ${{ matrix.variant.builder_name }}-${{ matrix.variant.package_type }}
    runs-on: ${{ matrix.variant.runner }}
    needs:
      - build_dependencies
    strategy:
      matrix:
        variant:
          - { "builder_name": "aio-x86_64",  "package_type": "deb", runner: "ubuntu-22.04" }
          - { "builder_name": "aio-x86_64",  "package_type": "rpm", runner: "ubuntu-22.04" }
          - { "builder_name": "aio-aarch64", "package_type": "deb", runner: "aws-aarch64" }
          - { "builder_name": "aio-aarch64", "package_type": "rpm", runner: "aws-aarch64" }
          - { "builder_name": "non-aio",     "package_type": "deb", runner: "ubuntu-22.04" }
          - { "builder_name": "non-aio",     "package_type": "rpm", runner: "ubuntu-22.04" }
    steps:
      - name: Checkout repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: install python and requirements
        uses: actions/setup-python@v4
        continue-on-error: true
        with:
          python_version: ${{ inputs.python_version }}

      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
          platforms: all

      - name: Set up Docker Context for Buildx
        id: buildx-context
        run: |
          docker context create builders || echo Already exists

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3
        with:
          driver-opts: network=host
          endpoint: builders

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Build package
        shell: bash
        id: build
        env:
          USE_GHA_CACHE: "1"
          CACHE_VERSION: ${{ inputs.cache_version }}
        run: |
          python3 build_package_new.py package ${{ matrix.variant.builder_name }} build \
            --package-type ${{ matrix.variant.package_type }}

      - name: Save packages as artifact
        uses: actions/upload-artifact@v3
        with:
          name: linux-packages-${{ matrix.variant.builder_name }}
          path: build
          retention-days: 1

  test_packages:
    name: Test package on ${{ matrix.test_target.distro-name }}-${{ matrix.test_target.remote-machine-type }}, ${{ matrix.test_target.builder }}
    if: github.ref_name != '_release_build'
    needs:
      - build_packages

    runs-on: ${{ matrix.test_target.runner }}
    strategy:
      fail-fast: false
      matrix:
        test_target:
          - { "package_type": "deb", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "ubuntu2204",    "remote-machine-type": "ec2", runner: "ubuntu-22.04" }
          - { "package_type": "deb", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "ubuntu2004",    "remote-machine-type": "ec2", runner: "ubuntu-22.04" }
          - { "package_type": "deb", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "ubuntu1804",    "remote-machine-type": "ec2", runner: "ubuntu-22.04" }
          - { "package_type": "deb", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "ubuntu1604",    "remote-machine-type": "ec2", runner: "ubuntu-22.04" }
          - { "package_type": "deb", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "ubuntu1404",    "remote-machine-type": "docker", runner: "ubuntu-22.04" }
          - { "package_type": "deb", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "debian10",      "remote-machine-type": "ec2", runner: "ubuntu-22.04" }
          - { "package_type": "deb", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "debian11",      "remote-machine-type": "docker", runner: "ubuntu-22.04" }
          - { "package_type": "rpm", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "centos8",       "remote-machine-type": "docker", runner: "ubuntu-22.04" }
          - { "package_type": "rpm", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "centos7",       "remote-machine-type": "ec2", runner: "ubuntu-22.04" }
          - { "package_type": "rpm", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "centos6",       "remote-machine-type": "docker", runner: "ubuntu-22.04" }
          - { "package_type": "rpm", "builder": "aio-x86_64",  "arch": "x86_64", "distro-name": "amazonlinux2",  "remote-machine-type": "ec2", runner: "ubuntu-22.04" }
          - { "package_type": "deb", "builder": "aio-aarch64", "arch": "arm64",  "distro-name": "ubuntu1404",    "remote-machine-type": "docker", runner: "aws-aarch64" }
          - { "package_type": "rpm", "builder": "aio-aarch64", "arch": "arm64",  "distro-name": "centos7",       "remote-machine-type": "docker", runner: "aws-aarch64" }
            # - { "package_type": "deb", "builder": "non-aio",     "arch": "x86_64", "distro-name": "ubuntu1404",    "remote-machine-type": "ec2", runner: "ubuntu-22.04" }
          - { "package_type": "deb", "builder": "non-aio",     "arch": "x86_64", "distro-name": "ubuntu2204",    "remote-machine-type": "docker", runner: "ubuntu-22.04" }
            # - { "package_type": "rpm", "builder": "non-aio",     "arch": "x86_64", "distro-name": "centos7",       "remote-machine-type": "ec2", runner: "ubuntu-22.04" }
          - { "package_type": "rpm", "builder": "non-aio",     "arch": "x86_64", "distro-name": "amazonlinux2",  "remote-machine-type": "docker", runner: "ubuntu-22.04" }
    steps:
      - name: Checkout repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
          platforms: all

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3
        with:
          driver-opts: network=host

      - name: Download built packages.
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          name:  linux-packages-${{ matrix.test_target.builder }}
          path: /tmp/packages

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ inputs.python_version }}

      - name: Test '${{ matrix.test_target.builder }}' packages
        env:
          AWS_ACCESS_KEY: ${{ secrets.CT_AWS_DEV_EC2_ACCESS_KEY }}
          AWS_SECRET_KEY: ${{ secrets.CT_AWS_DEV_EC2_SECRET_KEY }}
          AWS_PRIVATE_KEY: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY }}
          AWS_PRIVATE_KEY_NAME: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY_NAME }}
          AWS_REGION: ${{ inputs.aws_region }}
          CICD_WORKFLOW: ${{ inputs.cicd_workflow }}
          CICD_JOB: ${{ matrix.test_target.builder }}-${{ matrix.test_target.package_type }}-${{ matrix.test_target.distro-name }}
          USE_GHA_CACHE: "1"
          CACHE_VERSION: ${{ inputs.cache_version }}

          SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
          READ_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
          TEST_SESSION_SUFFIX: ${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
        run: |
          python3 tests/end_to_end_tests/managed_packages_tests/remote_machine_tests/run_test_remotely.py \
            tests/end_to_end_tests/managed_packages_tests \
            -s \
            --builder-name ${{ matrix.test_target.builder }} \
            --distro-name ${{ matrix.test_target.distro-name }} \
            --remote-machine-type ${{ matrix.test_target.remote-machine-type }} \
            --package-type ${{ matrix.test_target.package_type }} \
            --packages-source-type dir \
            --packages-source /tmp/packages/${{ matrix.test_target.package_type }}


agent-build-windows

name: agent-build-windows

on:
  workflow_call:
    secrets:
        SLACK_WEBHOOK_URL:
          required: true

jobs:
  build-windows-package:
    name: Build windows MSI installer
    runs-on: windows-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Setup Python
        uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: '3.10'

      - name: Install Requirements
        run: |
          python -m pip install --upgrade pip
          pip install -r dev-requirements-new.txt

      - name: Build agent package and installer
        run: python build_package.py win32

      - name: Save installer as artifact
        uses: actions/upload-artifact@v3
        with:
          name: windows-msi
          path: ScalyrAgentInstaller-*.msi

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'


TruffleHog Secrets Scan

name: TruffleHog Secrets Scan
on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master
  schedule:
    - cron: '0 4 * * *'

permissions:
  contents: read

jobs:
  TruffleHog:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
        with:
          fetch-depth: 0

      # Special check which ensures that the clone performed above is not shallow. We need the
      # complete git history for scanning to work correctly in all the situations. In some cases
      # if a shallow clone is used, trufflehog won't not fail with an error, but it would simply
      # not detect any files and that could be dangerous.
      - name: Shallow repo check
        run: |
          if git rev-parse --is-shallow-repository | grep -q "true"; then
            echo "Encountered a shallow repository, trufflehog may not work as expected!"
            exit 1
          fi

      - name: scan-pr
        uses: trufflesecurity/trufflehog@main
        if: ${{ github.event_name == 'pull_request' }}
        with:
          path: ./
          base: ${{ github.event.repository.default_branch }}
          head: HEAD
          extra_args: --debug --only-verified

      - name: scan-push
        uses: trufflesecurity/trufflehog@main
        if: ${{ github.event_name == 'push' }}
        with:
          path: ./
          base: ""
          head: ${{ github.ref_name }}
          extra_args: --debug --only-verified

      # As part of cron trigger we scan the whole repo directory.
      # NOTE: Since trufflehog GHA is meant to be used in context of push / pr it can't be
      # used dorectly to scan the whole repo directory. This may take a while, but it's good idea
      # to run it on a daily basis.
      - name: scan-cron
        if: ${{ github.event_name == 'schedule' }}
        run: |
          docker run --rm -v "$PWD:/workdir" trufflesecurity/trufflehog:latest git \
            file:///workdir --fail --no-update --debug --only-verified

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-o11y'


Kubernetes End to End Tests

name: "Kubernetes End to End Tests"

on:
  workflow_call:
    inputs:
      python_version:
        description: "Version of Python to use."
        type: string

      cache_version:
        description: "Version suffix for the cache key. May be useful when it is needed to invalidate the cache."
        type: string

      aws_region:
        description: "AWS region for remote builds."
        type: string

      cicd_workflow:
        description: "Additional string that has to differentiate AWS object from another workflows."
        type: string

      builder_name:
        description: "Builder name used to build docker images. Accepted values: ubuntu, alpine, ubuntu-fips."
        type: string

      base_image:
        description: "Base image to be used when bulding the docker image, i.e. ubuntu:22.04, alpine:latest."
        type: string

      architectures:
        description: "JSON representation of a list of architectures used in a stragety matrix in some jobs. Accepted values: x86_64, aarch64, armv7."
        type: string

    secrets:
      CT_AWS_DEV_EC2_PRIVATE_KEY:
        required: true
      CT_AWS_DEV_EC2_PRIVATE_KEY_NAME:
        required: true
      CT_AWS_DEV_EC2_ACCESS_KEY:
        required: true
      CT_AWS_DEV_EC2_SECRET_KEY:
        required: true
      CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE:
        required: true
      CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ:
        required: true
      SCALYR_API_KEY_READ_2:
        required: true
      SCALYR_API_KEY_READ_3:
        required: true
      SCALYR_API_KEY_READ_4:
        required: true
      SCALYR_API_KEY_READ_5:
        required: true
      SCALYR_API_KEY_READ_6:
        required: true
      SCALYR_API_KEY_READ_7:
        required: true
      SCALYR_API_KEY_WRITE_2:
        required: true
      SCALYR_API_KEY_WRITE_3:
        required: true
      SCALYR_API_KEY_WRITE_4:
        required: true
      SCALYR_API_KEY_WRITE_5:
        required: true
      SCALYR_API_KEY_WRITE_6:
        required: true
      SCALYR_API_KEY_WRITE_7:
        required: true
      PULL_REGISTRY_USERNAME:
        required: true
      PULL_REGISTRY_PASSWORD:
        required: true
      PUBLISH_REGISTRY_USERNAME:
        required: true
      PUBLISH_REGISTRY_PASSWORD:
        required: true
      INTERNAL_BUILD_REGISTRY_USERNAME:
        required: true
      INTERNAL_BUILD_REGISTRY_PASSWORD:
        required: true

jobs:
  coverage:
    runs-on: ubuntu-latest
    if: ( success() || failure() ) && github.ref_name != '_release_build' && vars.UNIT_TESTS_DISABLED != 'true'
    needs:
      - k8s-smoketest
      - docker-smoketest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
      - name: Setup Python
        uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: 3.8
      - name: Install pycoverage
        run: pip install coverage==4.5.4
      - name: Download coverage reports
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          path: reports

      - name: Prepare Coverage Data for codecov.io
        run: |
          coverage combine reports/**/.coverage
          coverage xml -i -o coverage.xml

      - name: Upload Coverage to Codecov.io
        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4
        with:
          files: ./coverage.xml
          fail_ci_if_error: true

  # Build cacheable requirements of the images.
  build_image_requirements:
    name: requirements-${{ inputs.builder_name }}-${{ matrix.architecture }}
    runs-on: ${{ vars.RUNNER }}
    strategy:
      matrix:
        architecture: ${{ fromJson(inputs.architectures) }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ inputs.python_version }}

      - name: Login to DockerHub
        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
        with:
          username: ${{ secrets.PULL_REGISTRY_USERNAME }}
          password: ${{ secrets.PULL_REGISTRY_PASSWORD }}

      - name: Login to an internal build registry
        if: vars.INTERNAL_BUILD_REGISTRY
        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
        continue-on-error: true
        with:
          registry: ${{ vars.INTERNAL_BUILD_REGISTRY }}
          username: ${{ secrets.INTERNAL_BUILD_REGISTRY_USERNAME }}
          password: ${{ secrets.INTERNAL_BUILD_REGISTRY_PASSWORD }}

      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
          platforms: all

        # Context need to be created for a self-hosted runner
      - name: Set up Docker Context for Buildx
        id: buildx-context
        run: |
          docker context create builders || echo Already exists

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3 # v3
        with:
          driver-opts: network=host
          endpoint: builders

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Build requirements
        shell: bash
        id: build
        env:
          AWS_ACCESS_KEY: ${{ secrets.CT_AWS_DEV_EC2_ACCESS_KEY }}
          AWS_SECRET_KEY: ${{ secrets.CT_AWS_DEV_EC2_SECRET_KEY }}
          AWS_PRIVATE_KEY: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY }}
          AWS_PRIVATE_KEY_NAME: ${{ secrets.CT_AWS_DEV_EC2_PRIVATE_KEY_NAME }}
          AWS_REGION: ${{ inputs.aws_region }}
          CICD_WORKFLOW: ${{ inputs.cicd_workflow }}
          CICD_JOB: build_image_requirements-${{ inputs.builder_name }}-${{ matrix.architecture }}

          USE_GHA_CACHE: "1"
          ALLOW_FALLBACK_TO_REMOTE_BUILDER: "1"
          CACHE_VERSION: ${{ inputs.cache_version }}
        run: |
          python3 build_package_new.py \
            image \
            ${{ inputs.builder_name }} \
            --base-image ${{ inputs.base_image }} \
            cache-requirements \
            --architecture ${{ matrix.architecture }}

  build_image_tarball:
    name: Build image tarball ${{  matrix.variant.image_type }}-${{ inputs.builder_name }}
    runs-on: ${{ vars.RUNNER }}
    needs:
      - build_image_requirements
    strategy:
      matrix:
        variant:
          - { "image_type": "k8s" }
          - { "image_type": "docker-json" }
          - { "image_type": "docker-syslog" }
          - { "image_type": "docker-api" }

    steps:
      - name: Checkout repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ inputs.python_version }}

      - name: Login to DockerHub
        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
        with:
          username: ${{ secrets.PULL_REGISTRY_USERNAME }}
          password: ${{ secrets.PULL_REGISTRY_PASSWORD }}

      - name: Login to an internal build registry
        if: vars.INTERNAL_BUILD_REGISTRY
        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
        continue-on-error: true
        with:
          registry: ${{ vars.INTERNAL_BUILD_REGISTRY }}
          username: ${{ secrets.INTERNAL_BUILD_REGISTRY_USERNAME }}
          password: ${{ secrets.INTERNAL_BUILD_REGISTRY_PASSWORD }}

      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
          platforms: all

        # Context need to be created for a self-hosted runner
      - name: Set up Docker Context for Buildx
        id: buildx-context
        run: |
          docker context create builders || echo Already exists

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3
        with:
          driver-opts: network=host
          endpoint: builders

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Build image tarball
        shell: bash
        id: build
        env:
          CICD_WORKFLOW: ${{ inputs.cicd_workflow }}
          CICD_JOB: ${{  matrix.variant.image_type }}-${{ inputs.builder_name }}

          USE_GHA_CACHE: "1"
          CACHE_VERSION: ${{ inputs.cache_version }}
        run: |
          python3 build_package_new.py \
            image \
            ${{ inputs.builder_name }} \
            --base-image ${{ inputs.base_image }} \
            build-tarball \
            --image-type ${{  matrix.variant.image_type }} \
            --output-dir ./build

      - name: Save image OCI tarball as artifact
        uses: actions/upload-artifact@v3
        with:
          name: container-image-tarball-${{  matrix.variant.image_type }}-${{ inputs.builder_name }}
          path: build
          retention-days: 1

  # Jobs which performs basic sanity checks for the Kubernetes Monitor and Kubernetes Events Monitor
  k8s_kubernetes_monitor_tests:
    name: Kubernetes Monitors - k8s ${{ inputs.builder_name }} ${{ matrix.k8s_version.version }}-${{ matrix.k8s_version.runtime}}
    runs-on: ubuntu-20.04
    timeout-minutes: 15

    if: ${{ ! vars.K8S_TESTS_DISABLED }}

    needs:
      - build_image_tarball

    strategy:
      fail-fast: false
      matrix:
        k8s_version:
          # NOTE: We don't test under very old versions where CronJob was part of v2alpha1 since in
          # those versions CronJobs are not enabled by default and need to be installed extra
          - { "version": "v1.17.17", "driver": "", "runtime": "docker", "run_cronjob_checks": false }
          - { "version": "v1.20.15", "driver": "", "runtime": "docker", "run_cronjob_checks": false }
          - { "version": "v1.21.10", "driver": "", "runtime": "docker", "run_cronjob_checks": true }
          - { "version": "v1.22.7", "driver": "", "runtime": "docker", "run_cronjob_checks": true }
          # NOTE: Using containerd runtime in minikube on GHA only works with docker driver
          - { "version": "v1.23.17", "driver": "docker", "runtime": "containerd", "run_cronjob_checks": true }
          - { "version": "v1.24.14", "driver": "docker", "runtime": "containerd", "run_cronjob_checks": true }
          - { "version": "v1.25.10", "driver": "docker", "runtime": "containerd", "run_cronjob_checks": true }
          - { "version": "v1.26.5", "driver": "docker", "runtime": "containerd", "run_cronjob_checks": true }
          # CRI-O
          - { "version": "v1.27.4", "driver": "docker", "runtime": "cri-o", "run_cronjob_checks": true }

    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
        with:
          fetch-depth: 0

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ inputs.python_version }}

      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
          platforms: all

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3
        with:
          driver-opts: network=host

      - name: Login to DockerHub
        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
        with:
          username: ${{ secrets.PULL_REGISTRY_USERNAME }}
          password: ${{ secrets.PULL_REGISTRY_PASSWORD }}

      - name: Install Scalyr tool
        run: |
          curl https://raw.githubusercontent.com/scalyr/scalyr-tool/master/scalyr > scalyr
          chmod +x scalyr
          sudo mv scalyr /usr/local/bin

      - name: Setup minikube k8s cluster
        uses: ./.github/actions/setup-minikube-cluster/
        with:
          k8s_version: "${{ matrix.k8s_version.version }}"
          minikube_driver: "${{ matrix.k8s_version.driver }}"
          container_runtime: "${{ matrix.k8s_version.runtime }}"
          github_token: "${{ secrets.GITHUB_TOKEN }}"

      - name: Preload minikube images
        run: |
          # Since the minikube runtimes we use cannot be reasonably configured to use custom credentials for docker hub,
          # we need to pull the images and load them into minikube.
          docker pull busybox:1.28
          minikube image load busybox:1.28

      - name: Download OCI tarball of the built image.
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          name: container-image-tarball-k8s-${{ inputs.builder_name }}
          path: /tmp/image_tarball

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Build Agent k8s Test Docker Image
        env:
          USE_GHA_CACHE: "1"
          CACHE_VERSION: ${{ inputs.cache_version }}
        run: |
          python3 tests/end_to_end_tests/container_images_test/scripts/build_test_image.py \
            --image-builder-name ${{ inputs.builder_name }} \
            --base-image ${{ inputs.base_image }} \
            --architecture x86_64 \
            --image-oci-tarball /tmp/image_tarball/k8s-${{ inputs.builder_name }}.tar \
            --result-image-name k8s-image:test \
            --image-type k8s

          docker image ls

          # Needed for containerd or cri-o runtime
          if [ "${{ matrix.k8s_version.runtime }}" = "containerd" ] || [ "${{ matrix.k8s_version.runtime }}" = "cri-o" ]; then
            docker tag k8s-image:test docker.io/library/k8s-image:test
            minikube image load docker.io/library/k8s-image:test
          fi

      # Here we build the dummy container which continuously prints data to stdout and stderr
      - name: Build Dummy App Docker Images
        run: |
          docker build -f docker/Dockerfile.docker_monitor_testing_config -t std-printer scripts/
          docker build -f docker/Dockerfile.long_message_printer -t long-message-printer scripts/
          docker image ls

          # Needed for containerd or cri-o runtime
          if [ "${{ matrix.k8s_version.runtime }}" = "containerd" ] || [ "${{ matrix.k8s_version.runtime }}" = "cri-o" ]; then
            for IMAGE in std-printer long-message-printer; do
              docker tag ${IMAGE}:latest docker.io/library/${IMAGE}:latest
              minikube image load docker.io/library/${IMAGE}:latest
            done
          fi

      # Create pod for our mock std printer container which logs will be ingested by the agent
      - name: Create mock pods
        run: |
          minikube image ls
          kubectl apply -f tests/e2e/k8s_k8s_monitor/std_printer_deployment.yaml
          kubectl apply -f tests/e2e/k8s_k8s_monitor/long_message_printer_deployment.yaml

          kubectl wait --for=condition=ready pod -l app=std-printer
          kubectl wait --for=condition=ready pod -l app=long-message-printer
          kubectl get pods -A

          export APP_POD_NAME=$(kubectl get pod --namespace=default --selector=app=std-printer -o jsonpath="{.items[0].metadata.name}")
          echo "APP_POD_NAME=${APP_POD_NAME}" >> ${GITHUB_ENV}
          echo "APP_POD_NAME=${APP_POD_NAME}"

          echo ""
          kubectl logs "${APP_POD_NAME}"
          echo ""

      - name: Create mock pods for multi account printer
        env:
          scalyr_api_key_write_team_2: "${{ secrets.SCALYR_API_KEY_WRITE_2 }}"
          scalyr_api_key_write_team_3: "${{ secrets.SCALYR_API_KEY_WRITE_3 }}"
          scalyr_api_key_write_team_4: "${{ secrets.SCALYR_API_KEY_WRITE_4 }}"
          scalyr_api_key_write_team_5: "${{ secrets.SCALYR_API_KEY_WRITE_5 }}"
          scalyr_api_key_write_team_6: "${{ secrets.SCALYR_API_KEY_WRITE_6 }}"
          scalyr_api_key_write_team_7: "${{ secrets.SCALYR_API_KEY_WRITE_7 }}"
        run: |
          tests/e2e/k8s_k8s_monitor/multiple_account_printers/deploy.sh

      # We create mock CronJob so we can assert that Kubernetes Events monitor correctly handles
      # CronJob and Job Events.
      - name: Create mock CronJob (v1)
        if: ${{ matrix.k8s_version.run_cronjob_checks }}
        run: |
          # We need to use a different file for different K8s versions due to promotion (removal of
          # old v1beta alias) from v1beta to v1 in v1.25.0. v1 has been available since v1.21.0
          kubectl apply -f tests/e2e/k8s_events_monitor/cronjob_v1.yaml

          kubectl get cronjob -A
      - name: Wait random time to make it easier for the cloud github runners
        run: sleep $((RANDOM % 60))

      - name: Create scalyr-agent-2 daemonset
        uses: ./.github/actions/install-k8s-agent/
        with:
          scalyr_server: "agent.scalyr.com"
          scalyr_api_key: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}"
          scalyr_api_key_team_2: "${{ secrets.SCALYR_API_KEY_WRITE_2 }}"
          scalyr_api_key_team_3: "${{ secrets.SCALYR_API_KEY_WRITE_3 }}"
          scalyr_api_key_team_4: "${{ secrets.SCALYR_API_KEY_WRITE_4 }}"
          scalyr_cluster_name: "${K8S_CLUSTER_NAME}"
          scalyr_k8s_events_disable: "false"
          main_yaml_path: "tests/e2e/scalyr-agent-2-daemonset.yaml"

      - name: Create mock Jobs
        run: |
          kubectl apply -f tests/e2e/k8s_k8s_monitor/short-lived-jobs.yaml
          kubectl get job -A

      - name: Verify data has been ingested
        timeout-minutes: 14
        env:
          # Needed for scalyr-tool
          scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
          SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
          K8S_NODE_NAME: "${{ env.K8S_NODE_NAME }}"
        run: |
          export RETRY_ATTEMPTS="14"
          export SLEEP_DELAY="10"

          # Verify agent is running
          echo "Agent running checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting scalyr agent..."'

          # Verify Kubernetes monitor is running
          echo "Kubernetes Monitor running checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "kubernetes_monitor parameters: ignoring namespaces: "'
          echo ""

          # Verify Kubernetes events monitor is running
          echo "Kubernetes events monitor running checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting monitor kubernetes_events_monitor"'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Acting as Kubernetes event leader"'
          echo ""

          # Verify initial std-printer pod data has been ingested

          # 1. First we want for some data to be ingested using "log.config.scalyr.com/attributes.parser"
          # annotation set as part of the deployment YAML.
          # After a while, we change that dynamically using kubectl and verify that this change has
          # been correctly picked up by the agent.
          echo Waiting 20s for some data to be ingested
          sleep 20

          echo "Initial pod ingested data checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "stdout: line 2"'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "stderr: line 2"'
          echo ""

          kubectl describe pod ${APP_POD_NAME}
          kubectl annotate --overwrite pods ${APP_POD_NAME} log.config.scalyr.com/attributes.parser="changed"
          kubectl describe pod ${APP_POD_NAME}

          # Give agent some time to pick up the annotation change (by default we poll every 30 seconds
          # for pod metadata changes, but we use lower value for the tests)
          sleep 15

          echo ""
          echo "Post annotation change data checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="changed" stream="stdout" "stdout: line"'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="changed" stream="stderr" "stderr: line"'
          echo ""

          # Verify Kubernetes Events Monitor events are ingested
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" "\"kind\":\"Event\""'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" "\"kind\":\"Pod\""'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" "involvedObject"'
          # This event is not always available under Kubernetes >= 1.25
          #./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" "NodeReady"'

          agent_status=$(kubectl --namespace=scalyr exec -i ${SCALYR_AGENT_POD_NAME} --container scalyr-agent -- scalyr-agent-2 status -v)
          k8s_event_log_files_number=$(echo "$agent_status" | grep -c 'Path /var/log/scalyr-agent-2/kubernetes_events.log: copied')

          if [ "$k8s_event_log_files_number" != 1 ]; then
            echo "Kubernetes event monitor log has to be handled by the agent 1 time, but it got ${k8s_event_log_files_number}"
            exit 1
          fi

          echo "Wait for BEGIN DELAYED MESSAGE-END DELAYED MESSAGE lines. They should not be splitted."
          MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "BEGIN DELAYED MESSAGE stdout"'
          MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "BEGIN DELAYED MESSAGE stderr"'
          echo "Looking for partial lines."
          RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "BEGIN DELAYED MESSAGE stdout" !"BEGIN DELAYED MESSAGE stdout-END DELAYED MESSAGE stdout"' && echo -e $ERR_MSG && exit 1 || echo -e $SUCCESS_MSG
          RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "BEGIN DELAYED MESSAGE stderr" !"BEGIN DELAYED MESSAGE stderr-END DELAYED MESSAGE stderr"' && echo -e $ERR_MSG && exit 1 || echo -e $SUCCESS_MSG
          echo 'Looking for mixed streams'
          RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stdout" "stderr"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
          RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="std-printer" parser="test-parser-1" stream="stderr" "stdout"' && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG

          echo "Wait for BEGIN_LONG_MESSAGE .* END_LONG_MESSAGE lines. They should not be splitted."
          MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stdout" "stdout_BEGIN_LONG_MESSAGE"'
          MINIMUM_RESULTS=10 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stderr" "stderr_BEGIN_LONG_MESSAGE"'
          echo "Looking for partial lines."
          RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stdout" "stdout_BEGIN_LONG_MESSAGE" !(message matches "BEGIN_LONG_MESSAGE.*END_LONG_MESSAGE")'  && echo -e $ERR_MSG && exit 1 || echo -e $SUCCESS_MSG
          RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stderr" "stderr_BEGIN_LONG_MESSAGE" !(message matches "BEGIN_LONG_MESSAGE.*END_LONG_MESSAGE")'  && echo -e $ERR_MSG && exit 1 || echo -e $SUCCESS_MSG
          echo 'Looking for mixed streams'
          RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stdout" "stderr"'  && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG
          RETRY_ATTEMPTS=1 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" app="long-message-printer" parser="test-parser-2" stream="stderr" "stdout"'  && echo -e $MIXED_STREAMS_ERR_MSG && exit 1 || echo -e $MIXED_STREAMS_SUCCESS_MSG

          echo ""

      # We only assert this under newer Kubernetes versions since in the old ones CronJobs were not
      # available out of the box
      - name: Verify Job and CronJob events are ingested
        if: ${{ matrix.k8s_version.run_cronjob_checks }}
        timeout-minutes: 5
        env:
          # Needed for scalyr-tool
          scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
          SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
          K8S_NODE_NAME: "${{ env.K8S_NODE_NAME }}"
        run: |
          export RETRY_ATTEMPTS="14"
          export SLEEP_DELAY="10"

          # Wait for at least one hello-printer job to be scheduled and completed.
          RETRIES=1;
          until echo -n "Waiting for first job to be completed - retry $RETRIES: " && kubectl wait --for=condition=complete job -l app=hello-printer || [ $RETRIES -gt 60 ]; do sleep 1; RETRIES=$(($RETRIES+1)); done
          [ $RETRIES -gt 60 ] && echo "Job didn't start in time" && exit 1

          # Verify agent is running
          echo "Job events checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" k8s-kind="Job" watchEventType="ADDED" reason="Completed"'

          echo "CronJob events checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" k8s-kind="CronJob" involvedObjectKind="CronJob" involvedObjectName="hello" watchEventType="ADDED" reason="SawCompletedJob"'

          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Hello from the Kubernetes cluster"'

      - name: Check the short lived pods logs are ingested - 404 response handling
        if: ${{ matrix.k8s_version.runtime != 'docker' }}
        env:
          # Needed for scalyr-tool
          scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
          SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
        run: |
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" message matches "Pod default/short-lived-job-1.* not found in K8s API. Including pod based on SCALYR_K8S_INCLUDE_ALL_CONTAINERS=true."'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" message matches "Pod default/short-lived-job-2.* not found in K8s API. Including pod based on SCALYR_K8S_INCLUDE_ALL_CONTAINERS=true."'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" message matches "Pod default/short-lived-job-3.* not found in K8s API. Including pod based on SCALYR_K8S_INCLUDE_ALL_CONTAINERS=true."'

      - name: Check the short lived pods logs are ingested - log message
        env:
          # Needed for scalyr-tool
          scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
          SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
        run: |
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-1"'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-2"'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" "Message from short-lived-container-3"'

      - name: Install Testing Python Packages
        run: |
          python3 -m pip install kubernetes tabulate

      - name: Verify multiaccount records have been ingested
        timeout-minutes: 14
        env:
          SCALYR_API_KEY_READ_TEAM_1: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
          SCALYR_API_KEY_READ_TEAM_2: "${{ secrets.SCALYR_API_KEY_READ_2 }}"
          SCALYR_API_KEY_READ_TEAM_3: "${{ secrets.SCALYR_API_KEY_READ_3 }}"
          SCALYR_API_KEY_READ_TEAM_4: "${{ secrets.SCALYR_API_KEY_READ_4 }}"
          SCALYR_API_KEY_READ_TEAM_5: "${{ secrets.SCALYR_API_KEY_READ_5 }}"
          SCALYR_API_KEY_READ_TEAM_6: "${{ secrets.SCALYR_API_KEY_READ_6 }}"
          SCALYR_API_KEY_READ_TEAM_7: "${{ secrets.SCALYR_API_KEY_READ_7 }}"
          SERVER_HOST: "${{ env.SCALYR_AGENT_POD_NAME }}"
          ACCOUNT_NAME_1: "cloudtech-testing@scalyr.com"
          ACCOUNT_NAME_2: "cloudtech-testing-2@scalyr.com"
          ACCOUNT_NAME_3: "cloudtech-testing-3@scalyr.com"
          ACCOUNT_NAME_4: "cloudtech-testing-4@scalyr.com"
          ACCOUNT_NAME_5: "cloudtech-testing-5@scalyr.com"
          ACCOUNT_NAME_6: "cloudtech-testing-6@scalyr.com"
          ACCOUNT_NAME_7: "cloudtech-testing-7@scalyr.com"
        run: |
          echo 'Looking for multiple account logs'

          ./scripts/cicd/verify-multiaccount-records-have-been-ingested.py

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'

  k8s_open_metrics_monitor_tests:
    name: OpenMetrics Monitor - k8s ${{ inputs.builder_name }} ${{ matrix.k8s_version.version }}-${{ matrix.k8s_version.runtime}}
    runs-on: ubuntu-20.04
    if: vars.K8S_TESTS_DISABLED != 'true' && github.ref_name != '_release_build' && inputs.builder_name != 'alpine'
    timeout-minutes: 15

    needs:
      - build_image_tarball

    strategy:
      fail-fast: false
      matrix:
        k8s_version:
          - { "version": "v1.17.17", "driver": "", "runtime": "docker" }
          - { "version": "v1.20.15", "driver": "", "runtime": "docker" }
          - { "version": "v1.21.10", "driver": "", "runtime": "docker" }
          - { "version": "v1.22.7", "driver": "", "runtime": "docker" }
          # NOTE: Using containerd runtime in minikube on  GHA only works with docker driver
          - { "version": "v1.23.17", "driver": "docker", "runtime": "containerd" }
          - { "version": "v1.24.14", "driver": "docker", "runtime": "containerd" }
          - { "version": "v1.25.10", "driver": "docker", "runtime": "containerd" }
          - { "version": "v1.26.5", "driver": "docker", "runtime": "containerd" }
          # CRI-O
          - { "version": "v1.27.4", "driver": "docker", "runtime": "cri-o" }

    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
        with:
          fetch-depth: 0

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ inputs.python_version }}

      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
          platforms: all

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3
        with:
          driver-opts: network=host

      - name: Install Scalyr tool
        run: |
          curl https://raw.githubusercontent.com/scalyr/scalyr-tool/master/scalyr > scalyr
          chmod +x scalyr
          sudo mv scalyr /usr/local/bin

      - name: Setup minikube k8s cluster
        uses: ./.github/actions/setup-minikube-cluster/
        with:
          k8s_version: "${{ matrix.k8s_version.version }}"
          minikube_driver: "${{ matrix.k8s_version.driver }}"
          container_runtime: "${{ matrix.k8s_version.runtime }}"
          github_token: "${{ secrets.GITHUB_TOKEN }}"

      - name: Download OCI tarball of the built image.
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          name: container-image-tarball-k8s-${{ inputs.builder_name }}
          path: /tmp/image_tarball

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Build Agent k8s Test Docker Image
        env:
          USE_GHA_CACHE: "1"
          CACHE_VERSION: ${{ inputs.cache_version }}
        run: |
          python3 tests/end_to_end_tests/container_images_test/scripts/build_test_image.py \
            --image-builder-name ${{ inputs.builder_name }} \
            --base-image ${{ inputs.base_image }} \
            --architecture x86_64 \
            --image-oci-tarball /tmp/image_tarball/k8s-${{ inputs.builder_name }}.tar \
            --result-image-name k8s-image:test \
            --image-type k8s

          docker image ls

          # Needed for containerd or cri-o runtime
          if [ "${{ matrix.k8s_version.runtime }}" = "containerd" ] || [ "${{ matrix.k8s_version.runtime }}" = "cri-o" ]; then
            docker tag k8s-image:test docker.io/library/k8s-image:test
            minikube image load docker.io/library/k8s-image:test
          fi

      # Here we build the dummy Java app image which exposes JMX metrics via exporter
      - name: Build Test Java App Docker Image
        run: |
          pushd tests/e2e/k8s_om_monitor/java-hello-world
          docker build -t java-hello-world .
          popd

          docker image ls

          # Needed for containerd or cri-o runtime
          if [ "${{ matrix.k8s_version.runtime }}" = "containerd" ] || [ "${{ matrix.k8s_version.runtime }}" = "cri-o" ]; then
            docker tag java-hello-world:latest docker.io/library/java-hello-world:latest
            minikube image load docker.io/library/java-hello-world:latest
          fi

      # Create mock pods and exporters which will be scrapped by the monitor
      - name: Create mock pods and exporters
        run: |
          kubectl create namespace monitoring

          # 1. node exporter pod
          kubectl apply -f tests/e2e/k8s_om_monitor/node_exporter.yaml

          # 2. kube state metrics deployment
          kubectl apply -k tests/e2e/k8s_om_monitor/kube-state-metrics/

          # 3. Install dummy java app container with jmx exporter side
          kubectl apply -f tests/e2e/k8s_om_monitor/java_app_deployment.yaml

          sleep 20
          kubectl get pods -A

      - name: Create scalyr-agent-2 daemonset
        uses: ./.github/actions/install-k8s-agent/
        with:
          scalyr_server: "agent.scalyr.com"
          scalyr_api_key: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}"
          scalyr_cluster_name: "${K8S_CLUSTER_NAME}"
          main_yaml_path: "tests/e2e/k8s_om_monitor/scalyr-agent-2-daemonset.yaml"
          # Monitor is not enabled by default yet since it's still in preview and testing phase so
          # we expliticly enable it here
          extra_yaml_paths: "tests/e2e/k8s_om_monitor/scalyr-agent-extra-config-configmap.yaml"

      - name: Verify data has been ingested
        timeout-minutes: 14
        env:
          # Needed for scalyr-tool
          scalyr_readlog_token: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
          SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
          K8S_NODE_NAME: "${{ env.K8S_NODE_NAME }}"
          K8S_CLUSTER_NAME: "${{ env.K8S_CLUSTER_NAME }}"
        run: |
          export RETRY_ATTEMPTS="30"
          export SLEEP_DELAY="10"

          # Verify agent is running
          echo "Agent running checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Starting scalyr agent..."'

          # Verify monitor is running
          echo "Monitor running checks"
          kubectl get pods -A
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "Found 3 URL(s) to scrape for node"'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/agent.log" "There are currently 3 dynamic and 2 static open metrics monitors running"'
          echo ""

          # Kubernetes API metrics (static monitor)
          echo "Kubernetes API metrics monitor checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-metrics.log" "process_max_fds 1000000 k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-metrics.log" "process_open_fds "'

          # Kubernetes API cAdvisor metrics (static monitor)
          echo "Kubernetes API cAdvisor metrics monitor checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "machine_cpu_cores '$(nproc --all)'"'
          MINIMUM_RESULTS=2 ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "container_cpu_load_average_10s "'
          # Verify locally calculated rate metrics
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "container_cpu_usage_seconds_total_rate "'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "container_network_receive_bytes_total_rate "'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-kubernetes-api-cadvisor-metrics" "container_network_transmit_bytes_total_rate "'

          # 2. Verify node exporter metrics
          echo "Node exporter metrics monitor checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "process_max_fds "'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "process_open_fds "'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_vmstat_pswpin "'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_vmstat_pswpout "'
          # Verify locally calculated rate metrics
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_cpu_seconds_total_rate "'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_network_transmit_bytes_total_rate "'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_network_receive_bytes_total_rate "'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_disk_read_bytes_total_rate "'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'-node-exporter-" "node_disk_written_bytes_total_rate "'

          # 3. Verify kube state event metrics
          echo "Kube state events metrics monitor checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "kube-state-metrics" "kube_storageclass_labels 1 k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\" storageclass=\"standard\""'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "kube-state-metrics" "kube_secret_type 1 k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\" namespace=\"scalyr\" secret=\"scalyr-api-key\" type=\"Opaque\""'

          # 4. Verify java app JMX metrics
          echo "Java JMX metrics events metrics monitor checks"
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jmx_scrape_error 0.0 app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jmx_scrape_cached_beans 0.0 app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jvm_info 1.0 app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\" runtime="'

          # Client side calculated per second rate metrics
          # NOTE: There is no easy way to assert on the actual rate metric value here (we have unit tests for that)
          # so we just assert that the metric is present.
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jvm_threads_started_total_rate" "app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "jvm_memory_pool_allocated_bytes_total_rate" "app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'
          ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile contains "openmetrics_monitor-'${K8S_NODE_NAME}'" $logfile contains "java-hello-world" "process_cpu_seconds_total_rate" "app=\"java-hello-world\" app_instance=\"java-hello-world-1\" attribute1=\"value1\" k8s-cluster=\"'${K8S_CLUSTER_NAME}'\" k8s-node=\"'${K8S_NODE_NAME}'\""'

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'

  k8s-smoketest:
    runs-on: ${{ vars.RUNNER }}
    needs:
      - build_image_tarball
    name: K8s Smoketest - ${{ inputs.builder_name }}
    if: vars.K8S_TESTS_DISABLED != 'true' && github.ref_name != '_release_build' && (github.event_name != 'pull_request' || ! github.event.pull_request.draft)
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ inputs.python_version }}

      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
          platforms: all

      - name: Login to DockerHub
        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
        with:
          username: ${{ secrets.PULL_REGISTRY_USERNAME }}
          password: ${{ secrets.PULL_REGISTRY_PASSWORD }}

      - name: Set up Docker Context for Buildx
        id: buildx-context
        run: |
          docker context create builders || echo Already exists

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3
        with:
          driver-opts: network=host
          endpoint: builders

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Setup minikube k8s cluster
        uses: ./.github/actions/setup-minikube-cluster/
        with:
          k8s_version: v1.22.0
          minikube_driver: ""
          container_runtime: "docker"
          github_token: "${{ secrets.GITHUB_TOKEN }}"

      - name: Download OCI tarball of the built image.
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          name: container-image-tarball-k8s-${{ inputs.builder_name }}
          path: /tmp/image_tarball

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Build test image
        env:
          USE_GHA_CACHE: "1"
          CACHE_VERSION: ${{ inputs.cache_version }}
        run: |
          python3 tests/end_to_end_tests/container_images_test/scripts/build_test_image.py \
            --image-builder-name ${{ inputs.builder_name }} \
            --base-image ${{ inputs.base_image }} \
            --architecture x86_64 \
            --image-oci-tarball /tmp/image_tarball/k8s-${{ inputs.builder_name }}.tar \
            --result-image-name k8s-image:test \
            --image-type k8s

      - name: Run Tests (with coverage)
        env:
          CIRCLE_BUILD_NUM: ${{ github.run_number }}
          SCALYR_SERVER: https://agent.scalyr.com
          SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
          READ_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
        run: |
          source ./.circleci/smoketest_k8s.sh scalyr/scalyr-agent-ci-unittest:4 150 no_delete_existing_k8s_objs k8s-image:test

      - name: Upload test results
        uses: actions/upload-artifact@v3
        with:
          name: k8s-smoketest
          path: |
            .coverage
        if: ${{ success() || failure() }}

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'

  docker-smoketest:
    name: Docker Smoketest - ${{ inputs.builder_name }} ${{ matrix.variant.image_type }}
    if: github.ref_name != '_release_build'
    needs:
      - build_image_tarball
    runs-on: ${{ vars.RUNNER }}
    strategy:
      fail-fast: false
      matrix:
        variant:
          - image_type: docker-json
            timeout: 150
          - image_type: docker-syslog
            timeout: 150
          - image_type: docker-api
            timeout: 200
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ inputs.python_version }}

      - name: Set up QEMU
        id: qemu
        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
        with:
          image: tonistiigi/binfmt:qemu-v6.2.0
          platforms: all

      - name: Login to DockerHub
        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
        with:
          username: ${{ secrets.PULL_REGISTRY_USERNAME }}
          password: ${{ secrets.PULL_REGISTRY_PASSWORD }}

      - name: Login to an internal build registry
        if: vars.INTERNAL_BUILD_REGISTRY
        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
        continue-on-error: true
        with:
          registry: ${{ vars.INTERNAL_BUILD_REGISTRY }}
          username: ${{ secrets.INTERNAL_BUILD_REGISTRY_USERNAME }}
          password: ${{ secrets.INTERNAL_BUILD_REGISTRY_PASSWORD }}

        # Context need to be created for a self-hosted runner
      - name: Set up Docker Context for Buildx
        id: buildx-context
        run: |
          docker context create builders || echo Already exists

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3
        with:
          driver-opts: network=host
          endpoint: builders

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Download OCI tarball of the built image.
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          name: container-image-tarball-${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
          path: /tmp/image_tarball

      - name: Expose GitHub Runtime To Be Able to Use GHA Cache By Docker.
        uses: crazy-max/ghaction-github-runtime@715c25b40ccc0df9b62bfa8be3ccc57d09dbc4b1

      - name: Build test image
        env:
          USE_GHA_CACHE: "1"
          CACHE_VERSION: ${{ inputs.cache_version }}
        run: |
          python3 tests/end_to_end_tests/container_images_test/scripts/build_test_image.py \
            --image-builder-name ${{ inputs.builder_name }} \
            --base-image ${{ inputs.base_image }} \
            --architecture x86_64 \
            --image-oci-tarball /tmp/image_tarball/${{ matrix.variant.image_type }}-${{ inputs.builder_name }}.tar \
            --result-image-name ${{ matrix.variant.image_type }}-${{ inputs.builder_name }}:test \
            --image-type ${{ matrix.variant.image_type }}

      - name: Run Tests (with coverage)
        env:
          CIRCLE_BUILD_NUM: ${{ github.run_number }}
          SCALYR_SERVER: https://agent.scalyr.com
          SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
          READ_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
        run: |
          source ./.circleci/smoketest_docker.sh scalyr/scalyr-agent-ci-unittest:4 ${{ matrix.variant.image_type }} ${{ matrix.variant.timeout }} ${{ matrix.variant.image_type }}-${{ inputs.builder_name }}:test

      - name: Upload test results
        uses: actions/upload-artifact@v3
        with:
          name: docker-smoketest-${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
          path: |
            .coverage
        if: ${{ success() || failure() }}

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'

  publish_images:
    name: Publish image ${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
    runs-on: ubuntu-20.04
    if: github.ref_type == 'tag' || github.ref_name == 'master' || github.head_ref == 'release' || github.event_name == 'pull_request'
    needs:
      # k8s_kubernetes_monitor_tests, k8s_open_metrics_monitor_tests, k8s-smoketest checks are not needed before pushing a test version to dockerhub.
      # They need to pass to allow PR merge.
      - build_image_tarball
      - docker-smoketest
    strategy:
      matrix:
        variant:
          - { "image_type": "k8s" }
          - { "image_type": "docker-json" }
          - { "image_type": "docker-syslog" }
          - { "image_type": "docker-api" }
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: install python and requirements
        uses: ./.github/actions/install_python_and_requirements
        with:
          python_version: ${{ inputs.python_version }}

      - name: Download OCI tarball of the built image.
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
        with:
          name: container-image-tarball-${{ matrix.variant.image_type }}-${{ inputs.builder_name }}
          path: /tmp/image_tarball

      - name: Verify and Create Tags
        id: verify
        shell: bash
        run: |
          AGENT_VERSION="$(cat VERSION | tr -d '\n')"

          if [ "${{ github.ref_type }}" = "tag" ]; then
            if [ "${{ github.ref_name }}" = "v${AGENT_VERSION}" ]; then
              echo "tags=$AGENT_VERSION,latest" >> "${GITHUB_OUTPUT}"
            else
              echo "tags=empty" >> "${GITHUB_OUTPUT}"
            fi
          elif [ "${{ github.ref_name }}" = "master" ]; then
              echo "tags=${{ github.sha }}" >> "${GITHUB_OUTPUT}"
          elif [ "${{ github.head_ref }}" = "release" ]; then
              echo "tags=latest-release,pr-${{ github.event.number}}-${{ github.event.pull_request.commits}}-${{ github.sha }},\
              $AGENT_VERSION,latest" >> "${GITHUB_OUTPUT}"
          elif [ "${{ github.event_name }}" = "pull_request" ]; then
              echo "tags=latest-pr,pr-${{ github.event.number}}-${{ github.event.pull_request.commits}}-${{ github.sha }}" >> "${GITHUB_OUTPUT}"
          else
            echo "tags=empty" >> "${GITHUB_OUTPUT}"
          fi

      - name: Publish image
        if: steps.verify.outputs.tags != 'empty'
        run: |
          python3 build_package_new.py \
            image \
            ${{ inputs.builder_name }} \
            --base-image ${{ inputs.base_image }} \
            publish \
            --registry docker.io \
            --name-prefix "${{ secrets.PUBLISH_REGISTRY_USERNAME }}" \
            --tags "${{ steps.verify.outputs.tags }}" \
            --from-oci-layout /tmp/image_tarball/${{ matrix.variant.image_type }}-${{ inputs.builder_name }}.tar \
            --registry-username "${{ secrets.PUBLISH_REGISTRY_USERNAME }}" \
            --registry-password "${{ secrets.PUBLISH_REGISTRY_PASSWORD }}" \
            --image-type ${{ matrix.variant.image_type }}


Benchmarks

name: Benchmarks

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

jobs:
  # Special job which automatically cancels old runs for the same branch, prevents runs for the
  # same file set which has already passed, etc.
  pre_job:
    if: github.ref_name != '_release_build' && vars.UNIT_TESTS_DISABLED != 'true'
    name: Skip Duplicate Jobs Pre Job
    runs-on: ubuntu-latest
    permissions:
      actions: write  # Needed for skip-duplicate-jobs job
      contents: read
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
      - id: skip_check
        uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v4.0.0
        with:
          cancel_others: 'true'
          github_token: ${{ github.token }}

  memory-usage:
    name: Memory Usage - Syslog Monitor - Python ${{ matrix.python-version }}
    runs-on: ubuntu-20.04

    needs: pre_job
    # NOTE: We always want to run job on master branch
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}

    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "2.7"
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
      - name: Install loggen
        run: |
          sudo apt update
          sudo apt install -y syslog-ng-core
      - name: Run memory test
        run: |
          ./scripts/cicd/test_syslog_monitor_memory_usage/run_test.sh 10 10 600
        env:
          SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}