Integrations Testing

name: Integrations Testing
on:
  pull_request:
    branches:
      - main
      - 'release/*'

jobs:
  test-setup:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git branch --show-current
      - name: Get current branch
        id: get-branch
        run: >
          (git branch --show-current | grep -E "release/")
          && echo "::set-output name=branch::$(git branch --show-current)"
          || echo "::set-output name=branch::main"
      - name: "Checking if sparseml.transformers was changed"
        id: transformers-check
        run: >
          ((git diff --name-only origin/main HEAD | grep -E "[src/sparseml|tests/integrations]/transformers|setup.py")
          || (echo $GITHUB_REF | grep -E "refs/heads/[release/|main]"))
          && echo "::set-output name=output::transformers" || echo "::set-output name=output::"
      - name: "Checking if sparseml.yolov5 was changed"
        id: yolov5-check
        run: >
          ((git diff --name-only origin/main HEAD | grep -E "[src/sparseml|tests/integrations]/yolov5|setup.py")
          || (echo $GITHUB_REF | grep -E "refs/heads/[release/|main]"))
          && echo "::set-output name=output::yolov5" || echo "::set-output name=output::"
      - name: "Checking if sparseml.pytorch.image_classification was changed"
        id: image-classification-check
        run: >
          ((git diff --name-only origin/main HEAD | grep -E "[src/sparseml/pytorch|tests/integrations]/image_classification|setup.py")
          || (echo $GITHUB_REF | grep -E "refs/heads/[release/|main]"))
          && echo "::set-output name=output::image_classification" || echo "::set-output name=output::"

  integrations-tests:
    runs-on: ubuntu-22.04
    needs: test-setup
    env:
      SPARSEZOO_TEST_MODE: "true"
      SPARSEML_TEST_CADENCE: "pre-commit"
      TRANSFORMERS: needs.test-setup.outputs.transformers
      YOLOV5: needs.test-setup.outputs.yolov5
      IMAGE_CLASSIFICATION: needs.test-setup.outputs.image_classification
    steps:
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/compressed-tensors"
          path: "compressed-tensors"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install compressed-tensors dependencies"
        run: pip3 install -U pip && pip3 install setuptools compressed-tensors/
      - name: "Clean compressed-tensors directory"
        run: rm -r compressed-tensors/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,torchvision,deepsparse,onnxruntime,transformers,yolov5]
      - name: "🔬 Running integrations tests (cadence: pre-commit}})"
        run: make testinteg TARGETS=$TRANSFORMERS,$YOLOV5,$IMAGE_CLASSIFICATION


Check Markdown links

name: Check Markdown links

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  markdown-link-check:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: gaurav-nelson/github-action-markdown-link-check@v1
      with:
        use-quiet-mode: 'yes'
        config-file: '.github/workflows/mlc_config.json'


Quality Checks

name: Quality Checks
on:
  push:
    branches:
      - main
      - 'release/*'
  pull_request:
    branches:
      - main
      - 'release/*'
jobs:
  quality-check:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev]
      - name: "🧹 Running quality checks"
        run: make quality


Test Checks

name: Test Checks
on:
  push:
    branches:
      - main
      - 'release/*'
  pull_request:
    branches:
      - main
      - 'release/*'

jobs:
  test-setup:
    runs-on: ubuntu-22.04
    outputs:
      branch: ${{ steps.get-branch.outputs.branch }}
      base: ${{ steps.base-check.outputs.output }}
      deepsparse: ${{ steps.deepsparse-check.outputs.output }}
      onnx: ${{ steps.onnx-check.outputs.output }}
      pytorch: ${{ steps.pytorch-check.outputs.output }}
      export: ${{ steps.export-check.outputs.output }}
      transformers: ${{ steps.transformers-check.outputs.output }}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - run: git branch --show-current
      - name: Get current branch
        id: get-branch
        run: >
          (git branch --show-current | grep -E "release/")
          && echo "::set-output name=branch::$(git branch --show-current)"
          || echo "::set-output name=branch::main"
      - name: "Checking if sparseml was changed"
        id: base-check
        run: >
          ((git diff --name-only origin/main HEAD | grep -E "[src|tests]/sparseml|setup.py|.github")
          || (echo $GITHUB_REF | grep -E "refs/heads/[release/|main]"))
          && echo "::set-output name=output::1" || echo "::set-output name=output::0"
      - name: "Checking if sparseml.deepsparse was changed"
        id: deepsparse-check
        run: >
          ((git diff --name-only origin/main HEAD | grep -E "[src|tests]/sparseml/deepsparse|setup.py|.github")
          || (echo $GITHUB_REF | grep -E "refs/heads/[release/|main]"))
          && echo "::set-output name=output::1" || echo "::set-output name=output::0"
      - name: "Checking if sparseml.onnx was changed"
        id: onnx-check
        run: >
          ((git diff --name-only origin/main HEAD | grep -E "[src|tests]/sparseml/onnx|setup.py|.github")
          || (echo $GITHUB_REF | grep -E "refs/heads/[release/|main]"))
          && echo "::set-output name=output::1" || echo "::set-output name=output::0"
      - name: "Checking if sparseml.pytorch was changed"
        id: pytorch-check
        run: >
          ((git diff --name-only origin/main HEAD | grep -E "[src|tests]/sparseml/modifiers|pytorch|setup.py|.github")
          || (echo $GITHUB_REF | grep -E "refs/heads/[release/|main]"))
          && echo "::set-output name=output::1" || echo "::set-output name=output::0"
      - name: "Checking if sparseml.export was changed"
        id: export-check
        run: >
          ((git diff --name-only origin/main HEAD | grep -E "[src|tests]/sparseml/export|setup.py|.github")
          || (echo $GITHUB_REF | grep -E "refs/heads/[release/|main]"))
          && echo "::set-output name=output::1" || echo "::set-output name=output::0"
      - name: "Checking if sparseml.transformers was changed"
        id: transformers-check
        run: >
          ((git diff --name-only origin/main HEAD | grep -E "[src|tests]/sparseml/transformers|setup.py|.github")
          || (echo $GITHUB_REF | grep -E "refs/heads/[release/|main]"))
          && echo "::set-output name=output::1" || echo "::set-output name=output::0"
  base-tests:
    runs-on: ubuntu-22.04
    env:
      SPARSEZOO_TEST_MODE: "true"
    needs: test-setup
    if: ${{needs.test-setup.outputs.base == 1}}
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,onnxruntime]
      - name: "🔬 Running base tests"
        run: make test
  deepsparse-tests:
    runs-on: ubuntu-22.04
    env:
      SPARSEZOO_TEST_MODE: "true"
    needs: test-setup
    if: ${{needs.test-setup.outputs.deepsparse == 1}}
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,deepsparse,onnxruntime]
      - name: "🔬 Running deepsparse tests"
        run: make test TARGETS=deepsparse
  onnx-tests:
    runs-on: ubuntu-22.04
    env:
      SPARSEZOO_TEST_MODE: "true"
    needs: test-setup
    if: ${{needs.test-setup.outputs.onnx == 1}}
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,torchvision,onnxruntime]
      - name: "🔬 Running onnx tests"
        run: make test TARGETS=onnx
  pytorch-tests:
    runs-on: k8s-eng-gpu-64G-v100-32G
    env:
      SPARSEZOO_TEST_MODE: "true"
      CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }}
      CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }}
      CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }}
      CLEARML_FILES_HOST:  ${{ secrets.CLEARML_FILES_HOST }}
      CLEARML_API_SECRET_KEY:  ${{ secrets.CLEARML_API_SECRET_KEY }}
    needs: test-setup
    if: ${{needs.test-setup.outputs.pytorch == 1}}
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/compressed-tensors"
          path: "compressed-tensors"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install compressed-tensors dependencies"
        run: pip3 install -U pip && pip3 install setuptools compressed-tensors/
      - name: "Clean compressed-tensors directory"
        run: rm -r compressed-tensors/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,torchvision,onnxruntime,transformers]
      - name: "🔬 Running pytorch tests"
        run: make test TARGETS=pytorch
  compat-pytorch-1_9-pytorch-tests:
    runs-on: k8s-eng-gpu-64G-v100-32G
    env:
      SPARSEZOO_TEST_MODE: "true"
      CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }}
      CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }}
      CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }}
      CLEARML_FILES_HOST:  ${{ secrets.CLEARML_FILES_HOST }}
      CLEARML_API_SECRET_KEY:  ${{ secrets.CLEARML_API_SECRET_KEY }}
    needs: test-setup
    if: ${{needs.test-setup.outputs.pytorch == 1}}
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/compressed-tensors"
          path: "compressed-tensors"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install compressed-tensors dependencies"
        run: pip3 install -U pip && pip3 install setuptools compressed-tensors/
      - name: "Clean compressed-tensors directory"
        run: rm -r compressed-tensors/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,torchvision,onnxruntime,transformers]
      - name: "🔬 Running pytorch tests"
        run: make test TARGETS=pytorch
  compat-pytorch-1_9-onnx-tests:
    runs-on: ubuntu-22.04
    env:
      SPARSEZOO_TEST_MODE: "true"
    needs: test-setup
    if: ${{needs.test-setup.outputs.onnx == 1}}
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,torchvision,onnxruntime] torch==1.9.1
      - name: "🔬 Running onnx tests"
        run: make test TARGETS=onnx
  transformers-tests:
    runs-on: k8s-eng-gpu-64G-v100-32G
    env:
      SPARSEZOO_TEST_MODE: "true"
      CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }}
      CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }}
      CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }}
      CLEARML_FILES_HOST:  ${{ secrets.CLEARML_FILES_HOST }}
      CLEARML_API_SECRET_KEY:  ${{ secrets.CLEARML_API_SECRET_KEY }}
    needs: test-setup
    if: ${{needs.test-setup.outputs.transformers == 1}}
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/compressed-tensors"
          path: "compressed-tensors"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install compressed-tensors dependencies"
        run: pip3 install -U pip && pip3 install setuptools compressed-tensors/
      - name: "Clean compressed-tensors directory"
        run: rm -r compressed-tensors/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,torch,transformers,onnxruntime]
      - name: "🔬 Running transformers tests"
        run: make test TARGETS=transformers
  export-tests:
    runs-on: ubuntu-22.04
    env:
      SPARSEZOO_TEST_MODE: "true"
    needs: test-setup
    if: ${{needs.test-setup.outputs.export == 1}}
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - uses: actions/checkout@v2
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/sparsezoo"
          path: "sparsezoo"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install sparsezoo dependencies"
        run: pip3 install -U pip && pip3 install setuptools sparsezoo/
      - name: "Clean sparsezoo directory"
        run: rm -r sparsezoo/
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/compressed-tensors"
          path: "compressed-tensors"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install compressed-tensors dependencies"
        run: pip3 install -U pip && pip3 install setuptools compressed-tensors/
      - name: "Clean compressed-tensors directory"
        run: rm -r compressed-tensors/
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,torch,transformers,torchvision,onnxruntime]
      - name: "🔬 Running export tests"
        run: make test TARGETS=export


Run Nightly Tests

name: Run Nightly Tests
on:
  workflow_dispatch:
  workflow_call:
jobs:
  test-nightly-tests:
    runs-on: k8s-mle-gpu-12-vcpu-225GB-ram-2-a6000-48G
    env:
      CADENCE: "nightly"
      CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }}
      CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }}
      CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }}
      CLEARML_FILES_HOST:  ${{ secrets.CLEARML_FILES_HOST }}
      CLEARML_API_SECRET_KEY:  ${{ secrets.CLEARML_API_SECRET_KEY }}
    steps:
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/compressed-tensors"
          path: "compressed-tensors"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install compressed-tensors dependencies"
        run: pip3 install -U pip && pip3 install setuptools compressed-tensors/
      - name: "Clean compressed-tensors directory"
        run: rm -r compressed-tensors/
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          fetch-depth: 1
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,onnxruntime,torch,torchvision,transformers]
      - name: Run oneshot tests
        run: |
          pytest tests/sparseml/transformers/obcq -m integration
      - name: Run finetune tests
        run: |
          pytest tests/sparseml/transformers/finetune -m integration


Run Weekly tests

name: Run Weekly tests
on:
  schedule:
    - cron: '0 20 * * 0'
  workflow_dispatch:
jobs:
  run-weekly-tests:
    runs-on: k8s-mle-gpu-12-vcpu-225GB-ram-2-a6000-48G
    env:
      CADENCE: "weekly"
      CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }}
      CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }}
      CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }}
      CLEARML_FILES_HOST:  ${{ secrets.CLEARML_FILES_HOST }}
      CLEARML_API_SECRET_KEY:  ${{ secrets.CLEARML_API_SECRET_KEY }}
    steps:
      - uses: actions/checkout@v2
        with:
          repository: "neuralmagic/compressed-tensors"
          path: "compressed-tensors"
          ref: ${{needs.test-setup.outputs.branch}}
      - name: "⚙ Install compressed-tensors dependencies"
        run: pip3 install -U pip && pip3 install setuptools compressed-tensors/
      - name: "Clean compressed-tensors directory"
        run: rm -r compressed-tensors/
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          fetch-depth: 1
      - name: "⚙ Install dependencies"
        run: pip3 install .[dev,onnxruntime,torch,torchvision,transformers]
      - name: Run oneshot tests
        run: |
          pytest tests/sparseml/transformers/obcq -m integration
      - name: Run finetune tests
        if: always()
        run: |
          pytest tests/sparseml/transformers/finetune -m integration


Test Wheel and Publish

name: Test Wheel and Publish
on:
  workflow_call:
    inputs:
      build-label:
        description: "requested runner label"
        type: string
        required: true
      whl:
        type: string
        required: true
      python:
        type: string
      dev:
        type: string
        required: true
      release:
        type: string
        required: true

jobs:
  test-wheel-and-publish:
    runs-on: ${{ inputs.build-label }}
    steps:
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ inputs.python }}

    - name: Login to s3
      uses: aws-actions/configure-aws-credentials@v2
      with:
        role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }}
        aws-region: us-east-1

    - name: Make directory for wheel
      run: |
          mkdir dist_s3

    - name: Pull from s3
      uses: neuralmagic/nm-actions/actions/s3_pull@main
      with:
        filename: ${{ inputs.whl }}
        dst: dist_s3

    - name: Fetch name of whl
      run: |
          echo "FILENAME=$(echo dist_s3/*.whl)" >> $GITHUB_ENV

    - name: Install whl
      run: |
          pip3 install $FILENAME[dev,onnxruntime,torch,torchvision,transformers]

    - name: Checkout code
      uses: actions/checkout@v3

    - name: Remove src files and run tests
      run: |
            pwd
            rm -rf src
            make test

    - name: Make directory for wheel
      run: |
          mkdir dist_s3

    - name: Pull from s3
      uses: neuralmagic/nm-actions/actions/s3_pull@main
      with:
        filename: ${{ inputs.whl }}
        dst: dist_s3

    - name: Publish Nightly Wheel
      if: ${{ inputs.DEV == 'false' && inputs.RELEASE == 'false'}}
      uses: neuralmagic/nm-actions/actions/publish-whl@main
      with:
        username: ${{ secrets.PYPI_PUBLIC_USER }}
        password: ${{ secrets.PYPI_PUBLIC_AUTH }}
        whl: ./$FILENAME


report-to-testmo

name: report-to-testmo
on:
  workflow_call:
    inputs:
      runs_on:
        description: "runner label specifying instance running the job"
        type: string
        required: true
      run_id:
        description: "run id provided by GHA"
        required: true
        type: string
      build_type:
        description: "build type: nightly or release"
        type: string
        required: true
      testmo_project_id:
        description: "testmo project id"
        type: string
        required: true

jobs:

  BUILDA_AND_REPORT:
    runs-on: ${{ inputs.runs_on }}
    outputs:
      status: ${{ steps.build.outputs.status }}
      commitid: ${{ steps.build.outputs.commitid }}
    permissions:
      id-token: write
      contents: read
    steps:

      - name: repo checkout
        uses: actions/checkout@v3

      - name: s3
        uses: aws-actions/configure-aws-credentials@v2
        with:
          role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }}
          aws-region: us-east-1

      - name: build
        id: build
        run: |
             pwd
             sudo apt-get -y install python3-pip
             pip3 --version
             sudo pip3 install virtualenv
             virtualenv venv
             source venv/bin/activate
             pip install -e .
             if [[ "${{ inputs.build_type }}" = release ]]; then
                 sed -i 's/is_release = False/is_release = True/g' src/${{ github.event.repository.name }}/version.py
             fi
             status=$(make -B build || echo 'FAILED')
             deactivate
             echo "=========== Build log ==========="
             echo "${status}"
             echo "commitid=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
             echo "=========== Build status ==========="
             if [[ "${status}" = "FAILED" ]]; then
                 echo "${{ github.event.repository.name }} build failed"
                 exitCode=1
             else
                 echo "${{ github.event.repository.name }} build success"
                 exitCode=0
             fi
             echo "=========== Generated build ==========="
             ls dist/
             echo "=========== Copy build to S3 ==========="
             aws s3 cp dist/*.whl s3://nm-github-actions/${{ github.event.repository.name }}/
             if [ $? -eq 0 ]; then
                 echo "ok: copied to s3://nm-github-actions/${{ github.event.repository.name }}/"
             else
                 echo "failed: copied to s3://nm-github-actions/${{ github.event.repository.name }}/"
                 exitCode=1
             fi
             if [ ${exitCode} -eq 1 ]; then
                 echo "status=failed" >> "$GITHUB_OUTPUT"
                 cp .github/workflows/result.xml.fail result.xml
             else
                 echo "status=success" >> "$GITHUB_OUTPUT"
                 cp .github/workflows/result.xml.success result.xml
             fi
             oldDate=`date --date='-2 month' +%Y%m%d`
             oldWhl=`(aws s3 ls s3://nm-github-actions/${{ github.event.repository.name }}/ | grep nightly | grep "${oldDate}") || echo "notfound"`
             if [[ "${oldWhl}" != 'notfound' ]]; then
                 for oldwhl in $(echo "${oldWhl}" | awk '{print $4}')
                 do
                     echo "Remove old build ${oldwhl} in S3"
                     aws s3 rm s3://nm-github-actions/${{ github.event.repository.name }}/${oldwhl}
                 done
             fi
             # TESTMO
             echo "node: $(node -v)"
             echo "npm: $(npm -v)"
             echo "Installing testmo cli..."
             sudo npm install -g @testmo/testmo-cli
             export TESTMO_TOKEN=${{ secrets.TESTMO_TEST_TOKEN }}
             TESTMO_URL="https://neuralmagic.testmo.net"
             todaytime=`date +%Y%m%d`
             name="${{ github.event.repository.name }} ${{ inputs.build_type }} ${todaytime} ${GITHUB_SHA:0:7} RunID:${{ inputs.run_id }}"
             echo "========== Build info ==========="
             echo "name: ${name}"
             echo "build: $GITHUB_OUTPUT"
             echo "echo \"GHA job $GITHUB_OUTPUT: https://github.com/neuralmagic/${{ github.event.repository.name }}/actions/runs/${{ inputs.run_id }}\"; exit ${exitCode}" > result.sh
             echo "========== Report to testmo ==========="
             echo "testmo automation:run:submit \\"
             echo "  --instance ${TESTMO_URL} \\"
             echo "  --project-id ${{ inputs.testmo_project_id }} \\"
             echo "  --name ${name} \\"
             echo "  --source ${{ github.event.repository.name }} \\"
             echo "  --results result.xml"
             testmo automation:run:submit \
                         --instance "${TESTMO_URL}" \
                         --project-id ${{ inputs.testmo_project_id }} \
                         --name "${name}" \
                         --source "${{ github.event.repository.name }}" \
                         --results result.xml \
                         -- bash result.sh


Robot server lint/test

# This workflow runs test and lint on branch pushes that touch the
# robot-server project or its dependencies

name: 'Robot server lint/test'

on:
  # Most of the time, we run on pull requests, which lets us handle external PRs
  push:
    paths:
      - 'api/**/*'
      - 'hardware/**/*'
      - 'Makefile'
      - 'shared-data/**/*'
      - 'server-utils/**/*'
      - '!shared-data/js/**/*'
      - 'robot-server/**/*'
      - 'scripts/**/*.mk'
      - 'scripts/**/*.py'
      - '.github/workflows/robot-server-lint-test.yaml'
      - '.github/actions/python/**'
    branches:
      - 'edge'
      - 'release'
      - '*hotfix*'
    tags-ignore:
      - '*'
  pull_request:
    paths:
      - 'api/**/*'
      - 'hardware/**/*'
      - 'Makefile'
      - 'shared-data/**/*'
      - 'server-utils/**/*'
      - '!shared-data/js/**/*'
      - 'robot-server/**/*'
      - 'scripts/**/*.mk'
      - 'scripts/**/*.py'
      - '.github/workflows/robot-server-lint-test.yaml'
      - '.github/actions/python/**'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.ref_name != 'edge' || github.run_id}}-${{ github.ref_type != 'tag' || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  lint-test:
    name: 'robot server package linting and tests'
    timeout-minutes: 40
    runs-on: 'ubuntu-22.04'
    strategy:
      matrix:
        with-ot-hardware: ['true', 'false']
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: 'actions/setup-python@v4'
        with:
          python-version: '3.10'

      - uses: './.github/actions/python/setup'
        with:
          project: 'robot-server'
      - if: ${{ matrix.with-ot-hardware == 'false' }}
        name: Remove OT-3 hardware package
        run: make -C robot-server setup-ot2
      - name: Lint
        run: make -C robot-server lint
      - if: ${{ matrix.with-ot-hardware == 'false' }}
        name: Test without opentrons_hardware
        run: make -C robot-server test-cov test_opts="-m 'not ot3_only'"
      - if: ${{ matrix.with-ot-hardware == 'true' }}
        name: Test with opentrons_hardware
        run: make -C robot-server test-cov
      - name: Ensure assets build
        run: make -C robot-server sdist wheel
      - name: Upload coverage report
        uses: 'codecov/codecov-action@v3'
        with:
          files: ./robot-server/coverage.xml
          flags: robot-server


Update Server test/lint

# This workflow runs tests and lint on non-release branches pushed that alter
# the update-server subpackage

name: 'Update Server test/lint'

on:
  push:
    paths:
      - 'update-server/**/*'
      - 'Makefile'
      - 'scripts/**/*.mk'
      - 'scripts/**/*.py'
      - '.github/workflows/update-server-lint-test.yaml'
      - '.github/actions/python/**'
    branches: # ignore any release-related thing (handled elsewhere)
      - 'edge'
    tags-ignore:
      - '*'
  pull_request:
    types: [opened, reopened, synchronize, ready_for_review]
    paths:
      - 'update-server/**/*'
      - 'Makefile'
      - 'scripts/**/*.mk'
      - 'scripts/**/*.py'
      - '.github/workflows/update-server-lint-test.yaml'
      - '.github/actions/python/**'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.ref_name != 'edge' || github.run_id}}-${{ github.ref_type != 'tag' || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  lint:
    name: 'update server linting'
    timeout-minutes: 10
    runs-on: 'ubuntu-22.04'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: 'actions/setup-python@v4'
        with:
          python-version: '3.10'

      - uses: './.github/actions/python/setup'
        with:
          project: 'update-server'
      - name: Lint
        run: make -C update-server lint
  test:
    name: 'update server package tests'
    timeout-minutes: 10
    needs: [lint]
    runs-on: 'ubuntu-22.04'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: 'actions/setup-python@v4'
        with:
          python-version: '3.10'
      - uses: './.github/actions/python/setup'
        with:
          project: 'update-server'
      - name: Test
        run: make -C update-server test
      - uses: 'codecov/codecov-action@v3'
        with:
          files: ./update-server/coverage.xml
          flags: update-server


API docs build

# This workflow builds API docs when either:
#
# * A commit is tagged like "docs@foo".
#
# * Someone pushes, to any branch, a commit that touches any of:
#   * The parts of the api/ project that are used by the docs.
#   * The docs source themselves;
#   * The CI/makefile tooling used to build the docs.

name: 'API docs build'

on:
  push:
    tags:
      - 'docs@*'
    branches:
      # We don't want to do any filtering based on branch name.
      # But because we specified `tags`, we need to specify `branches` too,
      # or else GitHub will only run this workflow for tag matches
      # and not for path matches.
      - '**'
    paths:
      - 'api/**'
      - '.github/workflows/docs-build.yaml'
      - '.github/actions/python/**'
      - '.github/actions/webstack/deploy-to-sandbox/**'
      - '.github/workflows/utils.js'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.ref_name != 'edge' || github.run_id}}-${{ github.ref_type != 'tag' || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  build:
    name: opentrons documentation build
    runs-on: 'ubuntu-22.04'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: 'actions/setup-python@v3'
        with:
          python-version: '3.10'
      - uses: './.github/actions/python/setup'
        with:
          project: 'api'
      - name: 'set complex environment variables'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars, } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - name: 'Build docs'
        run: |
          make -C api docs
      - name: 'Deploy docs to sandbox'
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.S3_SANDBOX_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SANDBOX_SECRET }}
          AWS_DEFAULT_REGION: us-east-2
        uses: './.github/actions/webstack/deploy-to-sandbox'
        with:
          domain: 'docs.opentrons.com'
          distPath: './api/docs/dist'
          destPrefix: ${{ env.OT_BRANCH }}


JS checks

# This workflow runs checks (types, formatting, style) for our js projects
# which cannot be meaningfully separated

name: 'JS checks'

on:
  pull_request:
    paths:
      - '**/*.js'
      - './.*.js'
      - '**/*.ts'
      - '**/*.tsx'
      - '**/*.json'
      - '**/*.css'
      - '**/*.md'
  push:
    paths:
      - '**/*.js'
      - './.*.js'
      - '**/*.ts'
      - '**/*.tsx'
      - '**/*.json'
      - '**/*.md'
      - '.github/workflows/js-check.yaml'
      - '**/*.css'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.ref_name != 'edge' || github.run_id}}-${{ github.ref_type != 'tag' || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

env:
  CI: true

jobs:
  checks:
    name: 'js checks'
    runs-on: 'ubuntu-22.04'
    timeout-minutes: 20
    steps:
      - uses: 'actions/checkout@v3'
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'set complex environment variables'
        id: 'set-vars'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - name: 'install libudev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.yarn-cache
            ${{ github.workspace }}/.npm-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-
      - name: 'setup-js'
        id: 'setup-js'
        run: |
          npm config set cache ${{ github.workspace }}/.npm-cache
          yarn config set cache-folder ${{ github.workspace }}/.yarn-cache
          make setup-js
        # Use the if to run all the lint checks even of some fail
        shell: bash
      - name: 'typechecks'
        if: always() && steps.setup-js.outcome == 'success'
        run: make check-js
      - name: 'lint js'
        if: always() && steps.setup-js.outcome == 'success'
        run: make lint-js
      - name: 'circular deps'
        if: always() && steps.setup-js.outcome == 'success'
        run: make circular-dependencies-js
      - name: 'lint json'
        if: always() && steps.setup-js.outcome == 'success'
        run: make lint-json
      - name: 'lint css'
        if: always() && steps.setup-js.outcome == 'success'
        run: make lint-css
      - name: 'test scripts'
        run: yarn vitest scripts


API test/lint/deploy

# This workflow runs tests and lint on pull requests that touch the api/
# project or its CI configuration.

name: 'API test/lint/deploy'

on:
  # Most of the time, we run on pull requests, which lets us handle external PRs
  pull_request:
    paths:
      - 'api/**/*'
      - 'Makefile'
      - 'shared-data/**/*'
      - '!shared-data/js/**'
      - 'hardware/**/*'
      - 'scripts/**/*.mk'
      - 'scripts/**/*.py'
      - '.github/workflows/api-test-lint-deploy.yaml'
      - '.github/actions/python/**/*'
      - '.github/workflows/utils.js'
  push:
    paths:
      - 'api/**'
      - 'Makefile'
      - 'shared-data/**/*'
      - '!shared-data/js/**/*'
      - 'hardware/**/*'
      - 'scripts/**/*.mk'
      - 'scripts/**/*.py'
      - '.github/workflows/api-test-lint-deploy.yaml'
      - '.github/actions/python/**/*'
      - '.github/workflows/utils.js'
    branches:
      - 'edge'
      - 'release'
      - '*hotfix*'
    tags:
      - 'v*'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.ref_name != 'edge' || github.run_id}}-${{ github.ref_type != 'tag' || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  lint:
    name: 'opentrons package linting'
    timeout-minutes: 10
    runs-on: 'ubuntu-22.04'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: 'actions/setup-python@v4'
        with:
          python-version: '3.10'

      - uses: './.github/actions/python/setup'
        with:
          project: 'api'
      - name: Lint with opentrons_hardware
        run: make -C api lint
  test:
    name: 'opentrons package tests on ${{ matrix.os }}, python ${{ matrix.python }}'
    timeout-minutes: 30
    needs: [lint]
    strategy:
      matrix:
        os: ['windows-2022', 'ubuntu-22.04', 'macos-latest']
        # TODO(mc, 2022-02-24): expand this matrix to 3.8 and 3.9,
        # preferably in a nightly cronjob on edge or something
        python: ['3.10']
        with-ot-hardware: ['true', 'false']
        exclude:
          - os: 'windows-2022'
            with-ot-hardware: 'true'
          - os: 'macos-latest'
            with-ot-hardware: 'true'
    runs-on: '${{ matrix.os }}'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: 'actions/setup-python@v4'
        with:
          python-version: ${{ matrix.python }}
      - name: 'set complex environment variables'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars, } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - uses: './.github/actions/python/setup'
        with:
          project: 'api'
          python-version: ${{ matrix.python }}
      - if: ${{ matrix.with-ot-hardware == 'false' }}
        name: Remove OT-3 hardware package
        run: make -C api setup-ot2
        env:
          OT_VIRTUALENV_VERSION: ${{ matrix.python }}
      - if: ${{ matrix.with-ot-hardware == 'false' }}
        name: Test without opentrons_hardware
        run: make -C api test-ot2
      - if: ${{ matrix.with-ot-hardware == 'true' }}
        name: Test with opentrons_hardware
        run: make -C api test-cov
      - name: Ensure assets build
        run: make -C api sdist wheel
      - name: Upload coverage report
        uses: 'codecov/codecov-action@v3'
        with:
          files: ./api/coverage.xml
          flags: api

  deploy:
    name: 'deploy opentrons package'
    needs: [test]
    runs-on: 'ubuntu-22.04'
    if: github.event_name == 'push'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: 'actions/setup-python@v4'
        with:
          python-version: '3.10'
      - name: 'set complex environment variables'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars, } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - uses: './.github/actions/python/setup'
        with:
          project: 'api'
      # creds and repository configuration for deploying python wheels
      - if: ${{ !env.OT_TAG }}
        name: 'upload to test pypi'
        uses: './.github/actions/python/pypi-deploy'
        with:
          project: 'api'
          repository_url: 'https://test.pypi.org/legacy/'
          password: '${{ secrets.TEST_PYPI_DEPLOY_TOKEN_OPENTRONS }}'
      - if: startsWith(env.OT_TAG, 'v')
        name: 'upload to real pypi'
        uses: './.github/actions/python/pypi-deploy'
        with:
          project: 'api'
          repository_url: 'https://upload.pypi.org/legacy/'
          password: '${{ secrets.PYPI_DEPLOY_TOKEN_OPENTRONS }}'


shared-data test/lint/deploy

# This workflow runs tests and lint on branch pushes that touch the
# shared-data project or its CI configuration.

name: 'shared-data test/lint/deploy'

on:
  push:
    paths:
      - 'Makefile'
      - 'shared-data/*/**'
      - 'scripts/**/*.mk'
      - 'scripts/**/*.py'
      - '.github/workflows/shared-data-test-lint-deploy.yaml'
      - '.github/actions/python/**/*'
      - '.github/workflows/utils.js'
    branches:
      - 'edge'
      - 'release'
      - '*hotfix*'
    tags:
      - 'v*'
      - 'shared-data*'
      - 'components*'
  pull_request:
    paths:
      - 'Makefile'
      - 'shared-data/*/**'
      - 'scripts/**/*.mk'
      - 'scripts/**/*.py'
      - '.github/workflows/shared-data-test-lint-deploy.yaml'
      - '.github/actions/python/**/*'
      - '.github/workflows/utils.js'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.ref_name != 'edge' || github.run_id}}-${{ github.ref_type != 'tag' || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  python-lint:
    name: 'shared-data package python lint'
    timeout-minutes: 10
    runs-on: 'ubuntu-22.04'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: 'actions/setup-python@v3'
        with:
          python-version: '3.10'

      - uses: './.github/actions/python/setup'
        with:
          project: 'shared-data/python'
      - name: Lint
        run: make -C shared-data/python lint

  python-test:
    name: 'shared-data package python ${{ matrix.python }} tests on ${{ matrix.os }}'
    timeout-minutes: 20
    needs: [python-lint]
    strategy:
      matrix:
        os: ['windows-2022', 'ubuntu-22.04', 'macos-latest']
        # TODO(mc, 2022-02-24): expand this matrix to 3.8 and 3.9,
        # preferably in a nightly cronjob on edge or something
        python: ['3.10']

    runs-on: '${{ matrix.os }}'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      - name: 'install udev for usb-detection'
        if: startsWith(matrix.os, 'ubuntu')
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - uses: 'actions/setup-node@v1'
        with:
          node-version: '18.19.0'
      - uses: 'actions/setup-python@v4'
        with:
          python-version: ${{ matrix.python }}
      - uses: './.github/actions/python/setup'
        with:
          project: 'shared-data/python'
          python-version: ${{ matrix.python }}
      - name: 'set complex environment variables'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars, } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - name: Test
        run: make -C shared-data/python test
      - name: Ensure assets build
        run: make -C shared-data/python sdist wheel
      - name: 'Upload coverage report'
        uses: codecov/codecov-action@v3
        with:
          files: ./shared-data/python/coverage.xml
          flags: shared-data

  js-test:
    name: 'shared-data JS tests'
    runs-on: 'ubuntu-22.04'
    timeout-minutes: 30
    steps:
      - uses: 'actions/checkout@v3'
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install udev'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.yarn-cache
            ${{ github.workspace }}/.npm-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-
      - name: 'js deps'
        run: |
          npm config set cache ./.npm-cache
          yarn config set cache-folder ./.yarn-cache
          make setup-js
      - name: 'run shared-data JS unit tests'
        run: make -C shared-data test-cov
      - name: 'Upload coverage report'
        uses: codecov/codecov-action@v3
        with:
          files: ./coverage/lcov.info
          flags: shared-data

  python-deploy:
    name: 'shared-data package deploy'
    needs: [python-test]
    runs-on: 'ubuntu-22.04'
    if: github.event_name == 'push'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install udev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - uses: 'actions/setup-python@v4'
        with:
          python-version: '3.10'
      - uses: './.github/actions/python/setup'
        with:
          project: 'shared-data/python'
      - name: 'set complex environment variables'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars, } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      # creds and repository configuration for deploying python wheels
      - if: ${{ !env.OT_TAG }}
        name: 'upload to test pypi'
        uses: './.github/actions/python/pypi-deploy'
        with:
          project: 'shared-data/python'
          repository_url: 'https://test.pypi.org/legacy/'
          password: '${{ secrets.TEST_PYPI_DEPLOY_TOKEN_OPENTRONS_SHARED_DATA }}'
      - if: startsWith(env.OT_TAG, 'v')
        name: 'upload to pypi'
        uses: './.github/actions/python/pypi-deploy'
        with:
          project: 'shared-data/python'
          repository_url: 'https://upload.pypi.org/legacy/'
          password: '${{ secrets.PYPI_DEPLOY_TOKEN_OPENTRONS_SHARED_DATA }}'

  publish-switch:
    runs-on: 'ubuntu-latest'
    name: 'Determine whether or not to publish artifacts'
    outputs:
      should_publish: ${{steps.publish-switch.outputs.should_publish}}
    steps:
      - id: publish-switch
        run: |
          echo "Determining whether to publish artifacts for event ${{github.event_type}} and ref ${{github.ref}}"
          if [ "${{ format('{0}', startsWith(github.ref, 'refs/tags/shared-data')) }}" = "true" ] ; then
            echo "Publishing builds for shared-data@ tags"
            echo 'should_publish=true' >> $GITHUB_OUTPUT
          elif [ "${{ format('{0}', startsWith(github.ref, 'refs/tags/components')) }}" = "true" ] ; then
            echo "Publishing builds for components@ tags"
            echo 'should_publish=true' >> $GITHUB_OUTPUT
          else
            echo "No publish for ref ${{github.ref}} and event ${{github.event_type}}"
            echo 'should_publish=false' >> $GITHUB_OUTPUT
          fi

  publish-to-npm:
    name: 'publish shared-data package to npm'
    runs-on: 'ubuntu-latest'
    needs: ['js-test', 'publish-switch']
    if: needs.publish-switch.outputs.should_publish == 'true'
    steps:
      - uses: 'actions/checkout@v3'
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
          registry-url: 'https://registry.npmjs.org'
      - name: 'install udev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.yarn-cache
            ${{ github.workspace }}/.npm-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-
      - name: 'js deps'
        run: |
          npm config set cache ./.npm-cache
          yarn config set cache-folder ./.yarn-cache
          make setup-js
      - name: 'build typescript'
        run: make build-ts
      - name: 'build library'
        run: |
          make -C shared-data lib-js
      # replace package.json stub version number with version from tag
      - name: 'set version number'
        run: |
          npm install -g json
          VERSION_STRING=$(echo ${{ github.ref }} | sed -E 's/refs\/tags\/(components|shared-data)@//')
          json -I -f ./shared-data/package.json -e "this.version=\"$VERSION_STRING\""
          cd ./shared-data
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
          registry-url: 'https://registry.npmjs.org'
      - name: 'publish to npm registry'
        env:
          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
        run: |
          cd ./shared-data && echo "//registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN}" > ./.npmrc && npm publish --access public


App test, build, and deploy

# Run tests, build the app, and deploy it cross platform

name: 'App test, build, and deploy'

on:
  push:
    paths:
      - 'Makefile'
      - 'app/**/*'
      - 'app-shell/**/*'
      - 'app-shell-odd/**/*'
      - 'components/**/*'
      - 'shared-data/**/*'
      - 'discovery-client/**/*'
      - '*.js'
      - 'scripts/**/*'
      - '*.json'
      - 'yarn.lock'
      - '.github/workflows/app-test-build-deploy.yaml'
      - '.github/workflows/utils.js'
    branches:
      - '**'
    tags:
      - 'v*'
      - 'ot3@*'
  pull_request:
    paths:
      - 'Makefile'
      - 'app/**/*'
      - 'app-shell/**/*'
      - 'app-shell-odd/**/*'
      - 'components/**/*'
      - 'shared-data/**/*'
      - 'discovery-client/**/*'
      - '*.js'
      - '*.json'
      - 'yarn.lock'
      - 'scripts/**/*'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.ref_name != 'edge' || github.run_id}}-${{ github.ref_type != 'tag' || github.run_id }}
  cancel-in-progress: true

env:
  CI: true
  _APP_DEPLOY_BUCKET_ROBOTSTACK: builds.opentrons.com
  _APP_DEPLOY_FOLDER_ROBOTSTACK: app
  _APP_DEPLOY_BUCKET_OT3: ot3-development.builds.opentrons.com
  _APP_DEPLOY_FOLDER_OT3: app

jobs:
  js-unit-test:
    # unit tests for the app's view layer (not the node layer)
    runs-on: 'ubuntu-22.04'
    name: 'opentrons app frontend unit tests'
    timeout-minutes: 60
    steps:
      - uses: 'actions/checkout@v3'
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install udev'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'set complex environment variables'
        id: 'set-vars'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.npm-cache/_prebuild
            ${{ github.workspace }}/.yarn-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
      - name: 'setup-js'
        run: |
          npm config set cache ${{ github.workspace }}/.npm-cache
          yarn config set cache-folder ${{ github.workspace }}/.yarn-cache
          make setup-js
      - name: 'test frontend packages'
        run: |
          make -C app test-cov
      - name: 'Upload coverage report'
        uses: codecov/codecov-action@v3
        with:
          files: ./coverage/lcov.info
          flags: app

  backend-unit-test:
    strategy:
      matrix:
        os: ['windows-2022', 'ubuntu-22.04', 'macos-11']
    name: 'opentrons app backend unit tests on ${{matrix.os}}'
    runs-on: ${{ matrix.os }}
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      - name: check make version
        run: make --version
      - name: 'install libudev and libsystemd'
        if: startsWith(matrix.os, 'ubuntu')
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'set complex environment variables'
        id: 'set-vars'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.npm-cache/_prebuild
            ${{ github.workspace }}/.yarn-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
      - name: setup-js
        run: |
          npm config set cache ${{ github.workspace }}/.npm-cache
          yarn config set cache-folder ${{ github.workspace }}/.yarn-cache
          make setup-js
      - name: 'test native(er) packages'
        run: make test-js-internal tests="app-shell/src app-shell-odd/src discovery-client/src" cov_opts="--coverage=true"
      - name: 'Upload coverage report'
        uses: 'codecov/codecov-action@v3'
        with:
          files: ./coverage/lcov.info
          flags: app

  determine-build-type:
    runs-on: 'ubuntu-latest'
    name: 'Determine build variant and type'
    outputs:
      variants: ${{steps.determine-build-type.outputs.variants}}
      type: ${{steps.determine-build-type.outputs.type}}
    steps:
      - id: determine-build-type
        run: |
          echo "Determining build type for event ${{github.event_type}} and ref ${{github.ref}}"
          if [ "${{ format('{0}', github.event_name == 'pull_request') }}" = "true" ] ; then
            echo "No builds for pull requests"
            echo 'variants=[]' >> $GITHUB_OUTPUT
            echo 'type=develop' >> $GITHUB_OUTPUT
          elif [ "${{ format('{0}', startsWith(github.ref, 'refs/tags/ot3')) }}" = "true" ] ; then
            echo "internal-release release builds for ot3 tags"
            echo 'variants=["internal-release"]' >> $GITHUB_OUTPUT
            echo 'type=release' >> $GITHUB_OUTPUT
          elif [ "${{ format('{0}', startsWith(github.ref, 'refs/tags/v')) }}" = "true" ] ; then
            echo "release release builds for v tags"
            echo 'variants=["release"]' >> $GITHUB_OUTPUT
            echo 'type=release' >> $GITHUB_OUTPUT
          elif [ "${{ format('{0}', startsWith(github.ref, 'refs/heads/internal-release')) }}" = "true" ] ; then
            echo "internal-release develop builds for internal-release branches"
            echo 'variants=["internal-release"]' >> $GITHUB_OUTPUT
            echo 'type=develop' >> $GITHUB_OUTPUT
          elif [ "${{ format('{0}', startsWith(github.ref, 'refs/heads/release') || startsWith(github.ref, 'refs/heads/chore_release')) }}" = "true" ] ; then
            echo "Release develop builds for release branches"
            echo 'variants=["release"]' >> $GITHUB_OUTPUT
            echo 'type=develop' >> $GITHUB_OUTPUT
          elif [ "${{ format('{0}', github.ref == 'refs/heads/edge') }}" = "true" ] ; then
            echo "both develop builds for edge"
            echo 'variants=["release", "internal-release"]' >> $GITHUB_OUTPUT
            echo 'type=develop' >> $GITHUB_OUTPUT
          elif [ "${{ format('{0}', endsWith(github.ref, 'app-build-internal')) }}" = "true" ] ; then
            echo "internal-release builds for app-build-internal suffixes"
            echo 'variants=["internal-release"]' >> $GITHUB_OUTPUT
            echo 'type=develop' >> $GITHUB_OUTPUT
          elif [ "${{ format('{0}', endsWith(github.ref, 'app-build')) }}" = "true" ] ; then
            echo "release develop builds for app-build suffixes"
            echo 'variants=["release"]' >> $GITHUB_OUTPUT
            echo 'type=develop' >> $GITHUB_OUTPUT
          elif [ "${{ format('{0}', endsWith(github.ref, 'app-build-both')) }}" = "true" ] ; then
            echo "Both develop builds for app-build-both suffixes"
            echo 'variants=["release", "internal-release"]' >> $GITHUB_OUTPUT
            echo 'type=develop' >> $GITHUB_OUTPUT
          else
            echo "No build for ref ${{github.ref}} and event ${{github.event_type}}"
            echo 'variants=[]' >> $GITHUB_OUTPUT
            echo 'type=develop' >> $GITHUB_OUTPUT
          fi

  build-app:
    needs: [determine-build-type]
    if: needs.determine-build-type.outputs.variants != '[]'
    strategy:
      matrix:
        os: ['windows-2022', 'ubuntu-22.04', 'macos-11']
        variant: ${{fromJSON(needs.determine-build-type.outputs.variants)}}
        target: ['desktop', 'odd']
        exclude:
          - os: 'windows-2022'
            target: 'odd'
          - os: 'macos-11'
            target: 'odd'

    runs-on: ${{ matrix.os }}
    name: 'Build ${{matrix.variant}} ${{matrix.target}} app on ${{matrix.os}}'
    steps:
      - name: 'Get project name for variant'
        id: project
        shell: bash
        run: |
          if [ "${{matrix.variant}}" = "release" ] ; then
             echo "Configuring project, bucket, and folder for robot-stack"
             echo "project=robot-stack" >> $GITHUB_OUTPUT
             echo "bucket=${{env._APP_DEPLOY_BUCKET_ROBOTSTACK}}" >> $GITHUB_OUTPUT
             echo "folder=${{env._APP_DEPLOY_FOLDER_ROBOTSTACK}}" >> $GITHUB_OUTPUT
          else
             echo "Configuring project, bucket, and folder for ot3"
             echo "project=ot3" >> $GITHUB_OUTPUT
             echo "bucket=${{env._APP_DEPLOY_BUCKET_OT3}}" >> $GITHUB_OUTPUT
             echo "folder=${{env._APP_DEPLOY_FOLDER_OT3}}" >> $GITHUB_OUTPUT
          fi
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      - name: check make version
        run: make --version
      - name: 'install libudev and libsystemd'
        if: startsWith(matrix.os, 'ubuntu')
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'set complex environment variables'
        id: 'set-vars'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.npm-cache/_prebuild
            ${{ github.workspace }}/.yarn-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
      - name: setup-js
        run: |
          npm config set cache ${{ github.workspace }}/.npm-cache
          yarn config set cache-folder ${{ github.workspace }}/.yarn-cache
          make setup-js
      # build the desktop app and deploy it
      - name: 'build ${{matrix.variant}} app for ${{ matrix.os }}'
        if: matrix.target == 'desktop'
        timeout-minutes: 60
        env:
          OT_APP_MIXPANEL_ID: ${{ secrets.OT_APP_MIXPANEL_ID }}
          OT_APP_INTERCOM_ID: ${{ secrets.OT_APP_INTERCOM_ID }}
          WIN_CSC_LINK: ${{ secrets.OT_APP_CSC_WINDOWS }}
          WIN_CSC_KEY_PASSWORD: ${{ secrets.OT_APP_CSC_KEY_WINDOWS }}
          CSC_LINK: ${{ secrets.OT_APP_CSC_MACOS }}
          CSC_KEY_PASSWORD: ${{ secrets.OT_APP_CSC_KEY_MACOS }}
          APPLE_ID: ${{ secrets.OT_APP_APPLE_ID }}
          APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.OT_APP_APPLE_ID_PASSWORD }}
          APPLE_TEAM_ID: ${{ secrets.OT_APP_APPLE_TEAM_ID }}
          HOST_PYTHON: python
          OPENTRONS_PROJECT: ${{ steps.project.outputs.project }}
          OT_APP_DEPLOY_BUCKET: ${{ steps.project.outputs.bucket }}
          OT_APP_DEPLOY_FOLDER: ${{ steps.project.outputs.folder }}

        run: |
          make -C app-shell dist-${{ matrix.os }} USE_HARD_LINKS=false

      - name: 'upload github artifact'
        if: matrix.target == 'desktop'
        uses: actions/upload-artifact@v3
        with:
          name: 'opentrons-${{matrix.variant}}-${{ matrix.os }}'
          path: app-shell/dist/publish

      # build the ODD app
      - if: matrix.target == 'odd'
        name: 'build ${{matrix.variant}} ODD app'
        timeout-minutes: 60
        env:
          OPENTRONS_PROJECT: ${{ steps.project.outputs.project }}
        run: |
          make -C app-shell-odd dist-ot3

  deploy-release-app:
    name: 'Deploy built app artifacts to S3'
    runs-on: 'ubuntu-22.04'
    needs:
      ['js-unit-test', 'backend-unit-test', 'build-app', 'determine-build-type']
    if: contains(fromJSON(needs.determine-build-type.outputs.variants), 'release') || contains(fromJSON(needs.determine-build-type.outputs.variants), 'internal-release')
    steps:
      - name: 'download run app builds'
        uses: 'actions/download-artifact@v3'
        with:
          path: ./artifacts
      - name: 'separate release and internal-release artifacts'
        run: |
          for variant in release internal-release ; do
              mkdir to_upload_${variant}
              variant_pattern="./artifacts/opentrons-${variant}-*"
              ls ${variant_pattern}/ 2>/dev/null 1>/dev/null || continue
              echo "Moving ${variant} builds ${variant_pattern}/* to to_upload_${variant}"
              cp ${variant_pattern}/* ./to_upload_${variant}/
              echo "Moved $(ls ./to_upload_${variant})"
          done
      - name: 'configure s3 deploy creds'
        run: |
          aws configure set aws_access_key_id ${{ secrets.S3_OT3_APP_DEPLOY_KEY_ID }} --profile identity
          aws configure set aws_secret_access_key ${{ secrets.S3_OT3_APP_DEPLOY_SECRET }} --profile identity
          aws configure set region us-east-2 --profile identity
          aws configure set output json --profile identity
          aws configure set region us-east-2 --profile deploy
          aws configure set role_arn ${{ secrets.OT_APP_OT3_DEPLOY_ROLE }} --profile deploy
          aws configure set source_profile identity --profile deploy
        shell: bash
      - name: 'deploy release builds to s3'
        run: |
          aws --profile=deploy s3 sync --acl=public-read to_upload_release/ s3://${{ env._APP_DEPLOY_BUCKET_ROBOTSTACK }}/${{ env._APP_DEPLOY_FOLDER_ROBOTSTACK }}
      - name: 'deploy internal-release release builds to s3'
        run: |
          aws s3 --profile=deploy sync --acl=public-read to_upload_internal-release/ s3://${{ env._APP_DEPLOY_BUCKET_OT3 }}/${{ env._APP_DEPLOY_FOLDER_OT3 }}

      - name: 'upload windows artifacts to GH release'
        uses: 'ncipollo/release-action@v1.12.0'
        if: needs.determine-build-type.outputs.type == 'release'
        with:
          allowUpdates: true
          omitBodyDuringUpdate: true
          omitDraftDuringUpdate: true
          omitNameDuringUpdate: true
          omitPrereleaseDuringUpdate: true
          artifacts: ./artifacts/*/*.exe
          artifactContentType: application/vnd.microsoft.portable-executable
      - name: 'upload macos artifacts to GH release'
        uses: 'ncipollo/release-action@v1.12.0'
        if: needs.determine-build-type.outputs.type == 'release'
        with:
          allowUpdates: true
          omitBodyDuringUpdate: true
          omitDraftDuringUpdate: true
          omitNameDuringUpdate: true
          omitPrereleaseDuringUpdate: true
          artifacts: ./artifacts/*/*.dmg
          artifactContentType: application/octet-stream
      - name: 'upload linux artifacts to GH release'
        uses: 'ncipollo/release-action@v1.12.0'
        if: needs.determine-build-type.outputs.type == 'release'
        with:
          allowUpdates: true
          omitBodyDuringUpdate: true
          omitDraftDuringUpdate: true
          omitNameDuringUpdate: true
          omitPrereleaseDuringUpdate: true
          artifacts: ./artifacts/*/*.AppImage
          artifactContentType: application/octet-stream
      - name: 'detect build data for notification'
        id: names
        shell: bash
        run: |
          for variant in release internal-release ; do
              dir=./to_upload_${variant}
              echo "Checking for ${variant} builds in ${dir}"
              ls ${dir}/Opentrons* || continue
              _windows_build=$(basename $(ls ./to_upload_${variant}/Opentrons*.exe))
              _mac_build=$(basename $(ls ./to_upload_${variant}/Opentrons*.dmg))
              _linux_build=$(basename $(ls ./to_upload_${variant}/Opentrons*.AppImage))
              echo "windows-${variant}=$_windows_build">>$GITHUB_OUTPUT
              echo "mac-${variant}=$_mac_build">>$GITHUB_OUTPUT
              echo "linux-${variant}=$_linux_build">>$GITHUB_OUTPUT
          done
      - name: 'slack notify internal-release'
        uses: slackapi/slack-github-action@v1.14.0
        if: contains(fromJSON(needs.determine-build-type.outputs.variants), 'internal-release')
        with:
          payload: '{"branch_or_tag":"${{ github.ref_name }}","build_type":"${{ needs.determine-build-type.outputs.type }}", "gh_linkback":"https://github.com/Opentrons/opentrons/tree/${{ github.ref_name }}", "windows_build":"${{ env._ACCESS_URL }}/${{steps.names.outputs.windows-internal-release}}", "mac_build":"${{ env._ACCESS_URL }}/${{steps.names.outputs.mac-internal-release}}", "linux_build":"${{ env._ACCESS_URL }}/${{steps.names.outputs.linux-internal-release}}"}'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.OT_APP_OT3_SLACK_NOTIFICATION_WEBHOOK_URL }}
          _ACCESS_URL: https://${{env._APP_DEPLOY_BUCKET_OT3}}/${{env._APP_DEPLOY_FOLDER_OT3}}
      - name: 'slack notify release'
        uses: slackapi/slack-github-action@v1.14.0
        if: contains(fromJSON(needs.determine-build-type.outputs.variants), 'release')
        with:
          payload: '{"branch_or_tag":"${{ github.ref_name }}","build_type":"${{ needs.determine-build-type.outputs.type }}", "gh_linkback":"https://github.com/Opentrons/opentrons/tree/${{ github.ref_name }}", "windows_build":"${{ env._ACCESS_URL }}/${{steps.names.outputs.windows-release}}", "mac_build":"${{ env._ACCESS_URL }}/${{steps.names.outputs.mac-release}}", "linux_build":"${{ env._ACCESS_URL }}/${{steps.names.outputs.linux-release}}"}'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.OT_APP_ROBOTSTACK_SLACK_NOTIFICATION_WEBHOOK_URL }}
          _ACCESS_URL: https://${{env._APP_DEPLOY_BUCKET_ROBOTSTACK}}/${{env._APP_DEPLOY_FOLDER_ROBOTSTACK}}

      - name: 'pull repo for scripts'
        uses: 'actions/checkout@v3'
        with:
          path: ./monorepo
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          cd ./monorepo
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install udev'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'set complex environment variables'
        id: 'set-vars'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars } = require(`${process.env.GITHUB_WORKSPACE}/monorepo/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.npm-cache/_prebuild
            ${{ github.workspace }}/.yarn-cache
      - name: 'setup-js'
        run: |
          npm config set cache ${{ github.workspace }}/.npm-cache
          yarn config set cache-folder ${{ github.workspace }}/.yarn-cache
          cd monorepo
          make setup-js
      - name: 'update internal-releases releases.json'
        if: needs.determine-build-type.outputs.type == 'release' && contains(fromJSON(needs.determine-build-type.outputs.variants), 'internal-release')
        run: |
          aws --profile=deploy s3 cp s3://${{ env._APP_DEPLOY_BUCKET_OT3 }}/${{ env._APP_DEPLOY_FOLDER_OT3 }}/releases.json ./to_upload_internal-release/releases.json
          node ./monorepo/scripts/update-releases-json ./to_upload_internal-release/releases.json ot3 ./to_upload_internal-release https://ot3-development.builds.opentrons.com/app/
          aws --profile=deploy s3 cp ./to_upload_internal-release/releases.json s3://${{ env._APP_DEPLOY_BUCKET_OT3 }}/${{ env._APP_DEPLOY_FOLDER_OT3 }}/releases.json

      - name: 'update release releases.json'
        if: needs.determine-build-type.outputs.type == 'release' && contains(fromJSON(needs.determine-build-type.outputs.variants), 'release')
        run: |
          aws --profile=deploy s3 cp s3://${{ env._APP_DEPLOY_BUCKET_ROBOTSTACK }}/${{ env._APP_DEPLOY_FOLDER_ROBOTSTACK }}/releases.json ./to_upload_release/releases.json
          node ./monorepo/scripts/update-releases-json ./to_upload_release/releases.json robot-stack ./to_upload_release https://builds.opentrons.com/app/
          aws --profile=deploy s3 cp ./to_upload_release/releases.json s3://${{ env._APP_DEPLOY_BUCKET_ROBOTSTACK }}/${{ env._APP_DEPLOY_FOLDER_ROBOTSTACK }}/releases.json


PD test, build, and deploy

# Run tests, build PD, and deploy it to sandbox

name: 'PD test, build, and deploy'

on:
  pull_request:
    paths:
      - 'protocol-designer/**'
      - 'step-generation/**'
      - 'shared-data/**'
      - 'components/**'
      - 'package.json'
      - '.github/workflows/pd-test-build-deploy.yaml'
  push:
    paths:
      - 'protocol-designer/**'
      - 'step-generation/**'
      - 'shared-data/**'
      - 'components/**'
      - 'package.json'
      - '.github/workflows/pd-test-build-deploy.yaml'
    branches:
      - '**'
    tags:
      - 'protocol-designer*'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.ref_name != 'edge' || github.run_id}}-${{ github.ref_type != 'tag' || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

env:
  CI: true

jobs:
  js-unit-test:
    name: 'protocol designer unit tests'
    runs-on: 'ubuntu-22.04'
    timeout-minutes: 30
    steps:
      - uses: 'actions/checkout@v3'
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install udev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'cache yarn cache'
        uses: actions/cache@v2
        with:
          path: |
            ${{ github.workspace }}/.yarn-cache
            ${{ github.workspace }}/.npm-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-
      - name: 'setup-js'
        run: |
          npm config set cache ./.npm-cache
          yarn config set cache-folder ./.yarn-cache
          make setup-js
      - name: 'run PD unit tests'
        run: make -C protocol-designer test-cov
      - name: 'Upload coverage report'
        uses: codecov/codecov-action@v3
        with:
          files: ./coverage/lcov.info
          flags: protocol-designer

  e2e-test:
    name: 'pd e2e tests'
    needs: ['js-unit-test']
    timeout-minutes: 30
    strategy:
      matrix:
        os: ['ubuntu-22.04']
    runs-on: '${{ matrix.os }}'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install udev for usb-detection'
        if: startsWith(matrix.os, 'ubuntu')
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.yarn-cache
            ${{ github.workspace }}/.npm-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-
      - name: 'setup-js'
        run: |
          npm config set cache ./.npm-cache
          yarn config set cache-folder ./.yarn-cache
          make setup-js
      - name: 'test-e2e'
        run: make -C protocol-designer test-e2e
  build-pd:
    name: 'build protocol designer artifact'
    needs: ['js-unit-test']
    runs-on: 'ubuntu-22.04'
    if: github.event_name != 'pull_request'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install udev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.yarn-cache
            ${{ github.workspace }}/.npm-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
      - name: 'setup-js'
        run: |
          npm config set cache ./.npm-cache
          yarn config set cache-folder ./.yarn-cache
          make setup-js
      - name: 'build PD'
        env:
          OT_PD_MIXPANEL_ID: ${{ secrets.OT_PD_MIXPANEL_ID }}
          OT_PD_MIXPANEL_DEV_ID: ${{ secrets.OT_PD_MIXPANEL_DEV_ID }}
        run: |
          make -C protocol-designer
      - name: 'upload github artifact'
        uses: actions/upload-artifact@v3
        with:
          name: 'pd-artifact'
          path: protocol-designer/dist
  deploy-pd:
    name: 'deploy PD artifact to S3'
    runs-on: 'ubuntu-22.04'
    needs: ['js-unit-test', 'build-pd']
    if: github.event_name != 'pull_request'
    steps:
      - uses: 'actions/checkout@v3'
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install udev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'set complex environment variables'
        id: 'set-vars'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - name: 'download PD build'
        uses: 'actions/download-artifact@v3'
        with:
          name: pd-artifact
          path: ./dist
      - name: 'configure ot3 s3 deploy creds and deploy'
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.PD_S3_SANDBOX_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.PD_S3_SANDBOX_SECRET }}
          AWS_DEFAULT_REGION: us-east-1
        run: |
          aws configure set aws_access_key_id ${{ secrets.PD_S3_SANDBOX_KEY_ID }} --profile identity
          aws configure set aws_secret_access_key ${{ secrets.PD_S3_SANDBOX_SECRET }} --profile identity
          aws configure set region us-east-2 --profile identity
          aws configure set output json --profile identity
          aws configure set region us-east-2 --profile deploy
          aws configure set role_arn ${{ secrets.OT_PD_DEPLOY_ROLE }} --profile deploy
          aws configure set source_profile identity --profile deploy
          aws s3 sync ./dist s3://sandbox.designer.opentrons.com/${{ env.OT_BRANCH }} --acl=public-read --profile=deploy
        shell: bash


Labware Library test, build, and deploy

# Run tests, build labware library, and deploy it to sandbox

name: 'Labware Library test, build, and deploy'

on:
  pull_request:
    paths:
      - 'labware-library/**'
      - 'shared-data/labware/**'
      - 'components/**'
      - 'package.json'
      - '.github/workflows/ll-test-build-deploy.yaml'
      - '.github/actions/webstack/deploy-to-sandbox/**'
      - '.github/workflows/utils.js'
  push:
    paths:
      - 'labware-library/**'
      - 'shared-data/labware/**'
      - 'components/**'
      - 'package.json'
      - '.github/workflows/ll-test-build-deploy.yaml'
    branches:
      - '**'
    tags:
      - 'labware-library*'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.ref_name != 'edge' || github.run_id}}-${{ github.ref_type != 'tag' || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

env:
  CI: true

jobs:
  js-unit-test:
    name: 'labware library unit tests'
    timeout-minutes: 20
    runs-on: 'ubuntu-20.04'
    steps:
      - uses: 'actions/checkout@v3'
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - name: 'install libudev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.yarn-cache
            ${{ github.workspace }}/.npm-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-
      - name: 'setup-js'
        run: |
          npm config set cache ./.npm-cache
          yarn config set cache-folder ./.yarn-cache
          make setup-js
      - name: 'run labware library unit tests'
        run: make -C labware-library test-cov
      - name: 'Upload coverage report'
        uses: codecov/codecov-action@v3
        with:
          files: ./coverage/lcov.info
          flags: labware-library

  e2e-test:
    name: 'labware library e2e tests'
    needs: ['js-unit-test']
    timeout-minutes: 30
    runs-on: 'ubuntu-20.04'
    steps:
      - uses: 'actions/checkout@v3'
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install libudev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.yarn-cache
            ${{ github.workspace }}/.npm-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-
      - name: 'setup-js'
        run: |
          npm config set cache ./.npm-cache
          yarn config set cache-folder ./.yarn-cache
          make setup-js
      - name: 'test-e2e'
        env:
          OT_LL_MIXPANEL_ID: ${{ secrets.OT_LL_MIXPANEL_ID }}
          OT_LL_MIXPANEL_DEV_ID: ${{ secrets.OT_LL_MIXPANEL_DEV_ID }}
        run: make -C labware-library test-e2e
  build-ll:
    name: 'build labware library artifact'
    needs: ['js-unit-test']
    timeout-minutes: 30
    runs-on: 'ubuntu-20.04'
    if: github.event_name != 'pull_request'
    steps:
      - uses: 'actions/checkout@v3'
        with:
          fetch-depth: 0
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install libudev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'cache yarn cache'
        uses: actions/cache@v3
        with:
          path: |
            ${{ github.workspace }}/.yarn-cache
            ${{ github.workspace }}/.npm-cache
          key: js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
          restore-keys: |
            js-${{ secrets.GH_CACHE_VERSION }}-${{ runner.os }}-yarn-
      - name: 'setup-js'
        run: |
          npm config set cache ./.npm-cache
          yarn config set cache-folder ./.yarn-cache
          make setup-js
      - name: 'build LL'
        env:
          OT_LL_MIXPANEL_ID: ${{ secrets.OT_LL_MIXPANEL_ID }}
          OT_LL_MIXPANEL_DEV_ID: ${{ secrets.OT_LL_MIXPANEL_DEV_ID }}
        run: |
          make -C labware-library
      - name: 'upload github artifact'
        uses: actions/upload-artifact@v3
        with:
          name: 'll-artifact'
          path: labware-library/dist
  deploy-ll:
    name: 'deploy LL artifact to S3'
    runs-on: 'ubuntu-20.04'
    needs: ['js-unit-test', 'e2e-test', 'build-ll']
    if: github.event_name != 'pull_request'
    steps:
      - uses: 'actions/checkout@v3'
      # https://github.com/actions/checkout/issues/290
      - name: 'Fix actions/checkout odd handling of tags'
        if: startsWith(github.ref, 'refs/tags')
        run: |
          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
          git checkout ${{ github.ref }}
      - uses: 'actions/setup-node@v3'
        with:
          node-version: '18.19.0'
      - name: 'install udev for usb-detection'
        run: |
          # WORKAROUND: Remove microsoft debian repo due to https://github.com/microsoft/linux-package-repositories/issues/130. Remove line below after it is resolved
          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
          sudo apt-get update && sudo apt-get install libudev-dev
      - name: 'set complex environment variables'
        id: 'set-vars'
        uses: actions/github-script@v6
        with:
          script: |
            const { buildComplexEnvVars } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/utils.js`)
            buildComplexEnvVars(core, context)
      - name: 'download LL build'
        uses: 'actions/download-artifact@v3'
        with:
          name: ll-artifact
          path: ./dist
      - name: 'deploy builds to s3'
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.S3_SANDBOX_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SANDBOX_SECRET }}
          AWS_DEFAULT_REGION: us-east-2
        uses: './.github/actions/webstack/deploy-to-sandbox'
        with:
          domain: 'labware.opentrons.com'
          distPath: './dist'
          destPrefix: ${{ env.OT_BRANCH }}