Container

name: Container

on: [push, pull_request, workflow_dispatch]

jobs:
    build-rootfs:
        name: Build cbuildrt rootfs
        runs-on: ubuntu-24.04
        container: fedora:40
        steps:
          - name: Install prerequisites
            run: |
                dnf -y install dnf-plugins-core wget docker skopeo rsync openssl stunnel
                dnf -y copr enable ganto/umoci
                dnf -y install umoci
          - name: Install rsync-ssl
            run: |
                wget https://raw.githubusercontent.com/WayneD/rsync/c3f7414c450faaf6a8281cc4a4403529aeb7d859/rsync-ssl
                sudo install -D rsync-ssl /usr/local/bin/rsync-ssl
          - name: Checkout
            uses: actions/checkout@v2
          - name: Build container
            run: |
                docker build -t managarm-buildenv docker/
          - name: Build OCI runtime bundle
            run: |
                set -x

                skopeo copy docker-daemon:managarm-buildenv:latest oci:buildenv-image:latest
                umoci unpack --rootless --image buildenv-image:latest buildenv-bundle/
                rm -rf buildenv-image/
          - name: Finalize cbuildrt rootfs
            run: |
                set -x

                # Perform some post-processing.
                mkdir buildenv-bundle/rootfs/dev/pts
                mkdir buildenv-bundle/rootfs/dev/shm
                ln -s /proc/self/fd/0 buildenv-bundle/rootfs/dev/stdin
                ln -s /proc/self/fd/1 buildenv-bundle/rootfs/dev/stdout
                ln -s /proc/self/fd/2 buildenv-bundle/rootfs/dev/stderr
                ln -s /proc/self/fd buildenv-bundle/rootfs/dev/fd
                touch buildenv-bundle/rootfs/dev/{full,null,zero}
                touch buildenv-bundle/rootfs/dev/tty
                touch buildenv-bundle/rootfs/dev/{random,urandom}

                # Make directories for bind mounts.
                mkdir -p buildenv-bundle/rootfs/var/lib/managarm-buildenv/{src,build}

                # Pack the resulting rootfs.
                tar -czf managarm-buildenv.tar.gz -C buildenv-bundle/ rootfs
          - name: Upload rootfs to mirrors.managarm.org
            if: github.repository_owner == 'managarm' && github.ref == 'refs/heads/master'
            env:
                RSYNC_PASSWORD: ${{ secrets.RSYNC_CBUILDRT_ROOTFS_PASSWD }}
            run: |
                sha256sum managarm-buildenv.tar.gz > managarm-buildenv.sum
                rsync-ssl -a --delete managarm-buildenv.{sum,tar.gz} buildenv@rsync.managarm.org::wbuildenv/


Check if PR has fixups

name: Check if PR has fixups

on:
  pull_request:

jobs:
  find-fixups:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - run: |
          set -e -o pipefail

          git rev-list 'HEAD^..HEAD' \
            | while read -r COMMIT; do
            printf 'pondering commit %s\n' "$COMMIT"
            git show -s "$COMMIT"
            if git show -s --format='%s' "$COMMIT" | grep -q '^fixup! '; then
              exit 1
            fi
          done
          printf 'ready to merge!\n'


Sanity Checks

name: Sanity Checks

on: [push, pull_request]

jobs:
    compute-graph:
        name: Compute graph
        runs-on: ubuntu-20.04
        steps:
          - name: Install prerequisites
            # Note: the default jsonschema is too old.
            #       xbstrap should fix this by demanding a recent version.
            run: |
                sudo apt-get install jq
                sudo pip3 install setuptools
                sudo pip3 install -U jsonschema
                sudo pip3 install xbstrap
          - name: Prepare directories
            run: |
                mkdir src/
                mkdir build/
          - name: Checkout
            uses: actions/checkout@v2
            with:
              path: src/
          - name: Compute xbstrap-pipeline graph
            run: |
                set -x
                cp ../src/ci/pipeline.yml .
                echo '{"pkg_management": {"format": "xbps"}}' > bootstrap-site.yml
                xbstrap init ../src
                xbstrap rolling-versions fetch
                xbstrap rolling-versions determine --json | jq '{"commits":map_values({"rolling_id":.})}' > new-bootstrap-commits.yml
                # Ensure that the previs xbstrap command does not see bootstrap-commits.yml
                # until the file is fully written.
                mv new-bootstrap-commits.yml ../src/bootstrap-commits.yml
                xbstrap-pipeline compute-graph --artifact
            working-directory: build/


publish-docs

name: publish-docs

on:
  push:
    # Publish `main` as latest, and when pushes are done to branches with "v-doc" prefix
    branches:
      - main

permissions:
  contents: write

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # fetch all commits/branches
      - uses: actions/setup-python@v5
        with:
          python-version: 3.x
      - uses: actions/cache@v4
        with:
          key: ${{ github.ref }}
          path: .cache
      - name: Install Python dependencies
        run: pip install -r ./mkdocs-requirements.txt
      - name: Configure git user
        run: |
          git config --local user.email "github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"

      - name: Deploy docs
        run: |
          python --version
          # Generate the content into the docs folder
          code/genSite.sh
          mike deploy --push --update-aliases main latest
          mike set-default latest


release apps

name: "release apps"

on:
  # TODO change to pull request
  push:
    branches:
    - main
    paths:
    - 'edge/publish-apps.json'

jobs:
  set-tag:
    runs-on: ubuntu-latest
    outputs:
      tag_value: ${{ steps.read_tag.outputs.tag_value }}
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: read input tag
      id: read_tag
      run: |
        echo "tag_value=`./scripts/read-input-tag.sh`" >> "$GITHUB_OUTPUT"
    - name: check tag existing
      uses: mukunku/tag-exists-action@v1.0.0
      id: checkTag
      with:
        tag: ${{steps.read_tag.outputs.tag_value}}
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    - name: create tag
      if: ${{ steps.checkTag.outputs.exists != 'true' }}
      run: ./scripts/create-tag.sh

  publish-apps:
    needs: set-tag
    runs-on: ubuntu-latest
    steps:
    - name: checkout specific tag version
      uses: actions/checkout@v2
      with:
        ref: ${{needs.set-tag.outputs.tag_value}}
    - name: install jq
      run: sudo apt-get install jq
    - name: Install Node
      uses: actions/setup-node@v1
      with:
        node-version: '16'
    - uses: actions/setup-python@v2
      with:
        python-version: '3.x'
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_SAR }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_SAR }}
        aws-region: us-east-1
    - name: SAM Package
      run: ./scripts/sam-package.sh
    - name: SAM Publish
      run: ./scripts/sam-publish.sh


website

name: website

on:
  push:
    branches:
      - main
    paths:
      - docs/**/*
      - docs-ig/**/*
      - website/**/*

jobs:
  deploy:
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repository and submodules
        uses: actions/checkout@v2
        with:
          submodules: recursive

      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.8'

      - name: Install dependencies
        run: |
          python3 -m pip install mkdocs            # install mkdocs
          python3 -m pip install mkdocs-material   # install material theme

      - name: Build mkdocs
        run: |
          mkdocs build -f ./docs-ig/mkdocs.en.yml
          mkdocs build -f ./docs-ig/mkdocs.zh.yml
          cp -av ./docs-ig/index.html ./docs-ig/site

      - name: Deploy IG
        uses: peaceiris/actions-gh-pages@v3.7.3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./docs-ig/site

      - name: Setup Hugo
        uses: peaceiris/actions-hugo@v2
        with:
          hugo-version: '0.79.1'
          extended: true

      - name: Build Hugo
        run: |
          cd website
          hugo --minify

      - name: Deploy workshop
        uses: peaceiris/actions-gh-pages@v3.7.3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./website/public
          destination_dir: workshop


Update Dependabot config

name: "Update Dependabot config"
on:
  push:
    branches:
    - main
    paths:
    - 'edge/**'

jobs:
  update-dependabot-config:
    name: update-dependabot-config
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: install python
      uses: actions/setup-python@v2
      with:
        python-version: '3.x'
    - name: update-dependabot-config
      run: ./scripts/update-dependabot-config.sh
    - name: Create Pull Request
      id: cpr
      uses: peter-evans/create-pull-request@v3
      with:
        delete-branch: true
        title: 'Update dependabot config'


pull request check

name: "pull request check"

on:
  pull_request:
    types: [opened, reopened, edited, synchronize]
permissions:
  contents: write
  id-token: write # required to use OIDC authentication

jobs:
  git-secrets:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: recursive
      - name: Checkout awslabs/git-secrets
        uses: actions/checkout@v2
        with:
          repository: awslabs/git-secrets
          ref: master
          path: git-secrets
      - name: Install git-secrets
        run: cd git-secrets && sudo make install && cd ..
      - name: Run git-secrets
        run: |
          git-secrets --register-aws
          git-secrets --scan

  opensource-license-review:
    needs: git-secrets
    runs-on: ubuntu-latest
    steps:
      - name: 'Checkout Repository'
        uses: actions/checkout@v3
      - name: Dependency Review
        uses: actions/dependency-review-action@v3
        with:
          fail-on-severity: moderate
          # Use comma-separated names to pass list arguments: LGPL-2.0,
          deny-licenses: BSD-2-Clause

  build:
    needs: opensource-license-review
    runs-on: ubuntu-latest
    env:
      CI: "false"
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
      - name: Use Node.js 18.x
        uses: actions/setup-node@v3
        with:
          node-version: '18.x'
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: '3.8'
      - name: Pre Build
        run: |
          npm install -g aws-cdk
          cdk --version
      - name: build
        run: |
          cd ${{ github.workspace }}/portal
          npm install --legacy-peer-deps
          npm run build
          cd ${{ github.workspace }}/templates/console/deployment
          ./pre-build.sh
          cd ${{ github.workspace }}/templates/console/source
          npm install --legacy-peer-deps
          cdk synth
          ls -LR ${{ github.workspace }}/templates/console/deployment/
      - name: archive production artifacts
        uses: actions/upload-artifact@v3
        with:
          name: build-artifacts
          path: |
              .git/**/*
              deployment/**/*
              docs/**/*
              templates/console/deployment/**/*
              deploy_vtag.yml
              CHANGELOG.md
              .cfnnagignore
              .cfnnag_global_suppress_list
              .cfnnag_*

  cfn-nag-scan:
    needs: build
    runs-on: ubuntu-latest
    env:
      CI: "false"
    steps:
      - name: Download all workflow run artifacts
        uses: actions/download-artifact@v3
      - name: Setup Ruby 2.6
        uses: ruby/setup-ruby@477b21f02be01bcb8030d50f37cfec92bfa615b6
        with:
          ruby-version: 2.6
      - name: cfn_nag
        run: |
          # install Ruby
          # sudo apt-get install ruby-full
          # ls -R
          gem install cfn-nag
          cfn_nag -v
          echo "CloudFormation template scanning utility installation complete `date`"
          echo "Starting CloudFormation template scanning `date` in `pwd`"
          cd build-artifacts/
          export CFN_NAG_OPTS="--template-pattern ..*.template.json --print-suppression --fail-on-warnings"
          if [ -f ".cfnnag_global_suppress_list" ]; then export CFN_NAG_OPTS="$CFN_NAG_OPTS -b .cfnnag_global_suppress_list"; fi
          set -x; cfn_nag_scan $CFN_NAG_OPTS --input-path templates/console/deployment/global-s3-assets/
          # cfn_nag_scan --input-path .
          # echo "Completed CloudFormation template scanning `date`"

  semgrep-scan:
    needs: opensource-license-review
    runs-on: ubuntu-latest
    container:
      # A Docker image with Semgrep installed. Do not change this.
      image: returntocorp/semgrep
    # Skip any PR created by dependabot to avoid permission issues:
    if: (github.actor != 'dependabot[bot]')
    permissions:
      actions: read
      contents: read
      security-events: write
    steps:
      # Fetch project source with GitHub Actions Checkout.
      - uses: actions/checkout@v3
      # Run the "semgrep ci" command on the command line of the docker image.
      - run: semgrep ci --config auto --sarif --output=semgrep.sarif
      - name: Upload SARIF file for GitHub Advanced Security Dashboard
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: semgrep.sarif
        if: always()

  viperlight:
    runs-on: ubuntu-latest
    needs: opensource-license-review
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Viperlight Scan
        run: |
          wget -q https://viperlight-scanner.s3.amazonaws.com/latest/.viperlightrc
          wget -q https://viperlight-scanner.s3.amazonaws.com/latest/viperlight.zip
          unzip -q viperlight.zip -d viperlight
          rm -r ./viperlight.zip
          cd viperlight
          ./install.sh
          viperlight --version
          echo "Content scanning utility installation complete `date`"
          echo "Starting content scanning `date` in `pwd`"
          cd ${{ github.workspace }}/templates/console/source
          viperlight scan -m files-contents -m files-aws -m files-binary -m files-entropy -m files-secrets
          echo "Completed content scanning `date`"

  cdk-unit-test:
    runs-on: ubuntu-latest
    needs: opensource-license-review
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Use Node.js 18.x
        uses: actions/setup-node@v3
        with:
          node-version: '18.x'
      - name: Run Typescript Unit Test
        run: |
          cd ${{ github.workspace }}/templates/console/source
          npm install --legacy-peer-deps
          npm run test
      - name: Sed lcov report
        if: always()
        run: |
          cd ${{ github.workspace }}/templates/console/source/coverage
          sed -i 's/source\/lib\//\/github\/workspace\/templates\/console\/source\/lib\//g' lcov.info
          sed -i 's/source\/test\//\/github\/workspace\/templates\/console\/source\/test\//g' lcov.info
      - name: archive coverage report
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: cdk-coverage-report
          path: |
            templates/console/source/coverage/lcov.info

  ssl-lambda-unit-test:
    runs-on: ubuntu-latest
    needs: opensource-license-review
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Run Saas Lambda Python Test
        env:
          AWS_DEFAULT_REGION: us-east-1
        run: |
          cd ${{ github.workspace }}/templates/console/source/lambda/ssl_for_saas/
          # pip install pytest
          pip install poetry
          poetry install
          export PYTHONPATH=../ssl_for_saas
          echo "PYTHONPATH=../ssl_for_saas" >> $GITHUB_ENV
          poetry run pytest --cov --cov-config=.coveragerc -cov-report=term-missing --cov-report="xml:./ssl-for-saas.coverage.xml" test

  config-deployer-unit-test:
    runs-on: ubuntu-latest
    needs: opensource-license-review
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Run Config and Deployer Lambda Python Test
        env:
          AWS_DEFAULT_REGION: us-east-1
        run: |
          cd ${{ github.workspace }}/templates/console/deployment
          ./run-lambda-ut.sh
      - name: Sed coverage report files
        if: always()
        run: |
          cd ${{ github.workspace }}/templates/console/source/all-coverages

  monitoring-unit-test:
    runs-on: ubuntu-latest
    needs: opensource-license-review
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Run Monitoring Lambda Python Test
        env:
          AWS_DEFAULT_REGION: us-east-1
        run: |
          cd ${{ github.workspace }}/templates/console/deployment
          ./run-unit-tests.sh
      - name: Sed coverage report files
        if: always()
        run: |
          cd ${{ github.workspace }}/templates/console/deployment/monitoring-coverages


pull request merge/push check

name: "pull request merge/push check"

on:
  push:
    branches:
      - 'main'
permissions:
  contents: write
  id-token: write # required to use OIDC authentication

jobs:
  git-secrets:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: recursive
      - name: Checkout awslabs/git-secrets
        uses: actions/checkout@v2
        with:
          repository: awslabs/git-secrets
          ref: master
          path: git-secrets
      - name: Install git-secrets
        run: cd git-secrets && sudo make install && cd ..
      - name: Run git-secrets
        run: |
          git-secrets --register-aws
          git-secrets --scan

  build:
    needs: git-secrets
    runs-on: ubuntu-latest
    env:
      CI: "false"
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}
      - name: Use Node.js 18.x
        uses: actions/setup-node@v3
        with:
          node-version: '18.x'
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: '3.8'
      - name: Pre Build
        run: |
          npm install -g aws-cdk
          cdk --version
          BSS_IMAGE_ASSET_REPOSITORY_NAME=$(echo "${{ vars.SOLUTION_NAME }}" | tr '[:upper:]' '[:lower:]' | tr ' ' '-')
          echo "BSS_IMAGE_ASSET_REPOSITORY_NAME=${BSS_IMAGE_ASSET_REPOSITORY_NAME}" >> $GITHUB_ENV
          BUILD_VERSION=v2.0.0_${{ github.run_number }}
          echo "BUILD_VERSION=${BUILD_VERSION}" >> $GITHUB_ENV
          if [ $(git tag -l "$BUILD_VERSION") ]; then
            git tag -d ${BUILD_VERSION}
          fi
          echo "CN_ASSETS=cn/" >> $GITHUB_ENV
          echo "GLOBAL_ASSETS=custom-domain/" >> $GITHUB_ENV
      - name: build
        run: |
          chmod +x ./templates/console/deployment/build-s3-dist.sh
          ${{ github.workspace }}/templates/console/deployment/build-s3-dist.sh ${{ secrets.DIST_OUTPUT_BUCKET }} ${{ vars.SOLUTION_NAME }} $BUILD_VERSION ${{ secrets.ECR }} ${{ secrets.ECR_CN }} ${{ vars.AWS_ASSET_ACCOUNT_ID }} ${{ vars.REGIONS }} ${{ secrets.AWS_ASSET_PUBLISH_ROLE }}
          mkdir -p ${{ github.workspace }}/templates/console/deployment/open-source/ && touch ./templates/console/deployment/open-source/.empty
          # cd edges/cdk/deployment
          # ./build-s3-dist.sh ${{ secrets.DIST_OUTPUT_BUCKET }} ${{ vars.SOLUTION_NAME }} $BUILD_VERSION
          # cp -r ${{ github.workspace }}/edges/cdk/deployment/global-s3-assets/* $${{ github.workspace }}/templates/console/deployment/global-s3-assets/custom-domain/
          ls -LR ${{ github.workspace }}/templates/console/deployment/

      - name: post_build
        run: |
          # aws s3 cp s3://solutions-build-assets/changelog-spec.yml buildspec_2.yml || true
          set -euxo pipefail
          __dir="${{ github.workspace }}/templates/console/deployment"
          function join_by { local IFS="$1"; shift; echo "$*"; }
          export ECR_REPOS=$(join_by , `cat "${__dir}/ecr-repos"`)
          echo "ECR_REPOS=${ECR_REPOS}" >> $GITHUB_ENV
          export CN_ECR_REPOS=$(join_by , `cat "${__dir}/cn-ecr-repos"`)
          echo "CN_ECR_REPOS=${CN_ECR_REPOS}" >> $GITHUB_ENV
          git config --global user.email "cloudfront-extns-helper@example.com"
          git config --global user.name "cloudfront-extns-helper"
          git tag -a $BUILD_VERSION -m "${{ github.run_number }} ${{ github.job_workflow_sha }}"
      - name: archive production artifacts
        uses: actions/upload-artifact@v3
        with:
          name: build-artifacts
          path: |
              .git/**/*
              deployment/**/*
              docs/**/*
              templates/console/deployment/**/*
              buildspec_2.yml
              deploy_vtag.yml
              CHANGELOG.md
              .cfnnagignore
              .cfnnag_global_suppress_list
              .cfnnag_*

  cfn-nag-scan:
    needs: build
    runs-on: ubuntu-latest
    env:
      CI: "false"
    steps:
      - name: Download all workflow run artifacts
        uses: actions/download-artifact@v3
      - name: Setup Ruby 2.6
        uses: ruby/setup-ruby@477b21f02be01bcb8030d50f37cfec92bfa615b6
        with:
          ruby-version: 2.6
      - name: cfn_nag
        run: |
          # install Ruby
          # sudo apt-get install ruby-full
          # ls -R
          gem install cfn-nag
          cfn_nag -v
          echo "CloudFormation template scanning utility installation complete `date`"
          echo "Starting CloudFormation template scanning `date` in `pwd`"
          cd build-artifacts/
          export CFN_NAG_OPTS="--template-pattern ..*.template.json --print-suppression --fail-on-warnings"
          if [ -f ".cfnnag_global_suppress_list" ]; then export CFN_NAG_OPTS="$CFN_NAG_OPTS -b .cfnnag_global_suppress_list"; fi
          set -x; cfn_nag_scan $CFN_NAG_OPTS --input-path templates/console/deployment/global-s3-assets/
          # cfn_nag_scan --input-path .
          # echo "Completed CloudFormation template scanning `date`"

#  sematic_validate:
#    name: Validate PR title
#    runs-on: ubuntu-latest
#    needs: opensource-license-review
#    permissions:
#      pull-requests: write
#    steps:
#      - uses: amannn/action-semantic-pull-request@v5.0.2
#        env:
#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
#        with :
#          types: |
#            feat
#            fix
#            docs
#            style
#            refactor
#            perf
#            test
#            build
#            ci
#            chore
#            revert

  semgrep-scan:
    needs: git-secrets
    runs-on: ubuntu-latest
    container:
      # A Docker image with Semgrep installed. Do not change this.
      image: returntocorp/semgrep
    # Skip any PR created by dependabot to avoid permission issues:
    if: (github.actor != 'dependabot[bot]')
    permissions:
      security-events: write
    steps:
      # Fetch project source with GitHub Actions Checkout.
      - uses: actions/checkout@v3
      # Run the "semgrep ci" command on the command line of the docker image.
      - run: semgrep ci --config auto --sarif --output=semgrep.sarif
      - name: Upload SARIF file for GitHub Advanced Security Dashboard
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: semgrep.sarif
        if: always()

  viperlight:
    runs-on: ubuntu-latest
    needs: git-secrets
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Viperlight Scan
        run: |
          wget -q https://viperlight-scanner.s3.amazonaws.com/latest/.viperlightrc
          wget -q https://viperlight-scanner.s3.amazonaws.com/latest/viperlight.zip
          unzip -q viperlight.zip -d viperlight
          rm -r ./viperlight.zip
          cd viperlight
          ./install.sh
          viperlight --version
          echo "Content scanning utility installation complete `date`"
          echo "Starting content scanning `date` in `pwd`"
          cd ${{ github.workspace }}/templates/console/source
          viperlight scan -m files-contents -m files-aws -m files-binary -m files-entropy -m files-secrets
          echo "Completed content scanning `date`"

  cdk-unit-test:
    runs-on: ubuntu-latest
    needs: git-secrets
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Use Node.js 18.x
        uses: actions/setup-node@v3
        with:
          node-version: '18.x'
      - name: Run Typescript Unit Test
        run: |
          cd ${{ github.workspace }}/templates/console/source
          npm install --legacy-peer-deps
          npm run test
      - name: Sed lcov report
        if: always()
        run: |
          cd ${{ github.workspace }}/templates/console/source/coverage
          sed -i 's/source\/lib\//\/github\/workspace\/templates\/console\/source\/lib\//g' lcov.info
          sed -i 's/source\/test\//\/github\/workspace\/templates\/console\/source\/test\//g' lcov.info
      - name: archive coverage report
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: cdk-coverage-report
          path: |
            templates/console/source/coverage/lcov.info

  ssl-lambda-unit-test:
    runs-on: ubuntu-latest
    needs: git-secrets
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Run Saas Lambda Python Test
        env:
          AWS_DEFAULT_REGION: us-east-1
        run: |
          cd ${{ github.workspace }}/templates/console/source/lambda/ssl_for_saas/
          # pip install pytest
          pip install poetry
          poetry install
          export PYTHONPATH=../ssl_for_saas
          echo "PYTHONPATH=../ssl_for_saas" >> $GITHUB_ENV
          poetry run pytest --cov --cov-config=.coveragerc -cov-report=term-missing --cov-report="xml:./ssl-for-saas.coverage.xml" test
          sed -i 's/\/home\/runner\/work\/aws-cloudfront-extensions\/aws-cloudfront-extensions\//\/github\/workspace\//g' ssl-for-saas.coverage.xml
      - name: archive coverage report
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: python-coverage-report
          path: |
            templates/console/source/lambda/ssl_for_saas/*.coverage.xml

  config-deployer-unit-test:
    runs-on: ubuntu-latest
    needs: git-secrets
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Run Config and Deployer Lambda Python Test
        env:
          AWS_DEFAULT_REGION: us-east-1
        run: |
          cd ${{ github.workspace }}/templates/console/deployment
          ./run-lambda-ut.sh
      - name: Sed coverage report files
        if: always()
        run: |
          cd ${{ github.workspace }}/templates/console/source/all-coverages
          sed -i 's/\/home\/runner\/work\/aws-cloudfront-extensions\/aws-cloudfront-extensions\//\/github\/workspace\//g' *.coverage.xml
      - name: archive coverage report
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: config-coverage-report
          path: |
            templates/console/source/all-coverages/*.coverage.xml

  monitoring-unit-test:
    runs-on: ubuntu-latest
    needs: git-secrets
    env:
      CI: "false"
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Run Monitoring Lambda Python Test
        env:
          AWS_DEFAULT_REGION: us-east-1
        run: |
          cd ${{ github.workspace }}/templates/console/deployment
          ./run-unit-tests.sh
      - name: Sed coverage report files
        if: always()
        run: |
          cd ${{ github.workspace }}/templates/console/deployment/monitoring-coverages
          sed -i 's/\/home\/runner\/work\/aws-cloudfront-extensions\/aws-cloudfront-extensions\//\/github\/workspace\//g' *.coverage.xml
      - name: archive coverage report
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: monitoring-coverage-report
          path: |
            templates/console/deployment/monitoring-coverages/*.coverage.xml

  sonarqube:
    needs: [ssl-lambda-unit-test, config-deployer-unit-test, monitoring-unit-test, cdk-unit-test]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - uses: actions/checkout@v3
        with:
          # Disabling shallow clone is recommended for improving relevancy of reporting
          fetch-depth: 0
      - name: Download python coverage report
        uses: actions/download-artifact@v3
        with:
          name: python-coverage-report
          path: ${{ github.workspace }}/templates/console/source/lambda/ssl_for_saas/
      - name: Download cdk coverage report
        uses: actions/download-artifact@v3
        with:
          name: cdk-coverage-report
          path: ${{ github.workspace }}/templates/console/source/coverage
      - name: Download config and deployer coverage report
        uses: actions/download-artifact@v3
        with:
          name: config-coverage-report
          path: ${{ github.workspace }}/templates/console/source/all-coverages
      - name: Download monitoring coverage report
        uses: actions/download-artifact@v3
        with:
          name: monitoring-coverage-report
          path: ${{ github.workspace }}/templates/console/source/monitoring-coverages
      - name: SonarQube Scan
        uses: sonarsource/sonarqube-scan-action@master
        env:
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}

  codeguru:
    needs: git-secrets
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0   # Required
      - name: configure aws credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          role-to-assume: arn:aws:iam::261557372226:role/CodguruGithubRole
          role-duration-seconds: 900 # the ttl of the session, in seconds.
          aws-region: us-west-2
      - name: CodeGuru Reviewer
        uses: aws-actions/codeguru-reviewer@v1.1
        with:
  #        build_path: target # build artifact(s) directory
          s3_bucket: codeguru-reviewer-cloudfront-extn-bucket2
      - name: Upload review result
        if: ${{ github.event_name != 'push' }}
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: codeguru-results.sarif.json


Greetings

name: Greetings

on: [pull_request, issues]

jobs:
  greeting:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/first-interaction@v1
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        issue-message: 'Hi there! 👋  Thank you for opening your first Lightkurve issue! 🙏  One of our maintainers will get back to you as soon as possible. 👩🚀  You can expect a response within 7 days. 📅  If you haven’t heard anything by then, feel free to ping this thread. 🛎  We love that you are using Lightkurve and appreciate your feedback. 👍'
        pr-message: 'Hi there! 👋  Thank you for opening your first Lightkurve pull request! 🙏  One of our maintainers will get back to you as soon as possible. 👩🚀  You can expect a response within 7 days. 📅  If you haven’t heard anything by then, feel free to ping this thread. 🛎  We love that you are using Lightkurve and appreciate your input. 👍'


Lightkurve-tests

# GitHub Actions workflow for Lightkurve's continuous integration.
#
# This file configures seven different jobs:
# 1) pytest on Linux, Python 3.8, with -m memtest (i.e. stress memory)
# 2) pytest on Linux, Python 3.8, with --remote-data enabled (i.e. get data from MAST).
# 3) pytest on Linux, Python 3.9.
# 4) pytest on Linux, Python 3.10.
# 5) pytest on Windows, Python 3.8.
# 6) pytest on OSX, Python 3.8.
# 7) flake8 syntax check.

name: Lightkurve-tests

on:
  # We always want to run tests on main
  push:
    branches:
      - main
  # And we'll also run tests on pull requests
  pull_request:

jobs:

  # Run unit tests on Linux
  pytest-linux:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        name: [3.8-remote-data, 3.8-memtest, 3.9, '3.10']
        include:
          - name: 3.8-remote-data
            python-version: 3.8
            pytest-command: poetry run pytest --remote-data --doctest-modules
          - name: 3.8-memtest
            python-version: 3.8
            pytest-command: poetry run pytest -m memtest --remote-data
          - name: 3.9
            python-version: 3.9
            pytest-command: poetry run pytest
          - name: '3.10'
            python-version: '3.10'
            pytest-command: poetry run pytest
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip 'poetry>=1.4.0'
        poetry config installer.modern-installation false
        poetry install
    - name: Test with ${{ matrix.pytest-command }}
      run: |
        ${{ matrix.pytest-command }}

  # Run unit tests on Windows
  pytest-windows:
    runs-on: windows-latest
    strategy:
      matrix:
        name: [3.8]
        python-version: [3.8]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip 'poetry>=1.4.0'
        poetry config installer.modern-installation false
        poetry install
    - name: Test with pytest
      run: |
        poetry run pytest

  # Run unit tests on Mac OSX
  pytest-osx:
    runs-on: macos-latest
    strategy:
      matrix:
        python-version: [3.8]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip 'poetry>=1.4.0'
        poetry config installer.modern-installation false
        poetry install
    - name: Test with pytest
      run: |
        poetry run pytest

  # Use the `flake8` tool to check for syntax errors
  flake8-linter:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.8]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install flake8
      run: |
        python -m pip install --upgrade pip 'poetry>=1.4.0'
        poetry config installer.modern-installation false
        poetry install
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude="setup.py,test_*"
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude="setup.py,test_*"


CI

name: CI

on:
  push:
    branches:
      - main
      - dev/*
  pull_request:
  workflow_dispatch:

jobs:
  test:
    # Based on pytest-blender workflow: https://github.com/mondeja/pytest-blender/blob/master/.github/workflows/ci.yml
    name: Test
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - blender-version: '4.0.2'
            python-version: '3.10'
            pytest-version: '7.2.0'
            pytest-blender-version: '3.0.5'

          - blender-version: '4.1.0'
            python-version: '3.11'
            pytest-version: '7.2.0'
            pytest-blender-version: '3.0.5'

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade PIP
        run: python -m pip install --upgrade pip

      - name: Cache Blender ${{ matrix.blender-version }}
        uses: actions/cache@v3
        id: cache-blender
        with:
          path: |
            _blender/blender-*
            _blender-executable-path.txt
          key: ${{ runner.os }}-${{ matrix.blender-version }}

      - name: Download Blender ${{ matrix.blender-version }}
        if: steps.cache-blender.outputs.cache-hit != 'true'
        run: |
          Set-PSDebug -Trace 2
          # Use blender-downloader fork with specifes a Sollumz User-Agent since the default python User-Agent is currently blocked
          # python -m pip install --upgrade blender-downloader
          python -m pip install git+https://github.com/Sollumz/blender-downloader.git
          python -m pip list
          mkdir _blender
          blender-downloader ${{ matrix.blender-version }} -d _blender --extract --remove-compressed --print-blender-executable --quiet | Out-File -FilePath _blender-executable-path.txt
          get-content _blender-executable-path.txt

      - name: Install Dependencies
        run: |
          Set-PSDebug -Trace 2
          python -m pip install pytest-blender==${{ matrix.pytest-blender-version }} pytest==${{ matrix.pytest-version }}
          $BLENDER = get-content _blender-executable-path.txt
          pytest-blender --blender-executable "$BLENDER" | Out-File -FilePath _blender-python-executable-path.txt
          $BLENDER_PYTHON = get-content _blender-python-executable-path.txt
          echo "$BLENDER_PYTHON"
          & "$BLENDER_PYTHON" -m pip install pytest-blender==${{ matrix.pytest-blender-version }} pytest==${{ matrix.pytest-version }}

      - name: Install Sollumz
        run: |
          $BLENDER = get-content _blender-executable-path.txt
          $BLENDER_RELEASE_VERSION = python -c "print('.'.join('${{ matrix.blender-version }}'.split('.')[:2]))" # extract first two version numbers
          New-Item -ItemType Directory -Path "$env:APPDATA\Blender Foundation\Blender\$BLENDER_RELEASE_VERSION\scripts\addons" -Force
          New-Item -ItemType SymbolicLink -Path "$env:APPDATA\Blender Foundation\Blender\$BLENDER_RELEASE_VERSION\scripts\addons" -Name Sollumz -Target $(Get-Location)
          & "$BLENDER" -b --python-expr "import bpy; bpy.ops.preferences.addon_enable(module='Sollumz'); bpy.ops.wm.save_userpref()"

      - name: Run Tests
        run: |
          $BLENDER = get-content _blender-executable-path.txt
          $BLENDER_PYTHON = get-content _blender-python-executable-path.txt
          $env:SOLLUMZ_TEST_TMP_DIR = $(mkdir _test_tmp)
          & "$BLENDER_PYTHON" -m pytest -svv --ignore _blender --ignore _test_tmp --blender-executable "$BLENDER"


build

name: build
on:
  workflow_dispatch:
  schedule:
  - cron: 0 0 * * 0     # weekly
  pull_request:
    branches:
    - main
# want pooch to raise an error if I've updated the file and forgot to update the
# hash.
env:
  POOCH_ALLOW_UPDATES: false

jobs:
  # based on https://slashgear.github.io/how-to-split-test-by-folder-with-github-action/
  notebooks:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.10', '3.11', '3.12']
      fail-fast: false
    name: Execute notebooks
    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
        cache-dependency-path: setup.py
    - name: Setup FFmpeg
      uses: FedericoCarboni/setup-ffmpeg@v3.1
    - name: Install dependencies
      run: |
        pip install --upgrade --upgrade-strategy eager .[dev,nb] papermill
    - name: Run notebooks
      run: |
        for file in examples/*ipynb; do
           # these first two notebooks take much longer than the rest to run (2
           # and 1 hours on laptop, respectively, longer on runners). So we use
           # papermill's parameters to reduce the max number of steps for
           # synthesis in them (we want to test that each cell runs, but we
           # don't need synthesis to go to completion).
           if [[ "$file" =~ "Metamer-Portilla-Simoncelli" ]]; then
              papermill $file $file_output.ipynb -p short_synth_max_iter 10 -p long_synth_max_iter 10 -p longest_synth_max_iter 10 -k python3 --cwd examples/
           elif [[ "$file" =~ "Demo_Eigendistortion" ]]; then
              papermill $file $file_output.ipynb -p max_iter_frontend 10 -p max_iter_vgg 10 -k python3 --cwd examples/
           else
              jupyter execute $file --kernel_name=python3
            fi
        done
  tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.10', '3.11', '3.12']
      fail-fast: false
    name: Run pytest scripts
    steps:
    - uses: actions/checkout@v4
    - name: Install Python 3
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
        cache-dependency-path: setup.py
    - name: Install dependencies
      run: |
        # using the --upgrade and --upgrade-strategy eager flags ensures that
        # pip will always install the latest allowed version of all
        # dependencies, to make sure the cache doesn't go stale
        pip install --upgrade --upgrade-strategy eager .[dev]
    - name: Run tests with pytest
      run: |
        pytest -n auto --cov-report xml
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@7598e39340e1dff4d6ebf7cf07a5e8184bde67e7 # v4.0.1
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
  all_tutorials_in_docs:
    runs-on: ubuntu-latest
    name: Check that all tutorial notebooks are included in docs
    steps:
    - uses: actions/checkout@v4
    - name: Check for file
      shell: bash
      # there are two levels that the notebooks can be in
      run: |
        for file in examples/*ipynb; do
           if [[ -z "$(grep $file docs/tutorials/*nblink)" && -z "$(grep $file docs/tutorials/*/*nblink)" ]] ; then
              exit 1
           fi
        done
  no_extra_nblinks:
    runs-on: ubuntu-latest
    name: Check that we don't have any extra nblink files
    steps:
    - uses: actions/checkout@v4
    - name: Check same number of nblink and notebooks
      shell: bash
      run: |
        n_nblink=0; for file in docs/tutorials/*nblink; do let "n_nblink+=1"; done;
        for file in docs/tutorials/*/*nblink; do let "n_nblink+=1"; done;
        n_ipynb=0; for file in examples/*ipynb; do let "n_ipynb+=1"; done;
        if [[ $n_nblink != $n_ipynb ]]; then exit 1; fi;
  check_urls:
    runs-on: ubuntu-latest
    name: Check all urls are valid
    steps:
    - uses: actions/checkout@v4
      # there are several cells in the notebook whose output includes links that
      # urlchecker htinks are invalid (though when I check them manually, they
      # look fine). Regardless, they're unimportant -- they're part of warning
      # messages and similar, so we don't want to check them.
    - name: strip notebook output
      run: |
        pip install nbstripout
        nbstripout examples/*ipynb
    - uses: urlstechie/urlchecker-action@b643b43e2ac605e1475331c7b67247d242b7dce4 # 0.0.34
      with:
        file_types: .md,.py,.rst,.ipynb
        print_all: false
        timeout: 5
        retry_count: 3

  check:
    if: always()
    needs:
    - notebooks
    - tests
    - all_tutorials_in_docs
    - no_extra_nblinks
    - check_urls
    runs-on: ubuntu-latest
    steps:
    - name: Decide whether all tests and notebooks succeeded
      uses: re-actors/alls-green@afee1c1eac2a506084c274e9c02c8e0687b48d9e # v1.2.2
      with:
        jobs: ${{ toJSON(needs) }}


deploy

name: deploy
on:
  workflow_dispatch:
  release:
    types: [published]

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        # this is necessary for setuptools_scm to work properly with github
        # actions, see https://github.com/pypa/setuptools_scm/issues/480 and
        # https://stackoverflow.com/a/68959339
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install build
      - name: Build package
        run: python -m build --outdir dist/ --sdist --wheel
      - name: Check there's only one sdist and one whl file created
        shell: bash
        # because the following two tests will be weird otherwise. see
        # https://askubuntu.com/a/454568 for why this is the right way to handle
        # it. using [[ BOOLEAN ]] || EXPR is a compact way of writing IF NOT
        # BOOLEAN THEN EXPR in bash
        run: |
          [[ $(find dist/ -type f -name "*whl" -printf x | wc -c) == 1 ]] || exit 1
          [[ $(find dist/ -type f -name "*tar.gz" -printf x | wc -c) == 1 ]] || exit 1
      - name: Check setuptools_scm version against git tag
        shell: bash
        # this won't work if we use workflow dispatch, so no sense in running
        # it.
        if: github.event_name == 'release' && github.event.action == 'published'
        run: |
          # we use the error code of this comparison: =~ is bash's regex
          # operator, so it checks whether the right side is contained in the
          # left side. In particular, we succeed if the path of the source code
          # ends in the most recent git tag, fail if it does not.
          [[ "$(ls dist/*tar.gz)" =~ "-$(git describe --tags).tar.gz" ]]
      - name: Check we can install from wheel
        # note that this is how this works in bash (different shells might be
        # slightly different). we've checked there's only one .whl file in an
        # earlier step, so the bit in `$()` will expand to that single file,
        # then we pass [dev] to get specify the optional dev dependencies, and
        # we wrap the whole thing in quotes so bash doesn't try to interpret the
        # square brackets but passes them directly to pip install
        shell: bash
        run: |
          pip install "$(ls dist/*whl)[dev,nb]" papermill
      - name: Run tests
        run: |
          pytest -n auto
      - name: Setup FFmpeg
        uses: FedericoCarboni/setup-ffmpeg@v3.1
      - name: Run notebooks
        run: |
          for file in examples/*ipynb; do
             # these first two notebooks take much longer than the rest to run (2
             # and 1 hours on laptop, respectively, longer on runners). So we use
             # papermill's parameters to reduce the max number of steps for
             # synthesis in them (we want to test that each cell runs, but we
             # don't need synthesis to go to completion).
             if [[ "$file" =~ "Metamer-Portilla-Simoncelli" ]]; then
                papermill $file $file_output.ipynb -p short_synth_max_iter 10 -p long_synth_max_iter 10 -p longest_synth_max_iter 10 -k python3 --cwd examples/
             elif [[ "$file" =~ "Demo_Eigendistortion" ]]; then
                papermill $file $file_output.ipynb -p max_iter_frontend 10 -p max_iter_vgg 10 -k python3 --cwd examples/
             else
                jupyter execute $file --kernel_name=python3
              fi
          done
      - uses: actions/upload-artifact@v4
        with:
          path: dist/*

  publish:
    name: Upload release to Test PyPI
    needs: [build]
    environment: pypi
    runs-on: ubuntu-latest
    permissions:
      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: artifact
          path: dist
      - name: Publish package to test pypi
        if: github.event_name != 'release'
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: https://test.pypi.org/legacy/
      - name: Publish package to real pypi
        if: github.event_name == 'release' && github.event.action == 'published'
        uses: pypa/gh-action-pypi-publish@release/v1


Build

name: Build

on:
  push:
    # branches: [master, develop]
    branches: [master]
  pull_request:
    # branches: [master, develop]
    branches: [master]

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2

      - name: Use Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x

      - name: Use Node.js
        uses: actions/setup-node@v2
        with:
          node-version: 16.x
          cache: npm

      - name: Install Pipenv
        run: pip install --upgrade pip && pip install pipenv

      - name: Install
        run: npm install

      - name: Check setup
        run: npm run check

      - name: Build
        run: npm run build


Lint

name: Lint

on:
  push:
    # branches: [master, develop]
    branches: [master]
  pull_request:
    # branches: [master, develop]
    branches: [master]

jobs:
  lint:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2

      - name: Use Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x

      - name: Use Node.js
        uses: actions/setup-node@v2
        with:
          node-version: 16.x
          cache: npm

      - name: Install Pipenv
        run: pip install --upgrade pip && pip install pipenv

      - name: Install
        run: npm install

      - name: Run linter
        run: npm run lint


Pre-release Node.js bridge

name: Pre-release Node.js bridge

on: workflow_dispatch

jobs:
  build:
    name: Build

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-20.04]

    runs-on: ${{ matrix.os }}

    steps:
      - name: Clone repository
        uses: actions/checkout@v3

      - name: Install Node.js
        uses: actions/setup-node@v3
        with:
          node-version: lts/*

      - name: Set Node.js bridge version
        working-directory: bridges/nodejs/src
        run: |
          echo "NODEJS_BRIDGE_VERSION=$(node --require fs --eval "const fs = require('node:fs'); const [, VERSION] = fs.readFileSync('version.ts', 'utf8').split(\"'\"); console.log(VERSION)")" >> $GITHUB_ENV

      - name: Display Node.js bridge version
        run: |
          echo "Node.js bridge version: ${{ env.NODEJS_BRIDGE_VERSION }}"

      - name: Install core
        run: npm install

      - name: Build Node.js bridge
        run: npm run build:nodejs-bridge

      - name: Upload Node.js bridge
        uses: actions/upload-artifact@v3
        with:
          path: bridges/nodejs/dist/*.zip

  draft-release:
    name: Draft-release
    needs: [build]
    runs-on: ubuntu-20.04

    steps:
      - name: Clone repository
        uses: actions/checkout@v3

      - name: Install Node.js
        uses: actions/setup-node@v3
        with:
          node-version: lts/*

      - name: Set Node.js bridge version
        working-directory: bridges/nodejs/src
        run: |
          echo "NODEJS_BRIDGE_VERSION=$(node --require fs --eval "const fs = require('node:fs'); const [, VERSION] = fs.readFileSync('version.ts', 'utf8').split(\"'\"); console.log(VERSION)")" >> $GITHUB_ENV

      - name: Download Node.js bridge
        uses: actions/download-artifact@v3
        with:
          path: bridges/nodejs/dist

      - uses: marvinpinto/action-automatic-releases@latest
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          automatic_release_tag: nodejs-bridge_v${{ env.NODEJS_BRIDGE_VERSION }}
          draft: true
          prerelease: false
          title: Node.js Bridge ${{ env.NODEJS_BRIDGE_VERSION }}
          files: bridges/nodejs/dist/artifact/*.zip


Pre-release Python bridge

name: Pre-release Python bridge

on: workflow_dispatch

env:
  PIPENV_PIPFILE: bridges/python/src
  PIPENV_VENV_IN_PROJECT: true

jobs:
  build:
    name: Build

    strategy:
      fail-fast: false
      matrix:
        # @see https://github.com/actions/runner-images/tree/main/images/macos
        # Use macos-12 instead of macos-latest because the latter use ARM64 (M1) architecture
        os: [ubuntu-20.04, macos-12]
        # Temporarily disable Windows release
        # os: [ubuntu-20.04, macos-12, windows-latest]

    runs-on: ${{ matrix.os }}

    steps:
      - name: Clone repository
        uses: actions/checkout@v3

      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11.9

      - name: Install Pipenv
        run: pip install --upgrade pip && pip install pipenv==2024.0.1

      - name: Install Node.js
        uses: actions/setup-node@v3
        with:
          node-version: lts/*

      - name: Set Python bridge version
        working-directory: bridges/python/src
        run: |
          echo "PYTHON_BRIDGE_VERSION=$(python -c "from version import __version__; print(__version__)")" >> $GITHUB_ENV

      - name: Display Python bridge version
        run: |
          echo "Python bridge version: ${{ env.PYTHON_BRIDGE_VERSION }}"

      - name: Install core
        run: npm install

      - name: Set up Python bridge
        run: npm run setup:python-bridge

      - name: Build Python bridge
        run: npm run build:python-bridge

      - name: Upload Python bridge
        uses: actions/upload-artifact@v3
        with:
          path: bridges/python/dist/*.zip

  draft-release:
    name: Draft-release
    needs: [build]
    runs-on: ubuntu-20.04

    steps:
      - name: Clone repository
        uses: actions/checkout@v3

      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11.9

      - name: Set Python bridge version
        working-directory: bridges/python/src
        run: |
          echo "PYTHON_BRIDGE_VERSION=$(python -c "from version import __version__; print(__version__)")" >> $GITHUB_ENV

      - name: Download Python bridge
        uses: actions/download-artifact@v3
        with:
          path: bridges/python/dist

      - uses: marvinpinto/action-automatic-releases@latest
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          automatic_release_tag: python-bridge_v${{ env.PYTHON_BRIDGE_VERSION }}
          draft: true
          prerelease: false
          title: Python Bridge ${{ env.PYTHON_BRIDGE_VERSION }}
          files: bridges/python/dist/artifact/*.zip


Pre-release TCP server

name: Pre-release TCP server

on: workflow_dispatch

env:
  PIPENV_PIPFILE: tcp_server/src
  PIPENV_VENV_IN_PROJECT: true

jobs:
  build:
    name: Build

    strategy:
      fail-fast: false
      matrix:
        # @see https://github.com/actions/runner-images/tree/main/images/macos
        # Use macos-12 instead of macos-latest because the latter use ARM64 (M1) architecture
        os: [ubuntu-20.04, macos-12, windows-latest]

    runs-on: ${{ matrix.os }}

    steps:
      - name: Clone repository
        uses: actions/checkout@v3

      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11.9

      - name: Install Pipenv
        run: pip install --upgrade pip && pip install pipenv==2024.0.1

      - name: Install Node.js
        uses: actions/setup-node@v3
        with:
          node-version: lts/*

      - name: Set TCP server version
        working-directory: tcp_server/src
        run: |
          echo "TCP_SERVER_VERSION=$(python -c "from version import __version__; print(__version__)")" >> $GITHUB_ENV

      - name: Display TCP server version
        run: |
          echo "TCP server version: ${{ env.TCP_SERVER_VERSION }}"

      - name: Install core
        run: npm install

      - name: Set up TCP server
        run: npm run setup:tcp-server

      - name: Build TCP server
        run: npm run build:tcp-server

      - name: Upload TCP server
        uses: actions/upload-artifact@v3
        with:
          path: tcp_server/dist/*.zip

  draft-release:
    name: Draft-release
    needs: [build]
    runs-on: ubuntu-20.04

    steps:
      - name: Clone repository
        uses: actions/checkout@v3

      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11.9

      - name: Set TCP server version
        working-directory: tcp_server/src
        run: |
          echo "TCP_SERVER_VERSION=$(python -c "from version import __version__; print(__version__)")" >> $GITHUB_ENV

      - name: Download TCP server
        uses: actions/download-artifact@v3
        with:
          path: tcp_server/dist

      - uses: marvinpinto/action-automatic-releases@latest
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          automatic_release_tag: tcp-server_v${{ env.TCP_SERVER_VERSION }}
          draft: true
          prerelease: false
          title: TCP Server ${{ env.TCP_SERVER_VERSION }}
          files: tcp_server/dist/artifact/*.zip


Tests

name: Tests

on:
  push:
    # branches: [master, develop]
    branches: [master]
  pull_request:
    # branches: [master, develop]
    branches: [master]

jobs:
  tests:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2

      - name: Use Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x

      - name: Use Node.js
        uses: actions/setup-node@v2
        with:
          node-version: 16.x
          cache: npm

      - name: Install Pipenv
        run: pip install --upgrade pip && pip install pipenv

      - name: Install
        run: npm install

      - name: Run JSON tests
        run: npm run test:json

      - name: Run over HTTP tests
        run: npm run test:over-http

      - name: Run E2E tests
        run: npm run test:e2e

      - name: Install offline STT
        run: npm run setup:offline-stt

      - name: Install offline TTS
        run: npm run setup:offline-tts

      - name: Run unit tests
        run: npm run test:unit


CI

name: CI
on:
  push:
    paths-ignore:
      - '.cookiecutter/*'
      - '.github/dependabot.yml'
      - '.github/workflows/deploy.yml'
      - '.github/workflows/redeploy.yml'
      - 'bin/logger'
      - 'bin/make_python'
      - 'bin/make_template'
      - 'conf/development.ini'
      - 'conf/production.ini'
      - 'conf/supervisord*.conf'
      - 'docs/*'
      - 'requirements/*.in'
      - 'requirements/dev.txt'
      - '.docker.env'
      - '**/.gitignore'
      - 'Dockerfile'
      - 'LICENSE'
      - '*.md'
      - 'docker-compose.yml'
  workflow_dispatch:
  workflow_call:
  schedule:
  - cron: '0 1 * * *'
jobs:
  Format:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - name: Cache the .tox dir
        uses: actions/cache@v3
        with:
          path: .tox
          key: format-${{ runner.os }}-tox-${{ hashFiles('tox.ini') }}-${{ hashFiles('requirements/*') }}
          restore-keys: |
            format-${{ runner.os }}-tox-
      - run: python -m pip install 'tox<4'
      - run: tox -e checkformatting
  Lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - name: Cache the .tox dir
        uses: actions/cache@v3
        with:
          path: .tox
          key: lint-${{ runner.os }}-tox-${{ hashFiles('tox.ini') }}-${{ hashFiles('requirements/*') }}
          restore-keys: |
            lint-${{ runner.os }}-tox-
      - run: python -m pip install 'tox<4'
      - run: tox -e lint
  Typecheck:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - name: Cache the .tox dir
        uses: actions/cache@v3
        with:
          path: .tox
          key: typecheck-${{ runner.os }}-tox-${{ hashFiles('tox.ini') }}-${{ hashFiles('requirements/*') }}
          restore-keys: |
            typecheck-${{ runner.os }}-tox-
      - run: python -m pip install 'tox<4'
      - run: tox -e typecheck
  Tests:
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:15.3-alpine
        ports:
        - 4433:5432
        env:
          POSTGRES_HOST_AUTH_METHOD: trust
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v3
      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - name: Cache the .tox dir
        uses: actions/cache@v3
        with:
          path: .tox
          key: tests-${{ runner.os }}-tox-${{ hashFiles('tox.ini') }}-${{ hashFiles('requirements/*') }}
          restore-keys: |
            tests-${{ runner.os }}-tox-
      - name: Create test database
        run: psql -U postgres -h localhost -p 4433 -c 'CREATE DATABASE via_tests'
      - run: yarn install --frozen-lockfile
      - run: yarn build
      - run: python -m pip install 'tox<4'
      - run: tox -e tests
        env:
          COVERAGE_FILE: .coverage.${{ matrix.python-version }}
      - name: Upload coverage file
        uses: actions/upload-artifact@v3
        with:
          name: coverage
          path: .coverage.*
  Coverage:
    needs: tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - name: Cache the .tox dir
        uses: actions/cache@v3
        with:
          path: .tox
          key: coverage-${{ runner.os }}-tox-${{ hashFiles('tox.ini') }}-${{ hashFiles('requirements/*') }}
          restore-keys: |
            coverage-${{ runner.os }}-tox-
      - name: Download coverage files
        uses: actions/download-artifact@v3
        with:
          name: coverage
      - run: python -m pip install 'tox<4'
      - run: tox -e coverage
  Functests:
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:15.3-alpine
        ports:
        - 4433:5432
        env:
          POSTGRES_HOST_AUTH_METHOD: trust
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v3
      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - name: Cache the .tox dir
        uses: actions/cache@v3
        with:
          path: .tox
          key: functests-${{ runner.os }}-tox-${{ hashFiles('tox.ini') }}-${{ hashFiles('requirements/*') }}
          restore-keys: |
            functests-${{ runner.os }}-tox-
      - name: Create test database
        run: psql -U postgres -h localhost -p 4433 -c 'CREATE DATABASE via_functests'
      - run: yarn install --frozen-lockfile
      - run: yarn build
      - run: python -m pip install 'tox<4'
      - run: tox -e functests
  Frontend:
    uses: ./.github/workflows/frontend.yml


Deploy

name: Deploy
concurrency:
  group: deploy
  cancel-in-progress: true
on:
  workflow_dispatch:
  push:
    branches:
      - main
    paths-ignore:
      - '.cookiecutter/*'
      - '.github/*'
      - 'bin/create-db'
      - 'bin/make_python'
      - 'bin/make_template'
      - 'conf/development.ini'
      - 'conf/supervisord-dev.conf'
      - 'docs/*'
      - 'requirements/*'
      - '!requirements/prod.txt'
      - 'tests/*'
      - '**/.gitignore'
      - '.python-version'
      - 'LICENSE'
      - '*.md'
      - 'docker-compose.yml'
      - 'tox.ini'
jobs:
  docker_hub:
    name: Docker Hub
    uses: hypothesis/workflows/.github/workflows/dockerhub.yml@main
    with:
      Application: ${{ github.event.repository.name }}
    secrets: inherit
  staging:
    name: Staging
    needs: [docker_hub]
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: deploy
      github_environment_name: Staging
      github_environment_url: https://via.staging.hypothes.is/https://en.wikipedia.org/api/rest_v1/page/pdf/Comet_Kohoutek
      aws_region: us-west-1
      elasticbeanstalk_application: via
      elasticbeanstalk_environment: staging
      docker_tag: ${{ needs.Docker_Hub.outputs.docker_tag }}
    secrets: inherit
  staging_edu:
    name: Staging (Edu)
    needs: [docker_hub]
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: deploy
      github_environment_name: Staging (Edu)
      github_environment_url: https://hypothesis.instructure.com/courses/125/assignments/878
      aws_region: us-west-1
      elasticbeanstalk_application: edu-via
      elasticbeanstalk_environment: staging
      docker_tag: ${{ needs.Docker_Hub.outputs.docker_tag }}
    secrets: inherit
  production:
    name: Production
    needs: [docker_hub, staging]
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: deploy
      github_environment_name: Production
      github_environment_url: https://via.hypothes.is/https://en.wikipedia.org/api/rest_v1/page/pdf/Comet_Kohoutek
      aws_region: us-west-1
      elasticbeanstalk_application: via
      elasticbeanstalk_environment: prod
      docker_tag: ${{ needs.Docker_Hub.outputs.docker_tag }}
    secrets: inherit
  production_edu:
    name: Production (Edu)
    needs: [docker_hub, staging_edu]
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: deploy
      github_environment_name: Production (Edu)
      github_environment_url: https://hypothesis.instructure.com/courses/125/assignments/882
      aws_region: us-west-1
      elasticbeanstalk_application: lms-via
      elasticbeanstalk_environment: prod
      docker_tag: ${{ needs.Docker_Hub.outputs.docker_tag }}
    secrets: inherit


Frontend

name: Frontend
on:
  workflow_dispatch:
  workflow_call:
jobs:
  Frontend:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v3

    - name: Cache the node_modules dir
      uses: actions/cache@v3
      with:
        path: node_modules
        key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }}

    - name: Install
      run: yarn install --frozen-lockfile

    - name: Format
      run: yarn checkformatting

    - name: Lint
      run: yarn lint

    - name: Typecheck
      run: yarn typecheck

    - name: Test
      run: yarn test


Keepalive

# Prevent scheduled workflows from being disabled due to inactivity.
#
# GitHub disables scheduled workflows after 60 days of repo inactivity:
#
# > Warning: To prevent unnecessary workflow runs, scheduled workflows may be
# > disabled automatically.
# > ... In a public repository, scheduled workflows are automatically disabled
# > when no repository activity has occurred in 60 days.
#
# https://docs.github.com/en/actions/using-workflows/disabling-and-enabling-a-workflow
#
# This keep-alive workflow triggers whenever one of the scheduled workflows
# listed below completes and prevents that scheduled workflow from being
# disabled.
name: Keepalive
on:
  workflow_run:
    workflows: [CI]
    types: [completed]
    branches: [main]
jobs:
  Keepalive:
    uses: hypothesis/workflows/.github/workflows/keepalive.yml@main


Redeploy

name: Redeploy
concurrency:
  group: deploy
  cancel-in-progress: true
on:
  workflow_dispatch:
    inputs:
      staging:
        type: boolean
        description: Redeploy Staging
      staging_edu:
        type: boolean
        description: Redeploy Staging (Edu)
      production:
        type: boolean
        description: Redeploy Production
      production_edu:
        type: boolean
        description: Redeploy Production (Edu)
jobs:
  staging:
    name: Staging
    if: inputs.staging
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: redeploy
      github_environment_name: Staging
      github_environment_url: https://via.staging.hypothes.is/https://en.wikipedia.org/api/rest_v1/page/pdf/Comet_Kohoutek
      aws_region: us-west-1
      elasticbeanstalk_application: via
      elasticbeanstalk_environment: staging
    secrets: inherit
  staging_edu:
    name: Staging (Edu)
    if: inputs.staging_edu
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: redeploy
      github_environment_name: Staging (Edu)
      github_environment_url: https://hypothesis.instructure.com/courses/125/assignments/878
      aws_region: us-west-1
      elasticbeanstalk_application: edu-via
      elasticbeanstalk_environment: staging
    secrets: inherit
  production:
    name: Production
    if: inputs.production
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: redeploy
      github_environment_name: Production
      github_environment_url: https://via.hypothes.is/https://en.wikipedia.org/api/rest_v1/page/pdf/Comet_Kohoutek
      aws_region: us-west-1
      elasticbeanstalk_application: via
      elasticbeanstalk_environment: prod
    secrets: inherit
  production_edu:
    name: Production (Edu)
    if: inputs.production_edu
    uses: hypothesis/workflows/.github/workflows/deploy.yml@main
    with:
      operation: redeploy
      github_environment_name: Production (Edu)
      github_environment_url: https://hypothesis.instructure.com/courses/125/assignments/882
      aws_region: us-west-1
      elasticbeanstalk_application: lms-via
      elasticbeanstalk_environment: prod
    secrets: inherit


Slack

name: Slack
on:
  workflow_run:
    workflows: [CI, Deploy]
    types: [completed]
    branches: [main]
jobs:
  on-failure:
    runs-on: ubuntu-latest
    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
    steps:
      - name: Post to Slack
        uses: slackapi/slack-github-action@v1.24.0
        with:
          channel-id: 'C4K6M7P5E'
          slack-message: "A workflow run failed\n*Repo:* `${{ github.event.repository.full_name }}` (${{ github.event.repository.html_url }})\n*Workflow:* ${{ github.event.workflow.name }} (${{ github.event.workflow.html_url }})\n*Branch:* `${{ github.event.workflow_run.head_branch }}`\n*Commit:* `${{ github.event.workflow_run.head_commit.id }}`\n*Run:* ${{ github.event.workflow_run.html_url }}\n*Conclusion:* ${{ github.event.workflow_run.conclusion }}"
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}


Tasks

name: Tasks

on:
  workflow_dispatch:
    inputs:
      Task:
        type: choice
        description: "The task to perform"
        required: true
        options:
          - 'db current revision'
          - 'db upgrade to head'
          - 'db upgrade to next'
          - 'db downgrade to previous'
      Application:
        type: choice
        description: "The via application to target"
        required: true
        default: 'via'
        options:
          - 'via'
          - 'edu-via'
          - 'lms-via'
      Environment:
        type: choice
        description: "The environment to target"
        required: true
        default: 'staging'
        options:
          - 'staging'
          - 'prod'
      Timeout:
        type: string
        description: "The timeout in seconds"
        default: '900'
      Region:
        type: choice
        description: "The AWS region to target"
        required: true
        default: 'us-west-1'
        options:
          - 'us-west-1'
      Destructive:
        description: Check to confirm you are happy to proceed with a destructive operation
        type: boolean
        required: false
        default: false

jobs:

  preflight:
    runs-on: ubuntu-latest
    steps:
    - name: catch unauthorised destructive operations
      run: |
        if [[ "${{ inputs.Task }}" = 'db upgrade to head' ]] &&
           [[ "${{ github.event.inputs.Destructive }}" = 'false' ]] ||
           [[ "${{ inputs.Task }}" = 'db upgrade to next' ]] &&
           [[ "${{ github.event.inputs.Destructive }}" = 'false' ]] ||
           [[ "${{ inputs.Task }}" = 'db downgrade to previous' ]] &&
           [[ "${{ github.event.inputs.Destructive }}" = 'false' ]] ; then
             echo "::error::'${{ inputs.Task }}' needs destructive option"
             exit 1
        else
             echo "::notice::Preparing '${{ inputs.Task }}' update in ${{ inputs.Environment }}"
             echo "::notice::Targeting regions - '${{ inputs.region }}'"
             echo "::notice::Timeout - '${{ inputs.Timeout }}'"
             exit 0
        fi

  current:
    if: inputs.Task == 'db current revision'
    needs: preflight
    uses: hypothesis/workflows/.github/workflows/eb-task.yml@main
    with:
      App: ${{ inputs.Application }}
      Env: ${{ inputs.Environment }}
      Timeout: ${{ inputs.Timeout }}
      Region: ${{ inputs.Region }}
      Command: 'alembic -c conf/alembic.ini current'
    secrets: inherit

  head:
    if: inputs.Task == 'db upgrade to head' && github.event.inputs.Destructive == 'true'
    needs: preflight
    uses: hypothesis/workflows/.github/workflows/eb-task.yml@main
    with:
      App: ${{ inputs.Application }}
      Env: ${{ inputs.Environment }}
      Timeout: ${{ inputs.Timeout }}
      Region: ${{ inputs.Region }}
      Command: 'alembic -c conf/alembic.ini upgrade head'
    secrets: inherit

  next:
    if: inputs.Task == 'db upgrade to next' && github.event.inputs.Destructive == 'true'
    needs: preflight
    uses: hypothesis/workflows/.github/workflows/eb-task.yml@main
    with:
      App: ${{ inputs.Application }}
      Env: ${{ inputs.Environment }}
      Timeout: ${{ inputs.Timeout }}
      Region: ${{ inputs.Region }}
      Command: 'alembic -c conf/alembic.ini upgrade +1'
    secrets: inherit

  downgrade:
    if: inputs.Task == 'db downgrade to previous' && github.event.inputs.Destructive == 'true'
    needs: preflight
    uses: hypothesis/workflows/.github/workflows/eb-task.yml@main
    with:
      App: ${{ inputs.Application }}
      Env: ${{ inputs.Environment }}
      Timeout: ${{ inputs.Timeout }}
      Region: ${{ inputs.Region }}
      Command: 'alembic -c conf/alembic.ini downgrade -1'
    secrets: inherit


Deploy to Server

name: Deploy to Server

on:
  push:
    branches:
      - master

jobs:
  build:
    name: Deploy
    runs-on: ubuntu-latest
    steps:
      - name: executing remote ssh commands using key
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ secrets.host }}
          username: ${{ secrets.user }}
          key: ${{ secrets.id }}
          port: ${{ secrets.port }}
          script: |
            cd ${{ secrets.deploy_dir }}
            . venv-prod/bin/activate
            cd website
            git pull
            python -m pip install --upgrade pip
            pip install --upgrade --force-reinstall -r prod_requirements.txt
            python manage.py migrate
            python manage.py collectstatic --noinput
            python manage.py compilemessages
            echo "${{ secrets.dingseboms_password }}" | sudo -S systemctl restart gunicorn


Python Integration

name: Python Integration

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install --upgrade --force-reinstall -r prod_requirements.txt
      - name:
          Initialize Django
          # Stop the build if migrations are not committed to repo. If action stops here,
          # run `pt manage.py makemigrations` and commit the generated files.
        run: |
          python manage.py makemigrations --check
          python manage.py migrate
      - name: Run pre-commit hooks
        run: |
          pre-commit run --all-files
      - name: Run Django tests
        run: |
          python manage.py test


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ "master", "dev" ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ "master", "dev" ]
  schedule:
    - cron: '42 18 * * 5'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'javascript', 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.

        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
        # queries: security-extended,security-and-quality

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v3

    # ℹ Command-line programs to run using the OS shell.
    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

    #   If the Autobuild fails above, remove it and uncomment the following three lines.
    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

    # - run: |
    #   echo "Run, Build Application using script"
    #   ./location_of_script_within_repo/buildscript.sh

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3


Lint

name: Lint
on: [push, pull_request]
jobs:
  build:
    name: Lint add-on configuration
    runs-on: ubuntu-latest
    steps:
      - name: ⤵ Check out code from GitHub
        uses: actions/checkout@v4
      - name: 🚀 Run Home Assistant Add-on Linter
        uses: frenck/action-addon-linter@v2.15
        with:
          path: "./hassio-google-drive-backup"


name: push_to_prod
on:
  workflow_dispatch:

jobs:
  deploy_to_staging:
    runs-on: ubuntu-latest
    steps:
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - name: Check out dev repo
      uses: actions/checkout@v4
      with:
        path: dev
        persist-credentials: false

    - name: Log in to the Container registry
      uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Publish Addon Image
      uses: home-assistant/builder@master
      with:
        args: |
          --all \
          --target dev/hassio-google-drive-backup \
          --docker-hub "ghcr.io/sabeechen"


run-pytest

name: run-pytest
on:
  push:
  pull_request:
  workflow_dispatch:
jobs:
  run-pytest:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r .devcontainer/requirements-dev.txt
    - name: Install fping
      run: sudo apt-get install fping
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-line-length=300 --statistics
    - name: Test with pytest
      run: |
        pip install pytest
        pip install pytest-cov
        pytest hassio-google-drive-backup/tests --junitxml=junit/test-results.xml --cov=hassio-google-drive-backup/backup --cov-report=xml --cov-report=html
    - name: Upload pytest test results
      uses: actions/upload-artifact@v4
      with:
        name: pytest-results
        path: junit/test-results.xml
      if: ${{ always() }}
    - name: Publish Unit Test Results
      uses: EnricoMi/publish-unit-test-result-action@v2
      if: always()
      with:
        files: junit/**/*.xml
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3.1.4
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: ./coverage.xml
        directory: ./coverage/reports/
        flags: unittests
        env_vars: OS,PYTHON
        name: codecov-umbrella
        fail_ci_if_error: true
        path_to_write_report: ./coverage/codecov_report.txt
        verbose: true


Server Image Push

name: Server Image Push

on:
  workflow_dispatch:

env:
  IMAGE_NAME: addon-server

jobs:
  # Push image to GitHub Packages.
  # See also https://docs.docker.com/docker-hub/builds/
  push:
    runs-on: ubuntu-latest
    permissions:
      packages: write
      contents: read

    steps:
      - uses: actions/checkout@v4

      - name: Build image
        run: docker build -f hassio-google-drive-backup/Dockerfile-server --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" hassio-google-drive-backup/.

      - name: Log in to registry
        # This is where you will update the PAT to GITHUB_TOKEN
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

      - name: Push image
        run: |
          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME

          # Change all uppercase to lowercase
          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
          # Strip git ref prefix from version
          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
          # Strip "v" prefix from tag name
          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
          # Use Docker `latest` tag convention
          [ "$VERSION" == "master" ] && VERSION=latest
          echo IMAGE_ID=$IMAGE_ID
          echo VERSION=$VERSION
          docker tag $IMAGE_NAME $IMAGE_ID:run-$GITHUB_RUN_NUMBER
          docker push $IMAGE_ID:run-$GITHUB_RUN_NUMBER


name: push_to_staging
concurrency:
  # Only run this workflow one at a time, and cancel any in progress
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  workflow_dispatch:
    branches:
      - dev
    inputs:
      name:
        # Friendly description to be shown in the UI instead of 'name'
        description: 'Staging Verison Override'
        # Default value if no value is explicitly provided
        default: 'increment'

jobs:
  run-pytest:
    runs-on: ubuntu-latest
    steps:
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    # Check out the current branch
    - uses: actions/checkout@v4
      with:
        path: dev
    # Check out the staging barnch
    - uses: actions/checkout@v4
      with:
        path: staging
        repository: sabeechen/hgdb-dev-staging
    - run: |
          python3 staging/update.py dev staging


name: push_to_staging
on:
  workflow_dispatch:
  push:
    branches:
      - dev

jobs:
  deploy_to_staging:
    runs-on: ubuntu-latest
    steps:
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - name: Check out dev repo
      uses: actions/checkout@v4
      with:
        path: dev
        persist-credentials: false

    - name: Checkout Staging Repo
      uses: actions/checkout@v4
      with:
        path: staging
        repository: sabeechen/hgdb-dev-staging
        persist-credentials: true
        token: ${{ secrets.STAGING_REPO_TOKEN }}

    - name: Update addon verison number
      run: |
          python3 staging/update.py dev staging

    - name: Log in to the Container registry
      uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Publish Staging Addon image
      uses: home-assistant/builder@master
      with:
        args: |
          --all \
          --target dev/hassio-google-drive-backup \
          --docker-hub "ghcr.io/sabeechen"

    - name: Publish Staging Addon Version
      run: |
          cd staging
          git config user.name github-actions
          git config user.email github-actions@github.com
          git add .
          git commit -m "Updating staging addon config"
          git push


ChangeDetection.io App Test

name: ChangeDetection.io App Test

# Triggers the workflow on push or pull request events
on: [push, pull_request]

jobs:
  lint-code:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Lint with flake8
        run: |
          pip3 install flake8
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

  test-application-3-10:
    needs: lint-code
    uses: ./.github/workflows/test-stack-reusable-workflow.yml
    with:
      python-version: '3.10'

  test-application-3-11:
    needs: lint-code
    uses: ./.github/workflows/test-stack-reusable-workflow.yml
    with:
      python-version: '3.11'
      skip-pypuppeteer: true

  test-application-3-12:
    needs: lint-code
    uses: ./.github/workflows/test-stack-reusable-workflow.yml
    with:
      python-version: '3.12'
      skip-pypuppeteer: true


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  schedule:
    - cron: '27 9 * * 4'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        language: [ 'javascript', 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v3

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3


Build and push containers

name: Build and push containers

on:
  # Automatically triggered by a testing workflow passing, but this is only checked when it lands in the `master`/default branch
#  workflow_run:
#    workflows: ["ChangeDetection.io Test"]
#    branches: [master]
#    tags: ['0.*']
#    types: [completed]

  # Or a new tagged release
  release:
    types: [published, edited]

  push:
    branches:
      - master

jobs:
  metadata:
    runs-on: ubuntu-latest
    steps:
    - name: Show metadata
      run: |
        echo SHA ${{ github.sha }}
        echo github.ref:  ${{ github.ref }}
        echo github_ref: $GITHUB_REF
        echo Event name: ${{ github.event_name }}
        echo Ref ${{ github.ref }}
        echo c: ${{ github.event.workflow_run.conclusion }}
        echo r: ${{ github.event.workflow_run }}
        echo tname: "${{ github.event.release.tag_name }}"
        echo headbranch: -${{ github.event.workflow_run.head_branch }}-
        set

  build-push-containers:
    runs-on: ubuntu-latest
    # If the testing workflow has a success, then we build to :latest
    # Or if we are in a tagged release scenario.
    if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install flake8 pytest
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

      - name: Create release metadata
        run: |
          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
          echo ${{ github.sha }} > changedetectionio/source.txt
          echo ${{ github.ref }} > changedetectionio/tag.txt

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          image: tonistiigi/binfmt:latest
          platforms: all

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Login to Docker Hub Container Registry
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          install: true
          version: latest
          driver-opts: image=moby/buildkit:master

      # master branch -> :dev container tag
      - name: Build and push :dev
        id: docker_build
        if: ${{ github.ref }} == "refs/heads/master"
        uses: docker/build-push-action@v5
        with:
          context: ./
          file: ./Dockerfile
          push: true
          tags: |
            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
          cache-from: type=gha
          cache-to: type=gha,mode=max

# Looks like this was disabled
#          provenance: false

      # A new tagged release is required, which builds :tag and :latest
      - name: Build and push :tag
        id: docker_build_tag_release
        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
        uses: docker/build-push-action@v5
        with:
          context: ./
          file: ./Dockerfile
          push: true
          tags: |
            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
            ghcr.io/dgtlmoon/changedetection.io:latest
          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
          cache-from: type=gha
          cache-to: type=gha,mode=max
# Looks like this was disabled
#          provenance: false

      - name: Image digest
        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}


ChangeDetection.io Container Build Test

name: ChangeDetection.io Container Build Test

# Triggers the workflow on push or pull request events

# This line doesnt work, even tho it is the documented one
#on: [push, pull_request]

on:
  push:
    paths:
      - requirements.txt
      - Dockerfile
      - .github/workflows/*
      - .github/test/Dockerfile*

  pull_request:
    paths:
      - requirements.txt
      - Dockerfile
      - .github/workflows/*
      - .github/test/Dockerfile*

  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
  # @todo: some kind of path filter for requirements.txt and Dockerfile
jobs:
  test-container-build:
    runs-on: ubuntu-latest
    steps:
        - uses: actions/checkout@v4
        - name: Set up Python 3.11
          uses: actions/setup-python@v5
          with:
            python-version: 3.11

        # Just test that the build works, some libraries won't compile on ARM/rPi etc
        - name: Set up QEMU
          uses: docker/setup-qemu-action@v3
          with:
            image: tonistiigi/binfmt:latest
            platforms: all

        - name: Set up Docker Buildx
          id: buildx
          uses: docker/setup-buildx-action@v3
          with:
            install: true
            version: latest
            driver-opts: image=moby/buildkit:master

        # https://github.com/dgtlmoon/changedetection.io/pull/1067
        # Check we can still build under alpine/musl
        - name: Test that the docker containers can build (musl via alpine check)
          id: docker_build_musl
          uses: docker/build-push-action@v5
          with:
            context: ./
            file: ./.github/test/Dockerfile-alpine
            platforms: linux/amd64,linux/arm64

        - name: Test that the docker containers can build
          id: docker_build
          uses: docker/build-push-action@v5
          # https://github.com/docker/build-push-action#customizing
          with:
            context: ./
            file: ./Dockerfile
            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
            cache-from: type=local,src=/tmp/.buildx-cache
            cache-to: type=local,dest=/tmp/.buildx-cache