Performance Benchmarks

name: Performance Benchmarks

on:
  push:
    branches:
      - develop
  pull_request:
    branches:
      - develop

jobs:
  benchmark:
    name: ${{ matrix.os }} x ${{ matrix.python }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python: ["3.8"]
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python }}

      - name: Install dependencies
        run: |
          pip install poetry
          poetry config virtualenvs.in-project true
          poetry install
          poetry run pip install pytest-benchmark

      - name: Run benchmark
        run: poetry run pytest dataprep/tests/benchmarks/eda.py --benchmark-json benchmark.json

      - name: Show benchmark result for pull request
        if: ${{ github.event_name == 'pull_request'}}
        uses: rhysd/github-action-benchmark@v1
        with:
          name: DataPrep.EDA Benchmarks
          tool: "pytest"
          output-file-path: benchmark.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: false
          save-data-file: false
          fail-threshold: "200%"
          comment-always: true
          fail-on-alert: true

      - name: Store benchmark result for push operator
        if: ${{ github.event_name == 'push'}}
        uses: rhysd/github-action-benchmark@v1
        with:
          name: DataPrep.EDA Benchmarks
          tool: "pytest"
          output-file-path: benchmark.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          fail-threshold: "200%"
          comment-always: true
          fail-on-alert: true


CI

name: CI

on:
  push:
    branches:
      - develop
      - release
  pull_request:
    branches:
      - develop

jobs:
  build:
    name: ${{ matrix.os }} x py${{ matrix.python }}
    strategy:
      fail-fast: false
      matrix:
        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        os: [ubuntu-latest, macos-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python }}

      - name: "Windows Graphviz install"
        if: runner.os == 'Windows'
        uses: crazy-max/ghaction-chocolatey@v3
        with:
          args: -h

      - name: Install Graphviz for Windows
        if: runner.os == 'Windows'
        run: |
          choco install graphviz --version=2.49.3

      - name: Install pygraphviz for Windows
        if: runner.os == 'Windows'
        run: |
          python -m pip install --use-pep517 --config-settings="--global-option=build_ext" --config-settings="--global-option=-IC:\\Program Files\\Graphviz\\include" --config-settings="--global-option=-LC:\\Program Files\\Graphviz\\lib" pygraphviz

      - name: Install Graphviz for other platforms
        if: runner.os != 'Windows'
        uses: ts-graphviz/setup-graphviz@v2
        with:
          macos-skip-brew-update: 'true'

      - name: Cache venv
        uses: actions/cache@v2
        with:
          path: ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-build-${{ matrix.python }}-${{ secrets.CACHE_VERSION }}-${{ hashFiles('poetry.lock') }}

      - name: Install dependencies
        run: |
          pip install poetry
          ${{ matrix.install_graphviz }}
          echo "Cache Version ${{ secrets.CACHE_VERSION }}"
          poetry install
          poetry run pip install ERAlchemy2
          poetry config --list

      - name: Print tool versions
        run: |
          poetry run pylint --version
          poetry run pytest --version
          poetry run black --version
          poetry run pyright --version

      - name: Check if the code is formatted
        run: poetry run black --check dataprep

      - name: Type check the project
        run: poetry run pyright dataprep || true

      - name: Build binary dependencies
        run: poetry build

      - name: Test the project
        run: poetry run pytest --cov-report xml --cov=dataprep dataprep/tests
        env:
          DATAPREP_BROWSER_TESTS: 0
          DATAPREP_CREDENTIAL_TESTS: 0
          DATAPREP_DATA_CONNECTOR_YELP_TOKEN: ""
          DATAPREP_DATA_CONNECTOR_YOUTUBE_TOKEN: ""

      - name: Style check the project
        run: poetry run pylint dataprep || true

      - uses: codecov/codecov-action@v2
        if: github.event.pull_request.head.repo.full_name == github.repository
        with:
          verbose: true
          token: ${{ secrets.CODECOV_TOKEN }}
          # path_to_write_report: ./coverage.xml

      - uses: codacy/codacy-coverage-reporter-action@v1
        if: github.event.pull_request.head.repo.full_name == github.repository
        with:
          project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
          coverage-reports: coverage.xml

  docs-build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2

      - name: Setup Graphviz
        uses: ts-graphviz/setup-graphviz@v1.2.0

      - name: Install dependencies
        run: |
          pip install poetry
          curl -L https://github.com/jgm/pandoc/releases/download/2.11.2/pandoc-2.11.2-1-amd64.deb -o /tmp/pandoc.deb && sudo dpkg -i /tmp/pandoc.deb

      - name: Cache venv
        uses: actions/cache@v2
        with:
          path: ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-build-${{ matrix.python }}-${{ secrets.CACHE_VERSION }}-${{ hashFiles('poetry.lock') }}

      - name: Install dependencies
        run: |
          pip install poetry
          poetry install
          poetry run pip install ERAlchemy2

      - name: Build docs
        run: poetry run sphinx-build -M html docs/source docs/build

      - name: Archive docs
        uses: actions/upload-artifact@v2
        with:
          name: docs
          path: docs/build/html


Clean GUI CI

name: Clean GUI CI

on:
  push:
    branches:
      - develop
      - release
  pull_request:
    branches:
      - develop

jobs:
  build:
    runs-on: ubuntu-latest

    strategy:
      matrix:
        node-version: [14.x]

    steps:
      - uses: actions/checkout@v2
      - name: Use Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v1
        with:
          node-version: ${{ matrix.node-version }}
      - name: Install Dependencies & Build Frontend
        run: |
          cd dataprep/clean/gui/clean_frontend
          rm -rf node_modules package-lock.json
          npm install
          npm run build


release

name: release

on:
  push:
    branches:
      - release

jobs:
  # docs-build:
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v2

  #     - name: Install dependencies
  #       run: |
  #         pip install poetry
  #         curl -L https://github.com/jgm/pandoc/releases/download/2.11.2/pandoc-2.11.2-1-amd64.deb -o /tmp/pandoc.deb && sudo dpkg -i /tmp/pandoc.deb

  #     - name: Cache venv
  #       uses: actions/cache@v2
  #       with:
  #         path: ~/.cache/pypoetry/virtualenvs
  #         key: ${{ runner.os }}-build-${{ matrix.python }}-${{ secrets.CACHE_VERSION }}-${{ hashFiles('poetry.lock') }}

  #     - name: Install dependencies
  #       run: |
  #         pip install poetry
  #         poetry install

  #     - name: Build docs
  #       run: poetry run sphinx-build -M html docs/source docs/build

  #     - name: Archive docs
  #       uses: actions/upload-artifact@v2
  #       with:
  #         name: docs
  #         path: docs/build/html

  build:
    runs-on: ubuntu-latest
    # needs: docs-build
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: "0"

      - uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python }}

      - name: Cache venv
        uses: actions/cache@v2
        with:
          path: ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-build-${{ matrix.python }}-${{ secrets.CACHE_VERSION }}-${{ hashFiles('poetry.lock') }}

      - name: Install dependencies
        run: |
          echo "Cache Version ${{ secrets.CACHE_VERSION }}"
          pip install poetry toml-cli
          poetry install
          poetry config --list

      - name: Print tool versions
        run: |
          poetry run pylint --version
          poetry run pytest --version
          poetry run black --version
          poetry run pyright --version

      - name: Build wheels
        run: poetry build

      - name: Parse version from pyproject.toml
        run: echo "DATAPREP_VERSION=`toml get --toml-path pyproject.toml tool.poetry.version`" >> $GITHUB_ENV

      - name: Create release note
        run: poetry run python scripts/release-note.py $(git rev-parse --short HEAD) > RELEASE.md

      - uses: ncipollo/release-action@v1
        with:
          artifacts: "dist/*.whl,dist/*.tar.gz"
          bodyFile: "RELEASE.md"
          token: ${{ secrets.GITHUB_TOKEN }}
          draft: true
          tag: v${{ env.DATAPREP_VERSION }}
          commit: ${{ env.GITHUB_SHA }}

      - name: Upload wheels
        run: poetry publish --username __token__ --password ${{ secrets.PYPI_TOKEN }}

  # docs-deploy:
  #   runs-on: ubuntu-latest
  #   needs: build
  #   if: ${{ github.event.ref == 'refs/heads/release' }}
  #   steps:
  #     - uses: actions/checkout@v2

  #     - name: Download docs
  #       uses: actions/download-artifact@v2
  #       with:
  #         name: docs
  #         path: docs/build/html

  #     - run: echo 'docs.dataprep.ai' > docs/build/html/CNAME

  #     - name: Deploy 🚀
  #       uses: JamesIves/github-pages-deploy-action@4.1.5
  #       with:
  #         branch: gh-pages # The branch the action should deploy to.
  #         folder: docs/build/html # The folder the action should deploy.
  #         clean-exclude: dev


Changelog check

name: Changelog check

on:
  pull_request:
    types: [labeled, unlabeled, opened, synchronize, reopened]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changelog:
    name: Check changelog entry
    runs-on: ubuntu-latest
    steps:
    - name: Check change log entry
      uses: scientific-python/action-check-changelogfile@064f2005176e1622e7c2bd9776140406609097d1
      env:
        CHANGELOG_FILENAME: CHANGES.rst
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        CHECK_MILESTONE: false


CI-crontests

name: CI-crontests

on:
  push:
    # Run this job on release tags, but not on pushes to the main branch
    tags:
    - '*'
  schedule:
    # run every Friday at 22:00 UTC
    - cron: '0 22 * * 5'
  workflow_dispatch:

permissions:
  contents: read

jobs:
  tests:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    if: github.repository == 'astropy/astroquery'
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: py3.12 pre-release all deps
            os: ubuntu-latest
            python: '3.12'
            toxenv: py312-test-alldeps-predeps
            toxargs: -v
            toxposargs: -v

          - name: Documentation build with linkcheck
            os: ubuntu-latest
            python: '3.10'
            toxenv: linkcheck

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python }}
    - name: Install Python dependencies
      run: python -m pip install --upgrade tox
    - name: Run tests
      run: tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }}


CI-devtest

# This test job is separated out into its own workflow to be able to obtain a separate badge for it
name: CI-devtest

on:
  push:
    branches:
    - main
    tags:
    - '*'
  pull_request:
    branches:
    - main
  schedule:
    # run every Monday at 5am UTC
    - cron: '0 5 * * 1'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  tests:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: true
      matrix:
        include:
          - name: dev dependencies with all dependencies with coverage
            os: ubuntu-latest
            python: '3.12'
            toxenv: py312-test-alldeps-devdeps-cov
            toxargs: -v

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python }}
    - name: Install Python dependencies
      run: python -m pip install --upgrade tox
    - name: Run tests
      run: tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }}
    - name: Upload coverage to codecov
      if: contains(matrix.toxenv,'-cov')
      uses: codecov/codecov-action@v3
      with:
        file: ./coverage.xml


CI-online-crontests

name: CI-online-crontests

on:
  push:
    # Run this job on release tags, but not on pushes to the main branch
    tags:
    - '*'
  schedule:
    # run every Friday at 23:00 UTC
    - cron: '0 23 * * 5'
  workflow_dispatch:

permissions:
  contents: read

jobs:
  tests:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    if: github.repository == 'astropy/astroquery'
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: py3.12 all dev deps online
            os: ubuntu-latest
            python: '3.12'
            toxenv: py312-test-alldeps-devdeps-online
            toxargs: -v
            toxposargs: -v --durations=50

          - name: Windows py3.9 all deps online
            os: windows-latest
            python: '3.9'
            toxenv: py39-test-alldeps-online
            toxargs: -v
            toxposargs: -v --durations=50

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python }}
    - name: Install Python dependencies
      run: python -m pip install --upgrade tox
    - name: Run tests
      run: tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }}


CI

# Developer version testing as well as cron testings are in separate workflows
name: CI

on:
  push:
    branches:
    - main
    tags:
    - '*'
  pull_request:
    branches:
    - main
  schedule:
    # run every Monday at 5am UTC
    - cron: '0 5 * * 1'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  tests:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:

          - name: Code style checks
            os: ubuntu-latest
            python: 3.x
            toxenv: codestyle

          - name: oldest version for all dependencies
            os: ubuntu-latest
            python: "3.9"
            toxenv: py39-test-oldestdeps-alldeps
            toxargs: -v

          - name: Python 3.10 with all optional dependencies (MacOS X)
            os: macos-latest
            python: "3.10"
            toxenv: py310-test-alldeps
            toxargs: -v

          - name: Python 3.11 with mandatory dependencies (Windows)
            os: windows-latest
            python: "3.11"
            toxenv: py311-test
            toxargs: -v

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python }}
    - name: Install Python dependencies
      run: python -m pip install --upgrade tox
    - name: Run tests
      run: tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }}
    - name: Upload coverage to codecov
      if: contains(matrix.toxenv,'-cov')
      uses: codecov/codecov-action@v3
      with:
        file: ./coverage.xml

  egg_info:
    name: egg_info with Python 3.9
    runs-on: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: "3.9"
    - name: Run egg_info
      run: python setup.py egg_info


CodeQL

name: "CodeQL"

on:
  schedule:
    - cron: '0 6 * * 1'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  analyze:
    permissions:
      actions: read  # for github/codeql-action/init to get workflow details
      contents: read  # for actions/checkout to fetch code
      security-events: write  # for github/codeql-action/autobuild to send a status report
    name: Analyze
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2


name: Milestone -- Add latest if missing on merge

on:
  pull_request_target:
    types:
      - closed
    branches:
      - 'main'

jobs:
  milestone_pr:
    name: attach missing milestone to PR
    if: github.event.pull_request.merged == true
    runs-on: ubuntu-latest
    steps:
      - uses: scientific-python/attach-next-milestone-action@a4889cfde7d2578c1bc7400480d93910d2dd34f6
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ main ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ main ]
  schedule:
    - cron: '41 23 * * 4'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://git.io/codeql-language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v3

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2


HACS validation

name: HACS validation

on:
  push:
  pull_request:

jobs:
  hacs:
    runs-on: "ubuntu-latest"
    steps:
      - uses: "actions/checkout@v2"
      - uses: "hacs/action@main"
        with: { category: "integration" }
  hassfest:
    runs-on: "ubuntu-latest"
    steps:
      - uses: "actions/checkout@v3"
      - uses: "home-assistant/actions/hassfest@master"


Pylint - FedML

name: Pylint - FedML

on:
  pull_request:
    branches: [ master, test/v0.7.0, dev/0.7.0 ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  build:
    runs-on: [self-hosted, Linux]
    strategy:
      matrix:
        os: [ ubuntu-latest ]
        arch: [ X64 ]
        python-version: ["3.8"]
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: Analysing the code with pylint
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          python3 -m pip install --upgrade pip
          pip install pylint
          pip install "fedml[gRPC]"
          pip install "fedml[tensorflow]"
          pip install "fedml[jax]"
          pip install "fedml[mxnet]"
          pip install tensorflow_federated
          pip install mxnet
          pip install jax
          pip install ptflops
          cd python
          pylint --rcfile=build_tools/lint/.pylintrc --disable=C,R,W,I ./


Build-Wheels-PyPi-Release

name: Build-Wheels-PyPi-Release
# https://github.com/pypa/cibuildwheel
# Controls when the workflow will run
on:
  workflow_run:
    workflows: [ LightSecAgg-Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  # Build the wheels for Linux, Windows and macOS for Python 3.8 and newer
  build_wheels:
    name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }}
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash
        working-directory: python

    strategy:
      # Ensure that a wheel builder finishes even if another fails
      fail-fast: false
      matrix:
        include:
          # Window 64 bit
          - os: windows-2019
            python: 38
            bitness: 64
            platform_id: win_amd64
          - os: windows-latest
            python: 39
            bitness: 64
            platform_id: win_amd64
          - os: windows-latest
            python: 310
            bitness: 64
            platform_id: win_amd64

          # Window 32 bit
          - os: windows-latest
            python: 38
            bitness: 32
            platform_id: win32
          - os: windows-latest
            python: 39
            bitness: 32
            platform_id: win32

          # Linux 64 bit manylinux2014
          - os: ubuntu-latest
            python: 38
            bitness: 64
            platform_id: manylinux_x86_64
            manylinux_image: manylinux2014
          - os: ubuntu-latest
            python: 39
            bitness: 64
            platform_id: manylinux_x86_64
            manylinux_image: manylinux2014

          # NumPy on Python 3.10 only supports 64bit and is only available with manylinux2014
          - os: ubuntu-latest
            python: 310
            bitness: 64
            platform_id: manylinux_x86_64
            manylinux_image: manylinux2014

          # MacOS x86_64
          - os: macos-latest
            bitness: 64
            python: 38
            platform_id: macosx_x86_64
          - os: macos-latest
            bitness: 64
            python: 39
            platform_id: macosx_x86_64
          - os: macos-latest
            bitness: 64
            python: 310
            platform_id: macosx_x86_64

          # MacOS arm64
          - os: macos-latest
            bitness: 64
            python: 38
            platform_id: macosx_arm64
          - os: macos-latest
            bitness: 64
            python: 39
            platform_id: macosx_arm64
          - os: macos-latest
            bitness: 64
            python: 310
            platform_id: macosx_arm64

    steps:
      - name: Checkout fedml
        uses: actions/checkout@v3

      - name: Setup Python
        uses: actions/setup-python@v4
        #with:
        #  python-version: '3.9'

      - name: Build and test wheels
        env:
          CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
          CIBW_ARCHS: all
          CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
          CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }}
          CIBW_TEST_SKIP: "*-macosx_arm64"
          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }}
          CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }}
          CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh
          CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }}
          CIBW_BUILD_VERBOSITY: 1

        #run: bash build_tools/github/build_wheels.sh
        run: |
          python -m pip install -U wheel setuptools
          python setup.py sdist bdist_wheel
          pwd
          ls dist/*.whl
          ls dist/*.tar.gz

      - name: Upload source zip file
        uses: actions/upload-artifact@v3
        with:
          path: python/dist/*.tar.gz

      - name: Upload Wheels
        uses: actions/upload-artifact@v3
        with:
          path: python/dist/*.whl

  # Build the source distribution under Linux
#  build_sdist:
#    name: Source distribution
#    needs: [ build_wheels ]
#    runs-on: ubuntu-latest
#    defaults:
#      run:
#        shell: bash
#        working-directory: python
#
#    steps:
#      - name: Checkout fedml
#        uses: actions/checkout@v3
#
#      - name: Setup Python
#        uses: actions/setup-python@v4
#        with:
#          python-version: '3.9'  # update once build dependencies are available
#
#      - name: Build source distribution
#        run: bash build_tools/github/build_source.sh
#
#      - name: Test source distribution
#        run: bash build_tools/github/test_source.sh
#        env:
#          OMP_NUM_THREADS: 2
#          OPENBLAS_NUM_THREADS: 2
#
#      - name: Store artifacts
#        uses: actions/upload-artifact@v3
#        with:
#          path: python/dist/*.tar.gz

  upload_pypi:
    name: Upload pypi
    needs: [ build_wheels ]
    runs-on: ubuntu-latest
    # upload to PyPI on every tag starting with 'v'
    #    if: github.event_name == 'push' && contains(github.event.comment, 'release v')
    # alternatively, to publish when a GitHub Release is created, use the following rule:
    # if: github.event_name == 'release' && github.event.action == 'published'
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: artifact
          path: python/dist

      - uses: pypa/gh-action-pypi-publish@v1.8.3
        with:
          skip_existing: true
          packages_dir: python/dist
          user: ${{ secrets.PYPI_USER_NAME }}
          password: ${{ secrets.PYPI_PASSWORD }}
          # To test: repository_url: https://test.pypi.org/legacy/


PIP, CLI, SP - On Windows

# This is a basic workflow to help you get started with Actions

name: PIP, CLI, SP - On Windows

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master2,  test/v0.7.0 ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  pip-install-fedml-and-test-sp:
    defaults:
      run:
        shell: powershell
        working-directory: fedml-devops\python
    strategy:
      fail-fast: false
      matrix:
        os: [ windows-2019 ]
        arch: [X64]
        python-version: ['3.8']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [self-hosted, runner-windows, devops]
    timeout-minutes: 15
    steps:
      - name: cleanup running processes
        continue-on-error: true
        run: |
          wmic.exe /interactive:off process where "name='python.exe'" call terminate
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: test "fedml login" and "fedml build"
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd tests/smoke_test/cli
          .\login.bat
          .\build.bat
      - name: test simulation-sp
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd quick_start/parrot
          python torch_fedavg_mnist_lr_one_line_example.py --cf fedml_config.yaml
          python torch_fedavg_mnist_lr_custum_data_and_model_example.py --cf fedml_config.yaml


PIP, CLI, SP - On Linux

# This is a basic workflow to help you get started with Actions

name: PIP, CLI, SP - On Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0 ]

  workflow_run:
    workflows: [ CodeQL ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

permissions: write-all

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  pip-install-fedml-and-test-sp:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [self-hosted, Linux]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: test "fedml login" and "fedml build"
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd ${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}/python
          cd tests/smoke_test/cli
          bash login.sh
          bash build.sh
      - name: test simulation-sp
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd ${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}/python
          cd quick_start/parrot
          python torch_fedavg_mnist_lr_one_line_example.py --cf fedml_config.yaml
          python torch_fedavg_mnist_lr_custum_data_and_model_example.py --cf fedml_config.yaml

      - name: test sp - sp_decentralized_mnist_lr_example
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd ${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}/python
          cd examples/simulation/sp_decentralized_mnist_lr_example
          python torch_fedavg_mnist_lr_step_by_step_example.py --cf fedml_config.yaml

      - name: test sp - sp_fednova_mnist_lr_example
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd ${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}/python
          cd examples/simulation/sp_fednova_mnist_lr_example
          python torch_fednova_mnist_lr_step_by_step_example.py --cf fedml_config.yaml

      - name: test sp - sp_fedopt_mnist_lr_example
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd ${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}/python
          cd examples/simulation/sp_fedopt_mnist_lr_example
          python torch_fedopt_mnist_lr_step_by_step_example.py --cf fedml_config.yaml

      - name: test sp - sp_hierarchicalfl_mnist_lr_example
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd ${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}/python
          cd examples/simulation/sp_hierarchicalfl_mnist_lr_example
          python torch_hierarchicalfl_mnist_lr_step_by_step_example.py --cf fedml_config.yaml

      - name: test sp - sp_turboaggregate_mnist_lr_example
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd ${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}/python
          cd examples/simulation/sp_turboaggregate_mnist_lr_example
          python torch_turboaggregate_mnist_lr_step_by_step_example.py --cf fedml_config.yaml

      - name: test sp - sp_vertical_mnist_lr_example
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd ${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}/python
          cd examples/simulation/sp_vertical_mnist_lr_example
          python torch_vertical_mnist_lr_step_by_step_example.py --cf fedml_config.yaml


CROSS-DEVICE-MNN-Linux

# This is a basic workflow to help you get started with Actions

name: CROSS-DEVICE-MNN-Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0 ]

  workflow_run:
    workflows: [ CROSS-SILO-HO-Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  cross-device-mnn-server:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest ]
        arch: [X64]
        python-version: ['3.8']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [ self-hosted, Linux ]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: Install MNN
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          pip install mnn==1.1.6

      - name: test server of cross-device
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd quick_start/beehive
          timeout 60 bash run_server.sh || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi


CROSS-SILO-HO-Win

# This is a basic workflow to help you get started with Actions

name: CROSS-SILO-HO-Win

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master2,  test/v0.7.0 ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  cross-silo-horizontal-test:
    defaults:
      run:
        shell: powershell
        working-directory: fedml-devops\python
    strategy:
      fail-fast: false
      matrix:
        os: [ windows-2019 ]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [self-hosted, runner-windows, devops]
    timeout-minutes: 15
    steps:
      - name: cleanup running processes
        continue-on-error: true
        run: |
          wmic.exe /interactive:off process where "name='python.exe'" call terminate
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: server - cross-silo - ho
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd quick_start/octopus
          .\run_server.bat ${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - cross-silo - ho
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd quick_start/octopus
          .\run_client.bat 1 ${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - cross-silo - ho
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd quick_start/octopus
          .\run_client.bat 2 ${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
        if: ${{ matrix.client-index == '2' }}


CROSS-SILO-HO-Linux

# This is a basic workflow to help you get started with Actions

name: CROSS-SILO-HO-Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0 ]

  workflow_run:
    workflows: [ PIP, CLI, SP - On Linux ]
    types:
      - completed

      # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  cross-silo-horizontal-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [self-hosted, Linux]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: server - cross-silo - ho
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd quick_start/octopus
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_server.sh $run_id
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - cross-silo - ho
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd quick_start/octopus
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 1 $run_id
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - cross-silo - ho
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd quick_start/octopus
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}


MPI - On Linux

# This is a basic workflow to help you get started with Actions

name: MPI - On Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0 ]

  workflow_run:
    workflows: [ CROSS-DEVICE-MNN-Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel

jobs:
  # https://github.com/mpi4py/mpi4py/actions/runs/34979774/workflow
  mpi_run:
    runs-on: [self-hosted, Linux]
    timeout-minutes: 15
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      matrix:
        python-version: [3.8]
        mpi: [mpich]
#        mpi: [mpich, openmpi]
        os: [ ubuntu-latest ]
        include:
        - os: ubuntu-latest
          mpi: mpich
          install-mpi: |
              sudo apt-get update
              sudo apt install -y mpich libmpich-dev
#        - os: ubuntu-latest
#          mpi: openmpi
#          install-mpi: sudo apt install -y openmpi-bin libopenmpi-dev
    steps:
    - name: Extract branch name
      shell: bash
      run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
      id: extract_branch
    - id: fedml_source_code_home
      name: cd to master or dev branch and git pull
      shell: bash
      run: |
        ls
        echo ${{ steps.extract_branch.outputs.branch }}
        if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
          echo "running on master"
          path=/home/actions-runner/fedml-master
          cd $path
          echo "dir=$path" >> $GITHUB_OUTPUT
        else
          echo "running on dev"
          path=/home/actions-runner/fedml-dev
          cd $path
          echo "dir=$path" >> $GITHUB_OUTPUT
        fi
    - name: sync git repo to local pip
      working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
      run: |
        homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
        echo $Homepath
        cd $homepath
        bash ./devops/scripts/sync-fedml-pip.sh

    - name: Test package - FedAvg
      working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
      run: |
        pwd
        cd python
        cd examples/simulation/mpi_torch_fedavg_mnist_lr_example
        sh run_custom_data_and_model_example.sh 4

    - name: Test package - Base
      working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
      run: |
        cd python
        cd examples/simulation/mpi_base_framework_example
        sh run.sh 4

    - name: Test package - Decentralized
      working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
      run: |
        cd python
        cd examples/simulation/mpi_decentralized_fl_example
        sh run.sh 4

    - name: Test package - FedOPT
      working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
      run: |
        cd python
        cd examples/simulation/mpi_fedopt_datasets_and_models_example
        sh run_step_by_step_example.sh 4 config/mnist_lr/fedml_config.yaml

    - name: Test package - FedProx
      working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
      run: |
        cd python
        cd examples/simulation/mpi_fedprox_datasets_and_models_example
        sh run_step_by_step_example.sh 4 config/mnist_lr/fedml_config.yaml

    - name: Test package - FedGAN
      working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
      run: |
        cd python
        cd examples/simulation/mpi_torch_fedgan_mnist_gan_example
        sh run_step_by_step_example.sh 4


Security(attack/defense) on Linux

# This is a basic workflow to help you get started with Actions

name: Security(attack/defense) on Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0  ]

  workflow_run:
    workflows: [ LDP-Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

permissions: write-all

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  security-attack-defense-tests:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [self-hosted, Linux]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: attack tests
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd tests/security
          sh run_attacker_tests.sh

      - name: defense tests
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd tests/security
          sh run_defender_tests.sh


LightSecAgg-Windows

# This is a basic workflow to help you get started with Actions

name: LightSecAgg-Windows

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master2,  test/v0.7.0 ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  cross-silo-horizontal-test:
    defaults:
      run:
        shell: powershell
        working-directory: fedml-devops\python
    strategy:
      fail-fast: false
      matrix:
        os: [ windows-2019 ]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [self-hosted, runner-windows, devops]
    timeout-minutes: 15
    steps:
      - name: cleanup running processes
        continue-on-error: true
        run: |
            wmic.exe /interactive:off process where "name='python.exe'" call terminate
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: server - cross-silo - ho
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd examples/cross_silo/light_sec_agg_example
          .\run_server.bat cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - cross-silo - ho
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd examples/cross_silo/light_sec_agg_example
          .\run_client.bat 1 cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - cross-silo - lightsecagg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd examples/cross_silo/light_sec_agg_example
          .\run_client.bat 2 cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
        if: ${{ matrix.client-index == '2' }}


LightSecAgg-Linux

# This is a basic workflow to help you get started with Actions

name: LightSecAgg-Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0 ]

  workflow_run:
    workflows: [ Security(attack/defense) on Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  cross-silo-horizontal-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [self-hosted, Linux]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: server - cross-silo - lightsecagg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/light_sec_agg_example
          run_id=cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_server.sh $run_id
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - cross-silo - lightsecagg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/light_sec_agg_example
          run_id=cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 1 $run_id
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - cross-silo - lightsecagg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/light_sec_agg_example
          run_id=cross-silo-lightsecagg-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}


Defender-Linux

# This is a basic workflow to help you get started with Actions

name: Defender-Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0  ]

  workflow_run:
    workflows: [ Attacker-Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  cross-silo-defense-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2', '3', '4']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [ self-hosted, Linux ]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: server - cross-silo - defense
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_defense_mnist_lr_example
          run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_server.sh $run_id
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - cross-silo - defense
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_defense_mnist_lr_example
          run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 1 $run_id
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - cross-silo - defense
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_defense_mnist_lr_example
          run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}

      - name: client 3 - cross-silo - defense
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_defense_mnist_lr_example
          run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 3 $run_id
        if: ${{ matrix.client-index == '3' }}

      - name: client 4 - cross-silo - defense
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_defense_mnist_lr_example
          run_id=cross-silo-defense-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 4 $run_id
        if: ${{ matrix.client-index == '4' }}


Attacker-Linux

# This is a basic workflow to help you get started with Actions

name: Attacker-Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master, test/v0.7.0, dev/v0.7.0  ]

  workflow_run:
    workflows: [ MPI - On Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  cross-silo-attack-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2', '3', '4']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [ self-hosted, Linux ]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: server - cross-silo - attack
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_attack_mnist_lr_example
          run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_server.sh $run_id
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - cross-silo - attack
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_attack_mnist_lr_example
          run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 1 $run_id
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - cross-silo - attack
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_attack_mnist_lr_example
          run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}

      - name: client 3 - cross-silo - attack
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_attack_mnist_lr_example
          run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 3 $run_id
        if: ${{ matrix.client-index == '3' }}

      - name: client 4 - cross-silo - attack
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/security/mqtt_s3_fedavg_attack_mnist_lr_example
          run_id=cross-silo-attack-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 4 $run_id
        if: ${{ matrix.client-index == '4' }}


Flow-Linux

# This is a basic workflow to help you get started with Actions

name: Flow-Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0 ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  Flow-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2']
    runs-on: [ self-hosted, Linux ]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: server - Flow
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd fedml/core/distributed/flow
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash test_run_server.sh $run_id
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - Flow
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd fedml/core/distributed/flow
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash test_run_client.sh 1 $run_id
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - Flow
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd fedml/core/distributed/flow
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash test_run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}


LDP-Linux

# This is a basic workflow to help you get started with Actions

name: LDP-Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0  ]

  workflow_run:
    workflows: [ CDP-Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  cross-silo-ldp-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [ self-hosted, Linux ]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: server - cross-silo - ldp
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/privacy/mqtt_s3_fedavg_ldp_mnist_lr_example
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_server.sh $run_id
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - cross-silo - ldp
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/privacy/mqtt_s3_fedavg_ldp_mnist_lr_example
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 1 $run_id
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - cross-silo - ldp
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/privacy/mqtt_s3_fedavg_ldp_mnist_lr_example
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}


CDP-Linux

# This is a basic workflow to help you get started with Actions

name: CDP-Linux

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0 ]

  workflow_run:
    workflows: [ Defender-Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  cross-silo-cdp-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [ self-hosted, Linux ]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh

      - name: server - cross-silo - cdp
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/privacy/mqtt_s3_fedavg_cdp_mnist_lr_example
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_server.sh $run_id
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - cross-silo - cdp
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/privacy/mqtt_s3_fedavg_cdp_mnist_lr_example
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 1 $run_id
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - cross-silo - cdp
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/privacy/mqtt_s3_fedavg_cdp_mnist_lr_example
          run_id=cross-silo-ho-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ "master" ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ "master" ]
  schedule:
    - cron: '34 20 * * 4'

  workflow_run:
    workflows: [ Pylint - FedML ]
    types:
      - completed

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v3

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.

        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
        # queries: security-extended,security-and-quality

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    # ℹ Command-line programs to run using the OS shell.
    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

    #   If the Autobuild fails above, remove it and uncomment the following three lines.
    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

    # - run: |
    #   echo "Run, Build Application using script"
    #   ./location_of_script_within_repo/buildscript.sh

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2
      with:
        category: "/language:${{matrix.language}}"


ML-Engines-Linux-JAX

# This is a basic workflow to help you get started with Actions

name: ML-Engines-Linux-JAX

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0 ]

  workflow_run:
    workflows: [ LightSecAgg-Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  jax-ml-engines-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest ]
        arch: [ X64 ]
        python-version: [ '3.8' ]
        client-index: [ '0', '1', '2' ]
    #        exclude:
    #          - os: macos-latest
    #            python-version: '3.8'
    #          - os: windows-latest
    #            python-version: '3.6'
    runs-on: [ self-hosted, Linux ]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh
          cd $homepath/python

      - name: server - jax - fedavg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/jax_haiku_mqtt_s3_fedavg_mnist_lr_example
          run_id=jax-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_server.sh $run_id
        if: ${{ matrix.client-index == '0' }}

      - name: client 1 - jax - fedavg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/jax_haiku_mqtt_s3_fedavg_mnist_lr_example
          run_id=jax-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 1 $run_id
        if: ${{ matrix.client-index == '1' }}

      - name: client 2 - jax - fedavg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/jax_haiku_mqtt_s3_fedavg_mnist_lr_example
          run_id=jax-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}


ML-Engines-Linux-Tensorflow

# This is a basic workflow to help you get started with Actions

name: ML-Engines-Linux-Tensorflow

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0 ]

  workflow_run:
    workflows: [ ML-Engines-Linux-MXNet ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  tf-ml-engines-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest]
        arch: [X64]
        python-version: ['3.8']
        client-index: ['0', '1', '2']
#        exclude:
#          - os: macos-latest
#            python-version: '3.8'
#          - os: windows-latest
#            python-version: '3.6'
    runs-on: [self-hosted, Linux]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh
          cd $homepath/python

      - name: server - tensorflow - fedavg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/tf_mqtt_s3_fedavg_mnist_lr_example
          run_id=tf-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_server.sh $run_id
        if:   ${{ matrix.client-index == '0' }}

      - name: client 1 - tensorflow - fedavg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/tf_mqtt_s3_fedavg_mnist_lr_example
          run_id=tf-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 1 $run_id
        if:   ${{ matrix.client-index == '1' }}

      - name: client 2 - tensorflow - fedavg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/tf_mqtt_s3_fedavg_mnist_lr_example
          run_id=tf-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}


Build-Wheels-PyPi-Test

name: Build-Wheels-PyPi-Test
# https://github.com/pypa/cibuildwheel
# Controls when the workflow will run
on:
  workflow_run:
    workflows: [ LightSecAgg-Linux ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  # Build the wheels for Linux, Windows and macOS for Python 3.8 and newer
  build_wheels:
    name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }}
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash
        working-directory: python

    strategy:
      # Ensure that a wheel builder finishes even if another fails
      fail-fast: false
      matrix:
        include:
          # Window 64 bit
          - os: windows-2019
            python: 38
            bitness: 64
            platform_id: win_amd64
          - os: windows-latest
            python: 39
            bitness: 64
            platform_id: win_amd64
          - os: windows-latest
            python: 310
            bitness: 64
            platform_id: win_amd64

          # Window 32 bit
          - os: windows-latest
            python: 38
            bitness: 32
            platform_id: win32
          - os: windows-latest
            python: 39
            bitness: 32
            platform_id: win32

          # Linux 64 bit manylinux2014
          - os: ubuntu-latest
            python: 38
            bitness: 64
            platform_id: manylinux_x86_64
            manylinux_image: manylinux2014
          - os: ubuntu-latest
            python: 39
            bitness: 64
            platform_id: manylinux_x86_64
            manylinux_image: manylinux2014

          # NumPy on Python 3.10 only supports 64bit and is only available with manylinux2014
          - os: ubuntu-latest
            python: 310
            bitness: 64
            platform_id: manylinux_x86_64
            manylinux_image: manylinux2014

          # MacOS x86_64
          - os: macos-latest
            bitness: 64
            python: 38
            platform_id: macosx_x86_64
          - os: macos-latest
            bitness: 64
            python: 39
            platform_id: macosx_x86_64
          - os: macos-latest
            bitness: 64
            python: 310
            platform_id: macosx_x86_64

          # MacOS arm64
          - os: macos-latest
            bitness: 64
            python: 38
            platform_id: macosx_arm64
          - os: macos-latest
            bitness: 64
            python: 39
            platform_id: macosx_arm64
          - os: macos-latest
            bitness: 64
            python: 310
            platform_id: macosx_arm64

    steps:
      - name: Checkout fedml
        uses: actions/checkout@v3

      - name: Setup Python
        uses: actions/setup-python@v4
        #with:
        #  python-version: '3.9'

      - name: Build and test wheels
        env:
          CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
          CIBW_ARCHS: all
          CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
          CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }}
          CIBW_TEST_SKIP: "*-macosx_arm64"
          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }}
          CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }}
          CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh
          CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }}
          CIBW_BUILD_VERBOSITY: 1

        #run: bash build_tools/github/build_wheels.sh
        run: |
          python -m pip install -U wheel setuptools
          python setup.py sdist bdist_wheel
          pwd
          ls dist/*.whl
          ls dist/*.tar.gz

      - name: Upload source zip file
        uses: actions/upload-artifact@v3
        with:
          path: python/dist/*.tar.gz

      - name: Upload Wheels
        uses: actions/upload-artifact@v3
        with:
          path: python/dist/*.whl

  # Build the source distribution under Linux
#  build_sdist:
#    name: Source distribution
#    needs: [ build_wheels ]
#    runs-on: ubuntu-latest
#    defaults:
#      run:
#        shell: bash
#        working-directory: python
#
#    steps:
#      - name: Checkout fedml
#        uses: actions/checkout@v3
#
#      - name: Setup Python
#        uses: actions/setup-python@v4
#        with:
#          python-version: '3.9'  # update once build dependencies are available
#
#      - name: Build source distribution
#        run: bash build_tools/github/build_source.sh
#
#      - name: Test source distribution
#        run: bash build_tools/github/test_source.sh
#        env:
#          OMP_NUM_THREADS: 2
#          OPENBLAS_NUM_THREADS: 2
#
#      - name: Store artifacts
#        uses: actions/upload-artifact@v3
#        with:
#          path: python/dist/*.tar.gz

  upload_pypi:
    name: Upload pypi
    needs: [ build_wheels ]
    runs-on: ubuntu-latest
    # upload to PyPI on every tag starting with 'v'
    #    if: github.event_name == 'push' && contains(github.event.comment, 'release v')
    # alternatively, to publish when a GitHub Release is created, use the following rule:
    # if: github.event_name == 'release' && github.event.action == 'published'
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: artifact
          path: python/dist

      - uses: pypa/gh-action-pypi-publish@v1.8.3
        with:
          skip_existing: true
          packages_dir: python/dist
          user: ${{ secrets.PYPI_API_USER_NAME }}
          password: ${{ secrets.PYPI_API_TOKEN }}
          repository_url: https://test.pypi.org/legacy/


ML-Engines-Linux-MXNet

# This is a basic workflow to help you get started with Actions

name: ML-Engines-Linux-MXNet

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  schedule:
    # Nightly build at 12:12 A.M.
    - cron: "12 12 */1 * *"
  pull_request:
    branches: [ master,  test/v0.7.0, dev/v0.7.0 ]

  workflow_run:
    workflows: [ ML-Engines-Linux-JAX ]
    types:
      - completed

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  mxnet-ml-engines-test:
    # defaults:
    #   run:
    #     shell: bash
    #     working-directory: python
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest ]
        arch: [ X64 ]
        python-version: [ '3.8' ]
        client-index: [ '0', '1', '2' ]
    #        exclude:
    #          - os: macos-latest
    #            python-version: '3.8'
    #          - os: windows-latest
    #            python-version: '3.6'
    runs-on: [ self-hosted, Linux ]
    timeout-minutes: 15
    steps:
      - name: Extract branch name
        shell: bash
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >>$GITHUB_OUTPUT
        id: extract_branch
      - id: fedml_source_code_home
        name: cd to master or dev branch and git pull
        shell: bash
        run: |
          ls
          echo ${{ steps.extract_branch.outputs.branch }}
          if [[ ${{ steps.extract_branch.outputs.branch }} == "master" ]]; then
            echo "running on master"
            path=/home/actions-runner/fedml-master
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          else
            echo "running on dev"
            path=/home/actions-runner/fedml-dev
            cd $path
            echo "dir=$path" >> $GITHUB_OUTPUT
          fi
      - name: sync git repo to local pip
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          homepath=${{ format('{0}', steps.fedml_source_code_home.outputs.dir) }}
          echo $Homepath
          cd $homepath
          bash ./devops/scripts/sync-fedml-pip.sh
          cd $homepath/python
          pip install mxnet==2.0.0b1

      - name: server - mxnet - fedavg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/mxnet_mqtt_s3_fedavg_mnist_lr_example
          run_id=mxnet-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_server.sh $run_id
        if: ${{ matrix.client-index == '0' }}

      - name: client 1 - mxnet - fedavg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/mxnet_mqtt_s3_fedavg_mnist_lr_example
          run_id=mxnet-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 1 $run_id
        if: ${{ matrix.client-index == '1' }}

      - name: client 2 - mxnet - fedavg
        working-directory: ${{ steps.fedml_source_code_home.outputs.dir }}
        run: |
          cd python
          cd examples/cross_silo/mxnet_mqtt_s3_fedavg_mnist_lr_example
          run_id=mxnet-ml-engine-${{ format('{0}{1}{2}{3}', github.run_id, matrix.os, matrix.arch, matrix.python-version) }}
          echo ${run_id}
          bash run_client.sh 2 $run_id
        if: ${{ matrix.client-index == '2' }}