benchmark-command.yml

# Workflow to trigger benchmarking, cleaning, aggregation of the PR and evaluating w.r.t master branch, results on dashboard
name: Benchmark Pull Request
on:
  workflow_dispatch:
    inputs:
      repository:
        description: 'The repository from which the slash command was dispatched'
        required: true
      comment-id:
        description: 'The comment-id of the slash command'
        required: true
      pr-sha:
        description: 'The pr-sha of which the slash command was dispatched'
        required: true
      module:
        description: 'Which module to run the benchmark on'
        required: true
        options:
          - tabular
          - timeseries
          - multimodal
      preset:
        description: 'Preset to run for tabular/timeseries/multimodal'
        required: true
        options:
          - tabular_best
          - tabular_high
          - tabular_good
          - tabular_medium
          - timeseries_best
          - multimodal_best
      benchmark:
        description: 'Benchmark to run'
        required: true
        options:
          - tabular_full
          - tabular_test
          - tabular_small
          - timeseries_small
          - automm-image
          - automm-text
          - automm-text-tabular
          - automm-text-tabular-image
      time_limit:
        description: 'Time limit for the benchmark to run'
        required: true
        options:
          - 1h
          - 4h
          - 8h
          - 16h
          - 24h
          - 10m4c
          - g4_12x
      folds:
        description: 'Number of folds to run'
        required: false
      branch_or_pr_number:
        description: 'Branch or PR number to run the benchmark on'
        required: true
      fork_info:
        description: 'Get the forked PR repository name and branch, e.g. username/autogluon|test_branch'
        required: true
      
permissions:
  id-token: write
  contents: read

jobs:
  setup:
    runs-on: ubuntu-latest
    steps:
      - name: Create URL to the run output
        if: (github.event_name == 'workflow_dispatch')
        id: vars
        run: echo ::set-output name=run-url::https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID

      - name: Create comment
        if: (github.event_name == 'workflow_dispatch')
        uses: peter-evans/create-or-update-comment@v4
        with:
          token: ${{ secrets.CICD_PAT }}
          repository: ${{ github.event.inputs.repository }}
          comment-id: ${{ github.event.inputs.comment-id }}
          body: |
            [Benchmark Output][1]

            [1]: ${{ steps.vars.outputs.run-url }}
  
  generate_amlb_user_dir:
    needs: setup
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Setup Env Vars
        uses: ./.github/actions/setup-env-vars
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::369469875935:role/AutoGluonCIBenchmarkConfig
          role-duration-seconds: 3600
          aws-region: us-east-1
      - name: Extract Fork Info
        id: parse_fork_info
        env:
          FORK_INFO: ${{ github.event.inputs.fork_info }}
        run: |
          IFS="|" read -r fork_name fork_branch <<< "$FORK_INFO"
          echo "FORK_NAME=$fork_name" >> $GITHUB_OUTPUT
          echo "FORK_BRANCH=$fork_branch" >> $GITHUB_OUTPUT
      - name: Generate AMLB User Dir - {{ github.event.inputs.module }}
        run: |
          /bin/bash CI/bench/generate_amlb_user_dir.sh ${{ github.event.inputs.module }} ${{ steps.parse_fork_info.outputs.FORK_NAME }} ${{ steps.parse_fork_info.outputs.FORK_BRANCH }} ${{ github.sha }} "" ${{ github.event.inputs.folds }}

  benchmark:
    needs: generate_amlb_user_dir
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - name: Free Disk Space (Ubuntu)
        # uses: jlumbroso/free-disk-space@v1.2.0
        uses: hirnidrin/free-disk-space@main  # revert back once fix in https://github.com/jlumbroso/free-disk-space/pull/11
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: true
          swap-storage: true
      - name: Checkout repository for PR
        if: (github.event_name == 'workflow_dispatch')
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.inputs.pr-sha }}
      - name: Checkout repository for nightly test
        if: (github.event_name == 'schedule')
        uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - name: Setup npm
        uses: actions/setup-node@v4
        with:
          node-version: 'latest'
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::369469875935:role/AutoGluonCIBenchmark
          role-duration-seconds: 36000
          aws-region: us-east-1
      - name: Extract Fork Info
        id: parse_fork_info
        env:
          FORK_INFO: ${{ github.event.inputs.fork_info }}
        run: |
          IFS="|" read -r fork_name fork_branch <<< "$FORK_INFO"
          echo "FORK_NAME=$fork_name" >> $GITHUB_OUTPUT
          echo "FORK_BRANCH=$fork_branch" >> $GITHUB_OUTPUT
      - name: Run benchmark
        shell: bash -l {0}
        run: |
          nvm install 20
          npm install -g aws-cdk
          /bin/bash ./.github/workflow_scripts/run_benchmark.sh ${{ github.event.inputs.module }} ${{ github.event.inputs.preset }} ${{ github.event.inputs.benchmark }} ${{ github.event.inputs.time_limit }} ${{ steps.parse_fork_info.outputs.FORK_BRANCH }} ${{ github.sha }}
      - name: Upload website.txt
        uses: actions/upload-artifact@v4
        with:
          name: dashboard-website
          path: ./website.txt
      - name: Upload final_eval.txt
        uses: actions/upload-artifact@v4
        with:
          name: evaluation-results
          path: ./final_eval.txt       
  
  dashboard: 
    needs: benchmark
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - name: Download final_eval.txt
        uses: actions/download-artifact@v4
        with:
          name: evaluation-results
      - name: get evaluation results
        id: eval_result
        run: |
          body="$(cat final_eval.txt)"
          echo ::set-output name=body::$body
      - name: Comment Evaluation Result of PR with Master
        uses: peter-evans/create-or-update-comment@v4
        with:
          token: ${{ secrets.CICD_PAT }}
          repository: ${{ github.event.inputs.repository }}
          comment-id: ${{ github.event.inputs.comment-id }}
          body: ${{ steps.eval_result.outputs.body }}
      - name: Download website.txt
        uses: actions/download-artifact@v4
        with:
          name: dashboard-website
      - name: get dashboard website
        id: website
        run: |
          body="$(cat website.txt)"
          echo ::set-output name=body::$body
      - name: Comment Dashboard Website Link on PR
        uses: peter-evans/create-or-update-comment@v4
        with:
          token: ${{ secrets.CICD_PAT }}
          repository: ${{ github.event.inputs.repository }}
          comment-id: ${{ github.event.inputs.comment-id }}
          body: ${{ steps.website.outputs.body }}