Added MS build benchmarking

.github/workflows/benchmark_msbuild.yaml

@@ -0,0 +1,163 @@
+name: msbuild benchmarks
+
+on:
+  workflow_dispatch:
+  push:
+  pull_request:
+
+env:
+  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
+  DOTNET_CLI_TELEMETRY_OPTOUT: true
+jobs:
+  msbuild_benchmarks_windows:
+    strategy:
+      matrix:
+        testcases:
+          - large_app_with_private_packages_centralised_ngbv_removed_windows
+          - orleans_windows
+          - orchardcore_windows
+          - nuget_windows
+      fail-fast: false
+    runs-on: windows-latest
+    steps:
+      #__________WE CHECK OUT TO WORKING REPO__________#
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      #__________WE GET THINGS READY TO RUN PYTHON SCRIPTS__________#
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      #__________FINALLY WE RUN BENCHMARK TEST AND MEASURE EXECUTION TIME__________#
+      - name: run benchmark for windows
+        working-directory: scripts/test_cases/
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          python "${{matrix.testcases}}.py"
+
+  msbuild_benchmarks_linux:
+    strategy:
+      matrix:
+        testcases:
+          - large_app_with_private_packages_centralised_ngbv_removed_linux
+          - orleans_linux
+          - orchardcore_linux
+          #- nuget_linux
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      #__________WE CHECK OUT TO WORKING REPO__________#
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      #__________WE GET THINGS READY TO RUN PYTHON SCRIPTS__________#
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      #__________FINALLY WE RUN BENCHMARK TEST AND MEASURE EXECUTION TIME__________#
+      - name: run benchmark for linux
+        working-directory: scripts/test_cases/
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          python "${{matrix.testcases}}.py"
+
+  process_data:
+    # Only run on main repository
+    # Scheduled workflows do not have information about fork status, hence the hardcoded check
+    if: github.repository == 'G-Research/DotNetPerfMonitor' && github.ref == 'refs/heads/main'
+    environment: updater
+    runs-on: ubuntu-latest
+    needs:
+      - msbuild_benchmarks_windows
+      - msbuild_benchmarks_linux
+    steps:
+      - name: Install Step CLI
+        env:
+          VERSION: 0.19.0
+        run: |
+          curl -sLO https://github.com/smallstep/cli/releases/download/v${VERSION}/step-cli_${VERSION}_amd64.deb
+          sudo dpkg -i step-cli_${VERSION}_amd64.deb
+          rm step-cli_${VERSION}_amd64.deb
+
+      - name: Create access token
+        id: token
+        env:
+          APP_ID: ${{ secrets.APP_ID }}
+          APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }}
+        run: |
+          jwt=$(step crypto jwt sign --key /dev/fd/3 --issuer $APP_ID --expiration $(date -d +5min +%s) --subtle 3<<< $APP_PRIVATE_KEY)
+          installation_id=$(curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer $jwt" https://api.github.com/app/installations | jq '.[] | select(.account.login == "${{ github.repository_owner }}") | .id')
+          token=$(curl -s -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer $jwt" https://api.github.com/app/installations/$installation_id/access_tokens | jq -r '.token')
+          echo "::add-mask::$token"
+          echo "token=$token" >> $GITHUB_OUTPUT
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          token: ${{ steps.token.outputs.token }}
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+
+      - name: Install python3 dependencies
+        working-directory: scripts/processors/
+        run: pip3 install -r requirements.txt
+
+      - name: Process data
+        shell: pwsh
+        run: |
+          Get-ChildItem -File -Path artifacts -Depth 1 | Foreach {. python3 scripts/processors/process_results.py $_.fullname data/msbuild.csv}
+
+      - name: Get alerts
+        id: alerts
+        run: |
+          python3 scripts/processors/generate_alert.py data/nuget.csv active_msbuild_regressions.txt
+          if test -f "active_msbuild_regressions.txt"; then
+            echo "file_exists=true" >> $GITHUB_OUTPUT
+            echo "file_content='$file_content'" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create new issue if necessary
+        if: steps.alerts.outputs.file_exists == 'true'
+        run: |
+          numOpenIssues="$(gh api graphql -F owner=$OWNER -F name=$REPO -f query='
+            query($name: String!, $owner: String!) {
+              repository(owner: $owner, name: $name) {
+                issues(states:OPEN, filterBy: { labels:  ["active_msbuild_regressions"]}){
+                  totalCount
+                }
+              }
+            }
+          ' --jq '.data.repository.issues.totalCount')"
+
+          if [ $numOpenIssues -eq 0 ]; then
+            echo "Creating new issue"
+            gh issue create --title "Active MS Build regression" --label "active_msbuild_regressions" --body "\`\`\` $(cat active_msbuild_regressions.txt)  \`\`\`" --repo $GITHUB_REPOSITORY
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OWNER: ${{ github.repository_owner }}
+          REPO: ${{ github.event.repository.name }}
+
+      - uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          file_pattern: "data/msbuild.csv"
+          commit_author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>"
+
+      - name: Upload data
+        if: steps.alerts.outputs.file_exists == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: active_msbuild_regressions
+          path: active_msbuild_regressions.txt

.gitignore

@@ -1 +1,2 @@
 .idea
+.DS_Store

.pylintrc

@@ -1,2 +1,2 @@
 [pylint.messages control]
-disable = unused-variable, line-too-long, invalid-name
+disable = unused-variable, line-too-long, invalid-name, redefined-outer-name, unused-argument, duplicate-code, too-many-locals, too-many-arguments

data/nuget.csv

@@ -16388,4 +16388,4 @@ version,base version,scenario,test case,timestamp,duration,base duration,relativ
 8.0.100-rc.1.23414.11,7.0.100,arctic,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,39.0085192,17.8316689,218.75977744292908
 8.0.100-rc.1.23414.11,7.0.100,cold,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,35.7928123,15.4269367,232.0150331594995
 8.0.100-rc.1.23414.11,7.0.100,force,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,6.2371047,5.4641512,114.14590247795486
-8.0.100-rc.1.23414.11,7.0.100,noop,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,4.7419945,4.0517558,117.03554542946542
+8.0.100-rc.1.23414.11,7.0.100,noop,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,4.7419945,4.0517558,117.03554542946542

scripts/benchmark_runner_linux.py

@@ -0,0 +1,189 @@
+"""Run a benchmark and return the results."""
+import argparse
+import tarfile
+import subprocess
+import time
+import os
+import urllib.request
+import benchmark_utils as utils
+
+
+def create_extract_destinations(path):
+    """ Create the extract destination directories if they do not exist"""
+    if not os.path.exists(path):
+        os.mkdir(path)
+    os.chdir(path)
+    if not os.path.exists("base"):
+        os.mkdir("base")
+    if not os.path.exists("daily"):
+        os.mkdir("daily")
+
+
+def download_file(url, filename):
+    """ Download file from url and save it to filename"""
+    with urllib.request.urlopen(url) as response, open(filename, 'wb') as out_file:
+        data = response.read()
+        out_file.write(data)
+
+
+def extract_tar_gz(file_path, extract_path):
+    """Extract a zip file to the specified destination path."""
+    with tarfile.open(file_path, 'r:gz') as tar:
+        tar.extractall(path=extract_path)
+
+
+def download_and_extract_dotnet_sdk(version_url, extract_path):
+    """ Download and extract the dotnet sdk"""
+
+    tar_gz_file = "dotnet-sdk.tar.gz"
+    download_file(version_url, tar_gz_file)
+
+    # Extract the tar.gz file
+    extract_tar_gz(tar_gz_file, extract_path)
+    # extract_command = f"tar -xzf {tar_gz_file} -C {extract_path}"
+    # subprocess.run(["tar", "-xzf", tar_gz_file,
+    #                "-C", extract_path], check=True)
+
+
+def run_build_to_restore_packages(dotnet_executable):
+    """_summary_
+
+    Args:
+        dotnet_executable (_type_): _description_
+    """
+    print('-----_restoting packages_-----')
+    subprocess.run([dotnet_executable, 'restore'], check=True)
+    subprocess.run([dotnet_executable, 'build'], check=True)
+
+
+def measure_execution_time(command):
+    """measure_execution_time runs build command and measure its execution time"""
+
+    # Record start time
+    start_time = time.time()
+
+    # Run the command
+    subprocess.call(command, shell=True)
+
+    # Calculate elapsed time
+    end_time = time.time()
+    elapsed_time = end_time - start_time
+
+    return elapsed_time
+
+
+def clone_repository(repo_url, test_repo_name, test_repo_path, nested, test_solution_dir, commit_hash):
+    """_summary_
+
+    Args:
+        repo_url (String): url of the repository to be cloned
+        repo_path (String): path containing test code
+    """
+    # Clone the repository containing the solution
+    os.chdir('..')
+    subprocess.run(['git', 'clone', '--recursive', repo_url], check=True)
+
+    # --- checkout to the commit hash if it is not empty ---- #
+    if commit_hash != '':
+        subprocess.run(['git', 'submodule', 'update',
+                       '--init', '--recursive'], check=True)
+        # os.chdir(test_repo_name)
+
+        subprocess.run(['git', 'checkout', commit_hash], check=True)
+
+    os.chdir(test_repo_name)
+    if nested:
+        os.chdir(test_repo_path)
+    os.chdir(test_solution_dir)
+    subprocess.run(['ls'], check=True)
+
+
+def run_benchamrk(args):
+    """_summary_
+        run_benchamrk()
+
+    """
+    # Access the arguments as attributes of the 'args' object
+    EXTRACT_PATH = args.extract_path
+    DOTNET_BASE_VERSION_URL_LINUX = args.dotnet_base_version_url_linux
+    DOTNET_DAILY_VERSION_URL_LINUX = args.dotnet_daily_version_url_linux
+    TEST_SOLUTION_REPO_URL = args.test_solution_repo_url
+    TEST_SOLUTION_CASE = args.test_solution_case
+    TEST_SOLUTION_DIR = args.test_solution_dir
+    TEST_SOLUTION_FILE = args.solution_file
+    COMMIT_HASH = args.commit_hash
+    DATABASE_FILE = args.database_file
+    NESTED = args.is_nested_solution == "True"
+    TEST_REPO_NAME = utils.get_repo_name_by_url(TEST_SOLUTION_REPO_URL)
+
+    # create the extract destination directories if they do not exist
+    create_extract_destinations(EXTRACT_PATH)
+
+    # download and extract the dotnet sdk
+
+    download_and_extract_dotnet_sdk(DOTNET_BASE_VERSION_URL_LINUX, "base")
+    download_and_extract_dotnet_sdk(DOTNET_DAILY_VERSION_URL_LINUX, "daily")
+
+    # clone the repository and navigate to the solution directory
+    clone_repository(TEST_SOLUTION_REPO_URL,
+                     TEST_REPO_NAME, TEST_SOLUTION_CASE, NESTED, TEST_SOLUTION_DIR, COMMIT_HASH)
+
+    # build the solution using the base version
+
+    msbuild_command = f"msbuild {TEST_SOLUTION_FILE}"
+    versions = ['base']
+    duration_in_seconds = 0
+    base_duration_in_seconds = 0
+    test_scenario = 'cold'
+    for version in versions:
+        # sub_dir = "/sdk" if version == 'daily' else ''
+        # subdirs = './../../../sdk' if NESTED else './../sdk'
+        subdirs = './../../../sdk' if NESTED else './../sdk'
+        exec_path = os.path.abspath(f"{subdirs}/{version}/dotnet")
+        run_build_to_restore_packages(exec_path)
+        simple_command = f"msbuild {TEST_SOLUTION_FILE}"
+        command = f"{exec_path} {simple_command}"
+        elapsed_time = measure_execution_time(command)
+        if version == 'base':
+            base_duration_in_seconds = elapsed_time
+        else:
+            duration_in_seconds = elapsed_time
+        print('-----LINUX BENCHMARK RESULT-----')
+        print(
+            f"Running '{command}' with {version} version took {elapsed_time}s to execute.")
+
+        # save benchmark results to a csv file
+        SDK_VERSION = subprocess.check_output(
+            [exec_path, '--version'], text=True, stderr=subprocess.STDOUT).strip()
+        SDK_DAILY_VERSION = "8.0.1xx"
+        utils.save_benchmark_results(
+            DATABASE_FILE, duration_in_seconds, base_duration_in_seconds, SDK_VERSION, SDK_DAILY_VERSION, TEST_SOLUTION_CASE, test_scenario
+        )
+
+
+if __name__ == '__main__':
+    # Parse arguments using argparse
+    parser = argparse.ArgumentParser(
+        description='---Benchmark Runner---')
+
+   # <-Parse arguments
+    parser.add_argument('--extract_path', help='Path for extraction')
+    parser.add_argument('--dotnet_base_version_url_linux',
+                        help='URL for the base version of .NET SDK on Linux')
+    parser.add_argument('--dotnet_daily_version_url_linux',
+                        help='URL for the daily version of .NET SDK on Linux')
+    parser.add_argument('--test_solution_repo_url',
+                        help='URL of the test solution repository')
+    parser.add_argument('--test_solution_case', help='Test solution case')
+    parser.add_argument('--test_solution_dir', help='Test solution directory')
+    parser.add_argument('--database_file', help='Path to the database file')
+    parser.add_argument('--solution_file', help='Path to the database file')
+    parser.add_argument('--is_nested_solution',
+                        help='Path to the database file')
+    parser.add_argument('--commit_hash', default='',
+                        help='Path to the database file')
+    # Parse arguments ->
+
+    args = parser.parse_args()
+
+    run_benchamrk(args)