diff --git a/.DS_Store b/.DS_Store index 0477eab..fca97e7 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.github/workflows/benchmark_msbuild.yaml b/.github/workflows/benchmark_msbuild.yaml new file mode 100644 index 0000000..40591c4 --- /dev/null +++ b/.github/workflows/benchmark_msbuild.yaml @@ -0,0 +1,163 @@ +name: msbuild benchmarks + +on: + workflow_dispatch: + push: + pull_request: + +env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + DOTNET_CLI_TELEMETRY_OPTOUT: true +jobs: + msbuild_benchmarks_windows: + strategy: + matrix: + testcases: + - large_app_with_private_packages_centralised_ngbv_removed_windows + - orleans_windows + - orchardcore_windows + - nuget_windows + fail-fast: false + runs-on: windows-latest + steps: + #__________WE CHECK OUT TO WORKING REPO__________# + - name: Checkout + uses: actions/checkout@v3 + + #__________WE GET THINGS READY TO RUN PYTHON SCRIPTS__________# + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + #__________FINALLY WE RUN BENCHMARK TEST AND MEASURE EXECUTION TIME__________# + - name: run benchmark for windows + working-directory: scripts/test_cases/ + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + python "${{matrix.testcases}}.py" + + msbuild_benchmarks_linux: + strategy: + matrix: + testcases: + - large_app_with_private_packages_centralised_ngbv_removed_linux + - orleans_linux + - orchardcore_linux + #- nuget_linux + fail-fast: false + runs-on: ubuntu-latest + steps: + #__________WE CHECK OUT TO WORKING REPO__________# + - name: Checkout + uses: actions/checkout@v3 + + #__________WE GET THINGS READY TO RUN PYTHON SCRIPTS__________# + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + #__________FINALLY WE RUN BENCHMARK TEST AND MEASURE EXECUTION TIME__________# + - name: run benchmark for linux + working-directory: scripts/test_cases/ + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + python "${{matrix.testcases}}.py" + + process_data: + # Only run on main repository + # Scheduled workflows do not have information about fork status, hence the hardcoded check + if: github.repository == 'G-Research/DotNetPerfMonitor' && github.ref == 'refs/heads/main' + environment: updater + runs-on: ubuntu-latest + needs: + - msbuild_benchmarks_windows + - msbuild_benchmarks_linux + steps: + - name: Install Step CLI + env: + VERSION: 0.19.0 + run: | + curl -sLO https://github.com/smallstep/cli/releases/download/v${VERSION}/step-cli_${VERSION}_amd64.deb + sudo dpkg -i step-cli_${VERSION}_amd64.deb + rm step-cli_${VERSION}_amd64.deb + + - name: Create access token + id: token + env: + APP_ID: ${{ secrets.APP_ID }} + APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }} + run: | + jwt=$(step crypto jwt sign --key /dev/fd/3 --issuer $APP_ID --expiration $(date -d +5min +%s) --subtle 3<<< $APP_PRIVATE_KEY) + installation_id=$(curl -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer $jwt" https://api.github.com/app/installations | jq '.[] | select(.account.login == "${{ github.repository_owner }}") | .id') + token=$(curl -s -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer $jwt" https://api.github.com/app/installations/$installation_id/access_tokens | jq -r '.token') + echo "::add-mask::$token" + echo "token=$token" >> $GITHUB_OUTPUT + + - name: Checkout + uses: actions/checkout@v3 + with: + token: ${{ steps.token.outputs.token }} + + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Install python3 dependencies + working-directory: scripts/processors/ + run: pip3 install -r requirements.txt + + - name: Process data + shell: pwsh + run: | + Get-ChildItem -File -Path artifacts -Depth 1 | Foreach {. python3 scripts/processors/process_results.py $_.fullname data/msbuild.csv} + + - name: Get alerts + id: alerts + run: | + python3 scripts/processors/generate_alert.py data/nuget.csv active_msbuild_regressions.txt + if test -f "active_msbuild_regressions.txt"; then + echo "file_exists=true" >> $GITHUB_OUTPUT + echo "file_content='$file_content'" >> $GITHUB_OUTPUT + fi + + - name: Create new issue if necessary + if: steps.alerts.outputs.file_exists == 'true' + run: | + numOpenIssues="$(gh api graphql -F owner=$OWNER -F name=$REPO -f query=' + query($name: String!, $owner: String!) { + repository(owner: $owner, name: $name) { + issues(states:OPEN, filterBy: { labels: ["active_msbuild_regressions"]}){ + totalCount + } + } + } + ' --jq '.data.repository.issues.totalCount')" + + if [ $numOpenIssues -eq 0 ]; then + echo "Creating new issue" + gh issue create --title "Active MS Build regression" --label "active_msbuild_regressions" --body "\`\`\` $(cat active_msbuild_regressions.txt) \`\`\`" --repo $GITHUB_REPOSITORY + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO: ${{ github.event.repository.name }} + + - uses: stefanzweifel/git-auto-commit-action@v4 + with: + file_pattern: "data/msbuild.csv" + commit_author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" + + - name: Upload data + if: steps.alerts.outputs.file_exists == 'true' + uses: actions/upload-artifact@v3 + with: + name: active_msbuild_regressions + path: active_msbuild_regressions.txt diff --git a/.gitignore b/.gitignore index 485dee6..1f1025f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .idea +.DS_Store \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index 84de4c0..40fa47f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,2 +1,2 @@ [pylint.messages control] -disable = unused-variable, line-too-long, invalid-name \ No newline at end of file +disable = unused-variable, line-too-long, invalid-name, redefined-outer-name, unused-argument, duplicate-code, too-many-locals, too-many-arguments \ No newline at end of file diff --git a/data/nuget.csv b/data/nuget.csv index 18b2bb2..b0ca39c 100644 --- a/data/nuget.csv +++ b/data/nuget.csv @@ -16388,4 +16388,4 @@ version,base version,scenario,test case,timestamp,duration,base duration,relativ 8.0.100-rc.1.23414.11,7.0.100,arctic,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,39.0085192,17.8316689,218.75977744292908 8.0.100-rc.1.23414.11,7.0.100,cold,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,35.7928123,15.4269367,232.0150331594995 8.0.100-rc.1.23414.11,7.0.100,force,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,6.2371047,5.4641512,114.14590247795486 -8.0.100-rc.1.23414.11,7.0.100,noop,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,4.7419945,4.0517558,117.03554542946542 +8.0.100-rc.1.23414.11,7.0.100,noop,Orleans-eda972a,2023-08-15T08:05:30.3948830Z,4.7419945,4.0517558,117.03554542946542 \ No newline at end of file diff --git a/scripts/benchmark_runner_linux.py b/scripts/benchmark_runner_linux.py new file mode 100644 index 0000000..987120a --- /dev/null +++ b/scripts/benchmark_runner_linux.py @@ -0,0 +1,189 @@ +"""Run a benchmark and return the results.""" +import argparse +import tarfile +import subprocess +import time +import os +import urllib.request +import benchmark_utils as utils + + +def create_extract_destinations(path): + """ Create the extract destination directories if they do not exist""" + if not os.path.exists(path): + os.mkdir(path) + os.chdir(path) + if not os.path.exists("base"): + os.mkdir("base") + if not os.path.exists("daily"): + os.mkdir("daily") + + +def download_file(url, filename): + """ Download file from url and save it to filename""" + with urllib.request.urlopen(url) as response, open(filename, 'wb') as out_file: + data = response.read() + out_file.write(data) + + +def extract_tar_gz(file_path, extract_path): + """Extract a zip file to the specified destination path.""" + with tarfile.open(file_path, 'r:gz') as tar: + tar.extractall(path=extract_path) + + +def download_and_extract_dotnet_sdk(version_url, extract_path): + """ Download and extract the dotnet sdk""" + + tar_gz_file = "dotnet-sdk.tar.gz" + download_file(version_url, tar_gz_file) + + # Extract the tar.gz file + extract_tar_gz(tar_gz_file, extract_path) + # extract_command = f"tar -xzf {tar_gz_file} -C {extract_path}" + # subprocess.run(["tar", "-xzf", tar_gz_file, + # "-C", extract_path], check=True) + + +def run_build_to_restore_packages(dotnet_executable): + """_summary_ + + Args: + dotnet_executable (_type_): _description_ + """ + print('-----_restoting packages_-----') + subprocess.run([dotnet_executable, 'restore'], check=True) + subprocess.run([dotnet_executable, 'build'], check=True) + + +def measure_execution_time(command): + """measure_execution_time runs build command and measure its execution time""" + + # Record start time + start_time = time.time() + + # Run the command + subprocess.call(command, shell=True) + + # Calculate elapsed time + end_time = time.time() + elapsed_time = end_time - start_time + + return elapsed_time + + +def clone_repository(repo_url, test_repo_name, test_repo_path, nested, test_solution_dir, commit_hash): + """_summary_ + + Args: + repo_url (String): url of the repository to be cloned + repo_path (String): path containing test code + """ + # Clone the repository containing the solution + os.chdir('..') + subprocess.run(['git', 'clone', '--recursive', repo_url], check=True) + + # --- checkout to the commit hash if it is not empty ---- # + if commit_hash != '': + subprocess.run(['git', 'submodule', 'update', + '--init', '--recursive'], check=True) + # os.chdir(test_repo_name) + + subprocess.run(['git', 'checkout', commit_hash], check=True) + + os.chdir(test_repo_name) + if nested: + os.chdir(test_repo_path) + os.chdir(test_solution_dir) + subprocess.run(['ls'], check=True) + + +def run_benchamrk(args): + """_summary_ + run_benchamrk() + + """ + # Access the arguments as attributes of the 'args' object + EXTRACT_PATH = args.extract_path + DOTNET_BASE_VERSION_URL_LINUX = args.dotnet_base_version_url_linux + DOTNET_DAILY_VERSION_URL_LINUX = args.dotnet_daily_version_url_linux + TEST_SOLUTION_REPO_URL = args.test_solution_repo_url + TEST_SOLUTION_CASE = args.test_solution_case + TEST_SOLUTION_DIR = args.test_solution_dir + TEST_SOLUTION_FILE = args.solution_file + COMMIT_HASH = args.commit_hash + DATABASE_FILE = args.database_file + NESTED = args.is_nested_solution == "True" + TEST_REPO_NAME = utils.get_repo_name_by_url(TEST_SOLUTION_REPO_URL) + + # create the extract destination directories if they do not exist + create_extract_destinations(EXTRACT_PATH) + + # download and extract the dotnet sdk + + download_and_extract_dotnet_sdk(DOTNET_BASE_VERSION_URL_LINUX, "base") + download_and_extract_dotnet_sdk(DOTNET_DAILY_VERSION_URL_LINUX, "daily") + + # clone the repository and navigate to the solution directory + clone_repository(TEST_SOLUTION_REPO_URL, + TEST_REPO_NAME, TEST_SOLUTION_CASE, NESTED, TEST_SOLUTION_DIR, COMMIT_HASH) + + # build the solution using the base version + + msbuild_command = f"msbuild {TEST_SOLUTION_FILE}" + versions = ['base'] + duration_in_seconds = 0 + base_duration_in_seconds = 0 + test_scenario = 'cold' + for version in versions: + # sub_dir = "/sdk" if version == 'daily' else '' + # subdirs = './../../../sdk' if NESTED else './../sdk' + subdirs = './../../../sdk' if NESTED else './../sdk' + exec_path = os.path.abspath(f"{subdirs}/{version}/dotnet") + run_build_to_restore_packages(exec_path) + simple_command = f"msbuild {TEST_SOLUTION_FILE}" + command = f"{exec_path} {simple_command}" + elapsed_time = measure_execution_time(command) + if version == 'base': + base_duration_in_seconds = elapsed_time + else: + duration_in_seconds = elapsed_time + print('-----LINUX BENCHMARK RESULT-----') + print( + f"Running '{command}' with {version} version took {elapsed_time}s to execute.") + + # save benchmark results to a csv file + SDK_VERSION = subprocess.check_output( + [exec_path, '--version'], text=True, stderr=subprocess.STDOUT).strip() + SDK_DAILY_VERSION = "8.0.1xx" + utils.save_benchmark_results( + DATABASE_FILE, duration_in_seconds, base_duration_in_seconds, SDK_VERSION, SDK_DAILY_VERSION, TEST_SOLUTION_CASE, test_scenario + ) + + +if __name__ == '__main__': + # Parse arguments using argparse + parser = argparse.ArgumentParser( + description='---Benchmark Runner---') + + # <-Parse arguments + parser.add_argument('--extract_path', help='Path for extraction') + parser.add_argument('--dotnet_base_version_url_linux', + help='URL for the base version of .NET SDK on Linux') + parser.add_argument('--dotnet_daily_version_url_linux', + help='URL for the daily version of .NET SDK on Linux') + parser.add_argument('--test_solution_repo_url', + help='URL of the test solution repository') + parser.add_argument('--test_solution_case', help='Test solution case') + parser.add_argument('--test_solution_dir', help='Test solution directory') + parser.add_argument('--database_file', help='Path to the database file') + parser.add_argument('--solution_file', help='Path to the database file') + parser.add_argument('--is_nested_solution', + help='Path to the database file') + parser.add_argument('--commit_hash', default='', + help='Path to the database file') + # Parse arguments -> + + args = parser.parse_args() + + run_benchamrk(args) diff --git a/scripts/benchmark_runner_windows.py b/scripts/benchmark_runner_windows.py new file mode 100644 index 0000000..d10f280 --- /dev/null +++ b/scripts/benchmark_runner_windows.py @@ -0,0 +1,185 @@ +"""Run a benchmark and return the results.""" +import argparse +import zipfile +import subprocess +import time +import os +import urllib.request +import benchmark_utils as utils + + +def create_extract_destinations(path): + """ Create the extract destination directories if they do not exist""" + if not os.path.exists(path): + os.mkdir(path) + os.chdir(path) + if not os.path.exists("base"): + os.mkdir("base") + if not os.path.exists("daily"): + os.mkdir("daily") + + +def download_file(url, filename): + """ Download file from url and save it to filename""" + with urllib.request.urlopen(url) as response, open(filename, 'wb') as out_file: + data = response.read() + out_file.write(data) + + +def extract_zip(zip_file, extract_path): + """Extract a zip file to the specified destination path.""" + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + zip_ref.extractall(extract_path) + + +def download_and_extract_dotnet_sdk(version_url, extract_path): + """ Download and extract the dotnet sdk""" + zip_file = "dotnet-sdk.zip" + download_file(version_url, zip_file) + + # Extract the zip file + extract_zip(zip_file, extract_path) + # subprocess.run(["powershell", "Expand-Archive", "-Path", zip_file, + # "-DestinationPath", extract_path], check=True, shell=True) + + +def run_build_to_restore_packages(dotnet_executable): + """_summary_ + + Args: + dotnet_executable (_type_): _description_ + """ + print('-----_restoting packages_-----') + subprocess.run([dotnet_executable, 'restore'], check=True) + subprocess.run([dotnet_executable, 'build'], check=True) + + +def measure_execution_time(command): + """measure_execution_time runs build command and measure its execution time""" + + # Record start time + start_time = time.time() + + # Run the command + subprocess.call(command, shell=True) + + # Calculate elapsed time + end_time = time.time() + elapsed_time = end_time - start_time + + return elapsed_time + + +def clone_repository(repo_url, test_repo_name, test_repo_path, nested, test_solution_dir, commit_hash): + """_summary_ + + Args: + repo_url (String): url of the repository to be cloned + repo_path (String): path containing test code + """ + # Clone the repository containing the solution + os.chdir('..') + subprocess.run(['git', 'clone', '--recursive', repo_url], check=True) + + # --- checkout to the commit hash if it is not empty ---- # + if commit_hash != '': + subprocess.run(['git', 'submodule', 'update', + '--init', '--recursive'], check=True) + # os.chdir(test_repo_name) + subprocess.run(['git', 'checkout', commit_hash], check=True) + + os.chdir(test_repo_name) + if nested: + os.chdir(test_repo_path) + os.chdir(test_solution_dir) + subprocess.run(['dir'], check=True) + + +def run_benchamrk(args): + """_summary_ + run_benchamrk() + + """ + # Access the arguments as attributes of the 'args' object + EXTRACT_PATH = args.extract_path + DOTNET_BASE_VERSION_URL_LINUX = args.dotnet_base_version_url_linux + DOTNET_DAILY_VERSION_URL_LINUX = args.dotnet_daily_version_url_linux + TEST_SOLUTION_REPO_URL = args.test_solution_repo_url + TEST_SOLUTION_CASE = args.test_solution_case + TEST_SOLUTION_DIR = args.test_solution_dir + TEST_SOLUTION_FILE = args.solution_file + DATABASE_FILE = args.database_file + NESTED = args.is_nested_solution == "True" + TEST_REPO_NAME = utils.get_repo_name_by_url(TEST_SOLUTION_REPO_URL) + COMMIT_HASH = args.commit_hash + # create the extract destination directories if they do not exist + create_extract_destinations(EXTRACT_PATH) + + # download and extract the dotnet sdk + + download_and_extract_dotnet_sdk(DOTNET_BASE_VERSION_URL_LINUX, "base") + download_and_extract_dotnet_sdk(DOTNET_DAILY_VERSION_URL_LINUX, "daily") + + # clone the repository and navigate to the solution directory + clone_repository(TEST_SOLUTION_REPO_URL, + TEST_REPO_NAME, TEST_SOLUTION_CASE, NESTED, TEST_SOLUTION_DIR, COMMIT_HASH) + + # build the solution using the base version + + msbuild_command = f"msbuild {TEST_SOLUTION_FILE}" + versions = ['base'] + duration_in_seconds = 0 + base_duration_in_seconds = 0 + test_scenario = 'cold' + for version in versions: + # sub_dir = "/sdk" if version == 'daily' else '' + # subdirs = './../../../sdk' if NESTED else './../sdk' + subdirs = './../../../sdk' if NESTED else './../sdk' + exec_path = os.path.abspath(f"{subdirs}/{version}/dotnet") + run_build_to_restore_packages(exec_path) + simple_command = f"msbuild {TEST_SOLUTION_FILE}" + command = f"{exec_path} {simple_command}" + elapsed_time = measure_execution_time(command) + if version == 'base': + base_duration_in_seconds = elapsed_time + else: + duration_in_seconds = elapsed_time + print('-----WINDOWS BENCHMARK RESULT-----') + print( + f"Running '{command}' with {version} version took {elapsed_time}s to execute.") + + # save benchmark results to a csv file + SDK_VERSION = subprocess.check_output( + [exec_path, '--version'], text=True, stderr=subprocess.STDOUT).strip() + SDK_DAILY_VERSION = "8.0.1xx" + utils.save_benchmark_results( + DATABASE_FILE, duration_in_seconds, base_duration_in_seconds, SDK_VERSION, SDK_DAILY_VERSION, TEST_SOLUTION_CASE, test_scenario + ) + + +if __name__ == '__main__': + # Parse arguments using argparse + parser = argparse.ArgumentParser( + description='---Benchmark Runner---') + + # <-Parse arguments + parser.add_argument('--extract_path', help='Path for extraction') + parser.add_argument('--dotnet_base_version_url_linux', + help='URL for the base version of .NET SDK on Linux') + parser.add_argument('--dotnet_daily_version_url_linux', + help='URL for the daily version of .NET SDK on Linux') + parser.add_argument('--test_solution_repo_url', + help='URL of the test solution repository') + parser.add_argument('--test_solution_case', help='Test solution case') + parser.add_argument('--test_solution_dir', help='Test solution directory') + parser.add_argument('--database_file', help='Path to the database file') + parser.add_argument('--solution_file', help='Path to the database file') + parser.add_argument('--is_nested_solution', + help='Path to the database file') + parser.add_argument('--commit_hash', default='', + help='Path to the database file') + # Parse arguments -> + + args = parser.parse_args() + + run_benchamrk(args) diff --git a/scripts/benchmark_utils.py b/scripts/benchmark_utils.py new file mode 100644 index 0000000..2ec155b --- /dev/null +++ b/scripts/benchmark_utils.py @@ -0,0 +1,49 @@ +"""Save benchmark results to a file.""" +import csv +from datetime import datetime + + +def camel_casify_solution_name(string): + """This method converts a string to camel case""" + parts = string.split('_') + camel_case_parts = [part.capitalize() for part in parts[:-1]] + return ''.join(camel_case_parts) + + +def print_csv_content(file_path): + """Prints content of csv file for debugging purposes""" + with open(file_path, 'r', encoding='utf-8') as csv_file: + reader = csv.reader(csv_file) + for row in reader: + print(row) + + +def save_benchmark_results(file_path, benchmark_duration, benchmark_base_duration, sdk_version, sdk_daily_version, test_case_name, test_scenario): + """Saves bencharmk results to a csv file + + Args: + file_path (_type_): _description_ + """ + # Get the benchmark results and create a row data + # row_data = [version,base version,scenario,test case,timestamp,duration,base duration,relative duration] + version = sdk_daily_version + base_version = sdk_version + timestamp = datetime.utcnow().isoformat(timespec='seconds') + scenario = test_scenario + test_case = test_case_name + duration = benchmark_duration + base_duration = benchmark_base_duration + relative_duration = abs(base_duration - duration) + benchmark_results = [version, base_version, scenario, test_case, + timestamp, duration, base_duration, relative_duration] + + with open(file_path, 'a', newline='', encoding='utf-8') as csv_file: + writer = csv.writer(csv_file) + writer.writerow(benchmark_results) + + print_csv_content(file_path) + + +def get_repo_name_by_url(repo_url): + """This method returns the name of the repository from its url""" + return repo_url.split('/')[-1] diff --git a/scripts/processors/.pylintrc b/scripts/processors/.pylintrc index fe98ef9..514fea8 100644 --- a/scripts/processors/.pylintrc +++ b/scripts/processors/.pylintrc @@ -1,2 +1,2 @@ [pylint.messages control] -disable = unused-variable, line-too-long, invalid-name +disable = unused-variable, line-too-long, invalid-name, redefined-outer-name, unused-argument, duplicate-code,too-many-locals, too-many-arguments \ No newline at end of file diff --git a/scripts/processors/requirements.txt b/scripts/processors/requirements.txt index 3ad1509..cecbdd6 100644 --- a/scripts/processors/requirements.txt +++ b/scripts/processors/requirements.txt @@ -1,3 +1,4 @@ plotly pandas -pylint \ No newline at end of file +pylint +argparse \ No newline at end of file diff --git a/scripts/test_cases/.pylintrc b/scripts/test_cases/.pylintrc new file mode 100644 index 0000000..40fa47f --- /dev/null +++ b/scripts/test_cases/.pylintrc @@ -0,0 +1,2 @@ +[pylint.messages control] +disable = unused-variable, line-too-long, invalid-name, redefined-outer-name, unused-argument, duplicate-code, too-many-locals, too-many-arguments \ No newline at end of file diff --git a/scripts/test_cases/large_app_with_private_packages_centralised_ngbv_removed_linux.py b/scripts/test_cases/large_app_with_private_packages_centralised_ngbv_removed_linux.py new file mode 100644 index 0000000..d47d876 --- /dev/null +++ b/scripts/test_cases/large_app_with_private_packages_centralised_ngbv_removed_linux.py @@ -0,0 +1,28 @@ +"""_summary_ +This script basically runs benchmark test for msbuild +Returns: + None: ..... +""" + +import subprocess + +EXTRACT_PATH = "sdk" +WORKING_DIR = "msbuild-performance-test" +DOTNET_BASE_VERSION_URL_LINUX = "https://download.visualstudio.microsoft.com/download/pr/dc930bff-ef3d-4f6f-8799-6eb60390f5b4/1efee2a8ea0180c94aff8f15eb3af981/dotnet-sdk-6.0.300-linux-x64.tar.gz" +DOTNET_DAILY_VERSION_URL_LINUX = "https://aka.ms/dotnet/8.0.1xx/daily/dotnet-sdk-linux-x64.tar.gz" +TEST_SOLUTION_REPO_URL = "https://github.com/marcin-krystianc/TestSolutions" +TEST_REPO_NAME = "TestSolutions" +TEST_SOLUTION_CASE = "LargeAppWithPrivatePackagesCentralisedSlim" +TEST_SOLUTION_DIR = "solution" +SDK_VERSION = "6.0.300" +SDK_DAILY_VERSION = "8.0.1xx" +DATABASE_FILE = "./../../../../../data/msbuild.csv" +NESTED = "True" +SOLUTION_FILE = "LargeAppWithPrivatePackagesCentralisedSlim.sln" +COMMIT_HASH = "142722bebfe90c4e5c98303fa1598db6a760adae" + +if __name__ == "__main__": + commands_chain = ["python3", "./../benchmark_runner_linux.py", "--extract_path", EXTRACT_PATH, "--dotnet_base_version_url_linux", + DOTNET_BASE_VERSION_URL_LINUX, "--dotnet_daily_version_url_linux", DOTNET_DAILY_VERSION_URL_LINUX, "--test_solution_repo_url", TEST_SOLUTION_REPO_URL, "--test_solution_case", TEST_SOLUTION_CASE, "--test_solution_dir", TEST_SOLUTION_DIR, "--solution_file", SOLUTION_FILE, "--commit_hash", COMMIT_HASH, "--database_file", DATABASE_FILE, "--is_nested_solution", NESTED] + + subprocess.run(commands_chain, check=True) diff --git a/scripts/test_cases/large_app_with_private_packages_centralised_ngbv_removed_windows.py b/scripts/test_cases/large_app_with_private_packages_centralised_ngbv_removed_windows.py new file mode 100644 index 0000000..66d5f20 --- /dev/null +++ b/scripts/test_cases/large_app_with_private_packages_centralised_ngbv_removed_windows.py @@ -0,0 +1,34 @@ +"""_summary_ +This script basically runs benchmark test for msbuild on windows +Returns: + None: ..... +""" + +import subprocess +# ___ EXTRACTION CONSTANTS ___ # +EXTRACT_PATH = "sdk" +WORKING_DIR = "msbuild-performance-test" + +# _____ BENCHMARK DEPENDENCIES CONSTANTS _____ # + +DOTNET_BASE_VERSION_URL_WINDOWS = "https://download.visualstudio.microsoft.com/download/pr/de1f99bb-4d6d-4dfe-9935-d24b1e8bca12/0b449d12398e45c62dce4b497e1b49bb/dotnet-sdk-6.0.316-win-x64.zip" + +DOTNET_DAILY_VERSION_URL_WINDOWS = "https://aka.ms/dotnet/8.0.1xx/daily/dotnet-sdk-win-x64.zip" + +TEST_SOLUTION_REPO_URL = "https://github.com/marcin-krystianc/TestSolutions" + +TEST_REPO_NAME = "TestSolutions" +TEST_SOLUTION_CASE = "LargeAppWithPrivatePackagesCentralisedSlim" +TEST_SOLUTION_DIR = "solution" +SDK_VERSION = "6.0.316" +SDK_DAILY_VERSION = "8.0.1xx" +DATABASE_FILE = "./../../../../../data/msbuild.csv" +NESTED = "True" +SOLUTION_FILE = "LargeAppWithPrivatePackagesCentralisedSlim.sln" +COMMIT_HASH = "142722bebfe90c4e5c98303fa1598db6a760adae" + +if __name__ == "__main__": + commands_chain = ["python3", "./../benchmark_runner_windows.py", "--extract_path", EXTRACT_PATH, "--dotnet_base_version_url_linux", + DOTNET_BASE_VERSION_URL_WINDOWS, "--dotnet_daily_version_url_linux", DOTNET_DAILY_VERSION_URL_WINDOWS, "--test_solution_repo_url", TEST_SOLUTION_REPO_URL, "--test_solution_case", TEST_SOLUTION_CASE, "--test_solution_dir", TEST_SOLUTION_DIR, "--solution_file", SOLUTION_FILE, "--commit_hash", COMMIT_HASH, "--database_file", DATABASE_FILE, "--is_nested_solution", NESTED] + + subprocess.run(commands_chain, check=True) diff --git a/scripts/test_cases/nuget_linux.py b/scripts/test_cases/nuget_linux.py new file mode 100644 index 0000000..53d508c --- /dev/null +++ b/scripts/test_cases/nuget_linux.py @@ -0,0 +1,27 @@ +"""_summary_ +This script basically runs benchmark test with [ORLEANS] solution for msbuild +Returns: + None: ..... +""" + +import subprocess + +EXTRACT_PATH = "sdk" +DOTNET_BASE_VERSION_URL_LINUX = "https://download.visualstudio.microsoft.com/download/pr/32f2c846-5581-4638-a428-5891dd76f630/ee8beef066f06c57998058c5af6df222/dotnet-sdk-8.0.100-preview.7.23376.3-linux-x64.tar.gz" +DOTNET_DAILY_VERSION_URL_LINUX = "https://aka.ms/dotnet/8.0.1xx/daily/dotnet-sdk-linux-x64.tar.gz" + +TEST_SOLUTION_REPO_URL = "https://github.com/NuGet/NuGet.Client" +TEST_REPO_NAME = "NuGet.Client" +TEST_SOLUTION_CASE = "NuGet.Client" +TEST_SOLUTION_DIR = "./" +SDK_VERSION = "8.0.100-preview.7.23376.3" +SOLUTION_FILE = "NuGet.sln" +SDK_DAILY_VERSION = "8.0.1xx" +DATABASE_FILE = "./../../../data/msbuild.csv" +NESTED = "False" + +if __name__ == "__main__": + commands_chain = ["python3", "./../benchmark_runner_linux.py", "--extract_path", EXTRACT_PATH, "--dotnet_base_version_url_linux", + DOTNET_BASE_VERSION_URL_LINUX, "--dotnet_daily_version_url_linux", DOTNET_DAILY_VERSION_URL_LINUX, "--test_solution_repo_url", TEST_SOLUTION_REPO_URL, "--test_repo_name", TEST_REPO_NAME, "--test_solution_case", TEST_SOLUTION_CASE, "--test_solution_dir", TEST_SOLUTION_DIR, "--solution_file", SOLUTION_FILE, "--sdk_version", SDK_VERSION, "--sdk_daily_version", SDK_DAILY_VERSION, "--database_file", DATABASE_FILE, "--is_nested_solution", NESTED] + + subprocess.run(commands_chain, check=True) diff --git a/scripts/test_cases/nuget_windows.py b/scripts/test_cases/nuget_windows.py new file mode 100644 index 0000000..4224692 --- /dev/null +++ b/scripts/test_cases/nuget_windows.py @@ -0,0 +1,35 @@ +"""_summary_ +This script basically runs benchmark test with [ORLEANS] solution for msbuild on windows +Returns: + None: ..... +""" + +import subprocess +# ___ EXTRACTION CONSTANTS ___ # +EXTRACT_PATH = "sdk" + +# _____ BENCHMARK DEPENDENCIES CONSTANTS _____ # + +# DOTNET_BASE_VERSION_URL_WINDOWS = "https://download.visualstudio.microsoft.com/download/pr/4ede0897-e03d-4d93-a50d-e06f2e430d9e/b5bd2605ce07ec7163d5b5b05dc2f1e0/dotnet-sdk-8.0.100-preview.7.23376.3-win-x64.zip" + +DOTNET_BASE_VERSION_URL_WINDOWS = "https://download.visualstudio.microsoft.com/download/pr/1fb808dc-d017-4460-94f8-bf1ac83e6cd8/756b301e714755e411b84684b885a516/dotnet-sdk-7.0.100-win-x64.zip" + +DOTNET_DAILY_VERSION_URL_WINDOWS = "https://aka.ms/dotnet/8.0.1xx/daily/dotnet-sdk-win-x64.zip" + +TEST_SOLUTION_REPO_URL = "https://github.com/NuGet/NuGet.Client" + +TEST_REPO_NAME = "NuGet.Client" +TEST_SOLUTION_CASE = "NuGet.Client" +TEST_SOLUTION_DIR = "./" +SDK_VERSION = "8.0.100-preview.7.23376.3" +SOLUTION_FILE = "NuGet.sln" +SDK_DAILY_VERSION = "8.0.1xx" +DATABASE_FILE = "./../../../data/msbuild.csv" +NESTED = "False" +COMMIT_HASH = "d76a117c590f8a91b844013bba7ea9b60e469aa1" + +if __name__ == "__main__": + commands_chain = ["python3", "./../benchmark_runner_windows.py", "--extract_path", EXTRACT_PATH, "--dotnet_base_version_url_linux", + DOTNET_BASE_VERSION_URL_WINDOWS, "--dotnet_daily_version_url_linux", DOTNET_DAILY_VERSION_URL_WINDOWS, "--test_solution_repo_url", TEST_SOLUTION_REPO_URL, "--test_solution_case", TEST_SOLUTION_CASE, "--test_solution_dir", TEST_SOLUTION_DIR, "--solution_file", SOLUTION_FILE, "--commit_hash", COMMIT_HASH, "--database_file", DATABASE_FILE, "--is_nested_solution", NESTED] + + subprocess.run(commands_chain, check=True) diff --git a/scripts/test_cases/orchardcore_linux.py b/scripts/test_cases/orchardcore_linux.py new file mode 100644 index 0000000..56efcdd --- /dev/null +++ b/scripts/test_cases/orchardcore_linux.py @@ -0,0 +1,26 @@ +"""_summary_ +This script basically runs benchmark test with [ORLEANS] solution for msbuild +Returns: + None: ..... +""" + +import subprocess + +EXTRACT_PATH = "sdk" +DOTNET_BASE_VERSION_URL_LINUX = "https://download.visualstudio.microsoft.com/download/pr/253e5af8-41aa-48c6-86f1-39a51b44afdc/5bb2cb9380c5b1a7f0153e0a2775727b/dotnet-sdk-7.0.100-linux-x64.tar.gz" +DOTNET_DAILY_VERSION_URL_LINUX = "https://aka.ms/dotnet/8.0.1xx/daily/dotnet-sdk-linux-x64.tar.gz" +TEST_SOLUTION_REPO_URL = "https://github.com/OrchardCMS/OrchardCore" +TEST_REPO_NAME = "OrchardCore" +TEST_SOLUTION_CASE = "OrchardCore" +TEST_SOLUTION_DIR = "./" +SDK_VERSION = "7.0.100" +SDK_DAILY_VERSION = "8.0.1xx" +SOLUTION_FILE = "OrchardCore.sln" +DATABASE_FILE = "./../../../data/msbuild.csv" +NESTED = "False" +COMMIT_HASH = "" +if __name__ == "__main__": + commands_chain = ["python3", "./../benchmark_runner_linux.py", "--extract_path", EXTRACT_PATH, "--dotnet_base_version_url_linux", + DOTNET_BASE_VERSION_URL_LINUX, "--dotnet_daily_version_url_linux", DOTNET_DAILY_VERSION_URL_LINUX, "--test_solution_repo_url", TEST_SOLUTION_REPO_URL, "--test_solution_case", TEST_SOLUTION_CASE, "--test_solution_dir", TEST_SOLUTION_DIR, "--commit_hash", COMMIT_HASH, "--solution_file", SOLUTION_FILE, "--database_file", DATABASE_FILE, "--is_nested_solution", NESTED] + + subprocess.run(commands_chain, check=True) diff --git a/scripts/test_cases/orchardcore_windows.py b/scripts/test_cases/orchardcore_windows.py new file mode 100644 index 0000000..820517b --- /dev/null +++ b/scripts/test_cases/orchardcore_windows.py @@ -0,0 +1,33 @@ +"""_summary_ +This script basically runs benchmark test with [ORLEANS] solution for msbuild on windows +Returns: + None: ..... +""" + +import subprocess +# ___ EXTRACTION CONSTANTS ___ # +EXTRACT_PATH = "sdk" + +# _____ BENCHMARK DEPENDENCIES CONSTANTS _____ # + +DOTNET_BASE_VERSION_URL_WINDOWS = "https://download.visualstudio.microsoft.com/download/pr/1fb808dc-d017-4460-94f8-bf1ac83e6cd8/756b301e714755e411b84684b885a516/dotnet-sdk-7.0.100-win-x64.zip" + +DOTNET_DAILY_VERSION_URL_WINDOWS = "https://aka.ms/dotnet/8.0.1xx/daily/dotnet-sdk-win-x64.zip" + +TEST_SOLUTION_REPO_URL = "https://github.com/OrchardCMS/OrchardCore" + +TEST_REPO_NAME = "OrchardCore" +TEST_SOLUTION_CASE = "OrchardCore" +TEST_SOLUTION_DIR = "./" +SDK_VERSION = "7.0.100" +SOLUTION_FILE = "OrchardCore.sln" +SDK_DAILY_VERSION = "8.0.1xx" +DATABASE_FILE = "./../../../data/msbuild.csv" +NESTED = "False" +COMMIT_HASH = "" + +if __name__ == "__main__": + commands_chain = ["python3", "./../benchmark_runner_windows.py", "--extract_path", EXTRACT_PATH, "--dotnet_base_version_url_linux", + DOTNET_BASE_VERSION_URL_WINDOWS, "--dotnet_daily_version_url_linux", DOTNET_DAILY_VERSION_URL_WINDOWS, "--test_solution_repo_url", TEST_SOLUTION_REPO_URL, "--test_solution_case", TEST_SOLUTION_CASE, "--test_solution_dir", TEST_SOLUTION_DIR, "--solution_file", SOLUTION_FILE, "--commit_hash", COMMIT_HASH, "--database_file", DATABASE_FILE, "--is_nested_solution", NESTED] + + subprocess.run(commands_chain, check=True) diff --git a/scripts/test_cases/orleans_linux.py b/scripts/test_cases/orleans_linux.py new file mode 100644 index 0000000..72aee4e --- /dev/null +++ b/scripts/test_cases/orleans_linux.py @@ -0,0 +1,26 @@ +"""_summary_ +This script basically runs benchmark test with [ORLEANS] solution for msbuild +Returns: + None: ..... +""" + +import subprocess + +EXTRACT_PATH = "sdk" +DOTNET_BASE_VERSION_URL_LINUX = "https://download.visualstudio.microsoft.com/download/pr/253e5af8-41aa-48c6-86f1-39a51b44afdc/5bb2cb9380c5b1a7f0153e0a2775727b/dotnet-sdk-7.0.100-linux-x64.tar.gz" +DOTNET_DAILY_VERSION_URL_LINUX = "https://aka.ms/dotnet/8.0.1xx/daily/dotnet-sdk-linux-x64.tar.gz" +TEST_SOLUTION_REPO_URL = "https://github.com/dotnet/orleans" +TEST_REPO_NAME = "orleans" +TEST_SOLUTION_CASE = "orleans" +TEST_SOLUTION_DIR = "./" +SOLUTION_FILE = "Orleans.sln" +SDK_VERSION = "7.0.100" +SDK_DAILY_VERSION = "8.0.1xx" +DATABASE_FILE = "./../../../data/msbuild.csv" +NESTED = "False" +COMMIT_HASH = "" +if __name__ == "__main__": + commands_chain = ["python3", "./../benchmark_runner_linux.py", "--extract_path", EXTRACT_PATH, "--dotnet_base_version_url_linux", + DOTNET_BASE_VERSION_URL_LINUX, "--dotnet_daily_version_url_linux", DOTNET_DAILY_VERSION_URL_LINUX, "--test_solution_repo_url", TEST_SOLUTION_REPO_URL, "--test_solution_case", TEST_SOLUTION_CASE, "--test_solution_dir", TEST_SOLUTION_DIR, "--solution_file", SOLUTION_FILE, "--commit_hash", COMMIT_HASH, "--database_file", DATABASE_FILE, "--is_nested_solution", NESTED] + + subprocess.run(commands_chain, check=True) diff --git a/scripts/test_cases/orleans_windows.py b/scripts/test_cases/orleans_windows.py new file mode 100644 index 0000000..411c5bb --- /dev/null +++ b/scripts/test_cases/orleans_windows.py @@ -0,0 +1,32 @@ +"""_summary_ +This script basically runs benchmark test with [ORLEANS] solution for msbuild on windows +Returns: + None: ..... +""" + +import subprocess +# ___ EXTRACTION CONSTANTS ___ # +EXTRACT_PATH = "sdk" + +# _____ BENCHMARK DEPENDENCIES CONSTANTS _____ # + +DOTNET_BASE_VERSION_URL_WINDOWS = "https://download.visualstudio.microsoft.com/download/pr/1fb808dc-d017-4460-94f8-bf1ac83e6cd8/756b301e714755e411b84684b885a516/dotnet-sdk-7.0.100-win-x64.zip" + +DOTNET_DAILY_VERSION_URL_WINDOWS = "https://aka.ms/dotnet/8.0.1xx/daily/dotnet-sdk-win-x64.zip" + +TEST_SOLUTION_REPO_URL = "https://github.com/dotnet/orleans" + +TEST_REPO_NAME = "orleans" +TEST_SOLUTION_CASE = "orleans" +TEST_SOLUTION_DIR = "./" +SDK_VERSION = "7.0.100" +SOLUTION_FILE = "Orleans.sln" +SDK_DAILY_VERSION = "8.0.1xx" +DATABASE_FILE = "./../../../data/msbuild.csv" +NESTED = "False" +COMMIT_HASH = "" +if __name__ == "__main__": + commands_chain = ["python3", "./../benchmark_runner_windows.py", "--extract_path", EXTRACT_PATH, "--dotnet_base_version_url_linux", + DOTNET_BASE_VERSION_URL_WINDOWS, "--dotnet_daily_version_url_linux", DOTNET_DAILY_VERSION_URL_WINDOWS, "--test_solution_repo_url", TEST_SOLUTION_REPO_URL, "--test_solution_case", TEST_SOLUTION_CASE, "--test_solution_dir", TEST_SOLUTION_DIR, "--solution_file", SOLUTION_FILE, "--commit_hash", COMMIT_HASH, "--database_file", DATABASE_FILE, "--is_nested_solution", NESTED] + + subprocess.run(commands_chain, check=True) diff --git a/scripts/test_cases/requirements.txt b/scripts/test_cases/requirements.txt new file mode 100644 index 0000000..826340a --- /dev/null +++ b/scripts/test_cases/requirements.txt @@ -0,0 +1,3 @@ +pandas +pylint +argparse \ No newline at end of file