diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e8f1eff03d..e1c06d5a611 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,6 +78,7 @@ tests-gen: artifacts: paths: - .gitlab/tests-gen.yml + - .gitlab/benchmarks/microbenchmarks-gen.yml run-tests-trigger: stage: tests @@ -134,6 +135,7 @@ serverless lambda tests: microbenchmarks: stage: benchmarks needs: + - job: tests-gen - job: "build linux" parallel: matrix: @@ -146,7 +148,9 @@ microbenchmarks: allow_failure: true - allow_failure: false trigger: - include: .gitlab/benchmarks/microbenchmarks.yml + include: + - artifact: .gitlab/benchmarks/microbenchmarks-gen.yml + job: tests-gen strategy: depend variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID diff --git a/.gitlab/benchmarks/microbenchmarks.yml b/.gitlab/benchmarks/microbenchmarks.yml index cdec0873e83..ac98a834054 100644 --- a/.gitlab/benchmarks/microbenchmarks.yml +++ b/.gitlab/benchmarks/microbenchmarks.yml @@ -159,31 +159,6 @@ candidate: paths: - "*.whl" -microbenchmarks: - extends: .benchmarks - parallel: - # DEV: The organization into these groups is mostly arbitrary, based on observed runtimes and - # trying to keep total runtime per job <10 minutes - matrix: - - CPUS_PER_RUN: "1" - SCENARIOS: - - "span tracer core_api set_http_meta telemetry_add_metric otel_span otel_sdk_span recursive_computation sampling_rule_matches" - - "http_propagation_extract http_propagation_inject rate_limiter appsec_iast_aspects appsec_iast_aspects_ospath appsec_iast_aspects_re_module appsec_iast_aspects_split appsec_iast_propagation" - - "packages_package_for_root_module_mapping packages_update_imported_dependencies" - - CPUS_PER_RUN: "2" - SCENARIOS: - - "django_simple flask_simple flask_sqli errortracking_django_simple errortracking_flask_sqli startup" - # Flaky timeouts on starting up - # - "appsec_iast_django_startup" - # They take a long time to run and frequently time out - # TODO: Make benchmarks faster, or run less frequently, or as macrobenchmarks - # - "appsec_iast_django_startup" - # Flaky. Timeout errors - # - "encoder" - # They take a long time to run, and now need the agent running - # TODO: Make benchmarks faster, or run less frequently, or as macrobenchmarks - # - "startup" - benchmarks-pr-comment: image: $MICROBENCHMARKS_CI_IMAGE tags: ["arch:amd64"] @@ -229,3 +204,8 @@ check-slo-breaches: DDOCTOSTS_POLICY: "gitlab.github-access.read" ARTIFACTS_DIR: "reports/" SLO_FILE: ".gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml" + +microbenchmarks: + extends: .benchmarks + parallel: + matrix: diff --git a/benchmarks/suitespec.yml b/benchmarks/suitespec.yml index 7cc7132021e..020bfb2ae9f 100644 --- a/benchmarks/suitespec.yml +++ b/benchmarks/suitespec.yml @@ -103,6 +103,7 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' tracer: paths: - '@bootstrap' @@ -110,6 +111,7 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' core_api: paths: - '@bootstrap' @@ -117,6 +119,7 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' set_http_meta: paths: - '@bootstrap' @@ -124,6 +127,7 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' telemetry_add_metric: paths: - '@bootstrap' @@ -131,6 +135,7 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' otel_span: paths: - '@bootstrap' @@ -139,6 +144,7 @@ suites: - '@vendor' - '@opentelemetry' cpus_per_run: 1 + type: 'microbenchmark' otel_sdk_span: paths: - '@bootstrap' @@ -147,6 +153,7 @@ suites: - '@vendor' - '@opentelemetry' cpus_per_run: 1 + type: 'microbenchmark' recursive_computation: paths: - '@bootstrap' @@ -154,6 +161,7 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' sampling_rule_matches: paths: - '@bootstrap' @@ -161,6 +169,7 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' http_propagation_extract: paths: - '@bootstrap' @@ -168,6 +177,7 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' http_propagation_inject: paths: - '@bootstrap' @@ -175,6 +185,7 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' rate_limiter: paths: - '@bootstrap' @@ -182,12 +193,14 @@ suites: - '@tracing' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' appsec_iast_aspects: paths: - '@bootstrap' - '@core' - '@vendor' cpus_per_run: 1 + type: 'microbenchmark' appsec_iast_aspects_ospath: paths: - '@bootstrap' @@ -195,6 +208,7 @@ suites: - '@vendor' - '@appsec_iast' cpus_per_run: 1 + type: 'microbenchmark' appsec_iast_aspects_re_module: paths: - '@bootstrap' @@ -202,6 +216,7 @@ suites: - '@vendor' - '@appsec_iast' cpus_per_run: 1 + type: 'microbenchmark' appsec_iast_aspects_split: paths: - '@bootstrap' @@ -209,6 +224,7 @@ suites: - '@vendor' - '@appsec_iast' cpus_per_run: 1 + type: 'microbenchmark' appsec_iast_propagation: paths: - '@bootstrap' @@ -216,14 +232,17 @@ suites: - '@vendor' - '@appsec_iast' cpus_per_run: 1 + type: 'microbenchmark' packages_package_for_root_module_mapping: paths: - '@core' cpus_per_run: 1 + type: 'microbenchmark' packages_update_imported_dependencies: paths: - '@core' cpus_per_run: 1 + type: 'microbenchmark' django_simple: paths: - '@bootstrap' @@ -232,6 +251,7 @@ suites: - '@vendor' - '@django' cpus_per_run: 2 + type: 'microbenchmark' flask_simple: paths: - '@bootstrap' @@ -240,6 +260,7 @@ suites: - '@vendor' - '@flask' cpus_per_run: 2 + type: 'microbenchmark' flask_sqli: paths: - '@bootstrap' @@ -248,6 +269,7 @@ suites: - '@vendor' - '@flask' cpus_per_run: 2 + type: 'microbenchmark' errortracking_django_simple: paths: - '@bootstrap' @@ -256,6 +278,7 @@ suites: - '@tracing' - '@django' cpus_per_run: 2 + type: 'microbenchmark' errortracking_flask_sqli: paths: - '@bootstrap' @@ -264,6 +287,7 @@ suites: - '@tracing' - '@flask' cpus_per_run: 2 + type: 'microbenchmark' startup: paths: - '@bootstrap' @@ -273,3 +297,4 @@ suites: - '@flask' - '@django' cpus_per_run: 2 + type: 'microbenchmark' diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index 430fedf67e9..eb58c2aba90 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -17,6 +17,7 @@ file. The function will be called automatically when this script is run. """ +from collections import defaultdict from dataclasses import dataclass import datetime import os @@ -25,6 +26,19 @@ import typing as t +MAX_BENCHMARKS_PER_GROUP = 8 + + +@dataclass +class BenchmarkSpec: + name: str + cpus_per_run: t.Optional[int] = 1 + pattern: t.Optional[str] = None + paths: t.Optional[t.Set[str]] = None # ignored + skip: bool = False + type: str = "benchmark" # ignored + + @dataclass class JobSpec: name: str @@ -43,6 +57,7 @@ class JobSpec: paths: t.Optional[t.Set[str]] = None # ignored only: t.Optional[t.Set[str]] = None # ignored gpu: bool = False + type: str = "test" # ignored def __str__(self) -> str: lines = [] @@ -199,7 +214,7 @@ def calculate_dynamic_parallelism(suite_name: str, suite_config: dict) -> t.Opti def gen_required_suites() -> None: - """Generate the list of test suites that need to be run.""" + """Generate the list of test and benchmark suites that need to be run.""" from needs_testrun import extract_git_commit_selections from needs_testrun import for_each_testrun_needed import suitespec @@ -220,6 +235,53 @@ def gen_required_suites() -> None: if any(suite in required_suites for suite in ci_visibility_suites): required_suites = sorted(suites.keys()) + _gen_tests(suites, required_suites) + _gen_benchmarks(suites, required_suites) + + +def _gen_benchmarks(suites: t.Dict, required_suites: t.List[str]) -> None: + suites = {k: v for k, v in suites.items() if "benchmark" in v.get("type", "test")} + required_suites = [a for a in required_suites if a in list(suites.keys())] + + # Copy the template file + MICROBENCHMARKS_GEN.write_text((GITLAB / "benchmarks/microbenchmarks.yml").read_text()) + + for suite_name, suite_config in suites.items(): + clean_name = suite_name.split("::")[-1] + suite_config["_clean_name"] = clean_name + + groups = defaultdict(list) + + for suite in required_suites: + suite_config = suites[suite].copy() + clean_name = suite_config.pop("_clean_name", suite) + + # Create JobSpec with clean name and explicit stage + jobspec = BenchmarkSpec(clean_name, **suite_config) + if jobspec.skip: + LOGGER.debug("Skipping suite %s", suite) + continue + + groups[jobspec.cpus_per_run].append(jobspec) + + with MICROBENCHMARKS_GEN.open("a") as f: + for cpus_per_run, jobspecs in groups.items(): + print(f' - CPUS_PER_RUN: "{cpus_per_run}"\n SCENARIOS:', file=f) + jobspecs = sorted(jobspecs, key=lambda s: s.name) + # DEV: The organization into these groups is mostly arbitrary, based on observed runtimes and + # trying to keep total runtime per job <10 minutes + for subgroup in [ + jobspecs[i : i + MAX_BENCHMARKS_PER_GROUP] for i in range(0, len(jobspecs), MAX_BENCHMARKS_PER_GROUP) + ]: + names = [i.name for i in subgroup] + group_spec = f' - "{" ".join(names)}"' + print(group_spec, file=f) + + +def _gen_tests(suites: t.Dict, required_suites: t.List[str]) -> None: + suites = {k: v for k, v in suites.items() if v.get("type", "test") == "test"} + required_suites = [a for a in required_suites if a in list(suites.keys())] + # Copy the template file TESTS_GEN.write_text((GITLAB / "tests.yml").read_text()) @@ -227,11 +289,12 @@ def gen_required_suites() -> None: stages = {"setup"} # setup is always needed for suite_name, suite_config in suites.items(): # Extract stage from suite name prefix if present - if "::" in suite_name: - stage, _, clean_name = suite_name.partition("::") + suite_parts = suite_name.split("::")[-2:] + if len(suite_parts) == 2: + stage, clean_name = suite_parts else: stage = "core" - clean_name = suite_name + clean_name = suite_parts[-1] stages.add(stage) # Store the stage in the suite config for later use @@ -482,6 +545,7 @@ def gen_detect_global_locks() -> None: GITLAB = ROOT / ".gitlab" TESTS = ROOT / "tests" TESTS_GEN = GITLAB / "tests-gen.yml" +MICROBENCHMARKS_GEN = GITLAB / "benchmarks/microbenchmarks-gen.yml" # Make the scripts and tests folders available for importing. sys.path.append(str(ROOT / "scripts")) sys.path.append(str(ROOT / "tests")) diff --git a/scripts/needs_testrun.py b/scripts/needs_testrun.py index 0ee09cc7631..06026d05400 100755 --- a/scripts/needs_testrun.py +++ b/scripts/needs_testrun.py @@ -148,10 +148,10 @@ def needs_testrun(suite: str, pr_number: int, sha: t.Optional[str] = None) -> bo ... "scripts/vcr/needs_testrun.yaml", ... filter_headers=["authorization", "user-agent"], ... record_mode="none"): - ... needs_testrun("debugger", 6485) - ... needs_testrun("debugger", 6388) - ... needs_testrun("foobar", 6412) - ... needs_testrun("profiling::profile", 11690) + ... needs_testrun("tests::debugger", 6485) + ... needs_testrun("tests::debugger", 6388) + ... needs_testrun("tests::foobar", 6412) + ... needs_testrun("tests::profiling::profile", 11690) True True True diff --git a/tests/suitespec.py b/tests/suitespec.py index 381f80ebf00..18d09de2760 100644 --- a/tests/suitespec.py +++ b/tests/suitespec.py @@ -6,17 +6,21 @@ TESTS = Path(__file__).parents[1] / "tests" +BENCHMARKS = Path(__file__).parents[1] / "benchmarks" +SEARCH_ROOTS = ((TESTS, ""), (BENCHMARKS, "benchmarks")) def _collect_suitespecs() -> dict: - # Recursively search for suitespec.yml in TESTS suitespec = {"components": {}, "suites": {}} - for s in TESTS.rglob("suitespec.yml"): - try: - namespace = ".".join(s.relative_to(TESTS).parts[:-1]) or None - except IndexError: - namespace = None + specfiles = [] + for root, ns_prefix in SEARCH_ROOTS: + for f in root.rglob("suitespec.yml"): + specfiles.append((f, root, ns_prefix)) + + for s, root, ns_prefix in specfiles: + path_parts = s.relative_to(root).parts[:-1] + namespace = "::".join(path_parts) if path_parts else ns_prefix or None with YAML() as yaml: data = yaml.load(s) suites = data.get("suites", {})