From b7b5a81beedb1bf319ebb9c7e2cc2551013dc798 Mon Sep 17 00:00:00 2001 From: Eric Conklin Date: Wed, 3 Jun 2026 20:04:08 -0500 Subject: [PATCH] add --- .../aws_accuracy_oracle_v1/README.md | 49 +++ .../binding_metadata.json | 175 +++++++++ .../expected_comparison.json | 296 +++++++++++++++ .../expected_findings.json | 240 ++++++++++++ .../aws_accuracy_oracle_v1/oracle_rows.json | 349 ++++++++++++++++++ .../aws_accuracy_oracle_v1/scenario.json | 146 ++++++++ ...t_prod_like_aws_accuracy_oracle_fixture.py | 249 +++++++++++++ 7 files changed, 1504 insertions(+) create mode 100644 tests/fixtures/prod_like/aws_accuracy_oracle_v1/README.md create mode 100644 tests/fixtures/prod_like/aws_accuracy_oracle_v1/binding_metadata.json create mode 100644 tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_comparison.json create mode 100644 tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_findings.json create mode 100644 tests/fixtures/prod_like/aws_accuracy_oracle_v1/oracle_rows.json create mode 100644 tests/fixtures/prod_like/aws_accuracy_oracle_v1/scenario.json create mode 100644 tests/test_prod_like_aws_accuracy_oracle_fixture.py diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/README.md b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/README.md new file mode 100644 index 0000000..8eac8eb --- /dev/null +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/README.md @@ -0,0 +1,49 @@ +# Prod-Like AWS Accuracy Oracle v1 + +Fixture id: `prod_like_aws_accuracy_oracle_v1` + +This is a local-only known-ground-truth oracle fixture for IAMScope's future prod-like AWS accuracy benchmark. It freezes 24 oracle rows for later Phase 5 comparison. It does not run IAMScope, does not run AWS, and does not prove IAMScope accuracy yet. + +## Files + +- `oracle_rows.json`: 24 frozen oracle rows and category breakdown. +- `scenario.json`: static descriptive local oracle support file, not replay-ready IAMScope output. +- `binding_metadata.json`: static descriptive binding notes, not an IAMScope replay sidecar. +- `expected_findings.json`: supported IAMScope-facing expected rows for future comparison; unsupported rows are separate static-only rows. +- `expected_comparison.json`: Phase 5 comparison skeleton with `not_run_yet` emitted categories. + +## Row Breakdown + +- `validated`: 6 +- `blocked`: 5 +- `precondition_only`: 4 +- `inconclusive`: 5 +- `unsupported`: 4 + +Unsupported rows are unsupported/static-only. They are not counted as false positives, false negatives, extra findings, or missing findings. + +## Local-Only Boundary + +- no live AWS +- no Terraform +- no AWS credentials +- no production accounts +- no raw AWS result JSON +- no real account IDs +- static fixture support only +- generated/replayed by IAMScope: false +- reasoners run: [] + +## Non-Claims + +- not broad IAMScope correctness +- not production readiness +- not real production AWS +- not exploitability proof +- not downstream authorization proof +- not Lambda invocation behavior +- not generic Deny correctness +- not resource-policy Deny support except unsupported/static-only row labeling +- not SCP Deny support beyond selected benchmark behavior +- no composite benchmark score +- no pass/fail benchmark label diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/binding_metadata.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/binding_metadata.json new file mode 100644 index 0000000..30b2603 --- /dev/null +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/binding_metadata.json @@ -0,0 +1,175 @@ +{ + "binding_notes": [ + { + "binding_kind": "validated", + "oracle_row_id": "oracle-v-001", + "reason": "None", + "static_fixture_support_only": true + }, + { + "binding_kind": "validated", + "oracle_row_id": "oracle-v-002", + "reason": "None", + "static_fixture_support_only": true + }, + { + "binding_kind": "validated", + "oracle_row_id": "oracle-v-003", + "reason": "None", + "static_fixture_support_only": true + }, + { + "binding_kind": "validated", + "oracle_row_id": "oracle-v-004", + "reason": "None", + "static_fixture_support_only": true + }, + { + "binding_kind": "validated", + "oracle_row_id": "oracle-v-005", + "reason": "None", + "static_fixture_support_only": true + }, + { + "binding_kind": "validated", + "oracle_row_id": "oracle-v-006", + "reason": "None", + "static_fixture_support_only": true + }, + { + "binding_kind": "blocked", + "oracle_row_id": "oracle-b-001", + "reason": "Permission boundary blocks `iam:PassRole` or `lambda:CreateFunction`", + "static_fixture_support_only": true + }, + { + "binding_kind": "blocked", + "oracle_row_id": "oracle-b-002", + "reason": "Permission boundary blocks second-hop action", + "static_fixture_support_only": true + }, + { + "binding_kind": "blocked", + "oracle_row_id": "oracle-b-003", + "reason": "SCP-like guardrail blocks `iam:PassRole`", + "static_fixture_support_only": true + }, + { + "binding_kind": "blocked", + "oracle_row_id": "oracle-b-004", + "reason": "Explicit identity Deny suppresses `sts:AssumeRole`", + "static_fixture_support_only": true + }, + { + "binding_kind": "blocked", + "oracle_row_id": "oracle-b-005", + "reason": "Explicit Deny blocks service action", + "static_fixture_support_only": true + }, + { + "binding_kind": "precondition_only", + "oracle_row_id": "oracle-p-001", + "reason": "Missing `iam:PassRole`", + "static_fixture_support_only": true + }, + { + "binding_kind": "precondition_only", + "oracle_row_id": "oracle-p-002", + "reason": "Missing target service trust", + "static_fixture_support_only": true + }, + { + "binding_kind": "precondition_only", + "oracle_row_id": "oracle-p-003", + "reason": "Missing service action permission", + "static_fixture_support_only": true + }, + { + "binding_kind": "precondition_only", + "oracle_row_id": "oracle-p-004", + "reason": "Missing `sts:AssumeRole` permission", + "static_fixture_support_only": true + }, + { + "binding_kind": "inconclusive", + "oracle_row_id": "oracle-i-001", + "reason": "Target resource scope cannot be proven specific enough", + "static_fixture_support_only": true + }, + { + "binding_kind": "inconclusive", + "oracle_row_id": "oracle-i-002", + "reason": "Condition key context unavailable", + "static_fixture_support_only": true + }, + { + "binding_kind": "inconclusive", + "oracle_row_id": "oracle-i-003", + "reason": "Runtime/session context missing", + "static_fixture_support_only": true + }, + { + "binding_kind": "inconclusive", + "oracle_row_id": "oracle-i-004", + "reason": "Account/OU guardrail scope unknown", + "static_fixture_support_only": true + }, + { + "binding_kind": "inconclusive", + "oracle_row_id": "oracle-i-005", + "reason": "Trust condition context unknown", + "static_fixture_support_only": true + }, + { + "binding_kind": "unsupported", + "oracle_row_id": "oracle-u-001", + "reason": "Generic resource-policy Deny outside v1 support", + "static_fixture_support_only": true + }, + { + "binding_kind": "unsupported", + "oracle_row_id": "oracle-u-002", + "reason": "Condition semantics outside v1 support", + "static_fixture_support_only": true + }, + { + "binding_kind": "unsupported", + "oracle_row_id": "oracle-u-003", + "reason": "Lambda invocation behavior outside v1 support", + "static_fixture_support_only": true + }, + { + "binding_kind": "unsupported", + "oracle_row_id": "oracle-u-004", + "reason": "Exploitability/downstream authorization outside v1 support", + "static_fixture_support_only": true + } + ], + "metadata": { + "aws_calls_made": 0, + "description": "Static descriptive binding notes for the local oracle, not an IAMScope replay sidecar.", + "fixture_id": "prod_like_aws_accuracy_oracle_v1", + "generated_or_replayed_by_iamscope": false, + "generation_mode": "local_prod_like_oracle_fixture", + "live_aws_used": false, + "local_only": true, + "non_claims": [ + "not broad IAMScope correctness", + "not production readiness", + "not real production AWS", + "not exploitability proof", + "not downstream authorization proof", + "not Lambda invocation behavior", + "not generic Deny correctness", + "not resource-policy Deny support except unsupported/static-only row labeling", + "not SCP Deny support beyond selected benchmark behavior", + "no composite benchmark score", + "no pass/fail benchmark label" + ], + "phase": "phase_2_local_prod_like_oracle_fixture", + "phase5_status": "not_run_yet", + "reasoners_run": [], + "schema_version": "prod_like_aws_accuracy_oracle_v1.binding_metadata_support.v1", + "source_tool": "static_fixture_authoring" + } +} diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_comparison.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_comparison.json new file mode 100644 index 0000000..c3191f7 --- /dev/null +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_comparison.json @@ -0,0 +1,296 @@ +{ + "comparison_method": { + "benchmark_label_policy": "no pass/fail benchmark label", + "counts_by_category_allowed": true, + "score_policy": "no composite benchmark score" + }, + "comparison_rows": [ + { + "blocker_precondition_uncertainty_reason": "None", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "validated", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-v-001", + "reviewer_note": "Mirrors the bounded service-mediated CreateFunction evidence shape without invoking Lambda." + }, + { + "blocker_precondition_uncertainty_reason": "None", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "validated", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-v-002", + "reviewer_note": "Keeps ECS as modeled service-mediated behavior, not runtime execution proof." + }, + { + "blocker_precondition_uncertainty_reason": "None", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "validated", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-v-003", + "reviewer_note": "Single-hop baseline." + }, + { + "blocker_precondition_uncertainty_reason": "None", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "validated", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-v-004", + "reviewer_note": "Bounded multi-hop case." + }, + { + "blocker_precondition_uncertainty_reason": "None", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "validated", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-v-005", + "reviewer_note": "Uses `synthetic-account-a` and `synthetic-account-b` aliases only." + }, + { + "blocker_precondition_uncertainty_reason": "None", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "validated", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-v-006", + "reviewer_note": "No downstream service action or invocation claim." + }, + { + "blocker_precondition_uncertainty_reason": "Permission boundary blocks `iam:PassRole` or `lambda:CreateFunction`", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "blocked", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-b-001", + "reviewer_note": "Boundary blocker should prevent validated promotion." + }, + { + "blocker_precondition_uncertainty_reason": "Permission boundary blocks second-hop action", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "blocked", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-b-002", + "reviewer_note": "Tests chain continuation blocker attribution." + }, + { + "blocker_precondition_uncertainty_reason": "SCP-like guardrail blocks `iam:PassRole`", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "blocked", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-b-003", + "reviewer_note": "Selected benchmark behavior only, not generic SCP Deny support." + }, + { + "blocker_precondition_uncertainty_reason": "Explicit identity Deny suppresses `sts:AssumeRole`", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "blocked", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-b-004", + "reviewer_note": "Static/report validation only unless future phase explicitly adds runtime proof." + }, + { + "blocker_precondition_uncertainty_reason": "Explicit Deny blocks service action", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "blocked", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-b-005", + "reviewer_note": "No generic Deny correctness claim." + }, + { + "blocker_precondition_uncertainty_reason": "Missing `iam:PassRole`", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "precondition_only", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-p-001", + "reviewer_note": "Aligns with denied controlled case expectation." + }, + { + "blocker_precondition_uncertainty_reason": "Missing target service trust", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "precondition_only", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-p-002", + "reviewer_note": "Target-first reasoners may emit no finding rather than precondition-only." + }, + { + "blocker_precondition_uncertainty_reason": "Missing service action permission", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "precondition_only", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-p-003", + "reviewer_note": "No live AWS call implied." + }, + { + "blocker_precondition_uncertainty_reason": "Missing `sts:AssumeRole` permission", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "precondition_only", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-p-004", + "reviewer_note": "Missing permission should not become a validated path." + }, + { + "blocker_precondition_uncertainty_reason": "Target resource scope cannot be proven specific enough", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "inconclusive", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-i-001", + "reviewer_note": "Keeps shared uncertainty visible." + }, + { + "blocker_precondition_uncertainty_reason": "Condition key context unavailable", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "inconclusive", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-i-002", + "reviewer_note": "Must not assume condition satisfaction." + }, + { + "blocker_precondition_uncertainty_reason": "Runtime/session context missing", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "inconclusive", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-i-003", + "reviewer_note": "Does not prove runtime authorization." + }, + { + "blocker_precondition_uncertainty_reason": "Account/OU guardrail scope unknown", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "inconclusive", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-i-004", + "reviewer_note": "Selected benchmark behavior only." + }, + { + "blocker_precondition_uncertainty_reason": "Trust condition context unknown", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "inconclusive", + "match_status": "phase5_not_run_yet", + "not_counted_as": [], + "oracle_row_id": "oracle-i-005", + "reviewer_note": "Uses sanitized account aliases only." + }, + { + "blocker_precondition_uncertainty_reason": "Generic resource-policy Deny outside v1 support", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "unsupported", + "match_status": "unsupported_behavior", + "not_counted_as": [ + "false_positive", + "false_negative", + "extra_finding", + "missing_finding" + ], + "oracle_row_id": "oracle-u-001", + "reviewer_note": "Not counted as false positive or false negative." + }, + { + "blocker_precondition_uncertainty_reason": "Condition semantics outside v1 support", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "unsupported", + "match_status": "unsupported_behavior", + "not_counted_as": [ + "false_positive", + "false_negative", + "extra_finding", + "missing_finding" + ], + "oracle_row_id": "oracle-u-002", + "reviewer_note": "Avoids overclaiming service-specific policy semantics." + }, + { + "blocker_precondition_uncertainty_reason": "Lambda invocation behavior outside v1 support", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "unsupported", + "match_status": "unsupported_behavior", + "not_counted_as": [ + "false_positive", + "false_negative", + "extra_finding", + "missing_finding" + ], + "oracle_row_id": "oracle-u-003", + "reviewer_note": "No Lambda invocation behavior claim." + }, + { + "blocker_precondition_uncertainty_reason": "Exploitability/downstream authorization outside v1 support", + "emitted_iamscope_category": "not_run_yet", + "evidence_used": [], + "expected_category": "unsupported", + "match_status": "unsupported_behavior", + "not_counted_as": [ + "false_positive", + "false_negative", + "extra_finding", + "missing_finding" + ], + "oracle_row_id": "oracle-u-004", + "reviewer_note": "Explicit non-claim row." + } + ], + "metadata": { + "aws_calls_made": 0, + "emitted_iamscope_category_default": "not_run_yet", + "fixture_id": "prod_like_aws_accuracy_oracle_v1", + "generated_or_replayed_by_iamscope": false, + "generation_mode": "local_prod_like_oracle_fixture", + "live_aws_used": false, + "local_only": true, + "non_claims": [ + "not broad IAMScope correctness", + "not production readiness", + "not real production AWS", + "not exploitability proof", + "not downstream authorization proof", + "not Lambda invocation behavior", + "not generic Deny correctness", + "not resource-policy Deny support except unsupported/static-only row labeling", + "not SCP Deny support beyond selected benchmark behavior", + "no composite benchmark score", + "no pass/fail benchmark label" + ], + "phase": "phase_2_local_prod_like_oracle_fixture", + "phase5_status": "not_run_yet", + "reasoners_run": [], + "schema_version": "prod_like_aws_accuracy_oracle_v1.expected_comparison.v1", + "source_tool": "static_fixture_authoring" + } +} diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_findings.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_findings.json new file mode 100644 index 0000000..a2f5aa1 --- /dev/null +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_findings.json @@ -0,0 +1,240 @@ +{ + "expected_supported_findings": [ + { + "evidence_required": "`lambda:CreateFunction`, `iam:PassRole`, Lambda trust, no selected blocker", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected `passrole_lambda` validated finding", + "oracle_row_id": "oracle-v-001", + "pattern": "PassRole-to-Lambda allowed scoped role", + "reviewer_note": "Mirrors the bounded service-mediated CreateFunction evidence shape without invoking Lambda." + }, + { + "evidence_required": "ECS task/run permission, `iam:PassRole`, ECS trust, no selected blocker", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected PassRole-to-ECS modeled finding if supported by current reasoner set", + "oracle_row_id": "oracle-v-002", + "pattern": "PassRole-to-ECS allowed scoped role", + "reviewer_note": "Keeps ECS as modeled service-mediated behavior, not runtime execution proof." + }, + { + "evidence_required": "`sts:AssumeRole` permission plus target trust", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected AssumeRole validated finding", + "oracle_row_id": "oracle-v-003", + "pattern": "Direct AssumeRole allowed", + "reviewer_note": "Single-hop baseline." + }, + { + "evidence_required": "First-hop permission/trust and second-hop permission/trust", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected chain finding", + "oracle_row_id": "oracle-v-004", + "pattern": "Two-hop AssumeRole chain allowed", + "reviewer_note": "Bounded multi-hop case." + }, + { + "evidence_required": "Source permission, target trust, modeled satisfied condition", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected cross-account trust-shaped finding if supported by current reasoner set", + "oracle_row_id": "oracle-v-005", + "pattern": "Cross-account trust-shaped allowed with modeled condition satisfied", + "reviewer_note": "Uses `synthetic-account-a` and `synthetic-account-b` aliases only." + }, + { + "evidence_required": "Service action permission, target role trust, service-mediated role use evidence", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected service-mediated modeled finding if supported by current reasoner set", + "oracle_row_id": "oracle-v-006", + "pattern": "Service-mediated role path with modeled trust and permission evidence", + "reviewer_note": "No downstream service action or invocation claim." + }, + { + "evidence_required": "PassRole/Lambda evidence plus boundary binding", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked PassRole-to-Lambda result", + "oracle_row_id": "oracle-b-001", + "pattern": "Permission boundary blocks PassRole-to-Lambda", + "reviewer_note": "Boundary blocker should prevent validated promotion." + }, + { + "evidence_required": "Chain evidence plus boundary binding on continuation edge", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked chain result", + "oracle_row_id": "oracle-b-002", + "pattern": "Permission boundary blocks AssumeRole chain continuation", + "reviewer_note": "Tests chain continuation blocker attribution." + }, + { + "evidence_required": "PassRole evidence plus SCP-like/account-guardrail binding", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked PassRole result", + "oracle_row_id": "oracle-b-003", + "pattern": "SCP-like guardrail blocks PassRole", + "reviewer_note": "Selected benchmark behavior only, not generic SCP Deny support." + }, + { + "evidence_required": "AssumeRole evidence plus identity-Deny evidence", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked/static suppression result where supported", + "oracle_row_id": "oracle-b-004", + "pattern": "Identity-Deny suppresses AssumeRole", + "reviewer_note": "Static/report validation only unless future phase explicitly adds runtime proof." + }, + { + "evidence_required": "Service permission evidence plus explicit Deny", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked service-mediated result where supported", + "oracle_row_id": "oracle-b-005", + "pattern": "Explicit Deny blocks service-mediated permission", + "reviewer_note": "No generic Deny correctness claim." + }, + { + "evidence_required": "Lambda create/service action exists; PassRole witness absent", + "expected_category": "precondition_only", + "expected_iamscope_behavior": "Emit no selected validated PassRole finding, or emit precondition-only where supported", + "oracle_row_id": "oracle-p-001", + "pattern": "Missing `iam:PassRole`", + "reviewer_note": "Aligns with denied controlled case expectation." + }, + { + "evidence_required": "Source permissions exist; target trust to required service absent", + "expected_category": "precondition_only", + "expected_iamscope_behavior": "Avoid validated service-mediated finding", + "oracle_row_id": "oracle-p-002", + "pattern": "Missing target service trust", + "reviewer_note": "Target-first reasoners may emit no finding rather than precondition-only." + }, + { + "evidence_required": "PassRole and trust exist; service action absent", + "expected_category": "precondition_only", + "expected_iamscope_behavior": "Avoid selected validated PassRole-to-Lambda finding", + "oracle_row_id": "oracle-p-003", + "pattern": "Missing `lambda:CreateFunction` or service action", + "reviewer_note": "No live AWS call implied." + }, + { + "evidence_required": "Target trust exists; source permission absent", + "expected_category": "precondition_only", + "expected_iamscope_behavior": "Avoid selected validated AssumeRole finding", + "oracle_row_id": "oracle-p-004", + "pattern": "Missing `sts:AssumeRole` permission", + "reviewer_note": "Missing permission should not become a validated path." + }, + { + "evidence_required": "Wildcard/resource scope evidence", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive or preserve uncertainty", + "oracle_row_id": "oracle-i-001", + "pattern": "Wildcard target resource scope unknown", + "reviewer_note": "Keeps shared uncertainty visible." + }, + { + "evidence_required": "Permission/trust evidence with unresolved condition key", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive where supported", + "oracle_row_id": "oracle-i-002", + "pattern": "Unresolved condition key", + "reviewer_note": "Must not assume condition satisfaction." + }, + { + "evidence_required": "Path evidence plus missing session/boundary context", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive or retain assumption where supported", + "oracle_row_id": "oracle-i-003", + "pattern": "Session policy or permission boundary context missing", + "reviewer_note": "Does not prove runtime authorization." + }, + { + "evidence_required": "Path evidence plus ambiguous SCP-like scope", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive where supported", + "oracle_row_id": "oracle-i-004", + "pattern": "SCP-like scope unknown", + "reviewer_note": "Selected benchmark behavior only." + }, + { + "evidence_required": "Cross-account trust-shaped evidence plus unresolved trust condition", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive where supported", + "oracle_row_id": "oracle-i-005", + "pattern": "Cross-account trust condition unknown", + "reviewer_note": "Uses sanitized account aliases only." + } + ], + "metadata": { + "aws_calls_made": 0, + "fixture_id": "prod_like_aws_accuracy_oracle_v1", + "generated_or_replayed_by_iamscope": false, + "generation_mode": "local_prod_like_oracle_fixture", + "live_aws_used": false, + "local_only": true, + "non_claims": [ + "not broad IAMScope correctness", + "not production readiness", + "not real production AWS", + "not exploitability proof", + "not downstream authorization proof", + "not Lambda invocation behavior", + "not generic Deny correctness", + "not resource-policy Deny support except unsupported/static-only row labeling", + "not SCP Deny support beyond selected benchmark behavior", + "no composite benchmark score", + "no pass/fail benchmark label" + ], + "phase": "phase_2_local_prod_like_oracle_fixture", + "phase5_status": "not_run_yet", + "reasoners_run": [], + "schema_version": "prod_like_aws_accuracy_oracle_v1.expected_findings.v1", + "source_tool": "static_fixture_authoring" + }, + "unsupported_static_only_rows": [ + { + "comparison_status_policy": "unsupported_static_only_not_false_positive_or_false_negative", + "expected_category": "unsupported", + "not_counted_as": [ + "false_positive", + "false_negative", + "extra_finding", + "missing_finding" + ], + "oracle_row_id": "oracle-u-001", + "reviewer_note": "Not counted as false positive or false negative." + }, + { + "comparison_status_policy": "unsupported_static_only_not_false_positive_or_false_negative", + "expected_category": "unsupported", + "not_counted_as": [ + "false_positive", + "false_negative", + "extra_finding", + "missing_finding" + ], + "oracle_row_id": "oracle-u-002", + "reviewer_note": "Avoids overclaiming service-specific policy semantics." + }, + { + "comparison_status_policy": "unsupported_static_only_not_false_positive_or_false_negative", + "expected_category": "unsupported", + "not_counted_as": [ + "false_positive", + "false_negative", + "extra_finding", + "missing_finding" + ], + "oracle_row_id": "oracle-u-003", + "reviewer_note": "No Lambda invocation behavior claim." + }, + { + "comparison_status_policy": "unsupported_static_only_not_false_positive_or_false_negative", + "expected_category": "unsupported", + "not_counted_as": [ + "false_positive", + "false_negative", + "extra_finding", + "missing_finding" + ], + "oracle_row_id": "oracle-u-004", + "reviewer_note": "Explicit non-claim row." + } + ] +} diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/oracle_rows.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/oracle_rows.json new file mode 100644 index 0000000..6e9fe51 --- /dev/null +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/oracle_rows.json @@ -0,0 +1,349 @@ +{ + "category_breakdown": { + "blocked": 5, + "inconclusive": 5, + "precondition_only": 4, + "unsupported": 4, + "validated": 6 + }, + "metadata": { + "aws_calls_made": 0, + "fixture_id": "prod_like_aws_accuracy_oracle_v1", + "generated_or_replayed_by_iamscope": false, + "generation_mode": "local_prod_like_oracle_fixture", + "live_aws_used": false, + "local_only": true, + "non_claims": [ + "not broad IAMScope correctness", + "not production readiness", + "not real production AWS", + "not exploitability proof", + "not downstream authorization proof", + "not Lambda invocation behavior", + "not generic Deny correctness", + "not resource-policy Deny support except unsupported/static-only row labeling", + "not SCP Deny support beyond selected benchmark behavior", + "no composite benchmark score", + "no pass/fail benchmark label" + ], + "phase": "phase_2_local_prod_like_oracle_fixture", + "phase5_status": "not_run_yet", + "reasoners_run": [], + "schema_version": "prod_like_aws_accuracy_oracle_v1.oracle_rows.v1", + "source_tool": "static_fixture_authoring" + }, + "oracle_rows": [ + { + "blocker_precondition_uncertainty_reason": "None", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "yes", + "evidence_required": "`lambda:CreateFunction`, `iam:PassRole`, Lambda trust, no selected blocker", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected `passrole_lambda` validated finding", + "oracle_row_id": "oracle-v-001", + "pattern": "PassRole-to-Lambda allowed scoped role", + "reviewer_note": "Mirrors the bounded service-mediated CreateFunction evidence shape without invoking Lambda.", + "source_principal_alias": "principal-ci-deployer", + "target_alias": "role-lambda-exec-scoped" + }, + { + "blocker_precondition_uncertainty_reason": "None", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "ECS task/run permission, `iam:PassRole`, ECS trust, no selected blocker", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected PassRole-to-ECS modeled finding if supported by current reasoner set", + "oracle_row_id": "oracle-v-002", + "pattern": "PassRole-to-ECS allowed scoped role", + "reviewer_note": "Keeps ECS as modeled service-mediated behavior, not runtime execution proof.", + "source_principal_alias": "principal-ecs-deployer", + "target_alias": "role-ecs-task-scoped" + }, + { + "blocker_precondition_uncertainty_reason": "None", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "yes", + "evidence_required": "`sts:AssumeRole` permission plus target trust", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected AssumeRole validated finding", + "oracle_row_id": "oracle-v-003", + "pattern": "Direct AssumeRole allowed", + "reviewer_note": "Single-hop baseline.", + "source_principal_alias": "principal-helpdesk", + "target_alias": "role-readonly-ops" + }, + { + "blocker_precondition_uncertainty_reason": "None", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "yes", + "evidence_required": "First-hop permission/trust and second-hop permission/trust", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected chain finding", + "oracle_row_id": "oracle-v-004", + "pattern": "Two-hop AssumeRole chain allowed", + "reviewer_note": "Bounded multi-hop case.", + "source_principal_alias": "principal-build", + "target_alias": "role-prod-observer" + }, + { + "blocker_precondition_uncertainty_reason": "None", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "Source permission, target trust, modeled satisfied condition", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected cross-account trust-shaped finding if supported by current reasoner set", + "oracle_row_id": "oracle-v-005", + "pattern": "Cross-account trust-shaped allowed with modeled condition satisfied", + "reviewer_note": "Uses `synthetic-account-a` and `synthetic-account-b` aliases only.", + "source_principal_alias": "principal-audit-a", + "target_alias": "role-audit-b" + }, + { + "blocker_precondition_uncertainty_reason": "None", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "Service action permission, target role trust, service-mediated role use evidence", + "expected_category": "validated", + "expected_iamscope_behavior": "Emit selected service-mediated modeled finding if supported by current reasoner set", + "oracle_row_id": "oracle-v-006", + "pattern": "Service-mediated role path with modeled trust and permission evidence", + "reviewer_note": "No downstream service action or invocation claim.", + "source_principal_alias": "principal-release-bot", + "target_alias": "role-service-mediated-target" + }, + { + "blocker_precondition_uncertainty_reason": "Permission boundary blocks `iam:PassRole` or `lambda:CreateFunction`", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "yes", + "evidence_required": "PassRole/Lambda evidence plus boundary binding", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked PassRole-to-Lambda result", + "oracle_row_id": "oracle-b-001", + "pattern": "Permission boundary blocks PassRole-to-Lambda", + "reviewer_note": "Boundary blocker should prevent validated promotion.", + "source_principal_alias": "principal-boundary-lambda", + "target_alias": "role-lambda-exec-boundary" + }, + { + "blocker_precondition_uncertainty_reason": "Permission boundary blocks second-hop action", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "Chain evidence plus boundary binding on continuation edge", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked chain result", + "oracle_row_id": "oracle-b-002", + "pattern": "Permission boundary blocks AssumeRole chain continuation", + "reviewer_note": "Tests chain continuation blocker attribution.", + "source_principal_alias": "principal-boundary-chain", + "target_alias": "role-chain-target" + }, + { + "blocker_precondition_uncertainty_reason": "SCP-like guardrail blocks `iam:PassRole`", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "PassRole evidence plus SCP-like/account-guardrail binding", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked PassRole result", + "oracle_row_id": "oracle-b-003", + "pattern": "SCP-like guardrail blocks PassRole", + "reviewer_note": "Selected benchmark behavior only, not generic SCP Deny support.", + "source_principal_alias": "principal-scp-passrole", + "target_alias": "role-scp-passrole-target" + }, + { + "blocker_precondition_uncertainty_reason": "Explicit identity Deny suppresses `sts:AssumeRole`", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "AssumeRole evidence plus identity-Deny evidence", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked/static suppression result where supported", + "oracle_row_id": "oracle-b-004", + "pattern": "Identity-Deny suppresses AssumeRole", + "reviewer_note": "Static/report validation only unless future phase explicitly adds runtime proof.", + "source_principal_alias": "principal-identity-deny", + "target_alias": "role-denied-assume" + }, + { + "blocker_precondition_uncertainty_reason": "Explicit Deny blocks service action", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "Service permission evidence plus explicit Deny", + "expected_category": "blocked", + "expected_iamscope_behavior": "Emit or preserve blocked service-mediated result where supported", + "oracle_row_id": "oracle-b-005", + "pattern": "Explicit Deny blocks service-mediated permission", + "reviewer_note": "No generic Deny correctness claim.", + "source_principal_alias": "principal-deny-service", + "target_alias": "role-service-denied" + }, + { + "blocker_precondition_uncertainty_reason": "Missing `iam:PassRole`", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "yes", + "evidence_required": "Lambda create/service action exists; PassRole witness absent", + "expected_category": "precondition_only", + "expected_iamscope_behavior": "Emit no selected validated PassRole finding, or emit precondition-only where supported", + "oracle_row_id": "oracle-p-001", + "pattern": "Missing `iam:PassRole`", + "reviewer_note": "Aligns with denied controlled case expectation.", + "source_principal_alias": "principal-missing-passrole", + "target_alias": "role-lambda-exec-missing-passrole" + }, + { + "blocker_precondition_uncertainty_reason": "Missing target service trust", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "Source permissions exist; target trust to required service absent", + "expected_category": "precondition_only", + "expected_iamscope_behavior": "Avoid validated service-mediated finding", + "oracle_row_id": "oracle-p-002", + "pattern": "Missing target service trust", + "reviewer_note": "Target-first reasoners may emit no finding rather than precondition-only.", + "source_principal_alias": "principal-missing-trust", + "target_alias": "role-no-lambda-trust" + }, + { + "blocker_precondition_uncertainty_reason": "Missing service action permission", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "yes", + "evidence_required": "PassRole and trust exist; service action absent", + "expected_category": "precondition_only", + "expected_iamscope_behavior": "Avoid selected validated PassRole-to-Lambda finding", + "oracle_row_id": "oracle-p-003", + "pattern": "Missing `lambda:CreateFunction` or service action", + "reviewer_note": "No live AWS call implied.", + "source_principal_alias": "principal-missing-service-action", + "target_alias": "role-lambda-exec-service-action" + }, + { + "blocker_precondition_uncertainty_reason": "Missing `sts:AssumeRole` permission", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "yes", + "evidence_required": "Target trust exists; source permission absent", + "expected_category": "precondition_only", + "expected_iamscope_behavior": "Avoid selected validated AssumeRole finding", + "oracle_row_id": "oracle-p-004", + "pattern": "Missing `sts:AssumeRole` permission", + "reviewer_note": "Missing permission should not become a validated path.", + "source_principal_alias": "principal-missing-assume", + "target_alias": "role-assume-target" + }, + { + "blocker_precondition_uncertainty_reason": "Target resource scope cannot be proven specific enough", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "yes", + "evidence_required": "Wildcard/resource scope evidence", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive or preserve uncertainty", + "oracle_row_id": "oracle-i-001", + "pattern": "Wildcard target resource scope unknown", + "reviewer_note": "Keeps shared uncertainty visible.", + "source_principal_alias": "principal-wildcard-scope", + "target_alias": "role-wildcard-target" + }, + { + "blocker_precondition_uncertainty_reason": "Condition key context unavailable", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "Permission/trust evidence with unresolved condition key", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive where supported", + "oracle_row_id": "oracle-i-002", + "pattern": "Unresolved condition key", + "reviewer_note": "Must not assume condition satisfaction.", + "source_principal_alias": "principal-condition-unknown", + "target_alias": "role-condition-target" + }, + { + "blocker_precondition_uncertainty_reason": "Runtime/session context missing", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "Path evidence plus missing session/boundary context", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive or retain assumption where supported", + "oracle_row_id": "oracle-i-003", + "pattern": "Session policy or permission boundary context missing", + "reviewer_note": "Does not prove runtime authorization.", + "source_principal_alias": "principal-session-context", + "target_alias": "role-session-target" + }, + { + "blocker_precondition_uncertainty_reason": "Account/OU guardrail scope unknown", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "Path evidence plus ambiguous SCP-like scope", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive where supported", + "oracle_row_id": "oracle-i-004", + "pattern": "SCP-like scope unknown", + "reviewer_note": "Selected benchmark behavior only.", + "source_principal_alias": "principal-scp-unknown", + "target_alias": "role-scp-unknown-target" + }, + { + "blocker_precondition_uncertainty_reason": "Trust condition context unknown", + "comparison_status_policy": "phase5_compare_to_oracle_expected_category", + "current_support_expectation": "partial", + "evidence_required": "Cross-account trust-shaped evidence plus unresolved trust condition", + "expected_category": "inconclusive", + "expected_iamscope_behavior": "Emit inconclusive where supported", + "oracle_row_id": "oracle-i-005", + "pattern": "Cross-account trust condition unknown", + "reviewer_note": "Uses sanitized account aliases only.", + "source_principal_alias": "principal-cross-condition-a", + "target_alias": "role-cross-condition-b" + }, + { + "blocker_precondition_uncertainty_reason": "Generic resource-policy Deny outside v1 support", + "comparison_status_policy": "unsupported_static_only_not_false_positive_or_false_negative", + "current_support_expectation": "no", + "evidence_required": "Resource-policy Deny note", + "expected_category": "unsupported", + "expected_iamscope_behavior": "Keep unsupported/static-only", + "oracle_row_id": "oracle-u-001", + "pattern": "Generic resource-policy Deny outside v1 support", + "reviewer_note": "Not counted as false positive or false negative.", + "source_principal_alias": "principal-resource-deny", + "target_alias": "resource-policy-target" + }, + { + "blocker_precondition_uncertainty_reason": "Condition semantics outside v1 support", + "comparison_status_policy": "unsupported_static_only_not_false_positive_or_false_negative", + "current_support_expectation": "no", + "evidence_required": "Service-specific condition note", + "expected_category": "unsupported", + "expected_iamscope_behavior": "Keep unsupported/static-only", + "oracle_row_id": "oracle-u-002", + "pattern": "Service-specific condition semantics outside v1 support", + "reviewer_note": "Avoids overclaiming service-specific policy semantics.", + "source_principal_alias": "principal-service-condition", + "target_alias": "role-service-condition-target" + }, + { + "blocker_precondition_uncertainty_reason": "Lambda invocation behavior outside v1 support", + "comparison_status_policy": "unsupported_static_only_not_false_positive_or_false_negative", + "current_support_expectation": "no", + "evidence_required": "Invocation behavior note", + "expected_category": "unsupported", + "expected_iamscope_behavior": "Keep unsupported/static-only", + "oracle_row_id": "oracle-u-003", + "pattern": "Downstream Lambda invocation behavior outside v1 support", + "reviewer_note": "No Lambda invocation behavior claim.", + "source_principal_alias": "principal-lambda-invocation", + "target_alias": "role-lambda-runtime-target" + }, + { + "blocker_precondition_uncertainty_reason": "Exploitability/downstream authorization outside v1 support", + "comparison_status_policy": "unsupported_static_only_not_false_positive_or_false_negative", + "current_support_expectation": "no", + "evidence_required": "Downstream authorization note", + "expected_category": "unsupported", + "expected_iamscope_behavior": "Keep unsupported/static-only", + "oracle_row_id": "oracle-u-004", + "pattern": "Broad exploitability/downstream authorization outside v1 support", + "reviewer_note": "Explicit non-claim row.", + "source_principal_alias": "principal-exploitability", + "target_alias": "role-downstream-target" + } + ] +} diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/scenario.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/scenario.json new file mode 100644 index 0000000..9d98f69 --- /dev/null +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/scenario.json @@ -0,0 +1,146 @@ +{ + "local_only_boundary": [ + "static fixture support", + "no live AWS", + "no generated/replayed IAMScope output", + "no broad correctness claim" + ], + "metadata": { + "aws_calls_made": 0, + "description": "Static descriptive local oracle support file, not replay-ready IAMScope output.", + "fixture_id": "prod_like_aws_accuracy_oracle_v1", + "generated_or_replayed_by_iamscope": false, + "generation_mode": "local_prod_like_oracle_fixture", + "live_aws_used": false, + "local_only": true, + "non_claims": [ + "not broad IAMScope correctness", + "not production readiness", + "not real production AWS", + "not exploitability proof", + "not downstream authorization proof", + "not Lambda invocation behavior", + "not generic Deny correctness", + "not resource-policy Deny support except unsupported/static-only row labeling", + "not SCP Deny support beyond selected benchmark behavior", + "no composite benchmark score", + "no pass/fail benchmark label" + ], + "phase": "phase_2_local_prod_like_oracle_fixture", + "phase5_status": "not_run_yet", + "reasoners_run": [], + "schema_version": "prod_like_aws_accuracy_oracle_v1.scenario_support.v1", + "source_tool": "static_fixture_authoring" + }, + "oracle_row_refs": [ + "oracle-v-001", + "oracle-v-002", + "oracle-v-003", + "oracle-v-004", + "oracle-v-005", + "oracle-v-006", + "oracle-b-001", + "oracle-b-002", + "oracle-b-003", + "oracle-b-004", + "oracle-b-005", + "oracle-p-001", + "oracle-p-002", + "oracle-p-003", + "oracle-p-004", + "oracle-i-001", + "oracle-i-002", + "oracle-i-003", + "oracle-i-004", + "oracle-i-005", + "oracle-u-001", + "oracle-u-002", + "oracle-u-003", + "oracle-u-004" + ], + "principals": [ + { + "alias": "principal-audit-a", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-boundary-chain", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-boundary-lambda", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-build", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-ci-deployer", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-condition-unknown", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-cross-condition-a", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-deny-service", + "synthetic_account_alias": "synthetic-account-a" + } + ], + "roles": [ + { + "alias": "role-lambda-exec-scoped", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "role-ecs-task-scoped", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "role-readonly-ops", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "role-prod-observer", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "role-audit-b", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "role-service-mediated-target", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "role-lambda-exec-boundary", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "role-chain-target", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "role-scp-passrole-target", + "synthetic_account_alias": "synthetic-account-b" + }, + { + "alias": "role-denied-assume", + "synthetic_account_alias": "synthetic-account-b" + } + ], + "synthetic_account_id_for_arn_like_values": "000000000000", + "synthetic_accounts": [ + { + "account_alias": "synthetic-account-a" + }, + { + "account_alias": "synthetic-account-b" + } + ] +} diff --git a/tests/test_prod_like_aws_accuracy_oracle_fixture.py b/tests/test_prod_like_aws_accuracy_oracle_fixture.py new file mode 100644 index 0000000..58b5b0c --- /dev/null +++ b/tests/test_prod_like_aws_accuracy_oracle_fixture.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import json +import re +from collections import Counter +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +FIXTURE_DIR = REPO_ROOT / "tests" / "fixtures" / "prod_like" / "aws_accuracy_oracle_v1" +FIXTURE_ID = "prod_like_aws_accuracy_oracle_v1" +REQUIRED_FILES = { + "README.md", + "oracle_rows.json", + "scenario.json", + "binding_metadata.json", + "expected_findings.json", + "expected_comparison.json", +} +REQUIRED_ROW_IDS = { + "oracle-v-001", + "oracle-v-002", + "oracle-v-003", + "oracle-v-004", + "oracle-v-005", + "oracle-v-006", + "oracle-b-001", + "oracle-b-002", + "oracle-b-003", + "oracle-b-004", + "oracle-b-005", + "oracle-p-001", + "oracle-p-002", + "oracle-p-003", + "oracle-p-004", + "oracle-i-001", + "oracle-i-002", + "oracle-i-003", + "oracle-i-004", + "oracle-i-005", + "oracle-u-001", + "oracle-u-002", + "oracle-u-003", + "oracle-u-004", +} +REQUIRED_ROW_FIELDS = { + "oracle_row_id", + "expected_category", + "pattern", + "source_principal_alias", + "target_alias", + "expected_iamscope_behavior", + "evidence_required", + "blocker_precondition_uncertainty_reason", + "current_support_expectation", + "reviewer_note", + "comparison_status_policy", +} +ALLOWED_CATEGORIES = {"validated", "blocked", "precondition_only", "inconclusive", "unsupported"} +ALLOWED_SUPPORT = {"yes", "partial", "no", "unknown"} +EXPECTED_BREAKDOWN = { + "validated": 6, + "blocked": 5, + "precondition_only": 4, + "inconclusive": 5, + "unsupported": 4, +} +NON_CLAIMS = { + "not broad IAMScope correctness", + "not production readiness", + "not real production AWS", + "not exploitability proof", + "not downstream authorization proof", + "not Lambda invocation behavior", + "not generic Deny correctness", + "not resource-policy Deny support except unsupported/static-only row labeling", + "not SCP Deny support beyond selected benchmark behavior", + "no composite benchmark score", + "no pass/fail benchmark label", +} + + +def _load(name: str) -> dict[str, Any]: + return json.loads((FIXTURE_DIR / name).read_text(encoding="utf-8")) + + +def _rows() -> list[dict[str, Any]]: + return list(_load("oracle_rows.json")["oracle_rows"]) + + +def _fixture_text() -> str: + return "\n".join(path.read_text(encoding="utf-8") for path in FIXTURE_DIR.iterdir() if path.is_file()) + + +def _metadata_payloads() -> list[dict[str, Any]]: + return [ + _load("oracle_rows.json")["metadata"], + _load("scenario.json")["metadata"], + _load("binding_metadata.json")["metadata"], + _load("expected_findings.json")["metadata"], + _load("expected_comparison.json")["metadata"], + ] + + +def test_required_fixture_files_exist() -> None: + assert {path.name for path in FIXTURE_DIR.iterdir() if path.is_file()} == REQUIRED_FILES + + +def test_fixture_id_and_metadata_contract() -> None: + for metadata in _metadata_payloads(): + assert metadata["fixture_id"] == FIXTURE_ID + assert metadata["source_tool"] == "static_fixture_authoring" + assert metadata["generation_mode"] == "local_prod_like_oracle_fixture" + assert metadata["local_only"] is True + assert metadata["live_aws_used"] is False + assert metadata["aws_calls_made"] == 0 + assert metadata["generated_or_replayed_by_iamscope"] is False + assert metadata["reasoners_run"] == [] + assert set(metadata["non_claims"]) == NON_CLAIMS + + +def test_oracle_rows_count_breakdown_and_ids() -> None: + rows = _rows() + row_ids = [row["oracle_row_id"] for row in rows] + + assert len(rows) == 24 + assert Counter(row["expected_category"] for row in rows) == EXPECTED_BREAKDOWN + assert set(row_ids) == REQUIRED_ROW_IDS + assert len(row_ids) == len(set(row_ids)) + assert _load("oracle_rows.json")["category_breakdown"] == EXPECTED_BREAKDOWN + + +def test_every_oracle_row_has_required_fields_and_allowed_values() -> None: + for row in _rows(): + assert set(row) == REQUIRED_ROW_FIELDS + assert row["expected_category"] in ALLOWED_CATEGORIES + assert row["current_support_expectation"] in ALLOWED_SUPPORT + assert str(row["evidence_required"]).strip() + assert str(row["reviewer_note"]).strip() + + +def test_unsupported_rows_are_static_only_and_not_counted_as_mismatches() -> None: + expected_findings = _load("expected_findings.json") + comparison = _load("expected_comparison.json") + unsupported_ids = {row["oracle_row_id"] for row in _rows() if row["expected_category"] == "unsupported"} + + assert {row["oracle_row_id"] for row in expected_findings["unsupported_static_only_rows"]} == unsupported_ids + assert unsupported_ids.isdisjoint( + {row["oracle_row_id"] for row in expected_findings["expected_supported_findings"]} + ) + for row in expected_findings["unsupported_static_only_rows"]: + assert row["comparison_status_policy"] == "unsupported_static_only_not_false_positive_or_false_negative" + assert set(row["not_counted_as"]) == { + "false_positive", + "false_negative", + "extra_finding", + "missing_finding", + } + for row in comparison["comparison_rows"]: + if row["oracle_row_id"] in unsupported_ids: + assert row["match_status"] == "unsupported_behavior" + assert set(row["not_counted_as"]) == { + "false_positive", + "false_negative", + "extra_finding", + "missing_finding", + } + + +def test_expected_comparison_is_phase5_not_run_table_shape() -> None: + comparison = _load("expected_comparison.json") + + assert comparison["metadata"]["phase5_status"] == "not_run_yet" + assert len(comparison["comparison_rows"]) == 24 + for row in comparison["comparison_rows"]: + assert { + "oracle_row_id", + "expected_category", + "emitted_iamscope_category", + "match_status", + "evidence_used", + "blocker_precondition_uncertainty_reason", + "reviewer_note", + "not_counted_as", + } == set(row) + assert row["emitted_iamscope_category"] == "not_run_yet" + assert isinstance(row["evidence_used"], list) + assert str(row["reviewer_note"]).strip() + + +def test_no_composite_score_or_pass_fail_label_fields() -> None: + text = json.dumps( + { + name: _load(name) + for name in ( + "oracle_rows.json", + "scenario.json", + "binding_metadata.json", + "expected_findings.json", + "expected_comparison.json", + ) + } + ).lower() + + assert "composite_score" not in text + assert "benchmark_passed" not in text + assert "pass_fail" not in text + assert _load("expected_comparison.json")["comparison_method"]["score_policy"] == "no composite benchmark score" + assert _load("expected_comparison.json")["comparison_method"]["benchmark_label_policy"] == ( + "no pass/fail benchmark label" + ) + + +def test_no_non_synthetic_account_ids_or_iam_arns() -> None: + text = _fixture_text() + account_ids = set(re.findall(r"\b[0-9]{12}\b", text)) + raw_arns = re.findall(r"arn:aws:iam::([0-9]{12})", text) + + assert account_ids <= {"000000000000"} + assert set(raw_arns) <= {"000000000000"} + assert "synthetic-account-a" in text + assert "synthetic-account-b" in text + + +def test_no_live_or_terraform_artifacts_committed_in_fixture() -> None: + forbidden_names = { + "result.json", + "terraform.tfstate", + "terraform.tfstate.backup", + ".terraform.lock.hcl", + "terraform-outputs.json", + } + forbidden_suffixes = (".tfplan",) + committed_names = {path.name for path in FIXTURE_DIR.rglob("*") if path.is_file()} + + assert forbidden_names.isdisjoint(committed_names) + assert not any(path.name.endswith(forbidden_suffixes) for path in FIXTURE_DIR.rglob("*") if path.is_file()) + + +def test_readme_includes_local_only_boundary_and_non_claims() -> None: + readme = (FIXTURE_DIR / "README.md").read_text(encoding="utf-8") + + assert "no live AWS" in readme + assert "no Terraform" in readme + assert "no AWS credentials" in readme + assert "generated/replayed by IAMScope: false" in readme + assert "reasoners run: []" in readme + for claim in NON_CLAIMS: + assert claim in readme