diff --git a/docs/case-studies/public-demo-review-runbook.md b/docs/case-studies/public-demo-review-runbook.md index 225f1bf..2e1c6be 100644 --- a/docs/case-studies/public-demo-review-runbook.md +++ b/docs/case-studies/public-demo-review-runbook.md @@ -26,6 +26,7 @@ The runner performs local checks only: - generated artifact hygiene scan for live result JSON, Terraform state/cache/plan/output files, and Terraform lock files; - report-only path-overcounting uncertainty grouping from the local synthetic fixture. - report-only summary of the complex synthetic benchmark fixture. +- narrow PassRole-to-Lambda replay subset probe from local replay-ready artifacts. The runner treats non-empty account, ARN, or artifact hygiene scan output as a failure. The current expected scan result is no output. @@ -36,6 +37,9 @@ The runner generates: - `/tmp/iamscope-public-demo-review/summary.md` - `/tmp/iamscope-public-demo-review/manifest.json` - `/tmp/iamscope-public-demo-review/path-overcounting-uncertainty-groups.json` +- `/tmp/iamscope-public-demo-review/passrole-lambda-replay-subset/replay-subset-summary.md` +- `/tmp/iamscope-public-demo-review/passrole-lambda-replay-subset/replay-subset-manifest.json` +- `/tmp/iamscope-public-demo-review/passrole-lambda-replay-subset/generated-findings.json` Generated outputs are local scratch outputs and are not committed by default. @@ -56,6 +60,29 @@ The complex synthetic benchmark section is not live AWS evidence, was not generated/replayed by IAMScope, and is not a composite score or pass/fail benchmark label. +## Narrow PassRole-to-Lambda Replay Subset + +The generated `summary.md` and `manifest.json` include a separate narrow +PassRole-to-Lambda replay subset section. That section is local-only and uses +existing IAMScope replay machinery against replay-ready subset artifacts. + +The current expected subset result is: + +- `replay_subset_status`: `replayed_selected_passrole_lambda_subset` +- `input_contract_status`: `replay_ready` +- `safe_replay_attempted`: `true` +- `reasoners_attempted`: `passrole_lambda` +- matched rows: `1` +- missing rows: `0` +- extra rows: `0` +- static-only rows: `1` +- live AWS used: `false` +- AWS calls made: `0` + +This subset does not prove replay-equivalence for the full complex synthetic +benchmark. It does not use live AWS, does not produce a score, and does not +produce a pass/fail benchmark label. + ## Claim boundary IAMScope currently has a local synthetic path-overcounting teaching demo and one two-sided controlled PassRole-to-Lambda validation pair. This is not broad IAMScope correctness. diff --git a/scripts/run_public_demo_review.py b/scripts/run_public_demo_review.py index 0fc4a76..6dd17b3 100644 --- a/scripts/run_public_demo_review.py +++ b/scripts/run_public_demo_review.py @@ -14,6 +14,11 @@ from typing import Callable, Sequence REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scripts.probe_complex_passrole_lambda_replay_subset import run_probe as run_passrole_lambda_replay_subset_probe + DEFAULT_OUT = Path("/tmp/iamscope-public-demo-review") COMPLEX_PASSROLE_SCOPE_CLASS = "shared_passrole_target_resource_scope_unknown" COMPLEX_TRUST_CONDITION_CLASS = "shared_cross_account_trust_condition_unknown" @@ -217,6 +222,28 @@ def _complex_synthetic_benchmark_summary() -> dict[str, object]: } +def _passrole_lambda_replay_subset_summary(output_dir: Path) -> dict[str, object]: + subset_out = output_dir / "passrole-lambda-replay-subset" + subset_manifest = run_passrole_lambda_replay_subset_probe(subset_out) + return { + "replay_subset_status": subset_manifest["replay_subset_status"], + "input_contract_status": subset_manifest["input_contract_status"], + "safe_replay_attempted": subset_manifest["safe_replay_attempted"], + "reasoners_attempted": subset_manifest["reasoners_attempted"], + "matched_row_count": len(subset_manifest["matched_rows"]), + "missing_row_count": len(subset_manifest["missing_rows"]), + "extra_row_count": len(subset_manifest["extra_rows"]), + "static_only_row_count": len(subset_manifest["static_only_rows"]), + "generated_findings_output": ( + "passrole-lambda-replay-subset/" + str(subset_manifest["generated_findings_output"]) + ), + "local_only": subset_manifest["local_only"], + "live_aws_used": subset_manifest["live_aws_used"], + "aws_calls_made": subset_manifest["aws_calls_made"], + "non_claims": subset_manifest["non_claims"], + } + + def _passrole_summary() -> dict[str, object]: selected = json.loads( ( @@ -260,6 +287,7 @@ def _render_summary( checks: list[CheckResult], path_summary: dict[str, object], complex_summary: dict[str, object], + replay_subset_summary: dict[str, object], passrole_summary: dict[str, object], ) -> str: check_lines = "\n".join( @@ -274,6 +302,8 @@ def _render_summary( complex_groups = complex_summary["uncertainty_group_counts"] assert isinstance(complex_verdicts, dict) assert isinstance(complex_groups, dict) + reasoners_attempted = replay_subset_summary["reasoners_attempted"] + assert isinstance(reasoners_attempted, list) allowed = passrole_summary["allowed"] denied = passrole_summary["denied"] assert isinstance(allowed, dict) @@ -316,6 +346,20 @@ def _render_summary( - This is not generated/replayed by IAMScope. - This is not a composite score or pass/fail benchmark label. +## Narrow PassRole-to-Lambda replay subset + +- replay subset status: `{replay_subset_summary["replay_subset_status"]}` +- input contract status: `{replay_subset_summary["input_contract_status"]}` +- safe replay attempted: `{str(replay_subset_summary["safe_replay_attempted"]).lower()}` +- reasoners attempted: `{", ".join(str(reasoner) for reasoner in reasoners_attempted)}` +- matched rows: {replay_subset_summary["matched_row_count"]} +- missing rows: {replay_subset_summary["missing_row_count"]} +- extra rows: {replay_subset_summary["extra_row_count"]} +- static-only rows: {replay_subset_summary["static_only_row_count"]} +- live AWS used: `{str(replay_subset_summary["live_aws_used"]).lower()}` +- AWS calls made: `{replay_subset_summary["aws_calls_made"]}` +- This does not prove replay-equivalence for the full complex synthetic benchmark. + ## PassRole-to-Lambda allowed-side summary - {allowed["summary"]}. @@ -362,6 +406,7 @@ def _manifest( generated_files: list[str], path_summary: dict[str, object], complex_summary: dict[str, object], + replay_subset_summary: dict[str, object], passrole_summary: dict[str, object], ) -> dict[str, object]: return { @@ -385,6 +430,7 @@ def _manifest( "output_files_generated": generated_files, "path_overcounting_demo": path_summary, "complex_synthetic_benchmark": complex_summary, + "passrole_lambda_replay_subset": replay_subset_summary, "passrole_lambda_controlled_pair": passrole_summary, "supported_claims": SUPPORTED_CLAIMS, "non_claims": NON_CLAIMS, @@ -399,16 +445,26 @@ def run_public_demo_review(output_dir: Path, *, check_runner: CheckRunner = run_ failed = [check for check in checks if not check.passed] path_summary = _path_overcounting_summary() complex_summary = _complex_synthetic_benchmark_summary() + replay_subset_summary = _passrole_lambda_replay_subset_summary(out_abs) passrole_summary = _passrole_summary() generated_files = ["summary.md", "manifest.json"] if (out_abs / "path-overcounting-uncertainty-groups.json").exists(): generated_files.append("path-overcounting-uncertainty-groups.json") + subset_generated_files = [ + "passrole-lambda-replay-subset/replay-subset-summary.md", + "passrole-lambda-replay-subset/replay-subset-manifest.json", + "passrole-lambda-replay-subset/generated-findings.json", + ] + generated_files.extend( + path for path in subset_generated_files if (out_abs / path).exists() + ) summary = _render_summary( output_dir=out_abs, checks=checks, path_summary=path_summary, complex_summary=complex_summary, + replay_subset_summary=replay_subset_summary, passrole_summary=passrole_summary, ) manifest = _manifest( @@ -417,6 +473,7 @@ def run_public_demo_review(output_dir: Path, *, check_runner: CheckRunner = run_ generated_files=generated_files, path_summary=path_summary, complex_summary=complex_summary, + replay_subset_summary=replay_subset_summary, passrole_summary=passrole_summary, ) diff --git a/tests/test_public_demo_review_runner.py b/tests/test_public_demo_review_runner.py index 0cd3514..7e9b9a1 100644 --- a/tests/test_public_demo_review_runner.py +++ b/tests/test_public_demo_review_runner.py @@ -47,6 +47,12 @@ def test_runner_writes_summary_and_manifest(tmp_path: Path) -> None: assert payload["aws_calls_made"] == 0 assert "summary.md" in payload["output_files_generated"] assert "manifest.json" in payload["output_files_generated"] + assert "passrole-lambda-replay-subset/replay-subset-summary.md" in payload["output_files_generated"] + assert "passrole-lambda-replay-subset/replay-subset-manifest.json" in payload["output_files_generated"] + assert "passrole-lambda-replay-subset/generated-findings.json" in payload["output_files_generated"] + assert (tmp_path / "passrole-lambda-replay-subset" / "replay-subset-summary.md").is_file() + assert (tmp_path / "passrole-lambda-replay-subset" / "replay-subset-manifest.json").is_file() + assert (tmp_path / "passrole-lambda-replay-subset" / "generated-findings.json").is_file() def test_summary_includes_claim_boundary_passrole_summaries_and_non_claims(tmp_path: Path) -> None: @@ -85,6 +91,24 @@ def test_summary_includes_complex_synthetic_benchmark_section(tmp_path: Path) -> assert "This is not a composite score or pass/fail benchmark label." in summary +def test_summary_includes_narrow_passrole_lambda_replay_subset(tmp_path: Path) -> None: + run_public_demo_review(tmp_path, check_runner=_passing_checks) + summary = (tmp_path / "summary.md").read_text() + + assert "## Narrow PassRole-to-Lambda replay subset" in summary + assert "replayed_selected_passrole_lambda_subset" in summary + assert "input contract status: `replay_ready`" in summary + assert "safe replay attempted: `true`" in summary + assert "reasoners attempted: `passrole_lambda`" in summary + assert "matched rows: 1" in summary + assert "missing rows: 0" in summary + assert "extra rows: 0" in summary + assert "static-only rows: 1" in summary + assert "live AWS used: `false`" in summary + assert "AWS calls made: `0`" in summary + assert "This does not prove replay-equivalence for the full complex synthetic benchmark." in summary + + def test_manifest_includes_supported_claims_non_claims_and_checks(tmp_path: Path) -> None: manifest = run_public_demo_review(tmp_path, check_runner=_passing_checks) @@ -132,6 +156,27 @@ def test_manifest_includes_complex_synthetic_benchmark_counts(tmp_path: Path) -> assert complex_summary["not_pass_fail_benchmark_label"] is True +def test_manifest_includes_narrow_passrole_lambda_replay_subset(tmp_path: Path) -> None: + manifest = run_public_demo_review(tmp_path, check_runner=_passing_checks) + subset = manifest["passrole_lambda_replay_subset"] + + assert subset["replay_subset_status"] == "replayed_selected_passrole_lambda_subset" + assert subset["input_contract_status"] == "replay_ready" + assert subset["safe_replay_attempted"] is True + assert subset["reasoners_attempted"] == ["passrole_lambda"] + assert subset["matched_row_count"] == 1 + assert subset["missing_row_count"] == 0 + assert subset["extra_row_count"] == 0 + assert subset["static_only_row_count"] == 1 + assert subset["generated_findings_output"] == "passrole-lambda-replay-subset/generated-findings.json" + assert subset["local_only"] is True + assert subset["live_aws_used"] is False + assert subset["aws_calls_made"] == 0 + assert "no full complex benchmark replay-equivalence" in subset["non_claims"] + assert "no composite benchmark score" in subset["non_claims"] + assert "no pass/fail benchmark label" in subset["non_claims"] + + def test_runner_fails_if_account_scan_produces_unexpected_output(tmp_path: Path) -> None: def checks(_: Path) -> list[CheckResult]: return [ @@ -174,9 +219,17 @@ def test_runner_does_not_require_aws_credentials(monkeypatch: pytest.MonkeyPatch def test_generated_outputs_are_not_committed() -> None: - forbidden = {"summary.md", "manifest.json", "path-overcounting-uncertainty-groups.json"} + forbidden = { + "summary.md", + "manifest.json", + "path-overcounting-uncertainty-groups.json", + "replay-subset-summary.md", + "replay-subset-manifest.json", + "generated-findings.json", + } fixture_dirs = [ REPO_ROOT / "tests" / "fixtures" / "demo" / "path_overcounting_shared_uncertainty", + REPO_ROOT / "tests" / "fixtures" / "demo" / "complex_replay_subset_passrole_lambda", REPO_ROOT / "tests" / "fixtures" / "live_binding" / "passrole_lambda_selected_finding", REPO_ROOT / "tests" / "fixtures" / "live_binding" / "passrole_lambda_denied_missing_passrole", ]