[codex] wire replay subset into public demo review

docs/case-studies/public-demo-review-runbook.md

@@ -26,6 +26,7 @@ The runner performs local checks only:
 - generated artifact hygiene scan for live result JSON, Terraform state/cache/plan/output files, and Terraform lock files;
 - report-only path-overcounting uncertainty grouping from the local synthetic fixture.
 - report-only summary of the complex synthetic benchmark fixture.
+- narrow PassRole-to-Lambda replay subset probe from local replay-ready artifacts.
 
 The runner treats non-empty account, ARN, or artifact hygiene scan output as a failure. The current expected scan result is no output.
 
@@ -36,6 +37,9 @@ The runner generates:
 - `/tmp/iamscope-public-demo-review/summary.md`
 - `/tmp/iamscope-public-demo-review/manifest.json`
 - `/tmp/iamscope-public-demo-review/path-overcounting-uncertainty-groups.json`
+- `/tmp/iamscope-public-demo-review/passrole-lambda-replay-subset/replay-subset-summary.md`
+- `/tmp/iamscope-public-demo-review/passrole-lambda-replay-subset/replay-subset-manifest.json`
+- `/tmp/iamscope-public-demo-review/passrole-lambda-replay-subset/generated-findings.json`
 
 Generated outputs are local scratch outputs and are not committed by default.
 
@@ -56,6 +60,29 @@ The complex synthetic benchmark section is not live AWS evidence, was not
 generated/replayed by IAMScope, and is not a composite score or pass/fail
 benchmark label.
 
+## Narrow PassRole-to-Lambda Replay Subset
+
+The generated `summary.md` and `manifest.json` include a separate narrow
+PassRole-to-Lambda replay subset section. That section is local-only and uses
+existing IAMScope replay machinery against replay-ready subset artifacts.
+
+The current expected subset result is:
+
+- `replay_subset_status`: `replayed_selected_passrole_lambda_subset`
+- `input_contract_status`: `replay_ready`
+- `safe_replay_attempted`: `true`
+- `reasoners_attempted`: `passrole_lambda`
+- matched rows: `1`
+- missing rows: `0`
+- extra rows: `0`
+- static-only rows: `1`
+- live AWS used: `false`
+- AWS calls made: `0`
+
+This subset does not prove replay-equivalence for the full complex synthetic
+benchmark. It does not use live AWS, does not produce a score, and does not
+produce a pass/fail benchmark label.
+
 ## Claim boundary
 
 IAMScope currently has a local synthetic path-overcounting teaching demo and one two-sided controlled PassRole-to-Lambda validation pair. This is not broad IAMScope correctness.

scripts/run_public_demo_review.py

@@ -14,6 +14,11 @@
 from typing import Callable, Sequence
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from scripts.probe_complex_passrole_lambda_replay_subset import run_probe as run_passrole_lambda_replay_subset_probe
+
 DEFAULT_OUT = Path("/tmp/iamscope-public-demo-review")
 COMPLEX_PASSROLE_SCOPE_CLASS = "shared_passrole_target_resource_scope_unknown"
 COMPLEX_TRUST_CONDITION_CLASS = "shared_cross_account_trust_condition_unknown"
@@ -217,6 +222,28 @@ def _complex_synthetic_benchmark_summary() -> dict[str, object]:
     }
 
 
+def _passrole_lambda_replay_subset_summary(output_dir: Path) -> dict[str, object]:
+    subset_out = output_dir / "passrole-lambda-replay-subset"
+    subset_manifest = run_passrole_lambda_replay_subset_probe(subset_out)
+    return {
+        "replay_subset_status": subset_manifest["replay_subset_status"],
+        "input_contract_status": subset_manifest["input_contract_status"],
+        "safe_replay_attempted": subset_manifest["safe_replay_attempted"],
+        "reasoners_attempted": subset_manifest["reasoners_attempted"],
+        "matched_row_count": len(subset_manifest["matched_rows"]),
+        "missing_row_count": len(subset_manifest["missing_rows"]),
+        "extra_row_count": len(subset_manifest["extra_rows"]),
+        "static_only_row_count": len(subset_manifest["static_only_rows"]),
+        "generated_findings_output": (
+            "passrole-lambda-replay-subset/" + str(subset_manifest["generated_findings_output"])
+        ),
+        "local_only": subset_manifest["local_only"],
+        "live_aws_used": subset_manifest["live_aws_used"],
+        "aws_calls_made": subset_manifest["aws_calls_made"],
+        "non_claims": subset_manifest["non_claims"],
+    }
+
+
 def _passrole_summary() -> dict[str, object]:
     selected = json.loads(
         (
@@ -260,6 +287,7 @@ def _render_summary(
     checks: list[CheckResult],
     path_summary: dict[str, object],
     complex_summary: dict[str, object],
+    replay_subset_summary: dict[str, object],
     passrole_summary: dict[str, object],
 ) -> str:
     check_lines = "\n".join(
@@ -274,6 +302,8 @@ def _render_summary(
     complex_groups = complex_summary["uncertainty_group_counts"]
     assert isinstance(complex_verdicts, dict)
     assert isinstance(complex_groups, dict)
+    reasoners_attempted = replay_subset_summary["reasoners_attempted"]
+    assert isinstance(reasoners_attempted, list)
     allowed = passrole_summary["allowed"]
     denied = passrole_summary["denied"]
     assert isinstance(allowed, dict)
@@ -316,6 +346,20 @@ def _render_summary(
 - This is not generated/replayed by IAMScope.
 - This is not a composite score or pass/fail benchmark label.
 
+## Narrow PassRole-to-Lambda replay subset
+
+- replay subset status: `{replay_subset_summary["replay_subset_status"]}`
+- input contract status: `{replay_subset_summary["input_contract_status"]}`
+- safe replay attempted: `{str(replay_subset_summary["safe_replay_attempted"]).lower()}`
+- reasoners attempted: `{", ".join(str(reasoner) for reasoner in reasoners_attempted)}`
+- matched rows: {replay_subset_summary["matched_row_count"]}
+- missing rows: {replay_subset_summary["missing_row_count"]}
+- extra rows: {replay_subset_summary["extra_row_count"]}
+- static-only rows: {replay_subset_summary["static_only_row_count"]}
+- live AWS used: `{str(replay_subset_summary["live_aws_used"]).lower()}`
+- AWS calls made: `{replay_subset_summary["aws_calls_made"]}`
+- This does not prove replay-equivalence for the full complex synthetic benchmark.
+
 ## PassRole-to-Lambda allowed-side summary
 
 - {allowed["summary"]}.
@@ -362,6 +406,7 @@ def _manifest(
     generated_files: list[str],
     path_summary: dict[str, object],
     complex_summary: dict[str, object],
+    replay_subset_summary: dict[str, object],
     passrole_summary: dict[str, object],
 ) -> dict[str, object]:
     return {
@@ -385,6 +430,7 @@ def _manifest(
         "output_files_generated": generated_files,
         "path_overcounting_demo": path_summary,
         "complex_synthetic_benchmark": complex_summary,
+        "passrole_lambda_replay_subset": replay_subset_summary,
         "passrole_lambda_controlled_pair": passrole_summary,
         "supported_claims": SUPPORTED_CLAIMS,
         "non_claims": NON_CLAIMS,
@@ -399,16 +445,26 @@ def run_public_demo_review(output_dir: Path, *, check_runner: CheckRunner = run_
     failed = [check for check in checks if not check.passed]
     path_summary = _path_overcounting_summary()
     complex_summary = _complex_synthetic_benchmark_summary()
+    replay_subset_summary = _passrole_lambda_replay_subset_summary(out_abs)
     passrole_summary = _passrole_summary()
     generated_files = ["summary.md", "manifest.json"]
     if (out_abs / "path-overcounting-uncertainty-groups.json").exists():
         generated_files.append("path-overcounting-uncertainty-groups.json")
+    subset_generated_files = [
+        "passrole-lambda-replay-subset/replay-subset-summary.md",
+        "passrole-lambda-replay-subset/replay-subset-manifest.json",
+        "passrole-lambda-replay-subset/generated-findings.json",
+    ]
+    generated_files.extend(
+        path for path in subset_generated_files if (out_abs / path).exists()
+    )
 
     summary = _render_summary(
         output_dir=out_abs,
         checks=checks,
         path_summary=path_summary,
         complex_summary=complex_summary,
+        replay_subset_summary=replay_subset_summary,
         passrole_summary=passrole_summary,
     )
     manifest = _manifest(
@@ -417,6 +473,7 @@ def run_public_demo_review(output_dir: Path, *, check_runner: CheckRunner = run_
         generated_files=generated_files,
         path_summary=path_summary,
         complex_summary=complex_summary,
+        replay_subset_summary=replay_subset_summary,
         passrole_summary=passrole_summary,
     )
 

tests/test_public_demo_review_runner.py

@@ -47,6 +47,12 @@ def test_runner_writes_summary_and_manifest(tmp_path: Path) -> None:
     assert payload["aws_calls_made"] == 0
     assert "summary.md" in payload["output_files_generated"]
     assert "manifest.json" in payload["output_files_generated"]
+    assert "passrole-lambda-replay-subset/replay-subset-summary.md" in payload["output_files_generated"]
+    assert "passrole-lambda-replay-subset/replay-subset-manifest.json" in payload["output_files_generated"]
+    assert "passrole-lambda-replay-subset/generated-findings.json" in payload["output_files_generated"]
+    assert (tmp_path / "passrole-lambda-replay-subset" / "replay-subset-summary.md").is_file()
+    assert (tmp_path / "passrole-lambda-replay-subset" / "replay-subset-manifest.json").is_file()
+    assert (tmp_path / "passrole-lambda-replay-subset" / "generated-findings.json").is_file()
 
 
 def test_summary_includes_claim_boundary_passrole_summaries_and_non_claims(tmp_path: Path) -> None:
@@ -85,6 +91,24 @@ def test_summary_includes_complex_synthetic_benchmark_section(tmp_path: Path) ->
     assert "This is not a composite score or pass/fail benchmark label." in summary
 
 
+def test_summary_includes_narrow_passrole_lambda_replay_subset(tmp_path: Path) -> None:
+    run_public_demo_review(tmp_path, check_runner=_passing_checks)
+    summary = (tmp_path / "summary.md").read_text()
+
+    assert "## Narrow PassRole-to-Lambda replay subset" in summary
+    assert "replayed_selected_passrole_lambda_subset" in summary
+    assert "input contract status: `replay_ready`" in summary
+    assert "safe replay attempted: `true`" in summary
+    assert "reasoners attempted: `passrole_lambda`" in summary
+    assert "matched rows: 1" in summary
+    assert "missing rows: 0" in summary
+    assert "extra rows: 0" in summary
+    assert "static-only rows: 1" in summary
+    assert "live AWS used: `false`" in summary
+    assert "AWS calls made: `0`" in summary
+    assert "This does not prove replay-equivalence for the full complex synthetic benchmark." in summary
+
+
 def test_manifest_includes_supported_claims_non_claims_and_checks(tmp_path: Path) -> None:
     manifest = run_public_demo_review(tmp_path, check_runner=_passing_checks)
 
@@ -132,6 +156,27 @@ def test_manifest_includes_complex_synthetic_benchmark_counts(tmp_path: Path) ->
     assert complex_summary["not_pass_fail_benchmark_label"] is True
 
 
+def test_manifest_includes_narrow_passrole_lambda_replay_subset(tmp_path: Path) -> None:
+    manifest = run_public_demo_review(tmp_path, check_runner=_passing_checks)
+    subset = manifest["passrole_lambda_replay_subset"]
+
+    assert subset["replay_subset_status"] == "replayed_selected_passrole_lambda_subset"
+    assert subset["input_contract_status"] == "replay_ready"
+    assert subset["safe_replay_attempted"] is True
+    assert subset["reasoners_attempted"] == ["passrole_lambda"]
+    assert subset["matched_row_count"] == 1
+    assert subset["missing_row_count"] == 0
+    assert subset["extra_row_count"] == 0
+    assert subset["static_only_row_count"] == 1
+    assert subset["generated_findings_output"] == "passrole-lambda-replay-subset/generated-findings.json"
+    assert subset["local_only"] is True
+    assert subset["live_aws_used"] is False
+    assert subset["aws_calls_made"] == 0
+    assert "no full complex benchmark replay-equivalence" in subset["non_claims"]
+    assert "no composite benchmark score" in subset["non_claims"]
+    assert "no pass/fail benchmark label" in subset["non_claims"]
+
+
 def test_runner_fails_if_account_scan_produces_unexpected_output(tmp_path: Path) -> None:
     def checks(_: Path) -> list[CheckResult]:
         return [
@@ -174,9 +219,17 @@ def test_runner_does_not_require_aws_credentials(monkeypatch: pytest.MonkeyPatch
 
 
 def test_generated_outputs_are_not_committed() -> None:
-    forbidden = {"summary.md", "manifest.json", "path-overcounting-uncertainty-groups.json"}
+    forbidden = {
+        "summary.md",
+        "manifest.json",
+        "path-overcounting-uncertainty-groups.json",
+        "replay-subset-summary.md",
+        "replay-subset-manifest.json",
+        "generated-findings.json",
+    }
     fixture_dirs = [
         REPO_ROOT / "tests" / "fixtures" / "demo" / "path_overcounting_shared_uncertainty",
+        REPO_ROOT / "tests" / "fixtures" / "demo" / "complex_replay_subset_passrole_lambda",
         REPO_ROOT / "tests" / "fixtures" / "live_binding" / "passrole_lambda_selected_finding",
         REPO_ROOT / "tests" / "fixtures" / "live_binding" / "passrole_lambda_denied_missing_passrole",
     ]