From e16a5a6fe9f34a37e5921591e2f128e3cd000907 Mon Sep 17 00:00:00 2001 From: Eric Conklin Date: Wed, 3 Jun 2026 17:31:02 -0500 Subject: [PATCH] add --- .../README.md | 53 ++ .../binding_metadata.json | 49 ++ .../expected_uncertainty_groups.json | 52 ++ .../findings.json | 828 ++++++++++++++++++ .../naive_candidates.json | 531 +++++++++++ .../scenario.json | 311 +++++++ ...est_complex_synthetic_benchmark_fixture.py | 300 +++++++ 7 files changed, 2124 insertions(+) create mode 100644 tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/README.md create mode 100644 tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/binding_metadata.json create mode 100644 tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/expected_uncertainty_groups.json create mode 100644 tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/findings.json create mode 100644 tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/naive_candidates.json create mode 100644 tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/scenario.json create mode 100644 tests/test_complex_synthetic_benchmark_fixture.py diff --git a/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/README.md b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/README.md new file mode 100644 index 0000000..5d11863 --- /dev/null +++ b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/README.md @@ -0,0 +1,53 @@ +# Complex Shared Uncertainty IAM Benchmark Fixture + +Fixture id: `complex_shared_uncertainty_iam_benchmark_001` + +This is a local-only synthetic fixture and frozen synthetic oracle for a future +IAMScope demo/benchmark slice. It uses only synthetic account `000000000000` +and does not contain live AWS evidence. + +## Fixture Files + +- `scenario.json` +- `binding_metadata.json` +- `findings.json` +- `naive_candidates.json` +- `expected_uncertainty_groups.json` + +## Oracle Counts + +- naive candidate rows: `42` +- findings/verdict rows: `18` +- `validated`: `4` +- `blocked`: `5` +- `precondition_only`: `3` +- `inconclusive`: `6` + +## Shared Uncertainty Groups + +- `shared_passrole_target_resource_scope_unknown`: `3` +- `shared_cross_account_trust_condition_unknown`: `2` +- `shared_boundary_or_session_policy_context_missing`: `1` + +## Boundary + +The fixture is manually authored as `frozen_synthetic_oracle` with +`source_tool: static_fixture_authoring`. It was not generated or replayed by +IAMScope reasoners, makes `0` AWS calls, and uses no live AWS. + +## Non-Claims + +- no live AWS +- no broad IAMScope correctness +- no broad PassRole correctness +- no generic Deny correctness +- no resource-policy Deny support +- no SCP Deny support beyond selected synthetic fixture behavior +- no exploitability proof +- no downstream authorization proof +- no Lambda invocation behavior +- no production readiness +- no correctness for real AWS environments +- no correctness for other principals, roles, accounts, regions, conditions, permission boundaries, SCPs, resource policies, or findings +- no composite benchmark score +- no pass/fail benchmark label diff --git a/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/binding_metadata.json b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/binding_metadata.json new file mode 100644 index 0000000..b31efba --- /dev/null +++ b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/binding_metadata.json @@ -0,0 +1,49 @@ +{ + "aws_calls_made": 0, + "findings_generation": { + "fixture_oracle_counts": { + "finding_count": 18, + "naive_candidate_count": 42, + "uncertainty_group_counts": { + "shared_boundary_or_session_policy_context_missing": 1, + "shared_cross_account_trust_condition_unknown": 2, + "shared_passrole_target_resource_scope_unknown": 3 + }, + "verdict_breakdown": { + "blocked": 5, + "inconclusive": 6, + "precondition_only": 3, + "validated": 4 + } + }, + "generated_or_replayed_by_iamscope": false, + "mode": "frozen_synthetic_oracle", + "reasoners_run": [], + "replay_equivalence_status": "not_claimed", + "source_tool": "static_fixture_authoring", + "stronger_demo_claims_allowed": false + }, + "fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "generated_or_replayed_by_iamscope": false, + "generation_mode": "frozen_synthetic_oracle", + "live_aws_used": false, + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "reasoners_run": [], + "schema_version": "complex_synthetic_benchmark_binding_metadata.v1", + "source_tool": "static_fixture_authoring" +} diff --git a/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/expected_uncertainty_groups.json b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/expected_uncertainty_groups.json new file mode 100644 index 0000000..1e1faa9 --- /dev/null +++ b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/expected_uncertainty_groups.json @@ -0,0 +1,52 @@ +{ + "fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "generation_mode": "frozen_synthetic_oracle", + "groups": [ + { + "count": 3, + "finding_ids": [ + "complex-finding-013", + "complex-finding-014", + "complex-finding-015" + ], + "reviewer_action": "Resolve target-role resource-scope coverage before treating these rows as independent validated risks.", + "uncertainty_class": "shared_passrole_target_resource_scope_unknown" + }, + { + "count": 2, + "finding_ids": [ + "complex-finding-016", + "complex-finding-017" + ], + "reviewer_action": "Resolve synthetic cross-account trust-condition context before promoting these rows.", + "uncertainty_class": "shared_cross_account_trust_condition_unknown" + }, + { + "count": 1, + "finding_ids": [ + "complex-finding-018" + ], + "reviewer_action": "Resolve boundary or session-policy context before treating this row as validated.", + "uncertainty_class": "shared_boundary_or_session_policy_context_missing" + } + ], + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "report_only": true, + "schema_version": "complex_synthetic_benchmark_uncertainty_groups.v1", + "source_tool": "static_fixture_authoring" +} diff --git a/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/findings.json b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/findings.json new file mode 100644 index 0000000..f1c3671 --- /dev/null +++ b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/findings.json @@ -0,0 +1,828 @@ +{ + "aws_calls_made": 0, + "findings": [ + { + "classification": "selected_synthetic_passrole_lambda_validated", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-001", + "naive_candidate_id": "cand-001", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_lambda", + "required_checks": [ + { + "name": "required_modeled_evidence_present", + "state": "pass" + } + ], + "severity": "high", + "source_principal": "AnalystUser", + "source_principal_arn": "arn:aws:iam::000000000000:user/AnalystUser", + "target": "LambdaExecutionScopedRole", + "target_role_arn": "arn:aws:iam::000000000000:role/LambdaExecutionScopedRole", + "verdict": "validated" + }, + { + "classification": "selected_synthetic_passrole_lambda_validated", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-002", + "naive_candidate_id": "cand-002", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_lambda", + "required_checks": [ + { + "name": "required_modeled_evidence_present", + "state": "pass" + } + ], + "severity": "high", + "source_principal": "CICDRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/CICDRole", + "target": "LambdaExecutionAdminLikeRole", + "target_role_arn": "arn:aws:iam::000000000000:role/LambdaExecutionAdminLikeRole", + "verdict": "validated" + }, + { + "classification": "selected_synthetic_passrole_ecs_validated", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-003", + "naive_candidate_id": "cand-003", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_ecs", + "required_checks": [ + { + "name": "required_modeled_evidence_present", + "state": "pass" + } + ], + "severity": "high", + "source_principal": "CICDRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/CICDRole", + "target": "ECSExecutionAdminLikeRole", + "target_role_arn": "arn:aws:iam::000000000000:role/ECSExecutionAdminLikeRole", + "verdict": "validated" + }, + { + "classification": "selected_synthetic_assumerole_chain_validated", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-004", + "naive_candidate_id": "cand-004", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "assume_role_chain", + "required_checks": [ + { + "name": "required_modeled_evidence_present", + "state": "pass" + } + ], + "severity": "medium", + "source_principal": "DeveloperRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/DeveloperRole", + "target": "CrossAccountOpsRole", + "target_role_arn": "arn:aws:iam::000000000000:role/CrossAccountOpsRole", + "verdict": "validated" + }, + { + "classification": "permission_boundary_blocked_path", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-005", + "naive_candidate_id": "cand-005", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_lambda", + "required_checks": [ + { + "name": "selected_blocker_present", + "state": "fail_closed" + } + ], + "severity": "medium", + "source_principal": "ReadOnlyAuditRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/ReadOnlyAuditRole", + "target": "BoundaryConstrainedRole", + "target_role_arn": "arn:aws:iam::000000000000:role/BoundaryConstrainedRole", + "verdict": "blocked" + }, + { + "classification": "scp_blocked_path", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-006", + "naive_candidate_id": "cand-006", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "assume_role_chain", + "required_checks": [ + { + "name": "selected_blocker_present", + "state": "fail_closed" + } + ], + "severity": "medium", + "source_principal": "DeveloperRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/DeveloperRole", + "target": "CrossAccountOpsRole", + "target_role_arn": "arn:aws:iam::000000000000:role/CrossAccountOpsRole", + "verdict": "blocked" + }, + { + "classification": "identity_deny_suppressed_path", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-007", + "naive_candidate_id": "cand-007", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "assume_role_chain", + "required_checks": [ + { + "name": "selected_blocker_present", + "state": "fail_closed" + } + ], + "severity": "medium", + "source_principal": "BreakGlassCandidateRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/BreakGlassCandidateRole", + "target": "BoundaryConstrainedRole", + "target_role_arn": "arn:aws:iam::000000000000:role/BoundaryConstrainedRole", + "verdict": "blocked" + }, + { + "classification": "permission_boundary_blocked_path", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-008", + "naive_candidate_id": "cand-008", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_ecs", + "required_checks": [ + { + "name": "selected_blocker_present", + "state": "fail_closed" + } + ], + "severity": "medium", + "source_principal": "AnalystUser", + "source_principal_arn": "arn:aws:iam::000000000000:user/AnalystUser", + "target": "ECSExecutionAdminLikeRole", + "target_role_arn": "arn:aws:iam::000000000000:role/ECSExecutionAdminLikeRole", + "verdict": "blocked" + }, + { + "classification": "scp_blocked_path", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-009", + "naive_candidate_id": "cand-009", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "cross_account_trust", + "required_checks": [ + { + "name": "selected_blocker_present", + "state": "fail_closed" + } + ], + "severity": "medium", + "source_principal": "CICDRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/CICDRole", + "target": "CrossAccountOpsRole", + "target_role_arn": "arn:aws:iam::000000000000:role/CrossAccountOpsRole", + "verdict": "blocked" + }, + { + "classification": "missing_iam_passrole_precondition", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-010", + "naive_candidate_id": "cand-010", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_lambda", + "required_checks": [ + { + "name": "required_precondition_missing", + "state": "missing" + } + ], + "severity": "low", + "source_principal": "ReadOnlyAuditRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/ReadOnlyAuditRole", + "target": "LambdaExecutionScopedRole", + "target_role_arn": "arn:aws:iam::000000000000:role/LambdaExecutionScopedRole", + "verdict": "precondition_only" + }, + { + "classification": "missing_target_trust_precondition", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-011", + "naive_candidate_id": "cand-011", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_ecs", + "required_checks": [ + { + "name": "required_precondition_missing", + "state": "missing" + } + ], + "severity": "low", + "source_principal": "DeveloperRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/DeveloperRole", + "target": "ECSExecutionAdminLikeRole", + "target_role_arn": "arn:aws:iam::000000000000:role/ECSExecutionAdminLikeRole", + "verdict": "precondition_only" + }, + { + "classification": "missing_target_trust_precondition", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-012", + "naive_candidate_id": "cand-012", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "assume_role_chain", + "required_checks": [ + { + "name": "required_precondition_missing", + "state": "missing" + } + ], + "severity": "low", + "source_principal": "AnalystUser", + "source_principal_arn": "arn:aws:iam::000000000000:user/AnalystUser", + "target": "CrossAccountOpsRole", + "target_role_arn": "arn:aws:iam::000000000000:role/CrossAccountOpsRole", + "verdict": "precondition_only" + }, + { + "classification": "shared_unknown_resource_scope", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-013", + "naive_candidate_id": "cand-013", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_lambda", + "required_checks": [ + { + "name": "shared_uncertainty_unresolved", + "state": "unknown" + } + ], + "severity": "medium", + "source_principal": "AnalystUser", + "source_principal_arn": "arn:aws:iam::000000000000:user/AnalystUser", + "target": "UnknownResourceScopeRole", + "target_role_arn": "arn:aws:iam::000000000000:role/UnknownResourceScopeRole", + "uncertainty_class": "shared_passrole_target_resource_scope_unknown", + "verdict": "inconclusive" + }, + { + "classification": "shared_unknown_resource_scope", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-014", + "naive_candidate_id": "cand-014", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_lambda", + "required_checks": [ + { + "name": "shared_uncertainty_unresolved", + "state": "unknown" + } + ], + "severity": "medium", + "source_principal": "CICDRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/CICDRole", + "target": "UnknownResourceScopeRole", + "target_role_arn": "arn:aws:iam::000000000000:role/UnknownResourceScopeRole", + "uncertainty_class": "shared_passrole_target_resource_scope_unknown", + "verdict": "inconclusive" + }, + { + "classification": "shared_unknown_resource_scope", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-015", + "naive_candidate_id": "cand-015", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_ecs", + "required_checks": [ + { + "name": "shared_uncertainty_unresolved", + "state": "unknown" + } + ], + "severity": "medium", + "source_principal": "DeveloperRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/DeveloperRole", + "target": "UnknownResourceScopeRole", + "target_role_arn": "arn:aws:iam::000000000000:role/UnknownResourceScopeRole", + "uncertainty_class": "shared_passrole_target_resource_scope_unknown", + "verdict": "inconclusive" + }, + { + "classification": "shared_cross_account_trust_condition_unknown", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-016", + "naive_candidate_id": "cand-016", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "cross_account_trust", + "required_checks": [ + { + "name": "shared_uncertainty_unresolved", + "state": "unknown" + } + ], + "severity": "medium", + "source_principal": "BreakGlassCandidateRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/BreakGlassCandidateRole", + "target": "CrossAccountOpsRole", + "target_role_arn": "arn:aws:iam::000000000000:role/CrossAccountOpsRole", + "uncertainty_class": "shared_cross_account_trust_condition_unknown", + "verdict": "inconclusive" + }, + { + "classification": "shared_cross_account_trust_condition_unknown", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-017", + "naive_candidate_id": "cand-017", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "assume_role_chain", + "required_checks": [ + { + "name": "shared_uncertainty_unresolved", + "state": "unknown" + } + ], + "severity": "medium", + "source_principal": "ReadOnlyAuditRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/ReadOnlyAuditRole", + "target": "CrossAccountOpsRole", + "target_role_arn": "arn:aws:iam::000000000000:role/CrossAccountOpsRole", + "uncertainty_class": "shared_cross_account_trust_condition_unknown", + "verdict": "inconclusive" + }, + { + "classification": "shared_boundary_or_session_policy_context_missing", + "evidence_boundary": { + "applies_only_to_fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "aws_calls_made": 0, + "frozen_synthetic_oracle": true, + "live_aws_used": false, + "local_only": true, + "synthetic_fixture": true + }, + "finding_id": "complex-finding-018", + "naive_candidate_id": "cand-018", + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_id": "passrole_lambda", + "required_checks": [ + { + "name": "shared_uncertainty_unresolved", + "state": "unknown" + } + ], + "severity": "medium", + "source_principal": "CICDRole", + "source_principal_arn": "arn:aws:iam::000000000000:role/CICDRole", + "target": "BoundaryConstrainedRole", + "target_role_arn": "arn:aws:iam::000000000000:role/BoundaryConstrainedRole", + "uncertainty_class": "shared_boundary_or_session_policy_context_missing", + "verdict": "inconclusive" + } + ], + "fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "generated_or_replayed_by_iamscope": false, + "generation_mode": "frozen_synthetic_oracle", + "live_aws_used": false, + "metadata": { + "aws_calls_made": 0, + "fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "generated_or_replayed_by_iamscope": false, + "generation_mode": "frozen_synthetic_oracle", + "live_aws_used": false, + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "reasoners_run": [], + "schema_version": "complex_synthetic_benchmark_findings_metadata.v1", + "source_tool": "static_fixture_authoring", + "verdict_breakdown": { + "blocked": 5, + "inconclusive": 6, + "precondition_only": 3, + "validated": 4 + } + }, + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "reasoners_run": [], + "schema_version": "complex_synthetic_benchmark_findings.v1", + "source_tool": "static_fixture_authoring", + "verdict_breakdown": { + "blocked": 5, + "inconclusive": 6, + "precondition_only": 3, + "validated": 4 + } +} diff --git a/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/naive_candidates.json b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/naive_candidates.json new file mode 100644 index 0000000..98fa62d --- /dev/null +++ b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/naive_candidates.json @@ -0,0 +1,531 @@ +{ + "candidate_count": 42, + "candidate_paths": [ + { + "candidate_id": "cand-001", + "candidate_interpretation": "Naive structural passrole_lambda row from AnalystUser to LambdaExecutionScopedRole.", + "expected_oracle_status": "validated", + "maps_to": { + "finding_id": "complex-finding-001" + }, + "pattern": "passrole_lambda", + "reason": "Frozen oracle maps this row to selected_synthetic_passrole_lambda_validated.", + "source_principal": "AnalystUser", + "target": "LambdaExecutionScopedRole" + }, + { + "candidate_id": "cand-002", + "candidate_interpretation": "Naive structural passrole_lambda row from CICDRole to LambdaExecutionAdminLikeRole.", + "expected_oracle_status": "validated", + "maps_to": { + "finding_id": "complex-finding-002" + }, + "pattern": "passrole_lambda", + "reason": "Frozen oracle maps this row to selected_synthetic_passrole_lambda_validated.", + "source_principal": "CICDRole", + "target": "LambdaExecutionAdminLikeRole" + }, + { + "candidate_id": "cand-003", + "candidate_interpretation": "Naive structural passrole_ecs row from CICDRole to ECSExecutionAdminLikeRole.", + "expected_oracle_status": "validated", + "maps_to": { + "finding_id": "complex-finding-003" + }, + "pattern": "passrole_ecs", + "reason": "Frozen oracle maps this row to selected_synthetic_passrole_ecs_validated.", + "source_principal": "CICDRole", + "target": "ECSExecutionAdminLikeRole" + }, + { + "candidate_id": "cand-004", + "candidate_interpretation": "Naive structural assume_role_chain row from DeveloperRole to CrossAccountOpsRole.", + "expected_oracle_status": "validated", + "maps_to": { + "finding_id": "complex-finding-004" + }, + "pattern": "assume_role_chain", + "reason": "Frozen oracle maps this row to selected_synthetic_assumerole_chain_validated.", + "source_principal": "DeveloperRole", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-005", + "candidate_interpretation": "Naive structural passrole_lambda row from ReadOnlyAuditRole to BoundaryConstrainedRole.", + "expected_oracle_status": "blocked", + "maps_to": { + "finding_id": "complex-finding-005" + }, + "pattern": "passrole_lambda", + "reason": "Frozen oracle maps this row to permission_boundary_blocked_path.", + "source_principal": "ReadOnlyAuditRole", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-006", + "candidate_interpretation": "Naive structural assume_role_chain row from DeveloperRole to CrossAccountOpsRole.", + "expected_oracle_status": "blocked", + "maps_to": { + "finding_id": "complex-finding-006" + }, + "pattern": "assume_role_chain", + "reason": "Frozen oracle maps this row to scp_blocked_path.", + "source_principal": "DeveloperRole", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-007", + "candidate_interpretation": "Naive structural assume_role_chain row from BreakGlassCandidateRole to BoundaryConstrainedRole.", + "expected_oracle_status": "blocked", + "maps_to": { + "finding_id": "complex-finding-007" + }, + "pattern": "assume_role_chain", + "reason": "Frozen oracle maps this row to identity_deny_suppressed_path.", + "source_principal": "BreakGlassCandidateRole", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-008", + "candidate_interpretation": "Naive structural passrole_ecs row from AnalystUser to ECSExecutionAdminLikeRole.", + "expected_oracle_status": "blocked", + "maps_to": { + "finding_id": "complex-finding-008" + }, + "pattern": "passrole_ecs", + "reason": "Frozen oracle maps this row to permission_boundary_blocked_path.", + "source_principal": "AnalystUser", + "target": "ECSExecutionAdminLikeRole" + }, + { + "candidate_id": "cand-009", + "candidate_interpretation": "Naive structural cross_account_trust row from CICDRole to CrossAccountOpsRole.", + "expected_oracle_status": "blocked", + "maps_to": { + "finding_id": "complex-finding-009" + }, + "pattern": "cross_account_trust", + "reason": "Frozen oracle maps this row to scp_blocked_path.", + "source_principal": "CICDRole", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-010", + "candidate_interpretation": "Naive structural passrole_lambda row from ReadOnlyAuditRole to LambdaExecutionScopedRole.", + "expected_oracle_status": "precondition_only", + "maps_to": { + "finding_id": "complex-finding-010" + }, + "pattern": "passrole_lambda", + "reason": "Frozen oracle maps this row to missing_iam_passrole_precondition.", + "source_principal": "ReadOnlyAuditRole", + "target": "LambdaExecutionScopedRole" + }, + { + "candidate_id": "cand-011", + "candidate_interpretation": "Naive structural passrole_ecs row from DeveloperRole to ECSExecutionAdminLikeRole.", + "expected_oracle_status": "precondition_only", + "maps_to": { + "finding_id": "complex-finding-011" + }, + "pattern": "passrole_ecs", + "reason": "Frozen oracle maps this row to missing_target_trust_precondition.", + "source_principal": "DeveloperRole", + "target": "ECSExecutionAdminLikeRole" + }, + { + "candidate_id": "cand-012", + "candidate_interpretation": "Naive structural assume_role_chain row from AnalystUser to CrossAccountOpsRole.", + "expected_oracle_status": "precondition_only", + "maps_to": { + "finding_id": "complex-finding-012" + }, + "pattern": "assume_role_chain", + "reason": "Frozen oracle maps this row to missing_target_trust_precondition.", + "source_principal": "AnalystUser", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-013", + "candidate_interpretation": "Naive structural passrole_lambda row from AnalystUser to UnknownResourceScopeRole.", + "expected_oracle_status": "inconclusive", + "maps_to": { + "finding_id": "complex-finding-013" + }, + "pattern": "passrole_lambda", + "reason": "Frozen oracle maps this row to shared_unknown_resource_scope.", + "source_principal": "AnalystUser", + "target": "UnknownResourceScopeRole" + }, + { + "candidate_id": "cand-014", + "candidate_interpretation": "Naive structural passrole_lambda row from CICDRole to UnknownResourceScopeRole.", + "expected_oracle_status": "inconclusive", + "maps_to": { + "finding_id": "complex-finding-014" + }, + "pattern": "passrole_lambda", + "reason": "Frozen oracle maps this row to shared_unknown_resource_scope.", + "source_principal": "CICDRole", + "target": "UnknownResourceScopeRole" + }, + { + "candidate_id": "cand-015", + "candidate_interpretation": "Naive structural passrole_ecs row from DeveloperRole to UnknownResourceScopeRole.", + "expected_oracle_status": "inconclusive", + "maps_to": { + "finding_id": "complex-finding-015" + }, + "pattern": "passrole_ecs", + "reason": "Frozen oracle maps this row to shared_unknown_resource_scope.", + "source_principal": "DeveloperRole", + "target": "UnknownResourceScopeRole" + }, + { + "candidate_id": "cand-016", + "candidate_interpretation": "Naive structural cross_account_trust row from BreakGlassCandidateRole to CrossAccountOpsRole.", + "expected_oracle_status": "inconclusive", + "maps_to": { + "finding_id": "complex-finding-016" + }, + "pattern": "cross_account_trust", + "reason": "Frozen oracle maps this row to shared_cross_account_trust_condition_unknown.", + "source_principal": "BreakGlassCandidateRole", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-017", + "candidate_interpretation": "Naive structural assume_role_chain row from ReadOnlyAuditRole to CrossAccountOpsRole.", + "expected_oracle_status": "inconclusive", + "maps_to": { + "finding_id": "complex-finding-017" + }, + "pattern": "assume_role_chain", + "reason": "Frozen oracle maps this row to shared_cross_account_trust_condition_unknown.", + "source_principal": "ReadOnlyAuditRole", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-018", + "candidate_interpretation": "Naive structural passrole_lambda row from CICDRole to BoundaryConstrainedRole.", + "expected_oracle_status": "inconclusive", + "maps_to": { + "finding_id": "complex-finding-018" + }, + "pattern": "passrole_lambda", + "reason": "Frozen oracle maps this row to shared_boundary_or_session_policy_context_missing.", + "source_principal": "CICDRole", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-019", + "candidate_interpretation": "Naive structural passrole_lambda row from AnalystUser to LambdaExecutionAdminLikeRole.", + "expected_oracle_status": "non_finding_structural_duplicate", + "maps_to": { + "non_finding_reason": "duplicate structural row folded into selected PassRole-to-Lambda fixture family" + }, + "pattern": "passrole_lambda", + "reason": "duplicate structural row folded into selected PassRole-to-Lambda fixture family", + "source_principal": "AnalystUser", + "target": "LambdaExecutionAdminLikeRole" + }, + { + "candidate_id": "cand-020", + "candidate_interpretation": "Naive structural passrole_ecs row from AnalystUser to UnknownResourceScopeRole.", + "expected_oracle_status": "non_finding_unknown_scope_duplicate", + "maps_to": { + "non_finding_reason": "unknown resource scope is represented by grouped inconclusive findings" + }, + "pattern": "passrole_ecs", + "reason": "unknown resource scope is represented by grouped inconclusive findings", + "source_principal": "AnalystUser", + "target": "UnknownResourceScopeRole" + }, + { + "candidate_id": "cand-021", + "candidate_interpretation": "Naive structural assume_role_chain row from AnalystUser to BoundaryConstrainedRole.", + "expected_oracle_status": "non_finding_blocker_duplicate", + "maps_to": { + "non_finding_reason": "permission-boundary blocker already represented by fixture oracle" + }, + "pattern": "assume_role_chain", + "reason": "permission-boundary blocker already represented by fixture oracle", + "source_principal": "AnalystUser", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-022", + "candidate_interpretation": "Naive structural cross_account_trust row from AnalystUser to CrossAccountOpsRole.", + "expected_oracle_status": "non_finding_missing_trust_context", + "maps_to": { + "non_finding_reason": "trust-condition context is intentionally unresolved" + }, + "pattern": "cross_account_trust", + "reason": "trust-condition context is intentionally unresolved", + "source_principal": "AnalystUser", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-023", + "candidate_interpretation": "Naive structural identity_deny_suppressed_path row from AnalystUser to BoundaryConstrainedRole.", + "expected_oracle_status": "non_finding_suppressed_duplicate", + "maps_to": { + "non_finding_reason": "identity-Deny suppression is represented by selected blocked finding" + }, + "pattern": "identity_deny_suppressed_path", + "reason": "identity-Deny suppression is represented by selected blocked finding", + "source_principal": "AnalystUser", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-024", + "candidate_interpretation": "Naive structural passrole_ecs row from CICDRole to UnknownResourceScopeRole.", + "expected_oracle_status": "non_finding_unknown_scope_duplicate", + "maps_to": { + "non_finding_reason": "unknown resource scope is represented by grouped inconclusive findings" + }, + "pattern": "passrole_ecs", + "reason": "unknown resource scope is represented by grouped inconclusive findings", + "source_principal": "CICDRole", + "target": "UnknownResourceScopeRole" + }, + { + "candidate_id": "cand-025", + "candidate_interpretation": "Naive structural assume_role_chain row from CICDRole to BoundaryConstrainedRole.", + "expected_oracle_status": "non_finding_boundary_context_duplicate", + "maps_to": { + "non_finding_reason": "boundary or session-policy context is represented by selected inconclusive finding" + }, + "pattern": "assume_role_chain", + "reason": "boundary or session-policy context is represented by selected inconclusive finding", + "source_principal": "CICDRole", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-026", + "candidate_interpretation": "Naive structural missing_target_trust_precondition row from CICDRole to ECSExecutionAdminLikeRole.", + "expected_oracle_status": "non_finding_precondition_duplicate", + "maps_to": { + "non_finding_reason": "missing trust precondition is represented by selected precondition-only finding" + }, + "pattern": "missing_target_trust_precondition", + "reason": "missing trust precondition is represented by selected precondition-only finding", + "source_principal": "CICDRole", + "target": "ECSExecutionAdminLikeRole" + }, + { + "candidate_id": "cand-027", + "candidate_interpretation": "Naive structural identity_deny_suppressed_path row from CICDRole to BoundaryConstrainedRole.", + "expected_oracle_status": "non_finding_suppressed_duplicate", + "maps_to": { + "non_finding_reason": "identity-Deny suppression is represented by selected blocked finding" + }, + "pattern": "identity_deny_suppressed_path", + "reason": "identity-Deny suppression is represented by selected blocked finding", + "source_principal": "CICDRole", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-028", + "candidate_interpretation": "Naive structural passrole_lambda row from DeveloperRole to UnknownResourceScopeRole.", + "expected_oracle_status": "non_finding_unknown_scope_duplicate", + "maps_to": { + "non_finding_reason": "unknown resource scope is represented by grouped inconclusive findings" + }, + "pattern": "passrole_lambda", + "reason": "unknown resource scope is represented by grouped inconclusive findings", + "source_principal": "DeveloperRole", + "target": "UnknownResourceScopeRole" + }, + { + "candidate_id": "cand-029", + "candidate_interpretation": "Naive structural cross_account_trust row from DeveloperRole to CrossAccountOpsRole.", + "expected_oracle_status": "non_finding_scp_duplicate", + "maps_to": { + "non_finding_reason": "SCP-like blocker is represented by selected blocked finding" + }, + "pattern": "cross_account_trust", + "reason": "SCP-like blocker is represented by selected blocked finding", + "source_principal": "DeveloperRole", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-030", + "candidate_interpretation": "Naive structural missing_iam_passrole_precondition row from DeveloperRole to LambdaExecutionScopedRole.", + "expected_oracle_status": "non_finding_precondition_duplicate", + "maps_to": { + "non_finding_reason": "missing iam:PassRole precondition is represented by selected precondition-only finding" + }, + "pattern": "missing_iam_passrole_precondition", + "reason": "missing iam:PassRole precondition is represented by selected precondition-only finding", + "source_principal": "DeveloperRole", + "target": "LambdaExecutionScopedRole" + }, + { + "candidate_id": "cand-031", + "candidate_interpretation": "Naive structural identity_deny_suppressed_path row from DeveloperRole to BoundaryConstrainedRole.", + "expected_oracle_status": "non_finding_suppressed_duplicate", + "maps_to": { + "non_finding_reason": "identity-Deny suppression is represented by selected blocked finding" + }, + "pattern": "identity_deny_suppressed_path", + "reason": "identity-Deny suppression is represented by selected blocked finding", + "source_principal": "DeveloperRole", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-032", + "candidate_interpretation": "Naive structural passrole_ecs row from ReadOnlyAuditRole to ECSExecutionAdminLikeRole.", + "expected_oracle_status": "non_finding_boundary_duplicate", + "maps_to": { + "non_finding_reason": "permission boundary blocker is represented by selected blocked finding" + }, + "pattern": "passrole_ecs", + "reason": "permission boundary blocker is represented by selected blocked finding", + "source_principal": "ReadOnlyAuditRole", + "target": "ECSExecutionAdminLikeRole" + }, + { + "candidate_id": "cand-033", + "candidate_interpretation": "Naive structural missing_iam_passrole_precondition row from ReadOnlyAuditRole to UnknownResourceScopeRole.", + "expected_oracle_status": "non_finding_precondition_duplicate", + "maps_to": { + "non_finding_reason": "missing iam:PassRole precondition is represented by selected precondition-only finding" + }, + "pattern": "missing_iam_passrole_precondition", + "reason": "missing iam:PassRole precondition is represented by selected precondition-only finding", + "source_principal": "ReadOnlyAuditRole", + "target": "UnknownResourceScopeRole" + }, + { + "candidate_id": "cand-034", + "candidate_interpretation": "Naive structural cross_account_trust row from ReadOnlyAuditRole to CrossAccountOpsRole.", + "expected_oracle_status": "non_finding_trust_uncertainty_duplicate", + "maps_to": { + "non_finding_reason": "cross-account trust uncertainty is represented by grouped inconclusive findings" + }, + "pattern": "cross_account_trust", + "reason": "cross-account trust uncertainty is represented by grouped inconclusive findings", + "source_principal": "ReadOnlyAuditRole", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-035", + "candidate_interpretation": "Naive structural passrole_lambda row from BreakGlassCandidateRole to LambdaExecutionAdminLikeRole.", + "expected_oracle_status": "non_finding_suppressed_duplicate", + "maps_to": { + "non_finding_reason": "identity-Deny suppression prevents promotion in this fixture oracle" + }, + "pattern": "passrole_lambda", + "reason": "identity-Deny suppression prevents promotion in this fixture oracle", + "source_principal": "BreakGlassCandidateRole", + "target": "LambdaExecutionAdminLikeRole" + }, + { + "candidate_id": "cand-036", + "candidate_interpretation": "Naive structural passrole_ecs row from BreakGlassCandidateRole to ECSExecutionAdminLikeRole.", + "expected_oracle_status": "non_finding_suppressed_duplicate", + "maps_to": { + "non_finding_reason": "identity-Deny suppression prevents promotion in this fixture oracle" + }, + "pattern": "passrole_ecs", + "reason": "identity-Deny suppression prevents promotion in this fixture oracle", + "source_principal": "BreakGlassCandidateRole", + "target": "ECSExecutionAdminLikeRole" + }, + { + "candidate_id": "cand-037", + "candidate_interpretation": "Naive structural missing_target_trust_precondition row from BreakGlassCandidateRole to BoundaryConstrainedRole.", + "expected_oracle_status": "non_finding_precondition_duplicate", + "maps_to": { + "non_finding_reason": "missing target trust precondition is represented by selected precondition-only finding" + }, + "pattern": "missing_target_trust_precondition", + "reason": "missing target trust precondition is represented by selected precondition-only finding", + "source_principal": "BreakGlassCandidateRole", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-038", + "candidate_interpretation": "Naive structural cross_account_trust row from BreakGlassCandidateRole to UnknownResourceScopeRole.", + "expected_oracle_status": "non_finding_trust_uncertainty_duplicate", + "maps_to": { + "non_finding_reason": "cross-account trust uncertainty is represented by grouped inconclusive findings" + }, + "pattern": "cross_account_trust", + "reason": "cross-account trust uncertainty is represented by grouped inconclusive findings", + "source_principal": "BreakGlassCandidateRole", + "target": "UnknownResourceScopeRole" + }, + { + "candidate_id": "cand-039", + "candidate_interpretation": "Naive structural scp_blocked_path row from CICDRole to CrossAccountOpsRole.", + "expected_oracle_status": "non_finding_scp_duplicate", + "maps_to": { + "non_finding_reason": "SCP-like blocker is represented by selected blocked finding" + }, + "pattern": "scp_blocked_path", + "reason": "SCP-like blocker is represented by selected blocked finding", + "source_principal": "CICDRole", + "target": "CrossAccountOpsRole" + }, + { + "candidate_id": "cand-040", + "candidate_interpretation": "Naive structural permission_boundary_blocked_path row from AnalystUser to BoundaryConstrainedRole.", + "expected_oracle_status": "non_finding_boundary_duplicate", + "maps_to": { + "non_finding_reason": "permission boundary blocker is represented by selected blocked finding" + }, + "pattern": "permission_boundary_blocked_path", + "reason": "permission boundary blocker is represented by selected blocked finding", + "source_principal": "AnalystUser", + "target": "BoundaryConstrainedRole" + }, + { + "candidate_id": "cand-041", + "candidate_interpretation": "Naive structural shared_unknown_resource_scope row from DeveloperRole to UnknownResourceScopeRole.", + "expected_oracle_status": "non_finding_unknown_scope_duplicate", + "maps_to": { + "non_finding_reason": "shared unknown resource scope is represented by grouped inconclusive findings" + }, + "pattern": "shared_unknown_resource_scope", + "reason": "shared unknown resource scope is represented by grouped inconclusive findings", + "source_principal": "DeveloperRole", + "target": "UnknownResourceScopeRole" + }, + { + "candidate_id": "cand-042", + "candidate_interpretation": "Naive structural shared_boundary_or_session_policy_context_missing row from ReadOnlyAuditRole to BoundaryConstrainedRole.", + "expected_oracle_status": "non_finding_boundary_context_duplicate", + "maps_to": { + "non_finding_reason": "boundary or session-policy context is represented by selected inconclusive finding" + }, + "pattern": "shared_boundary_or_session_policy_context_missing", + "reason": "boundary or session-policy context is represented by selected inconclusive finding", + "source_principal": "ReadOnlyAuditRole", + "target": "BoundaryConstrainedRole" + } + ], + "fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "generation_mode": "frozen_synthetic_oracle", + "is_iamscope_output": false, + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "reachability_evidence": false, + "schema_version": "complex_synthetic_benchmark_naive_candidates.v1", + "source_tool": "static_fixture_authoring" +} diff --git a/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/scenario.json b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/scenario.json new file mode 100644 index 0000000..1b0ff15 --- /dev/null +++ b/tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark/scenario.json @@ -0,0 +1,311 @@ +{ + "account_id": "000000000000", + "aws_calls_made": 0, + "edges": [ + { + "category": "PassRole-to-Lambda", + "dst": "LambdaExecutionScopedRole", + "edge_id": "edge-001", + "edge_type": "lambda:CreateFunction_permission", + "src": "AnalystUser" + }, + { + "category": "PassRole-to-Lambda", + "dst": "LambdaExecutionScopedRole", + "edge_id": "edge-002", + "edge_type": "iam:PassRole_permission", + "src": "AnalystUser" + }, + { + "category": "target_trust", + "dst": "LambdaExecutionScopedRole", + "edge_id": "edge-003", + "edge_type": "sts:AssumeRole_trust", + "src": "lambda.amazonaws.com" + }, + { + "category": "PassRole-to-Lambda", + "dst": "LambdaExecutionAdminLikeRole", + "edge_id": "edge-004", + "edge_type": "lambda:CreateFunction_permission", + "src": "CICDRole" + }, + { + "category": "PassRole-to-Lambda", + "dst": "LambdaExecutionAdminLikeRole", + "edge_id": "edge-005", + "edge_type": "iam:PassRole_permission", + "src": "CICDRole" + }, + { + "category": "target_trust", + "dst": "LambdaExecutionAdminLikeRole", + "edge_id": "edge-006", + "edge_type": "sts:AssumeRole_trust", + "src": "lambda.amazonaws.com" + }, + { + "category": "PassRole-to-ECS", + "dst": "ECSExecutionAdminLikeRole", + "edge_id": "edge-007", + "edge_type": "ecs:RegisterTaskDefinition_permission", + "src": "CICDRole" + }, + { + "category": "PassRole-to-ECS", + "dst": "ECSExecutionAdminLikeRole", + "edge_id": "edge-008", + "edge_type": "ecs:RunTask_permission", + "src": "CICDRole" + }, + { + "category": "PassRole-to-ECS", + "dst": "ECSExecutionAdminLikeRole", + "edge_id": "edge-009", + "edge_type": "iam:PassRole_permission", + "src": "CICDRole" + }, + { + "category": "target_trust", + "dst": "ECSExecutionAdminLikeRole", + "edge_id": "edge-010", + "edge_type": "sts:AssumeRole_trust", + "src": "ecs-tasks.amazonaws.com" + }, + { + "category": "AssumeRole chain", + "dst": "CrossAccountOpsRole", + "edge_id": "edge-011", + "edge_type": "sts:AssumeRole_permission", + "src": "DeveloperRole" + }, + { + "category": "Cross-account trust-shaped path", + "dst": "CrossAccountOpsRole", + "edge_id": "edge-012", + "edge_type": "sts:AssumeRole_trust", + "src": "DeveloperRole" + }, + { + "category": "permission boundary blocked path", + "dst": "BoundaryConstrainedRole", + "edge_id": "edge-013", + "edge_type": "permission_boundary_blocks", + "src": "ReadOnlyAuditRole" + }, + { + "category": "SCP blocked path", + "dst": "CrossAccountOpsRole", + "edge_id": "edge-014", + "edge_type": "scp_blocks", + "src": "DeveloperRole" + }, + { + "category": "identity-Deny suppressed path", + "dst": "BoundaryConstrainedRole", + "edge_id": "edge-015", + "edge_type": "identity_deny_suppresses", + "src": "BreakGlassCandidateRole" + }, + { + "category": "missing iam:PassRole precondition", + "dst": "LambdaExecutionScopedRole", + "edge_id": "edge-016", + "edge_type": "missing_iam:PassRole", + "src": "ReadOnlyAuditRole" + }, + { + "category": "missing target trust precondition", + "dst": "ECSExecutionAdminLikeRole", + "edge_id": "edge-017", + "edge_type": "missing_target_trust", + "src": "DeveloperRole" + }, + { + "category": "shared unknown resource scope", + "dst": "UnknownResourceScopeRole", + "edge_id": "edge-018", + "edge_type": "unknown_resource_scope", + "src": "AnalystUser" + } + ], + "fixture_id": "complex_shared_uncertainty_iam_benchmark_001", + "generation_mode": "frozen_synthetic_oracle", + "live_aws_used": false, + "nodes": [ + { + "node_id": "AnalystUser", + "node_type": "user", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:user/AnalystUser", + "region": "global" + }, + { + "node_id": "CICDRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/CICDRole", + "region": "global" + }, + { + "node_id": "DeveloperRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/DeveloperRole", + "region": "global" + }, + { + "node_id": "ReadOnlyAuditRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/ReadOnlyAuditRole", + "region": "global" + }, + { + "node_id": "BreakGlassCandidateRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/BreakGlassCandidateRole", + "region": "global" + }, + { + "node_id": "LambdaExecutionAdminLikeRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/LambdaExecutionAdminLikeRole", + "region": "global" + }, + { + "node_id": "LambdaExecutionScopedRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/LambdaExecutionScopedRole", + "region": "global" + }, + { + "node_id": "ECSExecutionAdminLikeRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/ECSExecutionAdminLikeRole", + "region": "global" + }, + { + "node_id": "CrossAccountOpsRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/CrossAccountOpsRole", + "region": "global" + }, + { + "node_id": "BoundaryConstrainedRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/BoundaryConstrainedRole", + "region": "global" + }, + { + "node_id": "UnknownResourceScopeRole", + "node_type": "role", + "provider": "aws", + "provider_id": "arn:aws:iam::000000000000:role/UnknownResourceScopeRole", + "region": "global" + }, + { + "node_id": "lambda.amazonaws.com", + "node_type": "service_principal", + "provider": "aws", + "provider_id": "lambda.amazonaws.com", + "region": "global" + }, + { + "node_id": "ecs-tasks.amazonaws.com", + "node_type": "service_principal", + "provider": "aws", + "provider_id": "ecs-tasks.amazonaws.com", + "region": "global" + }, + { + "node_id": "lambda-function:synthetic-review-function", + "node_type": "synthetic_resource", + "provider": "aws", + "provider_id": "lambda-function:synthetic-review-function", + "region": "us-east-1" + }, + { + "node_id": "ecs-service:synthetic-review-service", + "node_type": "synthetic_resource", + "provider": "aws", + "provider_id": "ecs-service:synthetic-review-service", + "region": "us-east-1" + }, + { + "node_id": "secretsmanager-secret:synthetic-review-secret", + "node_type": "synthetic_resource", + "provider": "aws", + "provider_id": "secretsmanager-secret:synthetic-review-secret", + "region": "us-east-1" + }, + { + "node_id": "s3-bucket:synthetic-review-bucket", + "node_type": "synthetic_resource", + "provider": "aws", + "provider_id": "s3-bucket:synthetic-review-bucket", + "region": "us-east-1" + } + ], + "non_claims": { + "no_broad_iamscope_correctness": true, + "no_broad_passrole_correctness": true, + "no_composite_benchmark_score": true, + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings": true, + "no_correctness_for_real_aws_environments": true, + "no_downstream_authorization_proof": true, + "no_exploitability_proof": true, + "no_generic_deny_correctness": true, + "no_lambda_invocation_behavior": true, + "no_live_aws": true, + "no_pass_fail_benchmark_label": true, + "no_production_readiness": true, + "no_resource_policy_deny_support": true, + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior": true + }, + "pattern_coverage": [ + "PassRole-to-Lambda", + "PassRole-to-ECS", + "AssumeRole chain", + "Cross-account trust-shaped path", + "permission boundary blocked path", + "SCP blocked path", + "identity-Deny suppressed path", + "missing iam:PassRole precondition", + "missing target trust precondition", + "shared unknown resource scope" + ], + "principals": [ + "AnalystUser", + "CICDRole", + "DeveloperRole", + "ReadOnlyAuditRole", + "BreakGlassCandidateRole" + ], + "schema_version": "complex_synthetic_benchmark_scenario.v1", + "service_principals": [ + "lambda.amazonaws.com", + "ecs-tasks.amazonaws.com" + ], + "source_tool": "static_fixture_authoring", + "synthetic_resources": [ + "lambda-function:synthetic-review-function", + "ecs-service:synthetic-review-service", + "secretsmanager-secret:synthetic-review-secret", + "s3-bucket:synthetic-review-bucket" + ], + "target_roles": [ + "LambdaExecutionAdminLikeRole", + "LambdaExecutionScopedRole", + "ECSExecutionAdminLikeRole", + "CrossAccountOpsRole", + "BoundaryConstrainedRole", + "UnknownResourceScopeRole" + ] +} diff --git a/tests/test_complex_synthetic_benchmark_fixture.py b/tests/test_complex_synthetic_benchmark_fixture.py new file mode 100644 index 0000000..6ebb459 --- /dev/null +++ b/tests/test_complex_synthetic_benchmark_fixture.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import json +import re +from collections import Counter +from pathlib import Path +from typing import Any + +FIXTURE_DIR = Path("tests/fixtures/demo/complex_shared_uncertainty_iam_benchmark") +FIXTURE_ID = "complex_shared_uncertainty_iam_benchmark_001" +SYNTHETIC_ACCOUNT = "000000000000" +REQUIRED_FILES = { + "README.md", + "scenario.json", + "binding_metadata.json", + "findings.json", + "naive_candidates.json", + "expected_uncertainty_groups.json", +} +EXPECTED_VERDICTS = { + "validated": 4, + "blocked": 5, + "precondition_only": 3, + "inconclusive": 6, +} +EXPECTED_UNCERTAINTY_GROUPS = { + "shared_passrole_target_resource_scope_unknown": 3, + "shared_cross_account_trust_condition_unknown": 2, + "shared_boundary_or_session_policy_context_missing": 1, +} +REQUIRED_PATTERN_CATEGORIES = { + "PassRole-to-Lambda", + "PassRole-to-ECS", + "AssumeRole chain", + "Cross-account trust-shaped path", + "permission boundary blocked path", + "SCP blocked path", + "identity-Deny suppressed path", + "missing iam:PassRole precondition", + "missing target trust precondition", + "shared unknown resource scope", +} +EXPECTED_PRINCIPALS = { + "AnalystUser", + "CICDRole", + "DeveloperRole", + "ReadOnlyAuditRole", + "BreakGlassCandidateRole", +} +EXPECTED_TARGET_ROLES = { + "LambdaExecutionAdminLikeRole", + "LambdaExecutionScopedRole", + "ECSExecutionAdminLikeRole", + "CrossAccountOpsRole", + "BoundaryConstrainedRole", + "UnknownResourceScopeRole", +} +EXPECTED_SERVICE_PRINCIPALS = { + "lambda.amazonaws.com", + "ecs-tasks.amazonaws.com", +} +REQUIRED_NON_CLAIMS = { + "no_live_aws", + "no_broad_iamscope_correctness", + "no_broad_passrole_correctness", + "no_generic_deny_correctness", + "no_resource_policy_deny_support", + "no_scp_deny_support_beyond_selected_synthetic_fixture_behavior", + "no_exploitability_proof", + "no_downstream_authorization_proof", + "no_lambda_invocation_behavior", + "no_production_readiness", + "no_correctness_for_real_aws_environments", + "no_correctness_for_other_principals_roles_accounts_regions_conditions_boundaries_scps_resource_policies_or_findings", + "no_composite_benchmark_score", + "no_pass_fail_benchmark_label", +} +GENERATED_ARTIFACT_NAMES = { + "result.json", + "terraform.tfstate", + "terraform.tfstate.backup", + ".terraform.lock.hcl", + "terraform-outputs.json", +} +FORBIDDEN_PROVENANCE_MARKERS = [ + "AccessKeyId", + "SecretAccessKey", + "SessionToken", + "raw_aws_log", + "raw_error", + "/home/ericc", + "/mnt/c/Users/ericc", + "Claude", + "ChatGPT", + "Anthropic", + "OpenAI", + "Toyota Financial", + "Red Team Lead", + "personal AWS Free Tier", + "gmail.com", + "aol.com", +] + + +def _load(name: str) -> dict[str, Any]: + return json.loads((FIXTURE_DIR / name).read_text()) + + +def _combined_fixture_text() -> str: + return "\n".join(path.read_text() for path in sorted(FIXTURE_DIR.iterdir()) if path.is_file()) + + +def test_required_fixture_files_exist() -> None: + assert FIXTURE_DIR.is_dir() + assert {path.name for path in FIXTURE_DIR.iterdir() if path.is_file()} == REQUIRED_FILES + + +def test_fixture_id_is_consistent() -> None: + for name in REQUIRED_FILES - {"README.md"}: + assert _load(name)["fixture_id"] == FIXTURE_ID + assert FIXTURE_ID in (FIXTURE_DIR / "README.md").read_text() + + +def test_only_synthetic_account_ids_and_arns_are_present() -> None: + combined = _combined_fixture_text() + account_ids = set(re.findall(r"(? None: + for name in ["scenario.json", "binding_metadata.json", "findings.json"]: + payload = _load(name) + assert payload["aws_calls_made"] == 0 + assert payload["live_aws_used"] is False + + +def test_synthetic_graph_shape_matches_design() -> None: + scenario = _load("scenario.json") + + assert set(scenario["principals"]) == EXPECTED_PRINCIPALS + assert set(scenario["target_roles"]) == EXPECTED_TARGET_ROLES + assert set(scenario["service_principals"]) == EXPECTED_SERVICE_PRINCIPALS + + +def test_findings_are_frozen_synthetic_oracle_not_iamscope_replay() -> None: + findings = _load("findings.json") + binding = _load("binding_metadata.json") + + assert findings["source_tool"] == "static_fixture_authoring" + assert findings["generation_mode"] == "frozen_synthetic_oracle" + assert findings["generated_or_replayed_by_iamscope"] is False + assert findings["reasoners_run"] == [] + assert findings["metadata"]["source_tool"] == "static_fixture_authoring" + assert findings["metadata"]["generation_mode"] == "frozen_synthetic_oracle" + assert findings["metadata"]["generated_or_replayed_by_iamscope"] is False + assert findings["metadata"]["reasoners_run"] == [] + assert binding["findings_generation"]["mode"] == "frozen_synthetic_oracle" + assert binding["findings_generation"]["generated_or_replayed_by_iamscope"] is False + assert binding["findings_generation"]["reasoners_run"] == [] + + +def test_naive_candidate_count_is_exact() -> None: + naive = _load("naive_candidates.json") + assert naive["candidate_count"] == 42 + assert len(naive["candidate_paths"]) == 42 + assert naive["is_iamscope_output"] is False + assert naive["reachability_evidence"] is False + + +def test_naive_candidates_have_required_shape_and_mapping() -> None: + finding_ids = {finding["finding_id"] for finding in _load("findings.json")["findings"]} + required = { + "candidate_id", + "source_principal", + "pattern", + "target", + "candidate_interpretation", + "expected_oracle_status", + "reason", + } + for candidate in _load("naive_candidates.json")["candidate_paths"]: + assert required <= set(candidate) + mapping = candidate["maps_to"] + has_finding = bool(mapping.get("finding_id")) + has_non_finding_reason = bool(mapping.get("non_finding_reason")) + assert has_finding != has_non_finding_reason + if has_finding: + assert mapping["finding_id"] in finding_ids + + +def test_finding_count_and_verdict_breakdown_are_exact() -> None: + findings_doc = _load("findings.json") + findings = findings_doc["findings"] + assert len(findings) == 18 + assert Counter(finding["verdict"] for finding in findings) == EXPECTED_VERDICTS + assert findings_doc["verdict_breakdown"] == EXPECTED_VERDICTS + assert findings_doc["metadata"]["verdict_breakdown"] == EXPECTED_VERDICTS + + +def test_uncertainty_group_counts_are_exact_and_reference_findings() -> None: + findings = _load("findings.json")["findings"] + finding_ids = {finding["finding_id"] for finding in findings} + inconclusive_ids = {finding["finding_id"] for finding in findings if finding["verdict"] == "inconclusive"} + groups = _load("expected_uncertainty_groups.json")["groups"] + referenced_ids = {finding_id for group in groups for finding_id in group["finding_ids"]} + + assert {group["uncertainty_class"]: group["count"] for group in groups} == EXPECTED_UNCERTAINTY_GROUPS + assert {group["uncertainty_class"]: len(group["finding_ids"]) for group in groups} == EXPECTED_UNCERTAINTY_GROUPS + assert referenced_ids <= finding_ids + assert referenced_ids == inconclusive_ids + + +def test_required_pattern_categories_are_present() -> None: + scenario = _load("scenario.json") + candidate_patterns = {candidate["pattern"] for candidate in _load("naive_candidates.json")["candidate_paths"]} + finding_patterns = {finding["pattern_id"] for finding in _load("findings.json")["findings"]} + + assert set(scenario["pattern_coverage"]) == REQUIRED_PATTERN_CATEGORIES + assert {"passrole_lambda", "passrole_ecs", "assume_role_chain", "cross_account_trust"} <= finding_patterns + assert { + "permission_boundary_blocked_path", + "scp_blocked_path", + "identity_deny_suppressed_path", + "missing_iam_passrole_precondition", + "missing_target_trust_precondition", + "shared_unknown_resource_scope", + } <= candidate_patterns + + +def test_each_finding_has_evidence_boundary_and_non_claims() -> None: + required_finding_fields = { + "finding_id", + "source_principal", + "target", + "pattern_id", + "verdict", + "classification", + "severity", + "required_checks", + "evidence_boundary", + "non_claims", + } + for finding in _load("findings.json")["findings"]: + assert required_finding_fields <= set(finding) + assert finding["required_checks"] + boundary = finding["evidence_boundary"] + assert boundary["local_only"] is True + assert boundary["synthetic_fixture"] is True + assert boundary["frozen_synthetic_oracle"] is True + assert boundary["live_aws_used"] is False + assert boundary["aws_calls_made"] == 0 + assert boundary["applies_only_to_fixture_id"] == FIXTURE_ID + assert set(finding["non_claims"]) == REQUIRED_NON_CLAIMS + assert all(finding["non_claims"].values()) + + +def test_top_level_non_claims_are_preserved() -> None: + for name in ["scenario.json", "binding_metadata.json", "findings.json", "naive_candidates.json"]: + non_claims = _load(name)["non_claims"] + assert set(non_claims) == REQUIRED_NON_CLAIMS + assert all(non_claims.values()) + + readme = (FIXTURE_DIR / "README.md").read_text() + for phrase in [ + "no live AWS", + "no broad IAMScope correctness", + "no exploitability proof", + "no production readiness", + "no composite benchmark score", + "no pass/fail benchmark label", + ]: + assert phrase in readme + + +def test_fixture_has_no_positive_live_or_score_claims() -> None: + combined = _combined_fixture_text().lower() + forbidden_positive_claims = [ + "live aws validation succeeded", + "proves exploitability", + "production ready", + "broad iamscope correctness evidence", + "composite_score", + "benchmark_passed", + "pass/fail score", + ] + for marker in forbidden_positive_claims: + assert marker not in combined + + +def test_no_generated_artifacts_or_provenance_markers_are_committed() -> None: + committed_names = {path.name for path in FIXTURE_DIR.iterdir() if path.is_file()} + assert committed_names.isdisjoint(GENERATED_ARTIFACT_NAMES) + assert not any(path.suffix == ".tfplan" for path in FIXTURE_DIR.iterdir()) + + combined = _combined_fixture_text() + for marker in FORBIDDEN_PROVENANCE_MARKERS: + assert marker not in combined