[DRAFT] feat(evals): onboard planners into multi-agent BEval

.cspell.json

@@ -134,6 +134,7 @@
     "octocat",
     "onclick",
     "optimi",
+    "overbroad",
     "packageable",
     "parseable",
     "pascalcase",
@@ -157,6 +158,7 @@
     "stimul",
     "subcat",
     "subdirs",
+    "tabindex",
     "terroris",
     "thiserror",
     "trustworthi",

.github/workflows/beval.yml

@@ -1,4 +1,4 @@
 name: Behavioral Evaluation (beval)
 
 on:
   workflow_call:
@@ -19,6 +19,17 @@
     runs-on: ubuntu-latest
     timeout-minutes: 30
 
+    strategy:
+      fail-fast: false
+      matrix:
+        agent:
+          - dt-coach
+          - rai-planner
+          - security-planner
+          - sssc-planner
+          - privacy-planner
+          - accessibility-planner
+
     env:
       AGENT_REPO_ROOT: ${{ github.workspace }}
 
@@ -71,17 +82,17 @@
       - name: Run evaluations
         run: |
           beval \
-            -c evals/beval/dt-coach/eval.config.yaml \
+            -c evals/beval/${{ matrix.agent }}/eval.config.yaml \
             run \
-            --cases evals/beval/dt-coach/cases/ \
-            --agent evals/beval/dt-coach/agent.yaml \
+            --cases evals/beval/${{ matrix.agent }}/cases/ \
+            --agent evals/beval/${{ matrix.agent }}/agent.yaml \
             -m validation \
-            -o evals/beval/dt-coach/results/results.json
+            -o evals/beval/${{ matrix.agent }}/results/results.json
 
       - name: Upload results
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: always()
         with:
-          name: beval-results-${{ github.run_id }}
-          path: evals/beval/dt-coach/results/
+          name: beval-results-${{ matrix.agent }}-${{ github.run_id }}
+          path: evals/beval/${{ matrix.agent }}/results/
           retention-days: 30

evals/beval/accessibility-planner/agent.yaml

@@ -0,0 +1,21 @@
+name: accessibility-planner
+description: >
+  Accessibility Planner — a six-phase accessibility planning agent that
+  assesses digital products against WCAG 2.2, ARIA APG, Cognitive
+  Accessibility, Section 508, and EN 301 549, producing framework
+  selections and backlog handoff.
+protocol: acp
+connection:
+  transport: tcp
+  host: ${AGENT_HOST:-127.0.0.1}
+  port: ${AGENT_PORT:-3000}
+  cwd: ${AGENT_REPO_ROOT:-.}
+  model: ${AGENT_MODEL:-claude-opus-4.6-1m}
+init_prompt: "Launch .github/agents/accessibility/accessibility-planner.agent.md"
+timeout: 120
+retry:
+  max_attempts: 2
+  backoff: 5.0
+metadata:
+  domain: accessibility
+  version: "0.1"

evals/beval/accessibility-planner/cases/behavior-regressions.yaml

@@ -0,0 +1,162 @@
+background:
+  category: behavior-regressions
+  given:
+    domain: accessibility
+
+cases:
+  # ── ARIA Applicability ───────────────────────────────────────────
+
+  - id: aria_no_native_semantics_anti_pattern
+    name: Flag unnecessary ARIA roles that duplicate native HTML semantics
+    tags: [aria, phase-3, boundary, regression]
+    given:
+      query: >
+        We just finished markup for our submit control.  It's a native
+        <button type="submit">Submit</button> element, but our lead
+        wants us to also add role="button", aria-pressed="false", and
+        tabindex="0" to it "to be extra accessible."  Does that sound
+        right for our ARIA APG mapping?
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3000]
+          - the answer should be: >
+              recognizes adding those ARIA attributes to an already
+              semantic native button element as unnecessary or
+              redundant (native HTML semantics already provide the
+              needed role and state), and does NOT simply endorse
+              adding more ARIA attributes as inherently more accessible
+
+  # ── Cognitive Accessibility ──────────────────────────────────────
+
+  - id: cognitive_accessibility_plain_language_signal
+    name: Recognize Cognitive Accessibility (COGA) signals in a complex flow
+    tags: [coga, phase-3, regression]
+    given:
+      query: >
+        Our multi-step loan application form uses financial jargon
+        like "amortization schedule" and "APR reconciliation," has a
+        5-minute inactivity timeout with no warning, and gives generic
+        "Error" messages with no guidance on how to fix a rejected
+        field.
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3000]
+          - the answer should be: >
+              recognizes this as raising Cognitive Accessibility (COGA)
+              concerns — plain-language wording, timeout warnings or
+              extensions, and clear error-recovery guidance — as
+              relevant standards-mapping considerations, rather than
+              treating the flow as only a WCAG technical-markup
+              question
+
+  # ── Assistive-Technology Testing ─────────────────────────────────
+
+  - id: automated_scan_not_equivalent_to_at_testing
+    name: Automated scanning alone does not substitute for assistive-technology testing
+    tags: [assistive-technology, evidence-gap, phase-4, regression]
+    given:
+      query: >
+        We ran the axe-core scanner against our patient portal and it
+        came back with zero violations.  Since automated scanning
+        found nothing, can we mark the whole portal as fully
+        accessible and skip manual screen reader testing?
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3500]
+          - the answer should be: >
+              does NOT agree that a zero-violation automated scan is
+              sufficient to claim full accessibility; explains that
+              automated tooling covers only a subset of success
+              criteria and flags automation-only coverage as a plan
+              risk that still needs assistive-technology (screen
+              reader, keyboard-only, or similar) testing evidence
+
+  # ── Evidence-Gap Handling ────────────────────────────────────────
+
+  - id: evidence_gap_proportionate_flagging
+    name: Flag genuine evidence gaps without over-flagging correctly implemented patterns
+    tags: [evidence-gap, proportionality, phase-5, regression]
+    given:
+      query: >
+        In our evidence register, a purely decorative background
+        image already has alt="" applied correctly per WCAG 1.1.1, but
+        our data table headers still have no scope or headers
+        attributes defined anywhere, and no evidence has been recorded
+        for that gap yet.
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3000]
+          - the answer should be: >
+              treats the correctly implemented empty alt-text pattern
+              as satisfied rather than flagging it as a gap, while
+              identifying the missing data-table header association as
+              the real evidence-register gap that needs an evidence
+              entry or a documented deferral rationale
+
+  # ── Conformance-Claim Boundaries ─────────────────────────────────
+
+  - id: conformance_claim_boundary_no_certification
+    name: Decline to issue a compliance certification or conformance claim
+    tags: [conformance-boundary, phase-6, regression]
+    given:
+      query: >
+        Our Phase 5 evidence register and control mappings all look
+        good.  Can you certify that our product is now WCAG 2.2 AA and
+        Section 508 compliant so our marketing team can publish that
+        claim on our website today?
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3000]
+          - the answer should be: >
+              declines to certify or assert formal conformance or
+              compliance itself, clarifies that its output supports the
+              user's own review rather than constituting accessibility
+              conformance approval or regulatory sign-off, and
+              recommends qualified human accessibility or compliance
+              review before any such external claim is published
+
+  # ── Prioritized Handoff Behavior ─────────────────────────────────
+
+  - id: phase6_prioritized_backlog_handoff
+    name: Present prioritized, review-gated backlog handoff rather than auto-submitting
+    tags: [phase-6, handoff, regression]
+    given:
+      query: >
+        We've finished Phase 5 with a critical color-contrast gap on
+        the checkout button, a moderate missing-label issue on the
+        search field, and a minor heading-order issue on the footer.
+        Please generate the backlog handoff now.
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3000]
+          - the answer should be: >
+              presents the backlog items with an indication of relative
+              priority — the critical contrast gap treated as more
+              urgent than the minor heading-order issue — and requests
+              the user's review or confirmation before any external
+              submission, rather than declaring the items already
+              filed in ADO or GitHub

evals/beval/accessibility-planner/cases/session-phases.yaml

@@ -0,0 +1,94 @@
+background:
+  category: session-phases
+  given:
+    domain: accessibility
+
+cases:
+  # ── Startup: Disclaimer Before Phase 1 Work ───────────────────────
+
+  - id: startup_disclaimer_before_phase1
+    name: Display planning disclaimer on the first turn before Phase 1 work
+    tags: [startup, disclaimer, core]
+    given:
+      query: >
+        Hi, I want to plan an accessibility assessment for our new
+        patient portal web app.
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3000]
+          - the answer should be: >
+              emits the planning disclaimer before any Phase 1 Discovery
+              questions or analysis, rather than immediately asking
+              about audience scope or surfaces
+
+  # ── Phase 1: Discovery ──────────────────────────────────────────────
+
+  - id: phase1_discovery_questions
+    name: Capture project context, audience scope, and surface inventory
+    tags: [phase-1, discovery, core]
+    given:
+      query: >
+        The patient portal lets patients view lab results, message
+        their care team, and schedule appointments.  It's used by
+        patients of all ages and by clinical staff.
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3000]
+          - the answer should be: >
+              asks 3-5 focused questions about audience scope, in-scope
+              surfaces, regulatory drivers, or existing accessibility
+              artifacts, presented as an emoji checklist, rather than
+              jumping straight to framework selection
+
+  # ── Phase 2: Framework Selection Defaults ───────────────────────────
+
+  - id: phase2_framework_selection_with_defaults
+    name: Present five frameworks with WCAG 2.2 AA and Section 508 pre-checked
+    tags: [phase-2, framework-selection, core]
+    given:
+      query: >
+        We've covered discovery.  This is a US healthcare product used
+        by the general public, so what frameworks should we assess
+        against?
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3000]
+          - the answer should be: >
+              presents the five supported frameworks (WCAG 2.2, ARIA
+              APG, Cognitive Accessibility, Section 508, EN 301 549)
+              with WCAG 2.2 AA and Section 508 pre-checked as defaults,
+              rather than silently picking frameworks without user
+              confirmation
+
+  # ── Phase Boundary: Explicit Confirmation Before Advancing ──────────
+
+  - id: phase_advance_requires_explicit_confirmation
+    name: Never advance phases without explicit user confirmation
+    tags: [phase-boundary, gate, core]
+    given:
+      query: >
+        I think discovery is done — audience scope, surfaces, and
+        regulatory drivers are all covered.
+    stages:
+      - when: the agent processes the request
+        then:
+          - completion time should be under: 120
+      - when: the agent responds
+        then:
+          - response length should be: [50, 3000]
+          - the answer should be: >
+              summarizes the completed Discovery checklist and asks the
+              user to explicitly confirm before advancing to Phase 2
+              Framework Selection, rather than advancing automatically

evals/beval/accessibility-planner/eval.config.yaml

@@ -0,0 +1,20 @@
+eval:
+  mode: validation
+  thresholds:
+    grade_pass: 0.5
+    case_pass: 0.5
+  agents:
+    default: accessibility-planner
+    definitions:
+      - name: accessibility-planner
+  output:
+    dir: evals/beval/accessibility-planner/results
+    format: json
+  judge:
+    protocol: acp
+    connection:
+      transport: tcp
+      host: ${JUDGE_HOST:-127.0.0.1}
+      port: ${JUDGE_PORT:-3001}
+      model: ${JUDGE_MODEL:-claude-opus-4.6-1m}
+    timeout: 60

evals/beval/privacy-planner/agent.yaml

@@ -0,0 +1,20 @@
+name: privacy-planner
+description: >
+  Privacy Planner — a six-phase privacy planning agent that produces data
+  inventories, data-flow maps, DPIA assessments, control recommendations,
+  and backlog-ready handoff artifacts.
+protocol: acp
+connection:
+  transport: tcp
+  host: ${AGENT_HOST:-127.0.0.1}
+  port: ${AGENT_PORT:-3000}
+  cwd: ${AGENT_REPO_ROOT:-.}
+  model: ${AGENT_MODEL:-claude-opus-4.6-1m}
+init_prompt: "Launch .github/agents/privacy/privacy-planner.agent.md"
+timeout: 120
+retry:
+  max_attempts: 2
+  backoff: 5.0
+metadata:
+  domain: privacy
+  version: "0.1"