diff --git a/.agents/skills/stuck-crewmate-recovery/SKILL.md b/.agents/skills/stuck-crewmate-recovery/SKILL.md index cf741b9d95f..b9b94b27d43 100644 --- a/.agents/skills/stuck-crewmate-recovery/SKILL.md +++ b/.agents/skills/stuck-crewmate-recovery/SKILL.md @@ -23,7 +23,7 @@ The target window's harness is recorded as `harness=` in `state/.meta`. This procedure covers ordinary `kind=ship` and `kind=scout` direct reports. Load `secondmate-provisioning` instead for `kind=secondmate` recovery. -For a REMOTE secondmate, `fm-crew-state`'s `unknown`/`worktree gone` and `fm-send`'s `remote send failed`/`delivery unconfirmed` verdicts are unreliable and routinely false-negative; do not conclude the mate is dead or the send failed from those alone, confirm against the actual remote pane first. +For a REMOTE secondmate, `fm-crew-state` and `fm-peek` read the actual remote endpoint over `fm-on.sh`, and `fm-send` reports a delivered-with-pending-confirmation steer as delivered (their headers own the contracts); an `unknown-remote` read or unreachable-host failure means the remote state could not be read, never that the mate is dead or the send failed. Recover a genuinely stuck remote mate only through `bin/fm-spawn.sh --secondmate`, never raw herdr pane close/kill surgery, which strands the endpoint binding. Treat the digest's endpoint result as a presence signal, not proof that the task's work or validation run is gone. diff --git a/AGENTS.md b/AGENTS.md index 6d6c9552288..256e1944407 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -329,7 +329,9 @@ After an autonomous merge, give the captain a one-line full-URL or local-main ou ### Validate -For a no-mistakes ship, trigger validation on the same worker after its implementation commit, using the harness invocation owned by `harness-adapters`. +For a no-mistakes ship, the same worker starts its own validation run immediately after the implementation commit and reports `done:` only with a PR. +The worker appends a nonterminal `working:` line when that run starts, and appends `failed:` or `blocked:` if the run dies mid-pipeline, so firstmate still learns start and failure without a handoff `done:`. +Firstmate does not send a start trigger after the implementation commit. The task worker that starts a no-mistakes run drives the pipeline and owns every `no-mistakes axi run` and `no-mistakes axi respond` call through the next gate or outcome. Firstmate never invokes `no-mistakes axi respond` for a crew-owned run. Once validation starts, prefer routing new requirements to follow-up work rather than expanding the current task, unless a new requirement completely invalidates the work being validated; however, the smallest downstream changes needed to keep already accepted product or engineering behavior correct, add behavioral tests where an executable contract exists, or keep documentation accurate remain within the current task even when they touch files not named at intake, and corrections required to satisfy already accepted intent are not new requirements. diff --git a/bin/fm-brief.sh b/bin/fm-brief.sh index 206e5a947ae..e88ce40705a 100755 --- a/bin/fm-brief.sh +++ b/bin/fm-brief.sh @@ -30,7 +30,9 @@ # For ship tasks, --mode is REQUIRED and shapes the definition of done. Firstmate # resolves it per task at intake (AGENTS.md section 7); data/projects.md holds the # captain's standing posture as context, and this script never reads it: -# no-mistakes implement -> /no-mistakes pipeline -> PR -> configured merge authority +# no-mistakes implement -> worker starts the no-mistakes pipeline (the `no-mistakes axi` +# CLI, not a skill) immediately after the implementation commit -> PR -> +# configured merge authority # direct-PR implement -> push + open PR via gh-axi (no pipeline) -> configured merge authority # local-only implement on branch, stop and report "ready in branch" (no push/PR); # the configured merge authority approves, firstmate merges to local main @@ -362,7 +364,7 @@ Delivery contract: mode=direct-PR This task ships **direct-PR**: you raise the PR yourself, without the no-mistakes pipeline. The task is complete only when committed on your branch. When it is implemented and committed, push your branch and open a PR with \`gh-axi\`, then append \`done: PR {url}\` to the status file and stop. -Do NOT run /no-mistakes. The configured merge authority decides whether to merge the PR; firstmate relays the outcome. +Do NOT run the no-mistakes pipeline. The configured merge authority decides whether to merge the PR; firstmate relays the outcome. EOF ;; local-only) @@ -385,12 +387,13 @@ EOF IFS= read -r -d '' DOD <"\` to start, and \`no-mistakes axi respond\` for each gate. +Do not append \`done:\` until there is a PR. You drive no-mistakes by responding to its gates, not by implementing fixes. -Follow the guidance no-mistakes itself provides for the mechanics: it loads when you invoke /no-mistakes, and \`no-mistakes axi run --help\` plus the \`help\` lines in each \`axi\` response are authoritative and version-matched to the installed binary. +Follow the guidance no-mistakes itself provides for the mechanics: \`no-mistakes axi run --help\` plus the \`help\` lines in each \`axi\` response are authoritative and version-matched to the installed binary. When starting no-mistakes, make \`--intent\` preserve all relevant content from this brief's \`# Task\` section plus every later accepted Firstmate requirement, clarification, constraint, exclusion, and supersession, carrying only each requirement's current accepted form; retain direct requirements instead of substituting a diff summary, and exclude generic operational, status, delivery, and other scaffold boilerplate unless it is task-specific. Do not hand-edit, commit, or fix findings yourself while a run is active - the pipeline applies every fix. @@ -400,7 +403,9 @@ Two firstmate-specific rules layer on top of that guidance: When the decision comes back, feed it to the gate with \`no-mistakes axi respond\` and let the pipeline apply it - do not route the question to "the user" or implement the fix yourself. - Avoid \`--yes\`: it would silently bypass firstmate's authority check and any required captain escalation. -After /no-mistakes reports CI green (the CI-ready return point - do not wait for it to keep monitoring in the background until merge), append \`done: PR {url} checks green\` and stop. You are finished. +If you cannot start or continue the run, append \`blocked: {the exact error}\` and stop, never \`done:\`. +If the run dies mid-pipeline, append \`failed: {the exact error}\` and stop, never stay silent. +After the run reports CI green (the CI-ready return point - do not wait for it to keep monitoring in the background until merge), append \`done: PR {url} checks green\` and stop. You are finished. EOF ;; esac diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index 2cb290373cb..df627b487f2 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -16,10 +16,17 @@ # fixed mapping logic, no heuristics and no LLM. Output is one stable, parseable, # token-tight line firstmate can read every heartbeat: # -# state: · source: · +# state: · source: · # # Logic, in order: -# 1. Resolve worktree + backend target + kind from state/.meta. +# 1. Resolve worktree + backend target + kind from state/.meta. A meta +# recording remote_host= is a remote secondmate: its worktree and endpoint +# live on that host, so the local worktree and pane reads are skipped and +# the remote host is asked for the endpoint's recovery-grade state +# (fm-on.sh + fm-remote-secondmate-control.sh state). alive falls through +# to the routed status log; dead/missing report the remote verdict; an +# unreachable or unreadable remote reports unknown-remote, never a false +# gone/dead. # 2. Matching no-mistakes run for this crew's branch AND current code identity, # active or terminal (from `axi status`, or the coarse `no-mistakes runs` # fallback)? Branch name alone is not enough: a historical run on a reused @@ -101,10 +108,13 @@ meta_value() { # WT=$(meta_value worktree) KIND=$(meta_value kind) HARNESS=$(meta_value harness) +REMOTE_HOST=$(meta_value remote_host) [ -n "$KIND" ] || KIND=ship -# A torn-down (or never-created) worktree has no current state to read. -if [ -z "$WT" ] || [ ! -d "$WT" ]; then +# A torn-down (or never-created) worktree has no current state to read. A +# remote secondmate's recorded worktree is a path on ITS host, so the local +# probe proves nothing for it - the remote arm below reads the true source. +if [ -z "$REMOTE_HOST" ] && { [ -z "$WT" ] || [ ! -d "$WT" ]; }; then emit unknown none "worktree gone (torn down?)" fi @@ -138,6 +148,45 @@ map_log_state() { # LOG_LINE=$(log_last_line || true) LOG_VERB=$(status_line_verb "$LOG_LINE") +# --- remote secondmate: the true source is the remote endpoint --------------- +# A remote mate's recorded worktree and backend target live on its own host, so +# the local worktree probe above and the local pane reads below would misreport +# a healthy remote mate as gone or dead. Ask the remote host for the endpoint's +# recovery-grade state over the same fm-on.sh transport fm-send uses, then read +# current activity from the routed status log exactly as for a local +# secondmate (an idle endpoint is healthy for a secondmate either way). An +# unreachable host or unreadable endpoint is reported as unknown-remote - +# explicitly NOT proof of death - so a transport blip never reads as a torn +# down or dead mate; only the remote host's own dead/missing verdict may say +# the endpoint is actually gone. +if [ -n "$REMOTE_HOST" ]; then + if ! REMOTE_STATE=$(FM_HOME="$FM_HOME" "$SCRIPT_DIR/fm-on.sh" "$ID" \ + fm-remote-secondmate-control.sh state "$ID" < /dev/null 2>/dev/null); then + REMOTE_STATE= + fi + REMOTE_STATE=$(printf '%s\n' "$REMOTE_STATE" | tail -1) + case "$REMOTE_STATE" in + alive) + if [ -n "$LOG_VERB" ]; then + LOG_STATE=$(map_log_state "$LOG_LINE") + if [ "$LOG_STATE" != unknown ]; then + emit "$LOG_STATE" status-log "$(status_line_note "$LOG_LINE")${SEP}remote endpoint alive on $REMOTE_HOST" + fi + fi + emit unknown remote-endpoint "alive on $REMOTE_HOST (an idle secondmate is healthy)" + ;; + dead|missing) + emit unknown remote-endpoint "remote endpoint $REMOTE_STATE on $REMOTE_HOST" + ;; + '') + emit unknown remote-endpoint "unknown-remote: $REMOTE_HOST unreachable or endpoint unreadable (not proof of death)" + ;; + *) + emit unknown remote-endpoint "unknown-remote: endpoint state '$REMOTE_STATE' on $REMOTE_HOST (not proof of death)" + ;; + esac +fi + # pane_readable is consulted ONLY in the no-run fallback below. The run-step path # stays authoritative regardless of pane liveness - judge by the run-step, not the # shell - so a finished crew whose endpoint has closed still reports its run-step diff --git a/bin/fm-peek.sh b/bin/fm-peek.sh index 97d2ffe2d25..e3156f66ed4 100755 --- a/bin/fm-peek.sh +++ b/bin/fm-peek.sh @@ -3,6 +3,11 @@ # Usage: fm-peek.sh [lines=40] # may be an exact task id, a legacy fm- task label resolved # through this home's state/.meta, or an explicit backend target. +# A selector whose meta records remote_host= is a remote secondmate: its pane +# lives on that host, so the capture routes over fm-on.sh to the host-local +# capture (fm-remote-secondmate-control.sh), clamped to that command's +# 100-line cap. An unreachable host or unreadable endpoint fails loudly naming +# the host; the local backend adapters are never asked to read a remote target. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -16,9 +21,25 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" "$SCRIPT_DIR/fm-guard.sh" || true RAW_TARGET=$1 -T=$(fm_backend_resolve_selector "$RAW_TARGET" "$STATE") N=${2:-40} +REMOTE_META=$(fm_backend_meta_for_selector "$RAW_TARGET" "$STATE" 2>/dev/null || true) +if [ -n "$REMOTE_META" ] && [ -n "$(fm_meta_get "$REMOTE_META" remote_host)" ]; then + REMOTE_ID=${REMOTE_META##*/} + REMOTE_ID=${REMOTE_ID%.meta} + REMOTE_HOST=$(fm_meta_get "$REMOTE_META" remote_host) + case "$N" in ''|*[!0-9]*|0) N=40 ;; esac + [ "$N" -le 100 ] || N=100 + if ! FM_HOME="$FM_HOME" "$SCRIPT_DIR/fm-on.sh" "$REMOTE_ID" \ + fm-remote-secondmate-control.sh capture "$REMOTE_ID" "$N" < /dev/null; then + echo "error: could not read the remote pane of $REMOTE_ID on $REMOTE_HOST (host unreachable or endpoint unreadable; the mate is not thereby dead)" >&2 + exit 1 + fi + exit 0 +fi + +T=$(fm_backend_resolve_selector "$RAW_TARGET" "$STATE") + BACKEND=$(fm_backend_of_selector "$RAW_TARGET" "$T" "$STATE") EXPECTED_LABEL=$(fm_backend_expected_label_of_selector "$RAW_TARGET" "$STATE") diff --git a/bin/fm-remote-secondmate-control.sh b/bin/fm-remote-secondmate-control.sh index f2edb32a7bb..aa17c952860 100755 --- a/bin/fm-remote-secondmate-control.sh +++ b/bin/fm-remote-secondmate-control.sh @@ -188,6 +188,12 @@ cmd_send() { validate_id "$id" validate_home "$id" remote_endpoint_require "$id" + # fm-send's exit status is the delivery verdict the parent home acts on + # (0 = confirmed, 3 = delivered with the submit read-back unconfirmed, other + # nonzero = failed; see bin/fm-send.sh's header). The job worker, entrypoint, + # and ssh all preserve it, so no mapping may happen here: flattening exit 3 + # into a generic failure is exactly the false-negative the parent's remote + # send path exists to avoid. FM_HOME="$TARGET_HOME" FM_ROOT_OVERRIDE="$FM_ROOT" FM_STATE_OVERRIDE="$TARGET_HOME/state" \ "$SCRIPT_DIR/fm-send.sh" "$REMOTE_ENDPOINT_TARGET" "$message" } diff --git a/bin/fm-send.sh b/bin/fm-send.sh index c46c55a340f..1da45d86f46 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -15,6 +15,12 @@ # submit or reports an inconclusive send. If a swallowed Enter is positively # confirmed, fm-send exits NON-ZERO so the caller knows the steer did not land # instead of silently leaving an unsubmitted instruction. +# Exit status contract: 0 = submit confirmed (or, for a remote secondmate +# target, delivered with confirmation pending - see the remote paragraph); +# 3 = the text was typed into the live endpoint and Enter was sent, but the +# submit read-back stayed unconfirmed (verify the pane before any resend, and +# never re-type blindly); any other nonzero = the send failed and nothing may +# be assumed delivered. # Submission dispatches through the target's recorded backend; the tmux adapter # shares its composer/submit core with the away-mode daemon via bin/fm-tmux-lib.sh. # Tune with FM_SEND_RETRIES (default 3) / FM_SEND_SLEEP (0.4). @@ -37,6 +43,20 @@ # re-sending a recovery request for an already-open expectation so a second # record is not created. Direct unmarked captain input never creates one. # +# Remote secondmate delivery: the send crosses fm-on.sh to a host-local leg +# (bin/fm-remote-secondmate-control.sh cmd_send) that runs this same verified +# submit against the recorded remote Herdr pane and relays its exit status +# unchanged. A leg that delivered the text into the live verified pane but +# could not synchronously confirm the submit (exit 3 - typically a busy mate +# whose harness queues the steer and keeps rendering it) is reported here as +# DELIVERED with confirmation pending: fm-send prints a non-error notice, +# exits 0, marks the pending-reply expectation delivered, and closes any +# --resolve-key decisions. Empirically that pattern is a delivered steer, a +# resend duplicates the instruction, and the parent's pending-reply +# recovery/escalation still surfaces the rare genuinely lost request. Transport +# loss (ssh exit 255, completion unknown) and every real remote failure keep +# failing loudly with the remote leg's own stderr attached. +# # Decision closure (answerer-closes): pass --resolve-key (repeatable, # before the message) when this send answers an open keyed needs-decision: or # blocked: record in the target task's state/.status. After the submit is @@ -63,7 +83,9 @@ # in this home's status log per status_open_decisions (bin/fm-classify-lib.sh), or # an active captain hold for the target task. A key in neither is refused before # sending, so a mistyped key cannot deliver an answer while silently orphaning the -# decision. A failed or unconfirmed send never closes a key; a +# decision. A failed or unconfirmed send never closes a key (a remote +# delivered-with-pending-confirmation outcome counts as delivered - see the +# remote paragraph above); a # delivered answer whose closing append fails exits nonzero with the exact # manual close command, leaving the decision open to re-surface (the safe # direction). A send without the flag never closes anything: a routine steer, @@ -537,12 +559,27 @@ else # Type once, submit, verify. Only exact empty confirms delivery; every other # verdict preserves the loud refusal boundary. send_rc=0 + REMOTE_DELIVERY_NOTICE=0 if [ "$TARGET_BACKEND" = remote ]; then - if "$SCRIPT_DIR/fm-on.sh" "$TARGET_REMOTE_ID" fm-remote-secondmate-control.sh send "$TARGET_REMOTE_ID" "$MESSAGE" < /dev/null >/dev/null; then + # The remote leg is this same script running host-locally against the + # recorded Herdr pane (cmd_send in fm-remote-secondmate-control.sh), so its + # submit verification IS the local one, and fm-on/the remote worker relay + # its exit status unchanged. Exit 3 is the delivered-unconfirmed contract + # (see this script's header) crossing the ssh boundary: the text reached + # the live verified pane and Enter was sent; only the synchronous read-back + # stayed unconfirmed. The remote stderr is held back and replayed only for + # a real failure, so a delivered outcome does not surface the inner leg's + # diagnostics as alarm. + remote_err=$("$SCRIPT_DIR/fm-on.sh" "$TARGET_REMOTE_ID" fm-remote-secondmate-control.sh send "$TARGET_REMOTE_ID" "$MESSAGE" < /dev/null 2>&1 >/dev/null) || send_rc=$? + if [ "$send_rc" -eq 0 ]; then + verdict=empty + elif [ "$send_rc" -eq 3 ]; then verdict=empty + send_rc=0 + REMOTE_DELIVERY_NOTICE=1 else - send_rc=$? verdict=send-failed + [ -z "$remote_err" ] || printf '%s\n' "$remote_err" >&2 fi elif verdict=$(fm_backend_send_text_submit "$TARGET_BACKEND" "$T" "$MESSAGE" "$retries" "$sleep_s" "$settle" "$EXPECTED_LABEL"); then : @@ -571,6 +608,19 @@ else echo "error: text not sent to $T ($TARGET_BACKEND send failed; tried $RESOLUTION_TRIED)" >&2 exit 1 ;; + pending) + # The text was typed into the live target and Enter was sent; only the + # submit read-back stayed unconfirmed (e.g. a busy harness queues the + # steer and keeps rendering it). That is not a proven failure, so never + # re-type the message: verify the pane instead. Exit 3 is the documented + # delivered-unconfirmed status, and the remote send leg above depends on + # it crossing the ssh boundary intact. + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then + fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + fi + echo "fm-send: text delivered to $T but submission is unconfirmed (verdict=pending; tried $RESOLUTION_TRIED); do not retype or blindly resend - verify with fm-peek.sh, then re-send '--key Enter' only if the composer still holds the text" >&2 + exit 3 + ;; *) if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true @@ -600,6 +650,12 @@ else fm_send_close_resolved_keys "$RESOLVE_ANSWER_TEXT" || exit 1 fm_send_feed_resolved_holds "$RESOLVE_ANSWER_TEXT" || exit 1 fi + # Remote delivered-with-pending-confirmation: the outcome above is treated as + # delivered (expectation marked, keys closed), and this one non-error notice + # carries the remaining nuance so nobody re-sends the steer. + if [ "$REMOTE_DELIVERY_NOTICE" = 1 ]; then + echo "fm-send: delivered to remote secondmate $TARGET_REMOTE_ID; the remote pane accepted the text and Enter, and only the synchronous submit confirmation is still pending. This is not a failure - do not resend; the pending-reply expectation stays armed." >&2 + fi # Submit landed with exact empty. Confirmation only proves the text was # accepted; the harness still needs a beat to spin up the # turn before its busy footer shows. Pause so an immediate peek catches the diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index 5a38d48e52b..a1560e20b5d 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -168,6 +168,11 @@ Send routed requests normally: FM_HOME= bin/fm-send.sh fm- '' ``` +The [`fm-send.sh` header](../bin/fm-send.sh) owns the exact delivery-status contract. +When the verified remote endpoint accepts the text and Enter but synchronous submit confirmation remains pending, the primary reports the request as delivered rather than failed; do not resend it, because its pending-reply expectation remains armed. +`fm-peek.sh` and `fm-crew-state.sh` route remote-secondmate reads to the endpoint's host instead of consulting local worktree or backend state. +An unreachable or unreadable remote read is unknown, not evidence that the endpoint is dead. + Marked requests keep the existing correlation contract. The remote charter appends replies to `state/parent-replies.status` in the remote home. A process-event source performs a non-destructive, cursor-anchored delta read, fetches only referenced `data/*.md` documents through the confined reader, mirrors every content-bearing line at most once into the primary status channel, and does not carry blank separators. @@ -231,6 +236,9 @@ The lifecycle test covers seeding a registered project that this machine has nev ```sh bin/fm-test-run.sh tests/fm-on.test.sh +bin/fm-test-run.sh tests/fm-send-remote-delivery.test.sh +bin/fm-test-run.sh tests/fm-peek-remote.test.sh +bin/fm-test-run.sh tests/fm-crew-state.test.sh bin/fm-test-run.sh tests/fm-remote-job.test.sh bin/fm-test-run.sh tests/fm-remote-doctor.test.sh bin/fm-test-run.sh tests/fm-project-origin.test.sh diff --git a/docs/tmux-backend.md b/docs/tmux-backend.md index 4d8c3e75feb..c2acead0c2f 100644 --- a/docs/tmux-backend.md +++ b/docs/tmux-backend.md @@ -85,7 +85,7 @@ The supervisor guard selects only the detected primary harness's signature rathe It types a message once and retries Enter only until the composer clears. Only a proven empty composer is a positive delivery acknowledgement. Text left in established structure remains `pending`, text in ambiguous structure remains unproven, and unreadable or unsafe state remains unknown. -`fm-send.sh` reports every unconfirmed verdict as a failure instead of retyping or assuming delivery. +`fm-send.sh` never retypes or assumes a confirmed submit for an unconfirmed verdict; its header owns the distinct delivered-unconfirmed exit status and operator response. OpenCode 1.18.4 has one busy-queue exception. While OpenCode is mid-turn, Enter queues the message but leaves its text visible until the turn completes. diff --git a/tests/fm-brief.test.sh b/tests/fm-brief.test.sh index c5ee3d00f05..5c6fbd693f4 100755 --- a/tests/fm-brief.test.sh +++ b/tests/fm-brief.test.sh @@ -256,7 +256,7 @@ test_ship_mode_is_explicit_not_registry() { brief="$home/data/brief-explicit-a5/brief.md" grep -qx "Delivery contract: mode=no-mistakes" "$brief" \ || fail "registered direct-PR posture overrode the explicit --mode" - assert_grep "Firstmate will then instruct you to run /no-mistakes" "$brief" \ + assert_grep "start the no-mistakes pipeline yourself immediately" "$brief" \ "explicit no-mistakes brief did not render the pipeline definition of done" # An unregistered project is not a blocker either, because nothing is looked up. @@ -354,6 +354,86 @@ test_no_mistakes_dod_wording() { pass "fm-brief.sh: no-mistakes DOD keeps its apostrophe prose, now parse-safe" } +# Option (b): the worker starts its own no-mistakes run the moment +# implementation is committed. The generated brief is the worker-facing +# contract; this test pins both directions of that change through the +# public scaffold, not by reading fm-brief.sh source. +# RED if the old firstmate-triggered handoff returns (either the origin/main +# "instruct you to run" form or the later "triggers validation" / +# "not yet validated - no PR yet" form). +# RED if the start-your-own-run instruction disappears. +# A silent start plus a silent mid-pipeline death is worse than the +# supervisor round trip this removes, so start and failure reporting are +# part of the same contract. +test_no_mistakes_worker_starts_own_validation() { + local home id brief + home="$TMP_ROOT/self-start-home" + mkdir -p "$home/data" + id="brief-self-start-b1" + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" some-proj --mode no-mistakes >/dev/null 2>&1 \ + || fail "no-mistakes self-start brief should scaffold" + brief="$home/data/$id/brief.md" + + # Negative: the intermediate handoff must not come back. + assert_no_grep "Firstmate will then instruct you to run" "$brief" \ + "no-mistakes DOD restored the origin/main firstmate-instructs-validation handoff" + assert_no_grep "Firstmate then triggers validation" "$brief" \ + "no-mistakes DOD restored the firstmate-triggers-validation handoff" + assert_no_grep "not yet validated - no PR yet" "$brief" \ + "no-mistakes DOD restored a done: line with no PR" + assert_no_grep "When you believe it is complete, append \`done:" "$brief" \ + "no-mistakes DOD restored the pre-PR done: {summary} stop" + + # Positive: the worker must start the run itself and tell firstmate. + assert_grep "start the no-mistakes pipeline yourself immediately" "$brief" \ + "no-mistakes DOD lost the start-your-own-run instruction" + assert_grep "working: starting no-mistakes validation" "$brief" \ + "no-mistakes DOD lost the nonterminal working: append that tells firstmate validation started" + # shellcheck disable=SC2016 # single quotes are deliberate: the backticks must stay literal + assert_grep 'the `no-mistakes` CLI on your `PATH`' "$brief" \ + "no-mistakes DOD did not name the CLI as the interface the worker actually has" + # shellcheck disable=SC2016 # single quotes are deliberate: the backticks must stay literal + assert_grep '`no-mistakes axi run --intent "<...>"` to start' "$brief" \ + "no-mistakes DOD did not give the concrete run command" + # shellcheck disable=SC2016 # single quotes are deliberate: the backticks must stay literal + assert_grep '`no-mistakes axi respond` for each gate' "$brief" \ + "no-mistakes DOD did not give the concrete gate-response command" + assert_grep "Do not append \`done:\` until there is a PR." "$brief" \ + "no-mistakes DOD lost the rule that done: requires a PR" + assert_grep "This mode is complete only when the no-mistakes pipeline has shipped a PR whose checks are green." "$brief" \ + "no-mistakes DOD did not bind completion to a green PR" + assert_grep "stop, never \`done:\`." "$brief" \ + "no-mistakes DOD did not route an unstartable run to blocked: instead of done:" + assert_grep "If the run dies mid-pipeline, append \`failed:" "$brief" \ + "no-mistakes DOD did not surface a mid-pipeline death" + + # Skill form is absent in a crewmate worktree; no scaffold may instruct it. + for id_args in \ + "brief-self-start-nomistakes:some-proj --mode no-mistakes" \ + "brief-self-start-directpr:some-proj --mode direct-PR" \ + "brief-self-start-localonly:some-proj --mode local-only" \ + "brief-self-start-scout:some-proj --scout"; do + id=${id_args%%:*} + # shellcheck disable=SC2086 # the arg list is an intentional word split + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" ${id_args#*:} >/dev/null 2>&1 \ + || fail "$id: scaffold exited non-zero" + brief="$home/data/$id/brief.md" + assert_no_grep "run /no-mistakes" "$brief" \ + "$id: brief tells the worker to run a skill it cannot load" + assert_no_grep "invoke /no-mistakes" "$brief" \ + "$id: brief tells the worker to invoke a skill it cannot load" + done + FM_HOME="$home" FM_SECONDMATE_CHARTER='Handle routed domain work.' \ + "$ROOT/bin/fm-brief.sh" brief-self-start-secondmate --secondmate --no-projects >/dev/null 2>&1 \ + || fail "secondmate charter scaffold exited non-zero" + assert_no_grep "run /no-mistakes" "$home/data/brief-self-start-secondmate/brief.md" \ + "secondmate charter tells the worker to run a skill it cannot load" + assert_grep "Do NOT run the no-mistakes pipeline." "$home/data/brief-self-start-directpr/brief.md" \ + "direct-PR brief lost its pipeline refusal" + + pass "fm-brief.sh: no-mistakes DOD starts its own run and never emits a pre-PR done:" +} + test_ship_project_memory_wording() { local home id brief home="$TMP_ROOT/project-memory-home" @@ -721,6 +801,7 @@ test_ship_mode_is_explicit_not_registry test_delivery_flags_are_refused_where_they_do_not_apply test_faster_paths_use_configured_authority_without_stacked_review test_no_mistakes_dod_wording +test_no_mistakes_worker_starts_own_validation test_ship_project_memory_wording test_herdr_lab_contract_is_explicit_and_complete test_herdr_lab_contract_quotes_foreign_firstmate_path diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index 8f986b6139e..602b3e5cfc3 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -1150,6 +1150,105 @@ test_torn_down_worktree() { pass "torn-down worktree is handled gracefully" } +# --- remote secondmate arm --------------------------------------------------- +# A meta recording remote_host= must never be read through the local worktree +# probe or a local backend adapter: the recorded worktree and pane live on the +# remote host, and the old local reads misreported a healthy remote mate as +# "worktree gone". These cases drive the real helper over the real fm-on.sh +# route with a stubbed ssh transport (FM_SSH_BIN seam): the stub prints +# FM_FAKE_REMOTE_STATE_OUT as the remote endpoint's recovery-grade state and +# exits FM_FAKE_SSH_RC. + +setup_remote_case() { # -> echoes case dir with remote meta + registry + local d + d=$(new_case "$1") + mkdir -p "$d/data" "$d/fakebin" + fm_write_meta "$d/state/rsm.meta" \ + "window=remote:rsm" \ + "endpoint_task_id=rsm" \ + "worktree=/remote/home/never-locally-present" \ + "harness=claude" \ + "kind=secondmate" \ + "mode=secondmate" \ + "remote_host=remote-mac" \ + "remote_root=/remote/root" \ + "remote_backend=herdr" \ + "remote_herdr_session=fm-remote" \ + "remote_target=fm-remote:w1:p1" + cat > "$d/data/secondmates.md" < "$d/fakebin/fake-ssh" <<'SH' +#!/usr/bin/env bash +cat > /dev/null +[ -z "${FM_FAKE_REMOTE_STATE_OUT:-}" ] || printf '%s\n' "$FM_FAKE_REMOTE_STATE_OUT" +exit "${FM_FAKE_SSH_RC:-0}" +SH + chmod +x "$d/fakebin/fake-ssh" + printf '%s\n' "$d" +} + +run_remote_crew_state() { # + PATH="$1/fakebin:$PATH" FM_HOME="$1" FM_STATE_OVERRIDE="$1/state" \ + FM_SSH_BIN="$1/fakebin/fake-ssh" "$CREW_STATE" "$2" +} + +test_remote_alive_with_log_uses_status_log() { + reset_fakes + local d out rc + d=$(setup_remote_case remote-alive-log) + make_fakebin "$d" >/dev/null + printf 'working: refactoring the quota adapter\n' > "$d/state/rsm.status" + out=$(FM_FAKE_REMOTE_STATE_OUT=alive FM_FAKE_SSH_RC=0 run_remote_crew_state "$d" rsm); rc=$? + expect_code 0 "$rc" "remote alive exits 0" + assert_contains "$out" "state: working" "alive remote mate with a working log reads working" + assert_contains "$out" "source: status-log" "alive remote mate reads current activity from the routed log" + assert_contains "$out" "remote endpoint alive on remote-mac" "the remote liveness read should be visible" + assert_not_contains "$out" "worktree gone" "a healthy remote mate must never read as torn down" + pass "fm-crew-state remote: alive endpoint falls through to the routed status log" +} + +test_remote_alive_idle_is_healthy_not_gone() { + reset_fakes + local d out rc + d=$(setup_remote_case remote-alive-idle) + make_fakebin "$d" >/dev/null + out=$(FM_FAKE_REMOTE_STATE_OUT=alive FM_FAKE_SSH_RC=0 run_remote_crew_state "$d" rsm); rc=$? + expect_code 0 "$rc" "remote alive-idle exits 0" + assert_contains "$out" "source: remote-endpoint" "the remote endpoint is the reported source" + assert_contains "$out" "alive on remote-mac" "an idle remote mate reads alive" + assert_not_contains "$out" "worktree gone" "a healthy remote mate must never read as torn down" + assert_not_contains "$out" "backend target gone" "a healthy remote mate must never read as a dead target" + pass "fm-crew-state remote: an idle alive endpoint reads alive, never gone or dead" +} + +test_remote_unreachable_is_unknown_remote_not_dead() { + reset_fakes + local d out rc + d=$(setup_remote_case remote-unreachable) + make_fakebin "$d" >/dev/null + printf 'working: refactoring the quota adapter\n' > "$d/state/rsm.status" + out=$(FM_FAKE_SSH_RC=255 run_remote_crew_state "$d" rsm); rc=$? + expect_code 0 "$rc" "unreachable remote exits 0" + assert_contains "$out" "unknown-remote" "an unreachable remote must be labeled unknown-remote" + assert_contains "$out" "not proof of death" "an unreachable remote must not read as dead" + assert_not_contains "$out" "worktree gone" "an unreachable remote must never read as torn down" + assert_not_contains "$out" "backend target gone" "an unreachable remote must never read as a dead target" + pass "fm-crew-state remote: an unreachable host reads unknown-remote, never gone or dead" +} + +test_remote_dead_reports_remote_verdict() { + reset_fakes + local d out rc + d=$(setup_remote_case remote-dead) + make_fakebin "$d" >/dev/null + out=$(FM_FAKE_REMOTE_STATE_OUT=dead FM_FAKE_SSH_RC=0 run_remote_crew_state "$d" rsm); rc=$? + expect_code 0 "$rc" "remote dead exits 0" + assert_contains "$out" "remote endpoint dead on remote-mac" \ + "a genuinely dead remote endpoint reports the remote host's own verdict" + pass "fm-crew-state remote: the remote host's own dead verdict is reported truthfully" +} + test_missing_meta() { reset_fakes local d; d=$(new_case nometa) @@ -1350,6 +1449,10 @@ test_dead_window_still_reports_active_run_step test_no_timeout_uses_perl_bound test_scout_skips_run_lookup test_torn_down_worktree +test_remote_alive_with_log_uses_status_log +test_remote_alive_idle_is_healthy_not_gone +test_remote_unreachable_is_unknown_remote_not_dead +test_remote_dead_reports_remote_verdict test_missing_meta test_provably_working_via_runs_list_fallback test_not_provably_working_when_stopped diff --git a/tests/fm-daemon.test.sh b/tests/fm-daemon.test.sh index 2fe02fb4318..ad0925be4d4 100755 --- a/tests/fm-daemon.test.sh +++ b/tests/fm-daemon.test.sh @@ -1569,27 +1569,37 @@ test_inject_wedge_alarm_throttles_when_marker_cannot_be_written() { pass "in-process wedge throttle prevents alert spam when the marker cannot persist" } -test_fm_send_exits_nonzero_on_confirmed_swallow() { - # fm-send.sh must exit NON-ZERO when a steer's Enter is positively swallowed - # (text left in the composer), so firstmate learns the instruction did not land - # — and exit ZERO on a clean submit. - local dir fakebin err +test_fm_send_reports_delivered_unconfirmed_submit() { + # When text was typed and Enter sent but the submit read-back remains pending, + # fm-send must return its documented delivered-unconfirmed status and prevent + # a duplicate resend reflex. A synchronously confirmed submit remains zero. + local dir fakebin err rc dir=$(make_bordered_case send-swallow) fakebin="$dir/fakebin"; err="$dir/send.err" # Clean submit -> exit 0. PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$dir/state" FM_FAKE_COMPOSER="$dir/composer" \ FM_SEND_SLEEP=0.05 "$ROOT/bin/fm-send.sh" sess:win 'route this work' >/dev/null 2>"$err" \ || fail "fm-send exited non-zero on a clean submit: $(cat "$err")" - # Persistent swallow -> exit non-zero with a clear message. + # Persistent composer text after Enter -> delivered-unconfirmed exit 3 with + # a non-error warning that explicitly tells the operator not to resend. printf '╭─────╮\n│ > │\n╰─────╯\n' > "$dir/composer" touch "$dir/.swallow" if PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$dir/state" FM_FAKE_COMPOSER="$dir/composer" \ FM_FAKE_SWALLOW="$dir/.swallow" FM_FAKE_PERSIST_SWALLOW=1 FM_SEND_SLEEP=0.05 \ "$ROOT/bin/fm-send.sh" sess:win 'fix findings 1 and 3, skip 2' >/dev/null 2>"$err"; then - fail "fm-send exited zero despite a swallowed Enter (silent unsubmitted instruction)" + rc=0 + else + rc=$? + fi + [ "$rc" -eq 3 ] || fail "fm-send returned $rc instead of delivered-unconfirmed exit 3: $(cat "$err")" + grep -F 'submission is unconfirmed' "$err" >/dev/null \ + || fail "fm-send did not explain the pending confirmation: $(cat "$err")" + grep -F 'do not retype or blindly resend' "$err" >/dev/null \ + || fail "fm-send did not prevent a duplicate resend: $(cat "$err")" + if grep -F 'error:' "$err" >/dev/null; then + fail "fm-send mislabeled delivered-unconfirmed as an error: $(cat "$err")" fi - grep -F 'not submitted' "$err" >/dev/null || fail "fm-send did not explain the swallowed submit: $(cat "$err")" - pass "fm-send exits non-zero on a confirmed swallow, zero on a clean submit" + pass "fm-send returns 3 with a non-error no-resend warning when confirmation stays pending" } test_fm_send_exits_nonzero_on_initial_send_failure() { @@ -1916,7 +1926,7 @@ test_wedge_alarm_hung_override_times_out_and_falls_through test_wedge_alarm_shutdown_stops_active_notifier_group test_inject_wedge_alarm_fires_active_alert_on_non_tmux_backend test_inject_wedge_alarm_throttles_when_marker_cannot_be_written -test_fm_send_exits_nonzero_on_confirmed_swallow +test_fm_send_reports_delivered_unconfirmed_submit test_fm_send_exits_nonzero_on_initial_send_failure test_fm_send_exits_nonzero_on_unproven_submit test_discover_supervisor_backend_precedence diff --git a/tests/fm-peek-remote.test.sh b/tests/fm-peek-remote.test.sh new file mode 100755 index 00000000000..7ef7286fb23 --- /dev/null +++ b/tests/fm-peek-remote.test.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# fm-peek remote-secondmate capture routing. +# +# A remote secondmate's pane lives on its own host. The old path resolved the +# meta's "remote:" window through the local backend adapters and handed it +# to tmux, which failed with "can't find session: remote" - a healthy remote +# mate misreported as an unreadable endpoint. These tests drive the real +# fm-peek + fm-on executables with a stubbed ssh transport (FM_SSH_BIN seam) +# and a poisoned local tmux, pinning: +# 1. A remote selector routes the capture over the remote transport and +# prints the remote pane tail; the local adapters are never consulted. +# 2. An unreachable host fails loudly naming the host, without claiming the +# mate is dead. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +PEEK="$ROOT/bin/fm-peek.sh" + +TMP_ROOT=$(fm_test_tmproot fm-peek-remote) + +# fake-ssh prints the canned remote capture; the poisoned tmux records any +# local read attempt so the "never consulted" property is a real assertion. +make_stubs() { # -> echoes fakebin dir + local dir=$1 fb="$1/fakebin" + mkdir -p "$fb" + cat > "$fb/fake-ssh" <<'SH' +#!/usr/bin/env bash +cat > /dev/null +[ -z "${FM_FAKE_REMOTE_CAPTURE:-}" ] || printf '%s\n' "$FM_FAKE_REMOTE_CAPTURE" +exit "${FM_FAKE_SSH_RC:-0}" +SH + chmod +x "$fb/fake-ssh" + cat > "$fb/tmux" <<'SH' +#!/usr/bin/env bash +printf 'tmux\n' >> "${FM_FAKE_TMUX_TOUCHED:?}" +exit 1 +SH + chmod +x "$fb/tmux" + printf '%s\n' "$fb" +} + +setup_remote_home() { # -> echoes home dir with remote meta + registry + local home="$TMP_ROOT/$1-$RANDOM" + mkdir -p "$home/state" "$home/data" + fm_write_meta "$home/state/rsm.meta" \ + "window=remote:rsm" \ + "endpoint_task_id=rsm" \ + "harness=claude" \ + "kind=secondmate" \ + "mode=secondmate" \ + "remote_host=remote-mac" \ + "remote_root=/remote/root" \ + "remote_backend=herdr" \ + "remote_herdr_session=fm-remote" \ + "remote_target=fm-remote:w1:p1" + cat > "$home/data/secondmates.md" < "$touched" + + out=$(env PATH="$fb:$PATH" \ + FM_HOME="$home" FM_STATE_OVERRIDE="$home/state" \ + FM_SSH_BIN="$fb/fake-ssh" FM_FAKE_SSH_RC=0 \ + FM_FAKE_REMOTE_CAPTURE='● the remote mate is mid-refactor' \ + FM_FAKE_TMUX_TOUCHED="$touched" \ + "$PEEK" rsm 20 2>"$dir/err"); rc=$? + expect_code 0 "$rc" "a healthy remote peek should succeed" + assert_contains "$out" "the remote mate is mid-refactor" \ + "the remote pane tail should be printed" + assert_not_contains "$out" "can't find session" \ + "a remote peek must not fall into a local session lookup" + [ ! -s "$touched" ] || fail "the local tmux adapter was consulted for a remote target" + pass "fm-peek remote: the capture routes over the remote transport, local adapters untouched" +} + +test_remote_peek_unreachable_fails_loudly_without_death_claim() { + local dir fb home touched rc err + dir="$TMP_ROOT/peek-down"; mkdir -p "$dir" + fb=$(make_stubs "$dir") + home=$(setup_remote_home peek-down) + touched="$dir/tmux-touched"; : > "$touched" + + env PATH="$fb:$PATH" \ + FM_HOME="$home" FM_STATE_OVERRIDE="$home/state" \ + FM_SSH_BIN="$fb/fake-ssh" FM_FAKE_SSH_RC=255 \ + FM_FAKE_TMUX_TOUCHED="$touched" \ + "$PEEK" rsm >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + [ "$rc" -ne 0 ] || fail "an unreachable remote peek must exit nonzero" + assert_contains "$err" "remote pane of rsm on remote-mac" \ + "the failure must name the remote mate and host" + assert_contains "$err" "not thereby dead" \ + "an unreadable remote pane must not be presented as a dead mate" + pass "fm-peek remote: an unreachable host fails loudly without a false death claim" +} + +test_remote_peek_reads_remote_pane +test_remote_peek_unreachable_fails_loudly_without_death_claim + +echo "all fm-peek-remote tests passed" diff --git a/tests/fm-send-remote-delivery.test.sh b/tests/fm-send-remote-delivery.test.sh new file mode 100755 index 00000000000..af546fbb4a4 --- /dev/null +++ b/tests/fm-send-remote-delivery.test.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +# fm-send remote-secondmate delivery reporting. +# +# The remote send leg (fm-on.sh -> fm-remote-secondmate-control.sh cmd_send) +# runs fm-send's own verified submit host-locally on the remote machine and +# relays its exit status unchanged. A leg that delivered the text into the +# live verified pane but could not synchronously confirm the submit exits 3 +# (the delivered-unconfirmed contract in bin/fm-send.sh's header); flattening +# that into a generic failure produced the false "error: text not sent" +# report that tempted duplicate resends of steers that had actually landed. +# These tests pin the delivery-reporting contract over the real fm-send + +# fm-on executables with a stubbed ssh transport (FM_SSH_BIN seam - the same +# process boundary tests/fm-on.test.sh proves preserves exit status): +# 1. Remote delivered-unconfirmed (ssh exit 3) is NOT a failure: exit 0, a +# non-error delivered notice, the inner leg's stderr held back, and the +# pending-reply expectation marked delivered (awaiting_report). +# 2. A real remote failure (nonzero, not 3/255) still fails loudly with the +# remote stderr replayed and the undelivered expectation discarded. +# 3. Transport-unknown (ssh exit 255) still refuses loudly and preserves the +# expectation as delivery_unknown. +# 4. A delivered-unconfirmed remote answer still closes its --resolve-key +# decision (delivered-with-pending-confirmation counts as delivered). +# 5. A LOCAL send whose submit read-back stays pending exits 3 with an +# honest non-error message (text delivered, submission unconfirmed). +# 6. That local unconfirmed send still never closes a --resolve-key +# decision (the local ledger boundary is unchanged). +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +SEND="$ROOT/bin/fm-send.sh" +DRAIN="$ROOT/bin/fm-wake-drain.sh" + +TMP_ROOT=$(fm_test_tmproot fm-send-remote-delivery) + +# Stub tmux for the local legs: logs literal typed text to FM_SEND_LOG. The +# default composer reads empty (clean submit); FM_FAKE_TMUX_PENDING=1 keeps a +# proven pending composer with no busy footer, so the real submit core +# exhausts its Enter budget and reports the pending verdict. The ssh stub +# records the invocation, emits FM_FAKE_SSH_STDERR as the remote leg's stderr, +# and exits FM_FAKE_SSH_RC - the exact relay contract the real transport +# preserves. +make_stubs() { # -> echoes fakebin dir + local dir=$1 fb="$1/fakebin" + mkdir -p "$fb" + cat > "$fb/tmux" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-}" in + send-keys) + shift + literal=0 + while [ $# -gt 0 ]; do + case "$1" in + -t) shift 2 ;; + -l) literal=1; shift ;; + *) break ;; + esac + done + if [ "$literal" = 1 ]; then + printf '%s' "${1:-}" >> "$FM_SEND_LOG" + fi + exit 0 ;; + display-message) + for a in "$@"; do case "$a" in *cursor_y*) printf '1\n'; exit 0 ;; esac; done + printf 'fakepane\n'; exit 0 ;; + capture-pane) + if [ "${FM_FAKE_TMUX_PENDING:-0}" = 1 ]; then + printf '╭────────────╮\n│ > steer │\n╰────────────╯\n' + else + printf '╭────╮\n│ │\n╰────╯\n' + fi + exit 0 ;; + list-windows) exit 0 ;; +esac +exit 0 +SH + chmod +x "$fb/tmux" + cat > "$fb/sleep" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + chmod +x "$fb/sleep" + cat > "$fb/fake-ssh" <<'SH' +#!/usr/bin/env bash +cat > /dev/null +printf '%s\n' "$*" >> "$FM_SSH_LOG" +[ -z "${FM_FAKE_SSH_STDERR:-}" ] || printf '%s\n' "$FM_FAKE_SSH_STDERR" >&2 +exit "${FM_FAKE_SSH_RC:-0}" +SH + chmod +x "$fb/fake-ssh" + printf '%s\n' "$fb" +} + +setup_home() { # -> echoes a fresh home dir with an empty state/ + local home="$TMP_ROOT/$1-$RANDOM" + mkdir -p "$home/state" + printf '%s\n' "$home" +} + +# A home with a remote-secondmate task meta plus the registry row fm-on.sh +# resolves the ssh route from - the same shape a live remote mate records. +setup_remote_home() { # -> echoes home dir + local home + home=$(setup_home "$1") + mkdir -p "$home/data" + fm_write_meta "$home/state/rsm.meta" \ + "window=fm-remote:w1:p1" \ + "endpoint_task_id=rsm" \ + "harness=claude" \ + "kind=secondmate" \ + "mode=secondmate" \ + "yolo=off" \ + "remote_host=remote-mac" \ + "remote_root=/remote/root" \ + "remote_backend=herdr" \ + "remote_herdr_session=fm-remote" \ + "remote_target=fm-remote:w1:p1" + cat > "$home/data/secondmates.md" <, or empty. +pending_record() { # + find "$1/state/pending-replies" -maxdepth 1 -type f ! -name '.*' 2>/dev/null | head -1 +} + +drain_out() { # + FM_STATE_OVERRIDE="$1/state" "$DRAIN" 2>/dev/null +} + +test_remote_delivered_unconfirmed_is_not_failure() { + local dir fb log ssh_log home rc err rec + dir="$TMP_ROOT/remote-du"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log"; ssh_log="$dir/ssh.log"; : > "$ssh_log" + home=$(setup_remote_home remote-du) + + : > "$log" + env PATH="$fb:$PATH" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + FM_SSH_BIN="$fb/fake-ssh" FM_SSH_LOG="$ssh_log" FM_FAKE_SSH_RC=3 \ + FM_FAKE_SSH_STDERR='fm-send: text delivered to fm-remote:w1:p1 but submission is unconfirmed (verdict=pending; tried meta=/remote/home/state/fm-remote:w1:p1.meta; metadata window/terminal lookup; backend=herdr; endpoint=verified)' \ + "$SEND" rsm "please rename the metric" >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + expect_code 0 "$rc" "a delivered-unconfirmed remote send must not exit as a failure" + assert_grep 'fm-remote-entrypoint.sh' "$ssh_log" "the steer should cross the remote transport" + assert_contains "$err" "delivered to remote secondmate rsm" \ + "the outcome must be reported as delivered" + assert_not_contains "$err" "text not sent" "a delivered steer must not read as not sent" + assert_not_contains "$err" "not submitted" "a delivered steer must not read as not submitted" + assert_not_contains "$err" "error: text" "a delivered steer must not carry an error-styled report" + assert_not_contains "$err" "verdict=pending" \ + "the inner leg's unconfirmed diagnostics must be held back on a delivered outcome" + + rec=$(pending_record "$home") + [ -n "$rec" ] || fail "the pending-reply expectation must survive a delivered-unconfirmed send" + [ -n "$(grep '^delivered_epoch=' "$rec" | cut -d= -f2-)" ] \ + || fail "a delivered-unconfirmed send must mark the expectation delivered: $(cat "$rec")" + [ "$(grep '^phase=' "$rec" | tail -1 | cut -d= -f2-)" = awaiting_report ] \ + || fail "a delivered-unconfirmed send must leave the expectation awaiting its report: $(cat "$rec")" + pass "fm-send remote: delivered-unconfirmed reports delivered, exits 0, keeps the expectation armed" +} + +test_remote_real_failure_still_fails() { + local dir fb log ssh_log home rc err + dir="$TMP_ROOT/remote-fail"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log"; ssh_log="$dir/ssh.log"; : > "$ssh_log" + home=$(setup_remote_home remote-fail) + + : > "$log" + env PATH="$fb:$PATH" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + FM_SSH_BIN="$fb/fake-ssh" FM_SSH_LOG="$ssh_log" FM_FAKE_SSH_RC=1 \ + FM_FAKE_SSH_STDERR='error: remote secondmate rsm endpoint metadata is invalid; refusing access until it is explicitly migrated' \ + "$SEND" rsm "please rename the metric" >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + [ "$rc" -ne 0 ] || fail "a genuinely failed remote send must exit nonzero" + assert_contains "$err" "error: text not sent to remote:rsm" \ + "a real remote failure must still report a real error" + assert_contains "$err" "endpoint metadata is invalid" \ + "a real remote failure must replay the remote leg's own stderr" + [ -z "$(pending_record "$home")" ] \ + || fail "a failed send must discard its undelivered expectation" + pass "fm-send remote: a real remote failure still fails loudly with the remote diagnostics" +} + +test_remote_transport_unknown_preserves_expectation() { + local dir fb log ssh_log home rc err rec + dir="$TMP_ROOT/remote-255"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log"; ssh_log="$dir/ssh.log"; : > "$ssh_log" + home=$(setup_remote_home remote-255) + + : > "$log" + env PATH="$fb:$PATH" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + FM_SSH_BIN="$fb/fake-ssh" FM_SSH_LOG="$ssh_log" FM_FAKE_SSH_RC=255 \ + "$SEND" rsm "please rename the metric" >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + [ "$rc" -ne 0 ] || fail "an unknown-completion transport loss must exit nonzero" + assert_contains "$err" "delivery to remote secondmate rsm is unknown" \ + "transport loss must be reported as unknown delivery, not silently dropped" + rec=$(pending_record "$home") + [ -n "$rec" ] || fail "transport loss must preserve the expectation for reconciliation" + [ "$(grep '^phase=' "$rec" | tail -1 | cut -d= -f2-)" = delivery_unknown ] \ + || fail "transport loss must move the expectation to delivery_unknown: $(cat "$rec")" + pass "fm-send remote: ssh 255 still refuses loudly and preserves the expectation as delivery_unknown" +} + +test_remote_delivered_unconfirmed_closes_resolve_key() { + local dir fb log ssh_log home rc out + dir="$TMP_ROOT/remote-key"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log"; ssh_log="$dir/ssh.log"; : > "$ssh_log" + home=$(setup_remote_home remote-key) + printf 'needs-decision [key=upgrade-window]: tonight or the weekend\n' > "$home/state/rsm.status" + + : > "$log" + env PATH="$fb:$PATH" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + FM_SSH_BIN="$fb/fake-ssh" FM_SSH_LOG="$ssh_log" FM_FAKE_SSH_RC=3 \ + "$SEND" rsm --resolve-key upgrade-window "the weekend, freeze Friday" >/dev/null 2>&1; rc=$? + expect_code 0 "$rc" "a delivered-unconfirmed remote answer must not exit as a failure" + grep -F 'resolved [key=upgrade-window]: answered: the weekend, freeze Friday' "$home/state/rsm.status" >/dev/null \ + || fail "a delivered-unconfirmed remote answer must close the decision: $(cat "$home/state/rsm.status")" + out=$(drain_out "$home") + if printf '%s' "$out" | grep -F 'OPEN DECISIONS' >/dev/null; then + fail "the answered decision still lists as open after a delivered-unconfirmed answer: $out" + fi + pass "fm-send remote: a delivered-unconfirmed answer closes its --resolve-key decision" +} + +test_local_pending_reports_delivered_unconfirmed() { + local dir fb log home rc err + dir="$TMP_ROOT/local-pending"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log" + home=$(setup_home local-pending) + fm_write_meta "$home/state/t1.meta" "window=sess:fm-t1" "kind=ship" + + : > "$log" + env PATH="$fb:$PATH" FM_FAKE_TMUX_PENDING=1 \ + FM_ROOT_OVERRIDE="$home" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + "$SEND" t1 "steer text" >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + expect_code 3 "$rc" "an unconfirmed local submit must exit with the delivered-unconfirmed status" + assert_contains "$err" "submission is unconfirmed" \ + "the unconfirmed local submit must be described honestly" + assert_not_contains "$err" "not submitted" \ + "an unconfirmed local submit must not claim the text was not submitted" + assert_not_contains "$err" "error:" \ + "an unconfirmed local submit must not carry an error-styled report" + pass "fm-send local: an unconfirmed submit exits 3 with an honest non-error report" +} + +test_local_pending_does_not_close_resolve_key() { + local dir fb log home rc out + dir="$TMP_ROOT/local-pending-key"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log" + home=$(setup_home local-pending-key) + fm_write_meta "$home/state/t2.meta" "window=sess:fm-t2" "kind=ship" + printf 'blocked [key=creds]: need the deploy token\n' > "$home/state/t2.status" + + : > "$log" + env PATH="$fb:$PATH" FM_FAKE_TMUX_PENDING=1 \ + FM_ROOT_OVERRIDE="$home" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + "$SEND" t2 --resolve-key creds "token is in the vault now" >/dev/null 2>&1; rc=$? + expect_code 3 "$rc" "an unconfirmed local answer must exit with the delivered-unconfirmed status" + if grep -F 'resolved' "$home/state/t2.status" >/dev/null; then + fail "an unconfirmed local answer must not close the decision: $(cat "$home/state/t2.status")" + fi + out=$(drain_out "$home") + printf '%s' "$out" | grep -F '[key=creds]' >/dev/null \ + || fail "the blocker must stay open after an unconfirmed local answer: $out" + pass "fm-send local: an unconfirmed submit still never closes a --resolve-key decision" +} + +test_remote_delivered_unconfirmed_is_not_failure +test_remote_real_failure_still_fails +test_remote_transport_unknown_preserves_expectation +test_remote_delivered_unconfirmed_closes_resolve_key +test_local_pending_reports_delivered_unconfirmed +test_local_pending_does_not_close_resolve_key + +echo "all fm-send-remote-delivery tests passed"