Add product demos: example-driven, skill-file usage guides

CLAUDE.md

@@ -27,6 +27,11 @@ needs detail.
 - **The catalogue is signed and fail-closed.** Keep the catalogue at `version: 2`; the
   publisher pin in the entry must match the bundle manifest's `store.publisher`.
 - **One stable publisher key per app id, forever** — back it up; the update gate requires it.
+- **Every submission carries a product demo.** Author a `product_demo` in
+  `submission.json` — the example-first, skill-file-shaped usage guide shown at install,
+  injected as a `SKILL.md`, and rendered on the website. Required by policy for new
+  submissions; metered apps MUST show costs ≤ the per-user budget. Guide:
+  [`docs/PRODUCT-DEMOS.md`](docs/PRODUCT-DEMOS.md).
 
 ## Tool entry points
 

README.md

@@ -59,6 +59,12 @@ You only ever touch **one repo** (`app-template`); see
 > runbook across every backend (`http`/`cli`) and auth mode (`byo`/`managed`),
 > native-binary delivery, testing, and the submission → catalogue → website-card
 > steps, with edge cases and a pre-flight checklist.
+>
+> **Every submission carries a product demo.** Author a `product_demo` in
+> `submission.json` — a compact, example-first, skill-file-shaped usage guide shown at
+> install, injected as a `SKILL.md`, and rendered on the website as the "Full usage
+> demo". It drives correct first usage for autonomous agents. See
+> [`docs/PRODUCT-DEMOS.md`](docs/PRODUCT-DEMOS.md).
 
 ### Two ways to publish — same required fields, same result
 

cmd/pilot-app/demo_score.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/pilot-protocol/app-template/internal/demoeval"
+)
+
+// cmdDemoScore is the CLI entry point for `pilot-app demo-score`. It scores every
+// submission's product demo (the potential-uplift report + CI gate) and exits
+// nonzero when any demo scores below the -min threshold.
+func cmdDemoScore(args []string) {
+	os.Exit(runDemoScore(args, os.Stdout, os.Stderr))
+}
+
+// runDemoScore is the testable core: it prints the per-app table + summary to out
+// (errors to errOut) and returns the process exit code. It never calls os.Exit,
+// so tests can assert on the code and output.
+func runDemoScore(args []string, out, errOut io.Writer) int {
+	fs := flag.NewFlagSet("demo-score", flag.ContinueOnError)
+	fs.SetOutput(errOut)
+	min := fs.Float64("min", demoeval.DefaultMinScore, "fail if any demo scores below this (0–100)")
+	asJSON := fs.Bool("json", false, "emit the reports as JSON instead of a table")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	dir := fs.Arg(0)
+	if dir == "" {
+		dir = "submissions"
+	}
+
+	reports, err := demoeval.ScoreSubmissionsDir(dir)
+	if err != nil {
+		fmt.Fprintf(errOut, "demo-score: %v\n", err)
+		return 2
+	}
+	sum := demoeval.Summarize(reports, *min)
+
+	if *asJSON {
+		enc := json.NewEncoder(out)
+		enc.SetIndent("", "  ")
+		_ = enc.Encode(struct {
+			Reports []demoeval.Report `json:"reports"`
+			Summary demoeval.Summary  `json:"summary"`
+		}{reports, sum})
+		if len(sum.Below) > 0 {
+			return 1
+		}
+		return 0
+	}
+
+	printTable(out, reports, *min)
+	printSummary(out, sum)
+
+	if len(sum.Below) > 0 {
+		fmt.Fprintf(errOut, "\nDEMO-SCORE FAILED — %s below the %.0f threshold: %s\n",
+			plur(len(sum.Below), "demo"), *min, strings.Join(sum.Below, ", "))
+		return 1
+	}
+	return 0
+}
+
+// printTable renders the per-app potential-uplift table: quality score, first-call
+// proxy, metered flag, worked-$ vs budget, and the top issue.
+func printTable(out io.Writer, reports []demoeval.Report, min float64) {
+	tw := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0)
+	fmt.Fprintln(tw, "APP\tSCORE\t1ST-CALL\tMETERED\tWORKED$/BUDGET\tTOP ISSUE")
+	fmt.Fprintln(tw, "---\t-----\t--------\t-------\t--------------\t---------")
+	for _, r := range reports {
+		score := "  n/a"
+		if r.HasDemo {
+			mark := "  "
+			if r.Score < min {
+				mark = "✗ "
+			}
+			score = fmt.Sprintf("%s%5.1f", mark, r.Score)
+		}
+		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n",
+			r.AppID, score, firstCallCol(r), meteredCol(r), costCol(r), topIssue(r))
+	}
+	tw.Flush()
+}
+
+func firstCallCol(r demoeval.Report) string {
+	if !r.HasDemo {
+		return "—"
+	}
+	if r.FirstCall == nil || r.FirstCall.Skipped {
+		return "n/a"
+	}
+	return fmt.Sprintf("%.2f", r.FirstCall.Score)
+}
+
+func meteredCol(r demoeval.Report) string {
+	if !r.HasDemo {
+		return "—"
+	}
+	if r.Metered {
+		return "✓"
+	}
+	return "-"
+}
+
+func costCol(r demoeval.Report) string {
+	if !r.HasDemo || !r.Metered || r.BudgetUSD <= 0 {
+		return "—"
+	}
+	return fmt.Sprintf("$%.2f/$%.2f", r.WorkedUSD, r.BudgetUSD)
+}
+
+// topIssue returns the single most important human-readable issue, truncated.
+func topIssue(r demoeval.Report) string {
+	if !r.HasDemo {
+		return "no product_demo"
+	}
+	if len(r.Issues) == 0 {
+		if r.FirstCall != nil && len(r.FirstCall.Notes) > 0 && r.FirstCall.Score < 1.0 {
+			return truncate(r.FirstCall.Notes[0], 60)
+		}
+		return "ok"
+	}
+	return truncate(r.Issues[0], 60)
+}
+
+func printSummary(out io.Writer, s demoeval.Summary) {
+	fmt.Fprintf(out, "\n%d submissions · %d with demo · %d without · mean score %.1f (gate %.0f)\n",
+		s.Total, s.WithDemo, s.WithoutDemo, s.MeanScore, s.Threshold)
+	if s.WithoutDemo > 0 {
+		fmt.Fprintf(out, "coverage gap: %d app(s) ship no demo — installs that never convert to a first call\n", s.WithoutDemo)
+	}
+	if len(s.Below) > 0 {
+		sort.Strings(s.Below)
+		fmt.Fprintf(out, "below gate: %s\n", strings.Join(s.Below, ", "))
+	}
+}
+
+func truncate(s string, n int) string {
+	s = strings.TrimSpace(strings.ReplaceAll(s, "\n", " "))
+	if len(s) <= n {
+		return s
+	}
+	if n <= 1 {
+		return s[:n]
+	}
+	return s[:n-1] + "…"
+}
+
+func plur(n int, thing string) string {
+	if n == 1 {
+		return "1 " + thing
+	}
+	return fmt.Sprintf("%d %ss", n, thing)
+}

cmd/pilot-app/demo_score_test.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// a well-formed submission: real methods + a demo whose commands hit them with
+// valid params. Scores high and passes the gate.
+const goodSubmission = `{
+  "id": "io.pilot.demoapp",
+  "version": "1.0.0",
+  "description": "A demo app.",
+  "email": "a@b.co",
+  "backend": {"type": "cli", "command": ["demoapp"]},
+  "methods": [
+    {"name": "demoapp.query", "description": "run a query", "latency": "fast",
+     "params": [{"name": "sql", "type": "string", "required": true}],
+     "cli": {"args": ["-c", "${sql}"]}},
+    {"name": "demoapp.version", "description": "version", "latency": "fast",
+     "cli": {"args": ["--version"]}}
+  ],
+  "product_demo": {
+    "skill": "io.pilot.demoapp",
+    "when_to_use": "When you need to run a quick demo query locally.",
+    "metered": false,
+    "quickstart": {"goal": "first query",
+      "command": "pilotctl appstore call io.pilot.demoapp demoapp.query '{\"sql\":\"SELECT 1\"}'",
+      "expect": "{\"rows\":[[1]]}"},
+    "examples": [
+      {"title": "version", "command": "pilotctl appstore call io.pilot.demoapp demoapp.version '{}'", "expect": "1.0.0"},
+      {"title": "count", "command": "pilotctl appstore call io.pilot.demoapp demoapp.query '{\"sql\":\"SELECT count(*)\"}'", "expect": "{\"rows\":[[3]]}"}
+    ],
+    "next": ["io.pilot.demoapp demoapp.help '{}'"]
+  }
+}`
+
+// a broken submission: demo with the wrong command prefix, no expects, one
+// example, metered-but-no-cost, mismatched skill. Scores below the gate.
+const badSubmission = `{
+  "id": "io.pilot.badapp",
+  "version": "1.0.0",
+  "description": "A bad app.",
+  "email": "a@b.co",
+  "backend": {"type": "cli", "command": ["badapp"]},
+  "methods": [
+    {"name": "badapp.go", "description": "go", "latency": "fast", "cli": {"args": ["go"]}}
+  ],
+  "product_demo": {
+    "skill": "io.pilot.WRONG",
+    "when_to_use": "",
+    "metered": true,
+    "quickstart": {"command": "curl https://example.com"},
+    "examples": [{"command": "echo hi"}]
+  }
+}`
+
+func writeSubmissions(t *testing.T, m map[string]string) string {
+	t.Helper()
+	dir := t.TempDir()
+	for name, body := range m {
+		sub := filepath.Join(dir, name)
+		if err := os.MkdirAll(sub, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(sub, "submission.json"), []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return dir
+}
+
+// TestDemoScoreGood: a dir of only well-formed demos passes (exit 0) and shows a
+// per-app row + summary.
+func TestDemoScoreGood(t *testing.T) {
+	dir := writeSubmissions(t, map[string]string{"io.pilot.demoapp": goodSubmission})
+	var out, errOut bytes.Buffer
+	code := runDemoScore([]string{dir}, &out, &errOut)
+	if code != 0 {
+		t.Fatalf("expected exit 0, got %d; err=%s", code, errOut.String())
+	}
+	s := out.String()
+	if !strings.Contains(s, "io.pilot.demoapp") {
+		t.Errorf("table missing app row:\n%s", s)
+	}
+	if !strings.Contains(s, "1 with demo") {
+		t.Errorf("summary wrong:\n%s", s)
+	}
+}
+
+// TestDemoScoreGateFails: a below-threshold demo trips the gate (exit 1) and is
+// named in the failure line.
+func TestDemoScoreGateFails(t *testing.T) {
+	dir := writeSubmissions(t, map[string]string{
+		"io.pilot.demoapp": goodSubmission,
+		"io.pilot.badapp":  badSubmission,
+	})
+	var out, errOut bytes.Buffer
+	code := runDemoScore([]string{dir}, &out, &errOut)
+	if code != 1 {
+		t.Fatalf("expected exit 1 (gate fail), got %d", code)
+	}
+	if !strings.Contains(errOut.String(), "io.pilot.badapp") {
+		t.Errorf("failure line should name the bad app:\n%s", errOut.String())
+	}
+}
+
+// TestDemoScoreMinFlag: raising -min above even a good demo trips the gate; a
+// -min of 0 always passes.
+func TestDemoScoreMinFlag(t *testing.T) {
+	dir := writeSubmissions(t, map[string]string{"io.pilot.demoapp": goodSubmission})
+	var out, errOut bytes.Buffer
+	if code := runDemoScore([]string{"-min", "0", dir}, &out, &errOut); code != 0 {
+		t.Errorf("min 0 should pass, got %d", code)
+	}
+	out.Reset()
+	errOut.Reset()
+	if code := runDemoScore([]string{"-min", "101", dir}, &out, &errOut); code != 1 {
+		t.Errorf("min 101 should fail every demo, got %d", code)
+	}
+}
+
+// TestDemoScoreJSON: -json emits machine-readable output.
+func TestDemoScoreJSON(t *testing.T) {
+	dir := writeSubmissions(t, map[string]string{"io.pilot.demoapp": goodSubmission})
+	var out, errOut bytes.Buffer
+	runDemoScore([]string{"-json", dir}, &out, &errOut)
+	s := out.String()
+	if !strings.Contains(s, `"reports"`) || !strings.Contains(s, `"summary"`) {
+		t.Errorf("json output malformed:\n%s", s)
+	}
+}
+
+// TestDemoScoreLiveTree: runs over the repo's real submissions tree — must scan
+// without error (exit code != 2). Score/gate outcome isn't asserted because the
+// tree is authored concurrently.
+func TestDemoScoreLiveTree(t *testing.T) {
+	if _, err := os.Stat("../../submissions"); err != nil {
+		t.Skip("no submissions tree")
+	}
+	var out, errOut bytes.Buffer
+	if code := runDemoScore([]string{"-min", "0", "../../submissions"}, &out, &errOut); code == 2 {
+		t.Fatalf("scan error over live tree: %s", errOut.String())
+	}
+}

cmd/pilot-app/main.go

@@ -49,6 +49,8 @@ func main() {
 		cmdSubmit(os.Args[2:])
 	case "update":
 		cmdUpdate(os.Args[2:])
+	case "demo-score":
+		cmdDemoScore(os.Args[2:])
 	case "example":
 		fmt.Print(scaffold.ExampleSpec)
 	case "-h", "--help", "help":
@@ -74,6 +76,9 @@ Usage:
                                                   enforce the update gate: same publisher key + higher version
   pilot-app submit   -C <project-dir> --prepare <app-template-fork>
                                                   write a submission PR payload (the single front door)
+  pilot-app demo-score [submissions-dir] [-min N]
+                                                  score every product demo (quality + first-call proxy);
+                                                  exits nonzero if any demo scores below -min (default 60)
   pilot-app example                               print a starter pilot.app.yaml
 
 After init: