From 66b09f9a3e024092cdb5405f088da8a9955e39d7 Mon Sep 17 00:00:00 2001 From: beardthelion Date: Thu, 6 Aug 2026 16:48:44 -0500 Subject: [PATCH] Make every anagram word the only word its letters spell The grader accepts one answer, so a pool word that shares its letters with another real word fails a solver who thought of the other one. Against the live service, the letters of 'star' answered as 'rats' come back failed. 31 of the 70 words were affected, checked against hunspell en_US with affix rules expanded so inflections count. Short words collide most, so the easiest levels were the least reliable: cloud/could, garden/danger, bread/beard, door/odor, kitchen/thicken. Fixed in the pool rather than the grader. Teaching grade() to accept alternates widens what the gate says yes to, and since a scramble can equal an alternate, echoing the prompt letters back would pass 5.9% of level-1 challenges against 0% today (measured over 50k generated challenges). A gate should not get more permissive to fix a correctness bug. Replacements are verified by the same dictionary check, which ships as scripts/audit-anagram-pool.mjs to re-run when words are added. It reads a hunspell dictionary already on the machine and skips when there is none, so the runtime keeps its zero production dependencies. --- scripts/audit-anagram-pool.mjs | 113 +++++++++++++++++++++++++++++++++ src/generators/anagram.ts | 22 +++++-- test/anagram-pool.test.ts | 83 ++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 7 deletions(-) create mode 100644 scripts/audit-anagram-pool.mjs create mode 100644 test/anagram-pool.test.ts diff --git a/scripts/audit-anagram-pool.mjs b/scripts/audit-anagram-pool.mjs new file mode 100644 index 0000000..c695489 --- /dev/null +++ b/scripts/audit-anagram-pool.mjs @@ -0,0 +1,113 @@ +// Audit the anagram word pool: every word must be the ONLY dictionary word its +// letters spell. A word that shares its letters with another real word (star and +// rats, cloud and could) makes the challenge unanswerable-as-graded, because the +// grader accepts exactly one answer. +// +// Dev-only, and deliberately not a dependency: it reads a hunspell dictionary +// already on the machine and skips cleanly when there isn't one, so the runtime +// keeps its zero production dependencies. +// +// bun run scripts/audit-anagram-pool.mjs +// +// On Debian/Ubuntu the dictionary comes from `apt install hunspell-en-us`; +// on macOS, `brew install hunspell` plus an en_US dictionary. + +import { readFileSync, existsSync } from 'node:fs' + +const DICT_CANDIDATES = [ + '/usr/share/hunspell/en_US', + '/usr/share/myspell/en_US', + '/opt/homebrew/share/hunspell/en_US', + '/usr/local/share/hunspell/en_US', +] + +const base = DICT_CANDIDATES.find((p) => existsSync(`${p}.dic`) && existsSync(`${p}.aff`)) +if (!base) { + console.log('no hunspell en_US dictionary found; skipping audit') + console.log(`looked in: ${DICT_CANDIDATES.join(', ')}`) + process.exit(0) +} + +// Expand affix rules so inflections count. Without this, `rats` and `notes` look +// absent and the audit passes words that are in fact ambiguous. +function loadDictionary(prefix) { + const aff = readFileSync(`${prefix}.aff`, 'latin1').split('\n') + const rules = { SFX: new Map(), PFX: new Map() } + for (let i = 0; i < aff.length; i++) { + const p = aff[i].trim().split(/\s+/) + if ((p[0] === 'SFX' || p[0] === 'PFX') && /^\d+$/.test(p[3] ?? '')) { + const table = [] + for (let j = i + 1; j <= i + Number(p[3]) && j < aff.length; j++) { + const q = aff[j].trim().split(/\s+/) + if (q[0] === p[0] && q[1] === p[1]) { + table.push({ + strip: q[2] === '0' ? '' : q[2], + add: q[3] === '0' ? '' : (q[3] ?? '').split('/')[0], + cond: q[4] ?? '.', + }) + } + } + rules[p[0]].set(p[1], table) + i += Number(p[3]) + } + } + + const forms = new Set() + const dic = readFileSync(`${prefix}.dic`, 'latin1').split('\n').slice(1) + for (const line of dic) { + const [raw, flags = ''] = line.trim().split('/') + const word = (raw ?? '').trim().toLowerCase() + if (!word || !/^[a-z]+$/.test(word)) continue + forms.add(word) + for (const flag of flags.trim()) { + for (const { strip, add, cond } of rules.SFX.get(flag) ?? []) { + if (strip && !word.endsWith(strip)) continue + const stem = strip ? word.slice(0, -strip.length) : word + if (cond !== '.' && !new RegExp(`${cond}$`).test(stem)) continue + const form = stem + add + if (/^[a-z]+$/.test(form)) forms.add(form) + } + for (const { strip, add } of rules.PFX.get(flag) ?? []) { + if (strip) continue + const form = add + word + if (/^[a-z]+$/.test(form)) forms.add(form) + } + } + } + return forms +} + +const forms = loadDictionary(base) +const letters = (w) => [...w].sort().join('') + +const byLetters = new Map() +for (const w of forms) { + const k = letters(w) + if (!byLetters.has(k)) byLetters.set(k, []) + byLetters.get(k).push(w) +} + +// Read the pool straight out of the generator so the audit cannot drift from it. +const src = readFileSync(new URL('../src/generators/anagram.ts', import.meta.url), 'utf8') +const block = src.slice(src.indexOf('const WORDS'), src.indexOf('function lengthFor')) +const pool = [...block.matchAll(/'([a-z]+)'/g)].map((m) => m[1]) +if (pool.length === 0) { + console.error('could not read the word pool from src/generators/anagram.ts') + process.exit(1) +} + +let bad = 0 +for (const word of pool) { + const others = (byLetters.get(letters(word)) ?? []).filter((w) => w !== word) + if (others.length) { + console.error(` ${word} -> also spells: ${others.join(', ')}`) + bad++ + } +} + +console.log(`checked ${pool.length} words against ${forms.size.toLocaleString()} dictionary forms`) +if (bad) { + console.error(`\n${bad} word(s) have another valid answer. Replace them.`) + process.exit(1) +} +console.log('every word is the only dictionary word its letters spell') diff --git a/src/generators/anagram.ts b/src/generators/anagram.ts index 95ff253..7f4ee31 100644 --- a/src/generators/anagram.ts +++ b/src/generators/anagram.ts @@ -5,14 +5,22 @@ import { choice, shuffle } from '../rng.ts' import type { Generator, GeneratedChallenge } from '../types.ts' // Curated, lower-cased, grouped by length so difficulty maps to length. +// +// Every word here has exactly one arrangement that spells a dictionary word: +// itself. That invariant is what makes the challenge well posed, because the +// grader accepts one answer, so a word whose letters also spell another real +// word fails a solver who happened to think of the other one. `star` and `rats`, +// `cloud` and `could`, `garden` and `danger` were all in this list. +// +// Re-run `bun run scripts/audit-anagram-pool.mjs` after adding a word. const WORDS: Record = { - 3: ['cat', 'dog', 'sun', 'map', 'key', 'red', 'box', 'fox', 'cup', 'pen'], - 4: ['tree', 'book', 'fish', 'lamp', 'gold', 'rain', 'wind', 'star', 'door', 'leaf'], - 5: ['apple', 'house', 'water', 'plant', 'cloud', 'music', 'light', 'river', 'stone', 'bread'], - 6: ['garden', 'planet', 'silver', 'bridge', 'castle', 'forest', 'rocket', 'flower', 'orange', 'pencil'], - 7: ['journey', 'diamond', 'gravity', 'kitchen', 'machine', 'picture', 'rainbow', 'thunder', 'village', 'crystal'], - 8: ['mountain', 'elephant', 'sunshine', 'computer', 'hospital', 'language', 'treasure', 'umbrella', 'dinosaur', 'sandwich'], - 9: ['adventure', 'chocolate', 'telephone', 'butterfly', 'orchestra', 'discovery', 'wonderful', 'breakfast', 'invention', 'pineapple'], + 3: ['key', 'box', 'fox', 'pen', 'sky', 'toy', 'cow', 'log', 'ice', 'hat'], + 4: ['fish', 'gold', 'wind', 'bird', 'sand', 'cake', 'frog', 'milk', 'corn', 'desk'], + 5: ['house', 'water', 'plant', 'music', 'light', 'river', 'beach', 'bench', 'world', 'juice'], + 6: ['rocket', 'pencil', 'yellow', 'purple', 'summer', 'guitar', 'pocket', 'window', 'basket', 'hammer'], + 7: ['journey', 'diamond', 'gravity', 'machine', 'rainbow', 'thunder', 'village', 'crystal', 'morning', 'penguin'], + 8: ['mountain', 'elephant', 'sunshine', 'computer', 'hospital', 'language', 'umbrella', 'dinosaur', 'sandwich', 'airplane'], + 9: ['adventure', 'chocolate', 'butterfly', 'discovery', 'breakfast', 'invention', 'pineapple', 'crocodile', 'waterfall', 'astronaut'], } function lengthFor(d: number): number { diff --git a/test/anagram-pool.test.ts b/test/anagram-pool.test.ts new file mode 100644 index 0000000..efef54b --- /dev/null +++ b/test/anagram-pool.test.ts @@ -0,0 +1,83 @@ +// The anagram challenge grades against one answer, so the pool must not contain +// a word whose letters spell another real word. The full check needs a +// dictionary and lives in scripts/audit-anagram-pool.mjs; these are the parts +// that hold with no dictionary present, so they run everywhere CI does. + +import { expect, test, describe } from 'bun:test' +import { anagram } from '../src/generators/anagram.ts' +import { gradeDefault } from '../src/grader.ts' +import type { Generator } from '../src/types.ts' + +function gradeWith(gen: Generator, response: string, answer: string): boolean | Promise { + return gen.grade ? gen.grade(response, answer) : gradeDefault(response, answer) +} + +const letters = (w: string) => [...w].sort().join('') + +// Sample every level rather than reading the private WORDS table, so the tests +// exercise what a caller actually receives. +async function poolFromGenerator(): Promise> { + const seen = new Map() + for (let difficulty = 1; difficulty <= 10; difficulty++) { + for (let i = 0; i < 500; i++) { + const c = await anagram.generate(difficulty) + seen.set(c.answer, difficulty) + } + } + return seen +} + +describe('anagram pool', async () => { + const pool = await poolFromGenerator() + + test('samples a meaningful share of the pool', () => { + expect(pool.size).toBeGreaterThanOrEqual(60) + }) + + test('no two words in the pool are anagrams of each other', () => { + const byLetters = new Map() + for (const word of pool.keys()) { + const k = letters(word) + byLetters.set(k, [...(byLetters.get(k) ?? []), word]) + } + const collisions = [...byLetters.values()].filter((ws) => ws.length > 1) + expect(collisions).toEqual([]) + }) + + test('the scrambled prompt is never the answer itself', async () => { + // Otherwise the answer is printed in the question and any script passes. + for (let difficulty = 1; difficulty <= 10; difficulty++) { + for (let i = 0; i < 300; i++) { + const c = await anagram.generate(difficulty) + const shown = c.prompt.split(':').pop()!.trim().replace(/\s/g, '') + expect(await gradeWith(anagram, shown, c.answer)).toBe(false) + } + } + }) + + test('the source word is still accepted', async () => { + for (const word of pool.keys()) { + expect(await gradeWith(anagram, word, word)).toBe(true) + expect(await gradeWith(anagram, ` ${word.toUpperCase()} `, word)).toBe(true) + } + }) + + test('every word in a difficulty bucket is the same length', () => { + // Word length is what maps a word to a difficulty, so a word filed under the + // wrong length silently makes that level easier or harder than intended. + const byDifficulty = new Map>() + for (const [word, difficulty] of pool) { + const lengths = byDifficulty.get(difficulty) ?? new Set() + lengths.add(word.length) + byDifficulty.set(difficulty, lengths) + } + for (const [difficulty, lengths] of byDifficulty) { + expect({ difficulty, lengths: [...lengths] }).toEqual({ difficulty, lengths: [...lengths].slice(0, 1) }) + } + }) + + test('a different word is still rejected', async () => { + expect(await gradeWith(anagram, 'zebra', 'house')).toBe(false) + expect(await gradeWith(anagram, '', 'house')).toBe(false) + }) +})