Pool DB acceptance test files to cut verification wall time

Run the 15 generated DB acceptance files with bounded concurrency
(default min(4, files), ACCEPTANCE_CONCURRENCY to override) instead of
strictly sequentially. Generation still runs first and sequentially.
Each generated file uses its own tmp/acceptance/level-* world, so the
pool is isolated. Full DB acceptance drops from ~430-790s to ~96s.

Also record the orphaned-mocha mutation-hang finding and remediation in
the architect process notes.

By architect.
This commit is contained in:
Chris Troutner
2026-09-17 09:22:13 -07:00
parent 92ce290873
commit 0072b81aac
2 changed files with 73 additions and 10 deletions
+24
View File
@@ -55,6 +55,30 @@ distinct from per-task verification results, which live in
sequentially, and use `--max-workers 8` to keep the mutation phase fast.
The DRY and soft-Gherkin-mutation steps are comparatively quick.
- **A mutated synchronous infinite loop can orphan a mocha process and wedge
later runs.** On 2026-09-17 a `mutate4javascript` worker on `helpers.js`
(`stripLeadingEmptyPushes`, `1 -> 0`) timed out but left a `c8`/`mocha`
process at ~100% CPU for 45+ minutes; the next mutation baseline then hung
behind it. The indexer and DB `npm test` mocha has no `--timeout`, so a
synchronous loop in mutated code cannot be interrupted by mocha — only the
tool's `--timeout-factor` (default 10x baseline) applies. Defenses: run each
mutation file under a shell-level `timeout`, redirect output to a file and
grep the `Mutation Report` section, and after any timeout `kill -9` orphaned
`mocha`/`mutate4javascript` processes before the next file. Tool-written
manifest updates in the source are expected; re-check `git status` to confirm
no mutant source remains applied.
- **DB acceptance was the dominant verification cost; the test phase is now
pooled.** Before 2026-09-17, `npm run acceptance` in psf-memo-db ran its 15
generated test files strictly sequentially (~430790s; 145 scenarios, each
opening/closing 21 LevelDB stores at ~850ms close apiece). `acceptance.js`
now runs the generated files with bounded concurrency (default
`min(4, files)`; override with `ACCEPTANCE_CONCURRENCY`, `=1` restores the
old sequential behavior). Measured 96s wall for the full DB suite. Generation
still runs sequentially before the pooled test phase, so the constitution's
"generation then tests" ordering is preserved. Pooling is safe because each
generated file creates its own uniquely named `tmp/acceptance/level-*` world.
- **`mutate4javascript` copies the whole project into each worker, including
`tmp/`.** The worker copy skips only `.git`, `node_modules`, and `target`.
A stale `tmp/acceptance` (LevelDB dirs from prior acceptance runs) can be
+49 -10
View File
@@ -6,7 +6,7 @@
generator -> generated test entry points -> node test runner
*/
import { execFileSync } from 'node:child_process'
import { execFile, execFileSync } from 'node:child_process'
import crypto from 'node:crypto'
import fs from 'node:fs'
import path from 'node:path'
@@ -29,6 +29,43 @@ function sh (cmd, args, opts = {}) {
}).toString()
}
// Async variant used to run generated acceptance test files concurrently. Each
// generated test opens its own isolated LevelDB world under tmp/acceptance, so
// running several at once does not share state.
function shAsync (cmd, args) {
return new Promise((resolve) => {
execFile(cmd, args, { maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => {
resolve({ ok: !error, stdout: stdout || '', stderr: stderr || '' })
})
})
}
// Run generated test files with bounded concurrency, preserving per-file
// output order. Generation stays sequential; only the test phase is pooled.
// Override the pool size with ACCEPTANCE_CONCURRENCY (1 restores the old
// sequential behavior).
async function runTests (tests) {
const requested = Number.parseInt(process.env.ACCEPTANCE_CONCURRENCY || '', 10)
const concurrency = Number.isFinite(requested) && requested > 0
? requested
: Math.min(4, tests.length)
const results = new Array(tests.length)
let next = 0
async function worker () {
while (true) {
const index = next++
if (index >= tests.length) return
const testFile = tests[index]
const result = await shAsync('node', [path.join(genDir, testFile)])
results[index] = { testFile, ...result }
}
}
await Promise.all(Array.from({ length: Math.min(concurrency, tests.length) }, worker))
return results
}
function ensureAps () {
if (fs.existsSync(path.join(apsDir, 'bb.edn'))) return
const shared = path.join(repoRoot, 'swarmforge', 'scripts', 'ensure-aps.sh')
@@ -94,7 +131,7 @@ function removeStaleGeneratedTests (features) {
}
}
function main () {
async function main () {
cleanStaleWorlds()
ensureAps()
@@ -131,16 +168,15 @@ function main () {
.filter((f) => f.endsWith('.acceptance.test.js'))
.sort()
const results = await runTests(tests)
let failures = 0
for (const testFile of tests) {
try {
const out = sh('node', [path.join(genDir, testFile)])
process.stdout.write(out)
for (const { testFile, ok, stdout, stderr } of results) {
process.stdout.write(stdout)
process.stderr.write(stderr)
if (ok) {
console.log(`ACCEPTANCE PASS: ${testFile}`)
} catch (err) {
} else {
failures++
process.stdout.write(err.stdout || '')
process.stderr.write(err.stderr || '')
console.error(`ACCEPTANCE FAIL: ${testFile}`)
}
}
@@ -153,4 +189,7 @@ function main () {
}
}
main()
main().catch((err) => {
console.error(err)
process.exit(1)
})