From 0072b81aac3e4fff235055aa2c8f131f5d48f01c Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Thu, 17 Sep 2026 09:22:13 -0700 Subject: [PATCH] Pool DB acceptance test files to cut verification wall time Run the 15 generated DB acceptance files with bounded concurrency (default min(4, files), ACCEPTANCE_CONCURRENCY to override) instead of strictly sequentially. Generation still runs first and sequentially. Each generated file uses its own tmp/acceptance/level-* world, so the pool is isolated. Full DB acceptance drops from ~430-790s to ~96s. Also record the orphaned-mocha mutation-hang finding and remediation in the architect process notes. By architect. --- docs/architect-process-notes.md | 24 +++++++++++ psf-memo-db/acceptance/acceptance.js | 59 +++++++++++++++++++++++----- 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/docs/architect-process-notes.md b/docs/architect-process-notes.md index 2ea4205..f415b8b 100644 --- a/docs/architect-process-notes.md +++ b/docs/architect-process-notes.md @@ -55,6 +55,30 @@ distinct from per-task verification results, which live in sequentially, and use `--max-workers 8` to keep the mutation phase fast. The DRY and soft-Gherkin-mutation steps are comparatively quick. +- **A mutated synchronous infinite loop can orphan a mocha process and wedge + later runs.** On 2026-09-17 a `mutate4javascript` worker on `helpers.js` + (`stripLeadingEmptyPushes`, `1 -> 0`) timed out but left a `c8`/`mocha` + process at ~100% CPU for 45+ minutes; the next mutation baseline then hung + behind it. The indexer and DB `npm test` mocha has no `--timeout`, so a + synchronous loop in mutated code cannot be interrupted by mocha — only the + tool's `--timeout-factor` (default 10x baseline) applies. Defenses: run each + mutation file under a shell-level `timeout`, redirect output to a file and + grep the `Mutation Report` section, and after any timeout `kill -9` orphaned + `mocha`/`mutate4javascript` processes before the next file. Tool-written + manifest updates in the source are expected; re-check `git status` to confirm + no mutant source remains applied. + +- **DB acceptance was the dominant verification cost; the test phase is now + pooled.** Before 2026-09-17, `npm run acceptance` in psf-memo-db ran its 15 + generated test files strictly sequentially (~430–790s; 145 scenarios, each + opening/closing 21 LevelDB stores at ~850ms close apiece). `acceptance.js` + now runs the generated files with bounded concurrency (default + `min(4, files)`; override with `ACCEPTANCE_CONCURRENCY`, `=1` restores the + old sequential behavior). Measured 96s wall for the full DB suite. Generation + still runs sequentially before the pooled test phase, so the constitution's + "generation then tests" ordering is preserved. Pooling is safe because each + generated file creates its own uniquely named `tmp/acceptance/level-*` world. + - **`mutate4javascript` copies the whole project into each worker, including `tmp/`.** The worker copy skips only `.git`, `node_modules`, and `target`. A stale `tmp/acceptance` (LevelDB dirs from prior acceptance runs) can be diff --git a/psf-memo-db/acceptance/acceptance.js b/psf-memo-db/acceptance/acceptance.js index 2dd45e6..684b481 100644 --- a/psf-memo-db/acceptance/acceptance.js +++ b/psf-memo-db/acceptance/acceptance.js @@ -6,7 +6,7 @@ generator -> generated test entry points -> node test runner */ -import { execFileSync } from 'node:child_process' +import { execFile, execFileSync } from 'node:child_process' import crypto from 'node:crypto' import fs from 'node:fs' import path from 'node:path' @@ -29,6 +29,43 @@ function sh (cmd, args, opts = {}) { }).toString() } +// Async variant used to run generated acceptance test files concurrently. Each +// generated test opens its own isolated LevelDB world under tmp/acceptance, so +// running several at once does not share state. +function shAsync (cmd, args) { + return new Promise((resolve) => { + execFile(cmd, args, { maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => { + resolve({ ok: !error, stdout: stdout || '', stderr: stderr || '' }) + }) + }) +} + +// Run generated test files with bounded concurrency, preserving per-file +// output order. Generation stays sequential; only the test phase is pooled. +// Override the pool size with ACCEPTANCE_CONCURRENCY (1 restores the old +// sequential behavior). +async function runTests (tests) { + const requested = Number.parseInt(process.env.ACCEPTANCE_CONCURRENCY || '', 10) + const concurrency = Number.isFinite(requested) && requested > 0 + ? requested + : Math.min(4, tests.length) + const results = new Array(tests.length) + let next = 0 + + async function worker () { + while (true) { + const index = next++ + if (index >= tests.length) return + const testFile = tests[index] + const result = await shAsync('node', [path.join(genDir, testFile)]) + results[index] = { testFile, ...result } + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, tests.length) }, worker)) + return results +} + function ensureAps () { if (fs.existsSync(path.join(apsDir, 'bb.edn'))) return const shared = path.join(repoRoot, 'swarmforge', 'scripts', 'ensure-aps.sh') @@ -94,7 +131,7 @@ function removeStaleGeneratedTests (features) { } } -function main () { +async function main () { cleanStaleWorlds() ensureAps() @@ -131,16 +168,15 @@ function main () { .filter((f) => f.endsWith('.acceptance.test.js')) .sort() + const results = await runTests(tests) let failures = 0 - for (const testFile of tests) { - try { - const out = sh('node', [path.join(genDir, testFile)]) - process.stdout.write(out) + for (const { testFile, ok, stdout, stderr } of results) { + process.stdout.write(stdout) + process.stderr.write(stderr) + if (ok) { console.log(`ACCEPTANCE PASS: ${testFile}`) - } catch (err) { + } else { failures++ - process.stdout.write(err.stdout || '') - process.stderr.write(err.stderr || '') console.error(`ACCEPTANCE FAIL: ${testFile}`) } } @@ -153,4 +189,7 @@ function main () { } } -main() +main().catch((err) => { + console.error(err) + process.exit(1) +})