Implement postHeights secondary index for efficient pagination

Adds a postHeights LevelDB store written by the indexer and used by
psf-memo-db to paginate recent posts and posts-by-address without full
scans. Includes unit tests and Gherkin acceptance pipelines for both
psf-memo-indexer and psf-memo-db.

By coder.
This commit is contained in:
Chris Troutner
2026-08-26 10:52:17 -07:00
parent 893e5c7513
commit fde0086385
28 changed files with 1553 additions and 195 deletions
+88
View File
@@ -0,0 +1,88 @@
/*
Normal acceptance runner for psf-memo-db.
Orchestrates the acceptance pipeline:
feature file -> bb gherkin-parser -> JSON IR -> acceptance entrypoint
generator -> generated test entry points -> node test runner
*/
import { execFileSync } from 'node:child_process'
import fs from 'node:fs'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const root = path.resolve(__dirname, '..')
const specsDir = path.join(root, 'specs')
const buildDir = path.join(root, 'build', 'acceptance')
const irDir = path.join(buildDir, 'ir')
const genDir = path.join(buildDir, 'generated')
const apsDir = path.join(root, '..', 'tmp', 'aps-spec')
function sh (cmd, args, opts = {}) {
return execFileSync(cmd, args, {
stdio: ['pipe', 'pipe', 'pipe'],
...opts
}).toString()
}
function ensureAps () {
if (fs.existsSync(apsDir)) return
fs.mkdirSync(path.dirname(apsDir), { recursive: true })
sh('git', ['clone', '--depth', '1',
'https://github.com/unclebob/Acceptance-Pipeline-Specification.git', apsDir])
}
function main () {
ensureAps()
const features = fs
.readdirSync(specsDir)
.filter((f) => f.endsWith('.feature'))
.sort()
if (features.length === 0) {
console.log('No feature files found under specs/.')
return
}
fs.mkdirSync(irDir, { recursive: true })
fs.mkdirSync(genDir, { recursive: true })
for (const featureFile of features) {
const base = featureFile.replace(/\.feature$/i, '')
const featurePath = path.join(specsDir, featureFile)
const irPath = path.join(irDir, `${base}.json`)
sh('bb', ['gherkin-parser', featurePath, irPath], { cwd: apsDir })
sh('node', [path.join(__dirname, 'lib', 'generate.js'), irPath, genDir])
}
const tests = fs
.readdirSync(genDir)
.filter((f) => f.endsWith('.acceptance.test.js'))
.sort()
let failures = 0
for (const testFile of tests) {
try {
const out = sh('node', [path.join(genDir, testFile)])
process.stdout.write(out)
console.log(`ACCEPTANCE PASS: ${testFile}`)
} catch (err) {
failures++
process.stdout.write(err.stdout || '')
process.stderr.write(err.stderr || '')
console.error(`ACCEPTANCE FAIL: ${testFile}`)
}
}
if (failures > 0) {
console.error(`ACCEPTANCE: ${failures} failing test file(s)`)
process.exit(1)
} else {
console.log(`ACCEPTANCE: all ${tests.length} generated test file(s) passed`)
}
}
main()
+100
View File
@@ -0,0 +1,100 @@
/*
Project-specific acceptance entrypoint generator for psf-memo-db.
Reads parser JSON IR and writes executable generated test entry points plus
per-feature metadata. Generated tests delegate all step behavior to the
acceptance runtime and project step handlers.
*/
import fs from 'node:fs'
import path from 'node:path'
import crypto from 'node:crypto'
import { fileURLToPath } from 'node:url'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
function metadataName (featureName) {
const slug = featureName
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
return `${slug || 'feature'}.json`
}
function relativeImport (fromDir, targetFile) {
let rel = path.relative(fromDir, targetFile).replace(/\\/g, '/')
if (!rel.startsWith('.')) rel = `./${rel}`
return rel
}
function main () {
const irArg = process.argv[2]
const outArg = process.argv[3]
if (!irArg || !outArg) {
console.error('usage: acceptance-entrypoint-generator <json-ir> <generated-test-output-dir>')
process.exit(2)
}
let ir
try {
ir = JSON.parse(fs.readFileSync(irArg, 'utf8'))
} catch (err) {
console.error(`Failed to read JSON IR "${irArg}": ${err.message}`)
process.exit(1)
}
const genDir = outArg
fs.mkdirSync(genDir, { recursive: true })
const featureKey = path.basename(irArg).replace(/\.json$/i, '')
const testFile = path.join(genDir, `${featureKey}.acceptance.test.js`)
const relRuntime = relativeImport(genDir, path.join(__dirname, 'runtime.js'))
const body = `import { runFeature } from '${relRuntime}'
const ir = ${JSON.stringify(ir, null, 2)}
async function main () {
const report = await runFeature(ir)
for (const r of report.results) {
console.log((r.status === 'passed' ? 'PASS ' : 'FAIL ') + r.name)
if (r.detail) console.log(' ' + r.detail)
}
if (report.failures > 0) {
console.error('ACCEPTANCE FAILURES: ' + report.failures + ' of ' + report.total)
process.exitCode = 1
}
}
main().catch((err) => { console.error(err); process.exit(1) })
`
try {
fs.writeFileSync(testFile, body)
} catch (err) {
console.error(`Failed to write generated test "${testFile}": ${err.message}`)
process.exit(1)
}
const metaDir = path.join(genDir, 'metadata')
fs.mkdirSync(metaDir, { recursive: true })
const hash = crypto
.createHash('sha256')
.update(fs.readFileSync(testFile))
.digest('hex')
const metadata = {
schema_version: 1,
feature_path: `${featureKey}.feature`,
ir_path: path.resolve(irArg),
implementation_hash: `sha256:${hash}`,
hash_scope: 'generated_files',
generated_files: [testFile]
}
fs.writeFileSync(
path.join(metaDir, metadataName(featureKey)),
JSON.stringify(metadata, null, 2)
)
process.exit(0)
}
main()
+289
View File
@@ -0,0 +1,289 @@
/*
Project step handlers for the psf-memo-db acceptance pipeline.
These handlers spin up a real psf-memo-db adapter set against a temporary
LevelDB directory, load the Gherkin fixtures, and exercise the real use cases
for recent posts and posts-by-address. Iterators and get calls are wrapped
so the efficiency steps can assert bounded reads.
*/
import path from 'node:path'
import fs from 'node:fs'
import { fileURLToPath } from 'node:url'
import { DB_NAMES } from '../../src/adapters/level-db.js'
import Adapters from '../../src/adapters/index.js'
import ListRecentPosts from '../../src/use-cases/list-recent-posts.js'
import ListPostsByAddr from '../../src/use-cases/list-posts-by-addr.js'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const tmpDir = path.resolve(__dirname, '..', '..', 'tmp', 'acceptance')
function resolveParam (value, example) {
const match = /^<([A-Za-z0-9_]+)>$/.exec(String(value).trim())
if (match) {
const param = match[1]
if (!(param in example)) {
throw new Error(`Missing example value for "${param}"`)
}
return example[param]
}
return String(value).trim()
}
function wrapIterator (db, counter) {
const original = db.iterator.bind(db)
db.iterator = function (...args) {
counter.calls++
return original(...args)
}
}
function wrapGet (db, counter) {
const original = db.get.bind(db)
db.get = async function (...args) {
counter.calls++
return original(...args)
}
}
async function createWorld () {
const levelDir = path.join(tmpDir, `level-${Date.now()}-${Math.random().toString(36).slice(2)}`)
const adapters = new Adapters()
// Force the LevelDB adapter to use a temporary directory for this scenario.
adapters.levelDb.openDbs = function () {
const dbs = {}
fs.mkdirSync(levelDir, { recursive: true })
for (const name of DB_NAMES) {
const prop = `${name}Db`
const storeDir = path.join(levelDir, name)
fs.mkdirSync(storeDir, { recursive: true })
dbs[prop] = adapters.levelDb.level(storeDir, {
valueEncoding: 'json',
cacheSize: name === 'posts' ? 512 * 1024 * 1024 : 64 * 1024 * 1024
})
this[prop] = dbs[prop]
}
return dbs
}
adapters.start()
const postHeightsIteratorCounter = { calls: 0 }
const postChildrenIteratorCounter = { calls: 0 }
const postsGetCounter = { calls: 0 }
wrapIterator(adapters.level.postHeightsDb, postHeightsIteratorCounter)
wrapIterator(adapters.level.postChildrenDb, postChildrenIteratorCounter)
wrapGet(adapters.level.postsDb, postsGetCounter)
const listRecentPosts = new ListRecentPosts({ adapters })
const listPostsByAddr = new ListPostsByAddr({ adapters })
let lastResponse = null
return {
adapters,
listRecentPosts,
listPostsByAddr,
postHeightsIteratorCounter,
postChildrenIteratorCounter,
postsGetCounter,
getLastResponse: () => lastResponse,
setLastResponse: (resp) => { lastResponse = resp },
close: async () => {
try {
await adapters.levelDb.closeDbs()
} catch (err) {
// ignore close errors
}
}
}
}
async function loadFixture (world, name) {
if (name !== 'three-top-level-posts-and-one-reply') {
throw new Error(`Unknown fixture: ${name}`)
}
const posts = [
{ txid: 'post-200-b', addr: 'bitcoincash:qaddr-b', text: 'b', seen: 200, blockHeight: 600200 },
{ txid: 'post-200-a', addr: 'bitcoincash:qaddr-a', text: 'a', seen: 100, blockHeight: 600200 },
{ txid: 'post-100', addr: 'bitcoincash:qaddr-a', text: 'c', seen: 50, blockHeight: 600100 }
]
const reply = { txid: 'reply-1', parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600050 }
for (const post of posts) {
await world.adapters.level.postsDb.put(post.txid, {
addr: post.addr,
text: post.text,
seen: post.seen,
blockHeight: post.blockHeight
})
await world.adapters.level.postHeightsDb.put(
String(post.blockHeight).padStart(12, '0') + ':' + post.txid,
{ txid: post.txid, blockHeight: post.blockHeight }
)
}
await world.adapters.level.postsDb.put('reply-1', {
addr: 'bitcoincash:qaddr-a',
text: 'reply body',
seen: 10,
blockHeight: 600050
})
await world.adapters.level.postHeightsDb.put(
'000000600050:reply-1',
{ txid: 'reply-1', blockHeight: 600050 }
)
await world.adapters.level.postParentsDb.put('reply-1', reply)
await world.adapters.level.postChildrenDb.put('post-200-a:reply-1', reply)
}
const handlers = [
{
name: 'db instance with posts and postHeights stores',
pattern: /^a psf-memo-db instance with a posts store and a postHeights secondary index$/,
async run () {
// World is already created with both stores.
}
},
{
name: 'load fixture',
pattern: /^the fixture "(.+)" is loaded into the posts store$/,
async run (m, example, world) {
await loadFixture(world, m[1])
}
},
{
name: 'request recent posts',
pattern: /^the client requests \/posts\/recent with limit (<limit>) and offset (<offset>)$/,
async run (m, example, world) {
const limit = parseInt(resolveParam(m[1], example), 10)
const offset = parseInt(resolveParam(m[2], example), 10)
const resp = await world.listRecentPosts.execute({ limit, offset })
world.setLastResponse(resp)
}
},
{
name: 'request posts by addr',
pattern: /^the client requests \/posts\/by\/(<addr>) with limit (<limit>) and offset (<offset>)$/,
async run (m, example, world) {
const addr = resolveParam(m[1], example)
const limit = parseInt(resolveParam(m[2], example), 10)
const offset = parseInt(resolveParam(m[3], example), 10)
const resp = await world.listPostsByAddr.execute({ addr, limit, offset })
world.setLastResponse(resp)
}
},
{
name: 'posts sorted by block height descending',
pattern: /^the response posts are sorted by block height descending$/,
run (m, example, world) {
const posts = world.getLastResponse().posts
for (let i = 1; i < posts.length; i++) {
if (posts[i].blockHeight > posts[i - 1].blockHeight) {
throw new Error(`Posts not sorted by descending block height at index ${i}`)
}
}
}
},
{
name: 'response contains expected txids',
pattern: /^the response contains the txids (<expected_txids>)$/,
run (m, example, world) {
const expected = resolveParam(m[1], example).split(',').map((s) => s.trim())
const actual = world.getLastResponse().posts.map((p) => p.txid)
if (expected.join(',') !== actual.join(',')) {
throw new Error(`Expected txids ${expected.join(',')}, got ${actual.join(',')}`)
}
}
},
{
name: 'response contains only posts by addr',
pattern: /^the response contains only posts by (<addr>)$/,
run (m, example, world) {
const addr = resolveParam(m[1], example)
const posts = world.getLastResponse().posts
for (const post of posts) {
if (post.addr !== addr) {
throw new Error(`Expected post by ${addr}, got ${post.addr}`)
}
}
}
},
{
name: 'response pagination metadata',
pattern: /^the response pagination shows total (<total>) and hasMore (<hasMore>)$/,
run (m, example, world) {
const expectedTotal = parseInt(resolveParam(m[1], example), 10)
const expectedHasMore = resolveParam(m[2], example) === 'true'
const pagination = world.getLastResponse().pagination
if (pagination.total !== expectedTotal) {
throw new Error(`Expected total ${expectedTotal}, got ${pagination.total}`)
}
if (pagination.hasMore !== expectedHasMore) {
throw new Error(`Expected hasMore ${expectedHasMore}, got ${pagination.hasMore}`)
}
}
},
{
name: 'bounded postHeights reads',
pattern: /^no more than (<limit>) postHeights entries are read after applying the offset$/,
run (m, example, world) {
const limit = parseInt(resolveParam(m[1], example), 10)
const reads = world.postHeightsIteratorCounter.calls
if (reads > limit) {
throw new Error(`Read ${reads} postHeights entries, expected at most ${limit}`)
}
}
},
{
name: 'bounded posts loaded by txid',
pattern: /^no more than (<limit>) posts are loaded by txid$/,
run (m, example, world) {
const limit = parseInt(resolveParam(m[1], example), 10)
const reads = world.postsGetCounter.calls
if (reads > limit) {
throw new Error(`Loaded ${reads} posts by txid, expected at most ${limit}`)
}
}
},
{
name: 'post has replyCount',
pattern: /^the response post with txid (<txid>) has replyCount (<replyCount>)$/,
run (m, example, world) {
const txid = resolveParam(m[1], example)
const expected = parseInt(resolveParam(m[2], example), 10)
const post = world.getLastResponse().posts.find((p) => p.txid === txid)
if (!post) {
throw new Error(`Post ${txid} not found in response`)
}
if (post.replyCount !== expected) {
throw new Error(`Expected replyCount ${expected} for ${txid}, got ${post.replyCount}`)
}
}
},
{
name: 'single postChildren scan',
pattern: /^the postChildren store was iterated exactly once$/,
run (m, example, world) {
const calls = world.postChildrenIteratorCounter.calls
if (calls !== 1) {
throw new Error(`Expected exactly one postChildren scan, got ${calls}`)
}
}
}
]
async function handleStep (step, example, world) {
for (const handler of handlers) {
const match = handler.pattern.exec(step.text)
if (match) {
await handler.run(match, example, world, step)
return
}
}
throw new Error(`Unsupported step: ${step.keyword} ${step.text}`)
}
export { createWorld, handleStep }
+64
View File
@@ -0,0 +1,64 @@
/*
Acceptance runtime for psf-memo-db.
Expands each scenario (and each example row) from parser JSON IR into
scenario executions, prepends background steps, and dispatches every step to
the project step handlers.
*/
import { createWorld, handleStep } from './handlers.js'
// Expand the IR scenarios into concrete executions.
function expandScenarios (ir) {
const executions = []
const background = ir.background || []
for (const scenario of ir.scenarios) {
const examples = (scenario.examples && scenario.examples.length > 0)
? scenario.examples.map((example, i) => ({ example, suffix: `example_${i + 1}` }))
: [{ example: {}, suffix: 'example_1' }]
for (const { example, suffix } of examples) {
executions.push({
name: `${scenario.name}/${suffix}`,
steps: [...background, ...scenario.steps],
example
})
}
}
return executions
}
async function runFeature (ir) {
const executions = expandScenarios(ir)
const results = []
let failures = 0
for (const ex of executions) {
const world = await createWorld()
let failure = null
for (const step of ex.steps) {
try {
await handleStep(step, ex.example, world)
} catch (err) {
failure = err.message
break
}
}
await world.close()
if (failure) {
failures++
results.push({ name: ex.name, status: 'failed', detail: failure })
} else {
results.push({ name: ex.name, status: 'passed' })
}
}
return { feature: ir.name, results, failures, total: results.length }
}
export { expandScenarios, runFeature }
+1
View File
@@ -8,6 +8,7 @@
"prestart": "npm run docs",
"start": "node index.js",
"test": "export SVC_ENV=test && c8 --reporter=text mocha --exit --timeout 15000 --recursive test/unit/",
"acceptance": "node acceptance/acceptance.js",
"lint": "standard --env mocha --fix",
"docs": "./node_modules/.bin/apidoc -i src/ -o docs"
},
+1
View File
@@ -24,6 +24,7 @@ class Adapters {
})
this.postQuery = new PostQuery({
postsDb: level.postsDb,
postHeightsDb: level.postHeightsDb,
postParentsDb: level.postParentsDb,
postChildrenDb: level.postChildrenDb
})
+1
View File
@@ -12,6 +12,7 @@ const dbDir = `${__dirname}/../../leveldb`
const DB_NAMES = [
'status',
'posts',
'postHeights',
'postParents',
'postChildren',
'likes',
+137 -75
View File
@@ -1,14 +1,18 @@
/*
Adapter for scanning posts with stored block height.
Excludes reply posts (txids present in postParentsDb).
Adapter for efficient post queries using a postHeights secondary index.
*/
const HEIGHT_PAD = 12
class PostQuery {
constructor (localConfig = {}) {
const { postsDb, postParentsDb, postChildrenDb } = localConfig
const { postsDb, postHeightsDb, postParentsDb, postChildrenDb } = localConfig
if (!postsDb) {
throw new Error('postsDb required when instantiating PostQuery adapter.')
}
if (!postHeightsDb) {
throw new Error('postHeightsDb required when instantiating PostQuery adapter.')
}
if (!postParentsDb) {
throw new Error('postParentsDb required when instantiating PostQuery adapter.')
}
@@ -16,12 +20,32 @@ class PostQuery {
throw new Error('postChildrenDb required when instantiating PostQuery adapter.')
}
this.postsDb = postsDb
this.postHeightsDb = postHeightsDb
this.postParentsDb = postParentsDb
this.postChildrenDb = postChildrenDb
this.scanPostsWithBlockHeight = this.scanPostsWithBlockHeight.bind(this)
this.scanPostsByAddr = this.scanPostsByAddr.bind(this)
this.scanRecentPostTxids = this.scanRecentPostTxids.bind(this)
this.scanPostsByAddrTxids = this.scanPostsByAddrTxids.bind(this)
this.loadPostsByTxids = this.loadPostsByTxids.bind(this)
this.countTopLevelPosts = this.countTopLevelPosts.bind(this)
this.countTopLevelPostsByAddr = this.countTopLevelPostsByAddr.bind(this)
this.loadReplyTxids = this.loadReplyTxids.bind(this)
this.buildReplyCountMap = this.buildReplyCountMap.bind(this)
this.txidFromPostHeight = this.txidFromPostHeight.bind(this)
}
static padHeight (height) {
return String(height).padStart(HEIGHT_PAD, '0')
}
static postHeightKey (blockHeight, txid) {
return `${PostQuery.padHeight(blockHeight)}:${txid}`
}
txidFromPostHeight (key, value) {
if (value && typeof value.txid === 'string') return value.txid
const parts = String(key).split(':')
return parts[parts.length - 1]
}
async loadReplyTxids () {
@@ -38,82 +62,120 @@ class PostQuery {
const counts = new Map()
for await (const [, child] of this.postChildrenDb.iterator()) {
const parentTxid = child.parentTxid
if (!parentTxid) continue
counts.set(parentTxid, (counts.get(parentTxid) || 0) + 1)
}
return counts
}
async scanPostsWithBlockHeight () {
const [replyTxids, replyCounts] = await Promise.all([
this.loadReplyTxids(),
this.buildReplyCountMap()
])
const posts = []
for await (const [txid, post] of this.postsDb.iterator()) {
if (replyTxids.has(txid)) continue
posts.push({
txid,
addr: post.addr,
text: post.text,
seen: post.seen,
blockHeight: post.blockHeight ?? 0,
replyCount: replyCounts.get(txid) ?? 0
})
}
return posts
}
async scanPostsByAddr (addr) {
const [replyTxids, replyCounts] = await Promise.all([
this.loadReplyTxids(),
this.buildReplyCountMap()
])
const posts = []
for await (const [txid, post] of this.postsDb.iterator()) {
if (post.addr !== addr) continue
if (replyTxids.has(txid)) continue
posts.push({
txid,
addr: post.addr,
text: post.text,
seen: post.seen,
blockHeight: post.blockHeight ?? 0,
replyCount: replyCounts.get(txid) ?? 0
})
}
return posts
}
async buildReplyCountMap () {
const counts = new Map()
let total = 0
for await (const [childTxid, child] of this.postChildrenDb.iterator()) {
total++
console.log('Indexed reply:', {
childTxid,
child,
parentTxid: child?.parentTxid
})
const parentTxid = child?.parentTxid
if (!parentTxid) continue
counts.set(parentTxid, (counts.get(parentTxid) || 0) + 1)
}
console.log(`Total postChildrenDb records: ${total}`)
return counts
}
}
async scanRecentPostTxids ({ limit, offset }) {
const replyTxids = await this.loadReplyTxids()
const txids = []
let skipped = 0
for await (const [key, value] of this.postHeightsDb.iterator({ reverse: true })) {
const txid = this.txidFromPostHeight(key, value)
if (replyTxids.has(txid)) continue
if (skipped < offset) {
skipped++
continue
}
txids.push(txid)
if (txids.length >= limit) break
}
return txids
}
async scanPostsByAddrTxids (addr, { limit, offset }) {
const replyTxids = await this.loadReplyTxids()
const txids = []
let skipped = 0
for await (const [key, value] of this.postHeightsDb.iterator({ reverse: true })) {
const txid = this.txidFromPostHeight(key, value)
if (replyTxids.has(txid)) continue
let post
try {
post = await this.postsDb.get(txid)
} catch (err) {
if (err.notFound || err.code === 'LEVEL_NOT_FOUND') continue
throw err
}
if (post.addr !== addr) continue
if (skipped < offset) {
skipped++
continue
}
txids.push(txid)
if (txids.length >= limit) break
}
return txids
}
async loadPostsByTxids (txids) {
const posts = []
for (const txid of txids) {
try {
const post = await this.postsDb.get(txid)
posts.push({
txid,
addr: post.addr,
text: post.text,
seen: post.seen,
blockHeight: post.blockHeight ?? 0
})
} catch (err) {
if (err.notFound || err.code === 'LEVEL_NOT_FOUND') continue
throw err
}
}
return posts
}
async countTopLevelPosts () {
const replyTxids = await this.loadReplyTxids()
let count = 0
for await (const [key, value] of this.postHeightsDb.iterator()) {
const txid = this.txidFromPostHeight(key, value)
if (replyTxids.has(txid)) continue
count++
}
return count
}
async countTopLevelPostsByAddr (addr) {
const replyTxids = await this.loadReplyTxids()
let count = 0
for await (const [key, value] of this.postHeightsDb.iterator()) {
const txid = this.txidFromPostHeight(key, value)
if (replyTxids.has(txid)) continue
try {
const post = await this.postsDb.get(txid)
if (post.addr === addr) count++
} catch (err) {
if (err.notFound || err.code === 'LEVEL_NOT_FOUND') continue
throw err
}
}
return count
}
}
export default PostQuery
@@ -36,6 +36,7 @@ export function makeCrudHandlers ({ dbProp, keyParam, bodyIdField, bodyDataField
export const ENTITY_CONFIG = [
{ route: 'post', dbProp: 'postsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'postData' },
{ route: 'postheight', dbProp: 'postHeightsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'postHeightData' },
{ route: 'postparent', dbProp: 'postParentsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'parentData' },
{ route: 'postchild', dbProp: 'postChildrenDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'childData' },
{ route: 'like', dbProp: 'likesDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'likeData' },
@@ -26,7 +26,7 @@ class PostsRouter {
attach (app) {
this.router.get('/recent', this.postsRESTController.getRecentPosts)
this.router.get('/by/:addr', this.postsRESTController.getPostsByAddr)
this.router.get('/:txid/thread',this.postsRESTController.getPostThread)
this.router.get('/:txid/thread', this.postsRESTController.getPostThread)
app.use(this.router.routes())
app.use(this.router.allowedMethods())
}
-1
View File
@@ -47,4 +47,3 @@ class UseCases {
}
export default UseCases
+14 -12
View File
@@ -1,5 +1,6 @@
/*
Use case: list posts for an address ordered by block height (most recent first), paginated.
Uses the postHeights secondary index for efficient sorting and pagination.
*/
const DEFAULT_LIMIT = 100
@@ -15,6 +16,7 @@ class ListPostsByAddr {
throw new Error('postQuery adapter required for ListPostsByAddr use case.')
}
this.execute = this.execute.bind(this)
this.attachReplyCounts = this.attachReplyCounts.bind(this)
}
parseLimit (limit) {
@@ -57,13 +59,11 @@ class ListPostsByAddr {
return addr
}
sortPosts (posts) {
return posts.sort((a, b) => {
if (b.blockHeight !== a.blockHeight) {
return b.blockHeight - a.blockHeight
}
return (b.seen || 0) - (a.seen || 0)
})
attachReplyCounts (posts, replyCounts) {
return posts.map((post) => ({
...post,
replyCount: replyCounts.get(post.txid) ?? 0
}))
}
async execute (inObj = {}) {
@@ -71,13 +71,15 @@ class ListPostsByAddr {
const limit = this.parseLimit(inObj.limit)
const offset = this.parseOffset(inObj.offset)
const allPosts = await this.adapters.postQuery.scanPostsByAddr(addr)
const sorted = this.sortPosts(allPosts)
const total = sorted.length
const posts = sorted.slice(offset, offset + limit)
const txids = await this.adapters.postQuery.scanPostsByAddrTxids(addr, { limit, offset })
const [posts, replyCounts, total] = await Promise.all([
this.adapters.postQuery.loadPostsByTxids(txids),
this.adapters.postQuery.buildReplyCountMap(),
this.adapters.postQuery.countTopLevelPostsByAddr(addr)
])
return {
posts,
posts: this.attachReplyCounts(posts, replyCounts),
pagination: {
limit,
offset,
+14 -12
View File
@@ -1,5 +1,6 @@
/*
Use case: list posts ordered by block height (most recent first), paginated.
Uses the postHeights secondary index for efficient sorting and pagination.
*/
const DEFAULT_LIMIT = 100
@@ -15,6 +16,7 @@ class ListRecentPosts {
throw new Error('postQuery adapter required for ListRecentPosts use case.')
}
this.execute = this.execute.bind(this)
this.attachReplyCounts = this.attachReplyCounts.bind(this)
}
parseLimit (limit) {
@@ -48,26 +50,26 @@ class ListRecentPosts {
return parsed
}
sortPosts (posts) {
return posts.sort((a, b) => {
if (b.blockHeight !== a.blockHeight) {
return b.blockHeight - a.blockHeight
}
return (b.seen || 0) - (a.seen || 0)
})
attachReplyCounts (posts, replyCounts) {
return posts.map((post) => ({
...post,
replyCount: replyCounts.get(post.txid) ?? 0
}))
}
async execute (inObj = {}) {
const limit = this.parseLimit(inObj.limit)
const offset = this.parseOffset(inObj.offset)
const allPosts = await this.adapters.postQuery.scanPostsWithBlockHeight()
const sorted = this.sortPosts(allPosts)
const total = sorted.length
const posts = sorted.slice(offset, offset + limit)
const txids = await this.adapters.postQuery.scanRecentPostTxids({ limit, offset })
const [posts, replyCounts, total] = await Promise.all([
this.adapters.postQuery.loadPostsByTxids(txids),
this.adapters.postQuery.buildReplyCountMap(),
this.adapters.postQuery.countTopLevelPosts()
])
return {
posts,
posts: this.attachReplyCounts(posts, replyCounts),
pagination: {
limit,
offset,
+184 -90
View File
@@ -8,11 +8,13 @@ describe('#PostQuery', () => {
let postsDb
let postParentsDb
let postChildrenDb
let postHeightsDb
beforeEach(() => {
sandbox = sinon.createSandbox()
postsDb = {
iterator: sandbox.stub()
iterator: sandbox.stub(),
get: sandbox.stub()
}
postParentsDb = {
iterator: sandbox.stub()
@@ -20,125 +22,217 @@ describe('#PostQuery', () => {
postChildrenDb = {
iterator: sandbox.stub()
}
postHeightsDb = {
iterator: sandbox.stub()
}
async function * emptyParents () {}
async function * emptyChildren () {}
async function * emptyHeights () {}
postParentsDb.iterator.returns(emptyParents())
postChildrenDb.iterator.returns(emptyChildren())
uut = new PostQuery({ postsDb, postParentsDb, postChildrenDb })
postHeightsDb.iterator.returns(emptyHeights())
uut = new PostQuery({ postsDb, postParentsDb, postChildrenDb, postHeightsDb })
})
afterEach(() => sandbox.restore())
it('should scan posts and read block height from stored document', async () => {
async function * mockIterator () {
yield ['tx1', { addr: 'addr1', text: 'hello', seen: 1000, blockHeight: 600100 }]
yield ['tx2', { addr: 'addr2', text: 'world', seen: 2000, blockHeight: 600200 }]
it('should throw when postHeightsDb is missing', () => {
try {
// eslint-disable-next-line no-new
new PostQuery({ postsDb, postParentsDb, postChildrenDb })
assert.fail('Expected error')
} catch (err) {
assert.include(err.message, 'postHeightsDb required')
}
postsDb.iterator.returns(mockIterator())
})
const result = await uut.scanPostsWithBlockHeight()
describe('#scanRecentPostTxids', () => {
it('should return top-level post txids sorted by block height descending', async () => {
async function * mockHeights () {
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
yield ['000000600100:post-100', { txid: 'post-100' }]
}
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
const result = await uut.scanRecentPostTxids({ limit: 2, offset: 0 })
assert.deepEqual(result, ['post-200-b', 'post-200-a'])
})
it('should skip replies when selecting recent posts', async () => {
async function * mockParents () {
yield ['reply-1', { parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 }]
}
async function * mockHeights () {
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
yield ['000000600150:reply-1', { txid: 'reply-1' }]
yield ['000000600100:post-100', { txid: 'post-100' }]
}
postParentsDb.iterator.returns(mockParents())
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
const result = await uut.scanRecentPostTxids({ limit: 2, offset: 0 })
assert.deepEqual(result, ['post-200-b', 'post-200-a'])
})
it('should apply offset after skipping replies', async () => {
async function * mockHeights () {
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
yield ['000000600100:post-100', { txid: 'post-100' }]
}
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
const result = await uut.scanRecentPostTxids({ limit: 2, offset: 1 })
assert.deepEqual(result, ['post-200-a', 'post-100'])
})
it('should stop reading after collecting limit top-level posts', async () => {
async function * mockHeights () {
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
yield ['000000600100:post-100', { txid: 'post-100' }]
}
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
await uut.scanRecentPostTxids({ limit: 2, offset: 0 })
assert.isTrue(postHeightsDb.iterator.calledOnce)
})
})
describe('#scanPostsByAddrTxids', () => {
it('should return txids for the address sorted by block height descending', async () => {
async function * mockHeights () {
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
yield ['000000600100:post-100', { txid: 'post-100' }]
}
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
postsDb.get.callsFake(async (txid) => {
const posts = {
'post-200-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
'post-200-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600200 },
'post-100': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600100 }
}
return posts[txid]
})
const result = await uut.scanPostsByAddrTxids('addr-a', { limit: 2, offset: 0 })
assert.deepEqual(result, ['post-200-a', 'post-100'])
})
it('should apply offset and limit for the address', async () => {
async function * mockHeights () {
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
yield ['000000600100:post-100', { txid: 'post-100' }]
}
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
postsDb.get.callsFake(async (txid) => {
const posts = {
'post-200-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
'post-200-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600200 },
'post-100': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600100 }
}
return posts[txid]
})
const result = await uut.scanPostsByAddrTxids('addr-a', { limit: 1, offset: 1 })
assert.deepEqual(result, ['post-100'])
})
})
describe('#loadPostsByTxids', () => {
it('should load posts by txid', async () => {
postsDb.get.withArgs('tx1').resolves({ addr: 'a1', text: 'hello', seen: 1000, blockHeight: 600100 })
postsDb.get.withArgs('tx2').resolves({ addr: 'a2', text: 'world', seen: 2000, blockHeight: 600200 })
const result = await uut.loadPostsByTxids(['tx1', 'tx2'])
assert.equal(result.length, 2)
assert.equal(result[0].txid, 'tx1')
assert.equal(result[0].blockHeight, 600100)
assert.equal(result[0].replyCount, 0)
assert.equal(result[1].txid, 'tx2')
assert.equal(result[1].blockHeight, 600200)
assert.equal(result[1].replyCount, 0)
})
it('should use block height 0 when field is missing', async () => {
async function * mockIterator () {
yield ['tx-missing', { addr: 'addr1', text: 'hi', seen: 1000 }]
}
postsDb.iterator.returns(mockIterator())
const result = await uut.scanPostsWithBlockHeight()
assert.equal(result[0].blockHeight, 0)
assert.equal(result[0].replyCount, 0)
})
it('should scan posts for a single address', async () => {
async function * mockIterator () {
yield ['tx1', { addr: 'addr-a', text: 'hello', seen: 1000, blockHeight: 600100 }]
yield ['tx2', { addr: 'addr-b', text: 'world', seen: 2000, blockHeight: 600200 }]
yield ['tx3', { addr: 'addr-a', text: 'again', seen: 3000, blockHeight: 600300 }]
}
postsDb.iterator.returns(mockIterator())
const result = await uut.scanPostsByAddr('addr-a')
assert.equal(result.length, 2)
assert.equal(result[0].txid, 'tx1')
assert.equal(result[1].txid, 'tx3')
})
it('should exclude reply posts from recent scan', async () => {
async function * mockParents () {
yield ['tx-reply', { parentTxid: 'tx1', childTxid: 'tx-reply', blockHeight: 600150 }]
}
async function * mockPosts () {
yield ['tx1', { addr: 'addr1', text: 'top', seen: 1000, blockHeight: 600100 }]
yield ['tx-reply', { addr: 'addr1', text: 'reply', seen: 1500, blockHeight: 600150 }]
yield ['tx2', { addr: 'addr2', text: 'other', seen: 2000, blockHeight: 600200 }]
}
postParentsDb.iterator.returns(mockParents())
postsDb.iterator.returns(mockPosts())
const result = await uut.scanPostsWithBlockHeight()
assert.equal(result.length, 2)
assert.equal(result[0].txid, 'tx1')
assert.equal(result[1].txid, 'tx2')
})
it('should exclude reply posts from address scan', async () => {
async function * mockParents () {
yield ['tx-reply', { parentTxid: 'tx1', childTxid: 'tx-reply', blockHeight: 600150 }]
}
async function * mockPosts () {
yield ['tx1', { addr: 'addr-a', text: 'top', seen: 1000, blockHeight: 600100 }]
yield ['tx-reply', { addr: 'addr-a', text: 'reply', seen: 1500, blockHeight: 600150 }]
}
postParentsDb.iterator.returns(mockParents())
postsDb.iterator.returns(mockPosts())
it('should skip missing posts', async () => {
const err = new Error('not found')
err.notFound = true
postsDb.get.withArgs('tx1').rejects(err)
postsDb.get.withArgs('tx2').resolves({ addr: 'a2', text: 'world', seen: 2000, blockHeight: 600200 })
const result = await uut.scanPostsByAddr('addr-a')
const result = await uut.loadPostsByTxids(['tx1', 'tx2'])
assert.equal(result.length, 1)
assert.equal(result[0].txid, 'tx1')
assert.equal(result[0].txid, 'tx2')
})
})
it('should include replyCount from postChildren scan', async () => {
describe('#countTopLevelPosts', () => {
it('should count top-level posts excluding replies', async () => {
async function * mockParents () {
yield ['reply-1', { parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 }]
}
async function * mockHeights () {
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
yield ['000000600150:reply-1', { txid: 'reply-1' }]
yield ['000000600100:post-100', { txid: 'post-100' }]
}
postParentsDb.iterator.returns(mockParents())
postHeightsDb.iterator.returns(mockHeights())
const result = await uut.countTopLevelPosts()
assert.equal(result, 3)
})
})
describe('#countTopLevelPostsByAddr', () => {
it('should count top-level posts for an address', async () => {
async function * mockHeights () {
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
yield ['000000600100:post-100', { txid: 'post-100' }]
}
postHeightsDb.iterator.returns(mockHeights())
postsDb.get.callsFake(async (txid) => {
const posts = {
'post-200-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
'post-200-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600200 },
'post-100': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600100 }
}
return posts[txid]
})
const result = await uut.countTopLevelPostsByAddr('addr-a')
assert.equal(result, 2)
})
})
describe('#buildReplyCountMap', () => {
it('should count replies per parent from postChildren', async () => {
async function * mockChildren () {
yield ['tx1:reply-a', { parentTxid: 'tx1', childTxid: 'reply-a', blockHeight: 600150 }]
yield ['tx1:reply-b', { parentTxid: 'tx1', childTxid: 'reply-b', blockHeight: 600160 }]
yield ['tx2:reply-c', { parentTxid: 'tx2', childTxid: 'reply-c', blockHeight: 600170 }]
}
async function * mockPosts () {
yield ['tx1', { addr: 'addr1', text: 'top', seen: 1000, blockHeight: 600100 }]
yield ['tx2', { addr: 'addr2', text: 'other', seen: 2000, blockHeight: 600200 }]
}
postChildrenDb.iterator.returns(mockChildren())
postsDb.iterator.returns(mockPosts())
const result = await uut.scanPostsWithBlockHeight()
const result = await uut.buildReplyCountMap()
assert.equal(result.length, 2)
assert.equal(result.find((p) => p.txid === 'tx1').replyCount, 2)
assert.equal(result.find((p) => p.txid === 'tx2').replyCount, 1)
assert.equal(result.get('tx1'), 2)
assert.equal(result.get('tx2'), 1)
})
it('should default replyCount to 0 when post has no replies', async () => {
async function * mockPosts () {
yield ['tx1', { addr: 'addr1', text: 'solo', seen: 1000, blockHeight: 600100 }]
}
postsDb.iterator.returns(mockPosts())
const result = await uut.scanPostsByAddr('addr1')
assert.equal(result.length, 1)
assert.equal(result[0].replyCount, 0)
})
})
@@ -19,7 +19,7 @@ describe('#LevelRESTController', () => {
sandbox = sinon.createSandbox()
uut = new LevelRESTControllerLib({
adapters: {
level: { postsDb: mockDb, statusDb: mockDb },
level: { postsDb: mockDb, postHeightsDb: mockDb, statusDb: mockDb },
dbBackup: { zipDb: sandbox.stub().resolves(true) }
},
useCases: {}
@@ -44,4 +44,15 @@ describe('#LevelRESTController', () => {
assert.equal(ctx.body.success, true)
assert.equal(ctx.body.txid, 'abc')
})
it('should expose a postheight entity handler', async () => {
const ctx = {
params: {},
request: { body: { key: '600000:abc', postHeightData: { txid: 'abc', blockHeight: 600000 } } },
body: null
}
await uut.entityHandlers.postheight.create(ctx)
assert.equal(ctx.body.success, true)
assert.equal(ctx.body.key, '600000:abc')
})
})
@@ -5,23 +5,32 @@ import ListPostsByAddr from '../../../src/use-cases/list-posts-by-addr.js'
describe('#ListPostsByAddr', () => {
let uut
let sandbox
let postQuery
const mockPosts = [
{ txid: 'tx-a', addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
{ txid: 'tx-b', addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
{ txid: 'tx-c', addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600200 }
]
const mockPosts = {
'tx-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
'tx-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
'tx-c': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600200 }
}
beforeEach(() => {
sandbox = sinon.createSandbox()
postQuery = {
scanPostsByAddrTxids: sandbox.stub().callsFake(async (addr, { limit, offset }) => {
const all = Object.entries(mockPosts)
.filter(([txid, post]) => post.addr === addr)
.sort((a, b) => b[1].blockHeight - a[1].blockHeight)
.map(([txid]) => txid)
return all.slice(offset, offset + limit)
}),
loadPostsByTxids: sandbox.stub().callsFake(async (txids) => {
return txids.map((txid) => ({ txid, ...mockPosts[txid] }))
}),
buildReplyCountMap: sandbox.stub().resolves(new Map()),
countTopLevelPostsByAddr: sandbox.stub().resolves(2)
}
uut = new ListPostsByAddr({
adapters: {
postQuery: {
scanPostsByAddr: sandbox.stub().callsFake(async (addr) => {
return mockPosts.filter((post) => post.addr === addr)
})
}
}
adapters: { postQuery }
})
})
@@ -45,4 +54,12 @@ describe('#ListPostsByAddr', () => {
assert.include(err.message, 'addr is required')
}
})
it('should pass addr, limit, and offset to postQuery', async () => {
await uut.execute({ addr: 'addr-a', limit: 5, offset: 10 })
assert.equal(postQuery.scanPostsByAddrTxids.calledOnce, true)
assert.equal(postQuery.scanPostsByAddrTxids.firstCall.args[0], 'addr-a')
assert.deepEqual(postQuery.scanPostsByAddrTxids.firstCall.args[1], { limit: 5, offset: 10 })
})
})
@@ -5,21 +5,26 @@ import ListRecentPosts from '../../../src/use-cases/list-recent-posts.js'
describe('#ListRecentPosts', () => {
let uut
let sandbox
let postQuery
const mockPosts = [
{ txid: 'tx-a', addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
{ txid: 'tx-b', addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
{ txid: 'tx-c', addr: 'addr-c', text: 'c', seen: 50, blockHeight: 600200 }
]
const mockPosts = {
'tx-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
'tx-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
'tx-c': { addr: 'addr-c', text: 'c', seen: 50, blockHeight: 600200 }
}
beforeEach(() => {
sandbox = sinon.createSandbox()
postQuery = {
scanRecentPostTxids: sandbox.stub().resolves(['tx-b', 'tx-c', 'tx-a']),
loadPostsByTxids: sandbox.stub().callsFake(async (txids) => {
return txids.map((txid) => ({ txid, ...mockPosts[txid] }))
}),
buildReplyCountMap: sandbox.stub().resolves(new Map([['tx-b', 1]])),
countTopLevelPosts: sandbox.stub().resolves(3)
}
uut = new ListRecentPosts({
adapters: {
postQuery: {
scanPostsWithBlockHeight: sandbox.stub().resolves([...mockPosts])
}
}
adapters: { postQuery }
})
})
@@ -32,11 +37,14 @@ describe('#ListRecentPosts', () => {
assert.equal(result.posts[0].txid, 'tx-b')
assert.equal(result.posts[1].txid, 'tx-c')
assert.equal(result.posts[2].txid, 'tx-a')
assert.equal(result.posts[0].replyCount, 1)
assert.equal(result.posts[1].replyCount, 0)
assert.equal(result.pagination.total, 3)
assert.equal(result.pagination.hasMore, false)
})
it('should paginate with limit and offset', async () => {
postQuery.scanRecentPostTxids.resolves(['tx-c'])
const result = await uut.execute({ limit: 1, offset: 1 })
assert.equal(result.posts.length, 1)
@@ -62,4 +70,11 @@ describe('#ListRecentPosts', () => {
assert.include(err.message, 'limit cannot exceed')
}
})
it('should pass limit and offset to postQuery', async () => {
await uut.execute({ limit: 5, offset: 10 })
assert.equal(postQuery.scanRecentPostTxids.calledOnce, true)
assert.deepEqual(postQuery.scanRecentPostTxids.firstCall.args[0], { limit: 5, offset: 10 })
})
})
+84
View File
@@ -0,0 +1,84 @@
/*
Normal acceptance runner for psf-memo-indexer.
*/
import { execFileSync } from 'node:child_process'
import fs from 'node:fs'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const root = path.resolve(__dirname, '..')
const specsDir = path.join(root, 'specs')
const buildDir = path.join(root, 'build', 'acceptance')
const irDir = path.join(buildDir, 'ir')
const genDir = path.join(buildDir, 'generated')
const apsDir = path.join(root, '..', 'tmp', 'aps-spec')
function sh (cmd, args, opts = {}) {
return execFileSync(cmd, args, {
stdio: ['pipe', 'pipe', 'pipe'],
...opts
}).toString()
}
function ensureAps () {
if (fs.existsSync(apsDir)) return
fs.mkdirSync(path.dirname(apsDir), { recursive: true })
sh('git', ['clone', '--depth', '1',
'https://github.com/unclebob/Acceptance-Pipeline-Specification.git', apsDir])
}
function main () {
ensureAps()
const features = fs
.readdirSync(specsDir)
.filter((f) => f.endsWith('.feature'))
.sort()
if (features.length === 0) {
console.log('No feature files found under specs/.')
return
}
fs.mkdirSync(irDir, { recursive: true })
fs.mkdirSync(genDir, { recursive: true })
for (const featureFile of features) {
const base = featureFile.replace(/\.feature$/i, '')
const featurePath = path.join(specsDir, featureFile)
const irPath = path.join(irDir, `${base}.json`)
sh('bb', ['gherkin-parser', featurePath, irPath], { cwd: apsDir })
sh('node', [path.join(__dirname, 'lib', 'generate.js'), irPath, genDir])
}
const tests = fs
.readdirSync(genDir)
.filter((f) => f.endsWith('.acceptance.test.js'))
.sort()
let failures = 0
for (const testFile of tests) {
try {
const out = sh('node', [path.join(genDir, testFile)])
process.stdout.write(out)
console.log(`ACCEPTANCE PASS: ${testFile}`)
} catch (err) {
failures++
process.stdout.write(err.stdout || '')
process.stderr.write(err.stderr || '')
console.error(`ACCEPTANCE FAIL: ${testFile}`)
}
}
if (failures > 0) {
console.error(`ACCEPTANCE: ${failures} failing test file(s)`)
process.exit(1)
} else {
console.log(`ACCEPTANCE: all ${tests.length} generated test file(s) passed`)
}
}
main()
@@ -0,0 +1,96 @@
/*
Project-specific acceptance entrypoint generator for psf-memo-indexer.
*/
import fs from 'node:fs'
import path from 'node:path'
import crypto from 'node:crypto'
import { fileURLToPath } from 'node:url'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
function metadataName (featureName) {
const slug = featureName
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
return `${slug || 'feature'}.json`
}
function relativeImport (fromDir, targetFile) {
let rel = path.relative(fromDir, targetFile).replace(/\\/g, '/')
if (!rel.startsWith('.')) rel = `./${rel}`
return rel
}
function main () {
const irArg = process.argv[2]
const outArg = process.argv[3]
if (!irArg || !outArg) {
console.error('usage: acceptance-entrypoint-generator <json-ir> <generated-test-output-dir>')
process.exit(2)
}
let ir
try {
ir = JSON.parse(fs.readFileSync(irArg, 'utf8'))
} catch (err) {
console.error(`Failed to read JSON IR "${irArg}": ${err.message}`)
process.exit(1)
}
const genDir = outArg
fs.mkdirSync(genDir, { recursive: true })
const featureKey = path.basename(irArg).replace(/\.json$/i, '')
const testFile = path.join(genDir, `${featureKey}.acceptance.test.js`)
const relRuntime = relativeImport(genDir, path.join(__dirname, 'runtime.js'))
const body = `import { runFeature } from '${relRuntime}'
const ir = ${JSON.stringify(ir, null, 2)}
async function main () {
const report = await runFeature(ir)
for (const r of report.results) {
console.log((r.status === 'passed' ? 'PASS ' : 'FAIL ') + r.name)
if (r.detail) console.log(' ' + r.detail)
}
if (report.failures > 0) {
console.error('ACCEPTANCE FAILURES: ' + report.failures + ' of ' + report.total)
process.exitCode = 1
}
}
main().catch((err) => { console.error(err); process.exit(1) })
`
try {
fs.writeFileSync(testFile, body)
} catch (err) {
console.error(`Failed to write generated test "${testFile}": ${err.message}`)
process.exit(1)
}
const metaDir = path.join(genDir, 'metadata')
fs.mkdirSync(metaDir, { recursive: true })
const hash = crypto
.createHash('sha256')
.update(fs.readFileSync(testFile))
.digest('hex')
const metadata = {
schema_version: 1,
feature_path: `${featureKey}.feature`,
ir_path: path.resolve(irArg),
implementation_hash: `sha256:${hash}`,
hash_scope: 'generated_files',
generated_files: [testFile]
}
fs.writeFileSync(
path.join(metaDir, metadataName(featureKey)),
JSON.stringify(metadata, null, 2)
)
process.exit(0)
}
main()
+268
View File
@@ -0,0 +1,268 @@
/*
Project step handlers for the psf-memo-indexer acceptance pipeline.
These handlers exercise the real Memo action handlers (handlePost, handleReply)
against an in-memory database that exposes the same CRUD surface as the
psf-memo-db entity routes used by the indexer.
*/
import crypto from 'node:crypto'
import { handlePost } from '../../src/use-cases/action-types/post.js'
import { handleReply } from '../../src/use-cases/action-types/reply.js'
function makeInMemoryDb () {
const store = new Map()
return {
async get (key) {
if (!store.has(key)) {
const err = new Error('not found')
err.notFound = true
throw err
}
return store.get(key)
},
async create (key, data) {
store.set(key, data)
return { success: true }
},
async update (key, data) {
store.set(key, data)
return { success: true }
},
async delete (key) {
store.delete(key)
return { success: true }
},
entries () {
return Array.from(store.entries())
}
}
}
function resolveParam (value, example) {
const match = /^\u003c([A-Za-z0-9_]+)\u003e$/.exec(String(value).trim())
if (match) {
const param = match[1]
if (!(param in example)) {
throw new Error(`Missing example value for "${param}"`)
}
return example[param]
}
return String(value).trim()
}
function deriveTxid (symbolic) {
// Produce a deterministic 32-byte buffer from a symbolic test txid and return
// its big-endian hex representation, which is what the indexer stores.
return crypto.createHash('sha256').update(symbolic).digest().toString('hex')
}
function resolveTxid (value, example, world) {
const resolved = resolveParam(value, example)
if (!world.txidMap) world.txidMap = new Map()
if (!world.txidMap.has(resolved)) {
world.txidMap.set(resolved, deriveTxid(resolved))
}
return world.txidMap.get(resolved)
}
async function createWorld () {
const postsDb = makeInMemoryDb()
const postHeightsDb = makeInMemoryDb()
const postParentsDb = makeInMemoryDb()
const postChildrenDb = makeInMemoryDb()
const adapters = {
postDb: postsDb,
postHeightDb: postHeightsDb,
postParentDb: postParentsDb,
postChildDb: postChildrenDb,
processErrorDb: makeInMemoryDb()
}
return {
adapters,
postsDb,
postHeightsDb,
postParentsDb,
postChildrenDb,
txidMap: new Map(),
lastTxid: null,
lastHeight: null,
lastAddr: null
}
}
const handlers = [
{
name: 'db instance with posts and postHeights stores',
pattern: /^a psf-memo-db instance with posts and postHeights stores$/,
async run () {
// World is already created with both stores.
}
},
{
name: 'indexer configured to write to db',
pattern: /^a psf-memo-indexer configured to write to that database$/,
async run () {
// Adapters object is already configured.
}
},
{
name: 'process a Memo post transaction',
pattern: /^the indexer processes a Memo post transaction (.+) from (.+) at block height (.+) with text "(.+)"$/,
async run (m, example, world) {
const txid = resolveTxid(m[1], example, world)
const addr = resolveParam(m[2], example)
const height = parseInt(resolveParam(m[3], example), 10)
const text = resolveParam(m[4], example)
world.lastTxid = txid
world.lastHeight = height
world.lastAddr = addr
const prefix = Buffer.from('6d02', 'hex')
const message = Buffer.from(text, 'utf8')
await handlePost({
adapters: world.adapters,
txid,
signerAddr: addr,
seen: Date.now(),
blockHeight: height,
decoded: {
action: 'post',
prefix,
pushDatas: [prefix, message]
}
})
}
},
{
name: 'process a Memo reply transaction',
pattern: /^the indexer processes a Memo reply transaction (.+) to parent (.+) from (.+) at block height (.+) with text "(.+)"$/,
async run (m, example, world) {
const txid = resolveTxid(m[1], example, world)
const parentTxid = resolveTxid(m[2], example, world)
const addr = resolveParam(m[3], example)
const height = parseInt(resolveParam(m[4], example), 10)
const text = resolveParam(m[5], example)
world.lastTxid = txid
world.lastHeight = height
world.lastAddr = addr
const prefix = Buffer.from('6d03', 'hex')
// handleReply expects the parent tx hash as a 32-byte buffer in the
// little-endian wire format; txHashFromPush reverses it to big-endian hex.
const parentHash = Buffer.from(parentTxid, 'hex').reverse()
const message = Buffer.from(text, 'utf8')
await handleReply({
adapters: world.adapters,
txid,
signerAddr: addr,
seen: Date.now(),
blockHeight: height,
decoded: {
action: 'reply',
prefix,
pushDatas: [prefix, parentHash, message]
}
})
}
},
{
name: 'process the same Memo post transaction again',
pattern: /^the indexer processes the same Memo post transaction (.+) again$/,
async run (m, example, world) {
const txid = resolveTxid(m[1], example, world)
const post = await world.postsDb.get(txid)
const prefix = Buffer.from('6d02', 'hex')
const message = Buffer.from(post.text, 'utf8')
await handlePost({
adapters: world.adapters,
txid,
signerAddr: post.addr,
seen: post.seen,
blockHeight: post.blockHeight,
decoded: {
action: 'post',
prefix,
pushDatas: [prefix, message]
}
})
}
},
{
name: 'posts store contains post document',
pattern: /^the posts store contains (.+) post document for (.+)$/,
run (m, example, world) {
const expectedCount = parseInt(resolveParam(m[1], example), 10)
const txid = resolveTxid(m[2], example, world)
const matching = world.postsDb.entries().filter(([key]) => key === txid)
if (matching.length !== expectedCount) {
throw new Error(`Expected ${expectedCount} post document(s) for ${txid}, got ${matching.length}`)
}
}
},
{
name: 'postHeights store contains entry',
pattern: /^the postHeights store contains (.+) entry whose key starts with the block height (.+) and ends with (.+)$/,
run (m, example, world) {
const expectedCount = parseInt(resolveParam(m[1], example), 10)
const height = resolveParam(m[2], example)
const txid = resolveTxid(m[3], example, world)
const prefix = String(height).padStart(12, '0')
const matching = world.postHeightsDb.entries().filter(([key, value]) => {
return key.startsWith(prefix) && (key.endsWith(`:${txid}`) || value?.txid === txid)
})
if (matching.length !== expectedCount) {
throw new Error(`Expected ${expectedCount} postHeights entry/entries for height ${height} txid ${txid}, got ${matching.length}`)
}
}
},
{
name: 'postParents store contains link',
pattern: /^the postParents store contains a link from (.+) to (.+)$/,
run (m, example, world) {
const childTxid = resolveTxid(m[1], example, world)
const parentTxid = resolveTxid(m[2], example, world)
const link = world.postParentsDb.entries().find(([key, value]) => {
return key === childTxid && value?.parentTxid === parentTxid
})
if (!link) {
throw new Error(`Expected postParents link from ${childTxid} to ${parentTxid}`)
}
}
},
{
name: 'postChildren store contains link',
pattern: /^the postChildren store contains a link from (.+) to (.+)$/,
run (m, example, world) {
const parentTxid = resolveTxid(m[1], example, world)
const childTxid = resolveTxid(m[2], example, world)
const link = world.postChildrenDb.entries().find(([key, value]) => {
return value?.parentTxid === parentTxid && value?.childTxid === childTxid
})
if (!link) {
throw new Error(`Expected postChildren link from ${parentTxid} to ${childTxid}`)
}
}
}
]
async function handleStep (step, example, world) {
for (const handler of handlers) {
const match = handler.pattern.exec(step.text)
if (match) {
await handler.run(match, example, world, step)
return
}
}
throw new Error(`Unsupported step: ${step.keyword} ${step.text}`)
}
export { createWorld, handleStep }
@@ -0,0 +1,57 @@
/*
Acceptance runtime for psf-memo-indexer.
*/
import { createWorld, handleStep } from './handlers.js'
function expandScenarios (ir) {
const executions = []
const background = ir.background || []
for (const scenario of ir.scenarios) {
const examples = (scenario.examples && scenario.examples.length > 0)
? scenario.examples.map((example, i) => ({ example, suffix: `example_${i + 1}` }))
: [{ example: {}, suffix: 'example_1' }]
for (const { example, suffix } of examples) {
executions.push({
name: `${scenario.name}/${suffix}`,
steps: [...background, ...scenario.steps],
example
})
}
}
return executions
}
async function runFeature (ir) {
const executions = expandScenarios(ir)
const results = []
let failures = 0
for (const ex of executions) {
const world = await createWorld()
let failure = null
for (const step of ex.steps) {
try {
await handleStep(step, ex.example, world)
} catch (err) {
failure = err.message
break
}
}
if (failure) {
failures++
results.push({ name: ex.name, status: 'failed', detail: failure })
} else {
results.push({ name: ex.name, status: 'passed' })
}
}
return { feature: ir.name, results, failures, total: results.length }
}
export { expandScenarios, runFeature }
+1
View File
@@ -7,6 +7,7 @@
"block-indexer": "node --max-old-space-size=8192 psf-memo-block-indexer.js",
"tx-indexer": "node --max-old-space-size=4096 psf-memo-tx-indexer.js",
"test": "c8 --reporter=text mocha --exit --recursive test/unit/",
"acceptance": "node acceptance/acceptance.js",
"lint": "standard --env mocha --fix"
},
"author": "Chris Troutner",
@@ -20,6 +20,7 @@ class Adapters {
this.dbCtrl = new DbCtrl()
this.postDb = createEntityDb('post', 'txid', 'postData')
this.postHeightDb = createEntityDb('postheight', 'key', 'postHeightData')
this.postParentDb = createEntityDb('postparent', 'txid', 'parentData')
this.postChildDb = createEntityDb('postchild', 'key', 'childData')
this.likeDb = createEntityDb('like', 'txid', 'likeData')
@@ -62,6 +62,11 @@ export function roomKey (roomName, txid) {
return `${roomName}:${txid}`
}
export function postHeightKey (blockHeight, txid) {
const padded = String(blockHeight).padStart(12, '0')
return `${padded}:${txid}`
}
export function postChildKey (parentTxid, childTxid) {
return `${parentTxid}:${childTxid}`
}
@@ -1,4 +1,4 @@
import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js'
import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas, postHeightKey } from './helpers.js'
import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
export async function handlePost (ctx) {
@@ -21,9 +21,16 @@ export async function handlePost (ctx) {
}
const postData = { addr: signerAddr, text, seen, blockHeight }
const heightKey = postHeightKey(blockHeight, txid)
try {
await adapters.postDb.get(txid)
} catch (err) {
await adapters.postDb.create(txid, postData)
}
try {
await adapters.postHeightDb.get(heightKey)
} catch (err) {
await adapters.postHeightDb.create(heightKey, { txid, blockHeight })
}
}
@@ -7,9 +7,12 @@ describe('#handlePost', () => {
it('should save a post to the database', async () => {
const create = sinon.stub().resolves({ success: true })
const get = sinon.stub().rejects(new Error('not found'))
const postHeightCreate = sinon.stub().resolves({ success: true })
const postHeightGet = sinon.stub().rejects(new Error('not found'))
const adapters = {
postDb: { create, get },
postHeightDb: { create: postHeightCreate, get: postHeightGet },
processErrorDb: { create: sinon.stub() }
}
@@ -31,5 +34,40 @@ describe('#handlePost', () => {
assert.equal(create.firstCall.args[0], 'abc123')
assert.equal(create.firstCall.args[1].text, 'hello memo')
assert.equal(create.firstCall.args[1].blockHeight, 600100)
assert.equal(postHeightCreate.callCount, 1)
assert.equal(postHeightCreate.firstCall.args[0], '000000600100:abc123')
assert.equal(postHeightCreate.firstCall.args[1].txid, 'abc123')
assert.equal(postHeightCreate.firstCall.args[1].blockHeight, 600100)
})
it('should not duplicate postHeight entries when reprocessing', async () => {
const create = sinon.stub().resolves({ success: true })
const get = sinon.stub().resolves({ addr: 'bitcoincash:qptest', text: 'hello memo', seen: 1000, blockHeight: 600100 })
const postHeightCreate = sinon.stub().resolves({ success: true })
const postHeightGet = sinon.stub().resolves({ txid: 'abc123', blockHeight: 600100 })
const adapters = {
postDb: { create, get },
postHeightDb: { create: postHeightCreate, get: postHeightGet },
processErrorDb: { create: sinon.stub() }
}
const message = Buffer.from('hello memo')
await handlePost({
adapters,
txid: 'abc123',
signerAddr: 'bitcoincash:qptest',
seen: 1000,
blockHeight: 600100,
decoded: {
action: 'post',
prefix: PREFIX_POST,
pushDatas: [PREFIX_POST, message]
}
})
assert.equal(create.callCount, 0)
assert.equal(postHeightCreate.callCount, 0)
})
})
@@ -0,0 +1,54 @@
import { assert } from 'chai'
import sinon from 'sinon'
import { handleReply } from '../../../../src/use-cases/action-types/reply.js'
import { PREFIX_REPLY } from '../../../../src/lib/memo-codes.js'
describe('#handleReply', () => {
it('should save a reply and its postHeight index entry', async () => {
const parentTxid = Buffer.alloc(32, 0xab)
const message = Buffer.from('hi there')
const postParentCreate = sinon.stub().resolves({ success: true })
const postChildCreate = sinon.stub().resolves({ success: true })
const postDbGet = sinon.stub().rejects(new Error('not found'))
const postDbCreate = sinon.stub().resolves({ success: true })
const postHeightGet = sinon.stub().rejects(new Error('not found'))
const postHeightCreate = sinon.stub().resolves({ success: true })
const adapters = {
postParentDb: { create: postParentCreate },
postChildDb: { create: postChildCreate },
postDb: { get: postDbGet, create: postDbCreate },
postHeightDb: { get: postHeightGet, create: postHeightCreate },
processErrorDb: { create: sinon.stub() }
}
await handleReply({
adapters,
txid: 'reply-abc',
signerAddr: 'bitcoincash:qptest',
seen: 1000,
blockHeight: 600150,
decoded: {
action: 'reply',
prefix: PREFIX_REPLY,
pushDatas: [PREFIX_REPLY, parentTxid, message]
}
})
assert.equal(postParentCreate.callCount, 1)
assert.equal(postParentCreate.firstCall.args[0], 'reply-abc')
assert.equal(postParentCreate.firstCall.args[1].parentTxid, parentTxid.toString('hex'))
assert.equal(postChildCreate.callCount, 1)
assert.include(postChildCreate.firstCall.args[0], 'reply-abc')
assert.equal(postDbCreate.callCount, 1)
assert.equal(postDbCreate.firstCall.args[0], 'reply-abc')
assert.equal(postDbCreate.firstCall.args[1].text, 'hi there')
assert.equal(postHeightCreate.callCount, 1)
assert.equal(postHeightCreate.firstCall.args[0], '000000600150:reply-abc')
assert.equal(postHeightCreate.firstCall.args[1].txid, 'reply-abc')
})
})