mirror of
https://github.com/Permissionless-Software-Foundation/psf-memo.git
synced 2026-09-21 16:52:01 -07:00
Implement postHeights secondary index for efficient pagination
Adds a postHeights LevelDB store written by the indexer and used by psf-memo-db to paginate recent posts and posts-by-address without full scans. Includes unit tests and Gherkin acceptance pipelines for both psf-memo-indexer and psf-memo-db. By coder.
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
Normal acceptance runner for psf-memo-db.
|
||||
|
||||
Orchestrates the acceptance pipeline:
|
||||
feature file -> bb gherkin-parser -> JSON IR -> acceptance entrypoint
|
||||
generator -> generated test entry points -> node test runner
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const root = path.resolve(__dirname, '..')
|
||||
const specsDir = path.join(root, 'specs')
|
||||
const buildDir = path.join(root, 'build', 'acceptance')
|
||||
const irDir = path.join(buildDir, 'ir')
|
||||
const genDir = path.join(buildDir, 'generated')
|
||||
const apsDir = path.join(root, '..', 'tmp', 'aps-spec')
|
||||
|
||||
function sh (cmd, args, opts = {}) {
|
||||
return execFileSync(cmd, args, {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
...opts
|
||||
}).toString()
|
||||
}
|
||||
|
||||
function ensureAps () {
|
||||
if (fs.existsSync(apsDir)) return
|
||||
fs.mkdirSync(path.dirname(apsDir), { recursive: true })
|
||||
sh('git', ['clone', '--depth', '1',
|
||||
'https://github.com/unclebob/Acceptance-Pipeline-Specification.git', apsDir])
|
||||
}
|
||||
|
||||
function main () {
|
||||
ensureAps()
|
||||
|
||||
const features = fs
|
||||
.readdirSync(specsDir)
|
||||
.filter((f) => f.endsWith('.feature'))
|
||||
.sort()
|
||||
|
||||
if (features.length === 0) {
|
||||
console.log('No feature files found under specs/.')
|
||||
return
|
||||
}
|
||||
|
||||
fs.mkdirSync(irDir, { recursive: true })
|
||||
fs.mkdirSync(genDir, { recursive: true })
|
||||
|
||||
for (const featureFile of features) {
|
||||
const base = featureFile.replace(/\.feature$/i, '')
|
||||
const featurePath = path.join(specsDir, featureFile)
|
||||
const irPath = path.join(irDir, `${base}.json`)
|
||||
|
||||
sh('bb', ['gherkin-parser', featurePath, irPath], { cwd: apsDir })
|
||||
sh('node', [path.join(__dirname, 'lib', 'generate.js'), irPath, genDir])
|
||||
}
|
||||
|
||||
const tests = fs
|
||||
.readdirSync(genDir)
|
||||
.filter((f) => f.endsWith('.acceptance.test.js'))
|
||||
.sort()
|
||||
|
||||
let failures = 0
|
||||
for (const testFile of tests) {
|
||||
try {
|
||||
const out = sh('node', [path.join(genDir, testFile)])
|
||||
process.stdout.write(out)
|
||||
console.log(`ACCEPTANCE PASS: ${testFile}`)
|
||||
} catch (err) {
|
||||
failures++
|
||||
process.stdout.write(err.stdout || '')
|
||||
process.stderr.write(err.stderr || '')
|
||||
console.error(`ACCEPTANCE FAIL: ${testFile}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (failures > 0) {
|
||||
console.error(`ACCEPTANCE: ${failures} failing test file(s)`)
|
||||
process.exit(1)
|
||||
} else {
|
||||
console.log(`ACCEPTANCE: all ${tests.length} generated test file(s) passed`)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
Project-specific acceptance entrypoint generator for psf-memo-db.
|
||||
|
||||
Reads parser JSON IR and writes executable generated test entry points plus
|
||||
per-feature metadata. Generated tests delegate all step behavior to the
|
||||
acceptance runtime and project step handlers.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import crypto from 'node:crypto'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
function metadataName (featureName) {
|
||||
const slug = featureName
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
return `${slug || 'feature'}.json`
|
||||
}
|
||||
|
||||
function relativeImport (fromDir, targetFile) {
|
||||
let rel = path.relative(fromDir, targetFile).replace(/\\/g, '/')
|
||||
if (!rel.startsWith('.')) rel = `./${rel}`
|
||||
return rel
|
||||
}
|
||||
|
||||
function main () {
|
||||
const irArg = process.argv[2]
|
||||
const outArg = process.argv[3]
|
||||
|
||||
if (!irArg || !outArg) {
|
||||
console.error('usage: acceptance-entrypoint-generator <json-ir> <generated-test-output-dir>')
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
let ir
|
||||
try {
|
||||
ir = JSON.parse(fs.readFileSync(irArg, 'utf8'))
|
||||
} catch (err) {
|
||||
console.error(`Failed to read JSON IR "${irArg}": ${err.message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const genDir = outArg
|
||||
fs.mkdirSync(genDir, { recursive: true })
|
||||
|
||||
const featureKey = path.basename(irArg).replace(/\.json$/i, '')
|
||||
const testFile = path.join(genDir, `${featureKey}.acceptance.test.js`)
|
||||
const relRuntime = relativeImport(genDir, path.join(__dirname, 'runtime.js'))
|
||||
|
||||
const body = `import { runFeature } from '${relRuntime}'
|
||||
const ir = ${JSON.stringify(ir, null, 2)}
|
||||
async function main () {
|
||||
const report = await runFeature(ir)
|
||||
for (const r of report.results) {
|
||||
console.log((r.status === 'passed' ? 'PASS ' : 'FAIL ') + r.name)
|
||||
if (r.detail) console.log(' ' + r.detail)
|
||||
}
|
||||
if (report.failures > 0) {
|
||||
console.error('ACCEPTANCE FAILURES: ' + report.failures + ' of ' + report.total)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
main().catch((err) => { console.error(err); process.exit(1) })
|
||||
`
|
||||
|
||||
try {
|
||||
fs.writeFileSync(testFile, body)
|
||||
} catch (err) {
|
||||
console.error(`Failed to write generated test "${testFile}": ${err.message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const metaDir = path.join(genDir, 'metadata')
|
||||
fs.mkdirSync(metaDir, { recursive: true })
|
||||
const hash = crypto
|
||||
.createHash('sha256')
|
||||
.update(fs.readFileSync(testFile))
|
||||
.digest('hex')
|
||||
|
||||
const metadata = {
|
||||
schema_version: 1,
|
||||
feature_path: `${featureKey}.feature`,
|
||||
ir_path: path.resolve(irArg),
|
||||
implementation_hash: `sha256:${hash}`,
|
||||
hash_scope: 'generated_files',
|
||||
generated_files: [testFile]
|
||||
}
|
||||
fs.writeFileSync(
|
||||
path.join(metaDir, metadataName(featureKey)),
|
||||
JSON.stringify(metadata, null, 2)
|
||||
)
|
||||
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,289 @@
|
||||
/*
|
||||
Project step handlers for the psf-memo-db acceptance pipeline.
|
||||
|
||||
These handlers spin up a real psf-memo-db adapter set against a temporary
|
||||
LevelDB directory, load the Gherkin fixtures, and exercise the real use cases
|
||||
for recent posts and posts-by-address. Iterators and get calls are wrapped
|
||||
so the efficiency steps can assert bounded reads.
|
||||
*/
|
||||
|
||||
import path from 'node:path'
|
||||
import fs from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { DB_NAMES } from '../../src/adapters/level-db.js'
|
||||
import Adapters from '../../src/adapters/index.js'
|
||||
import ListRecentPosts from '../../src/use-cases/list-recent-posts.js'
|
||||
import ListPostsByAddr from '../../src/use-cases/list-posts-by-addr.js'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const tmpDir = path.resolve(__dirname, '..', '..', 'tmp', 'acceptance')
|
||||
|
||||
function resolveParam (value, example) {
|
||||
const match = /^<([A-Za-z0-9_]+)>$/.exec(String(value).trim())
|
||||
if (match) {
|
||||
const param = match[1]
|
||||
if (!(param in example)) {
|
||||
throw new Error(`Missing example value for "${param}"`)
|
||||
}
|
||||
return example[param]
|
||||
}
|
||||
return String(value).trim()
|
||||
}
|
||||
|
||||
function wrapIterator (db, counter) {
|
||||
const original = db.iterator.bind(db)
|
||||
db.iterator = function (...args) {
|
||||
counter.calls++
|
||||
return original(...args)
|
||||
}
|
||||
}
|
||||
|
||||
function wrapGet (db, counter) {
|
||||
const original = db.get.bind(db)
|
||||
db.get = async function (...args) {
|
||||
counter.calls++
|
||||
return original(...args)
|
||||
}
|
||||
}
|
||||
|
||||
async function createWorld () {
|
||||
const levelDir = path.join(tmpDir, `level-${Date.now()}-${Math.random().toString(36).slice(2)}`)
|
||||
const adapters = new Adapters()
|
||||
|
||||
// Force the LevelDB adapter to use a temporary directory for this scenario.
|
||||
adapters.levelDb.openDbs = function () {
|
||||
const dbs = {}
|
||||
fs.mkdirSync(levelDir, { recursive: true })
|
||||
for (const name of DB_NAMES) {
|
||||
const prop = `${name}Db`
|
||||
const storeDir = path.join(levelDir, name)
|
||||
fs.mkdirSync(storeDir, { recursive: true })
|
||||
dbs[prop] = adapters.levelDb.level(storeDir, {
|
||||
valueEncoding: 'json',
|
||||
cacheSize: name === 'posts' ? 512 * 1024 * 1024 : 64 * 1024 * 1024
|
||||
})
|
||||
this[prop] = dbs[prop]
|
||||
}
|
||||
return dbs
|
||||
}
|
||||
|
||||
adapters.start()
|
||||
|
||||
const postHeightsIteratorCounter = { calls: 0 }
|
||||
const postChildrenIteratorCounter = { calls: 0 }
|
||||
const postsGetCounter = { calls: 0 }
|
||||
wrapIterator(adapters.level.postHeightsDb, postHeightsIteratorCounter)
|
||||
wrapIterator(adapters.level.postChildrenDb, postChildrenIteratorCounter)
|
||||
wrapGet(adapters.level.postsDb, postsGetCounter)
|
||||
|
||||
const listRecentPosts = new ListRecentPosts({ adapters })
|
||||
const listPostsByAddr = new ListPostsByAddr({ adapters })
|
||||
|
||||
let lastResponse = null
|
||||
|
||||
return {
|
||||
adapters,
|
||||
listRecentPosts,
|
||||
listPostsByAddr,
|
||||
postHeightsIteratorCounter,
|
||||
postChildrenIteratorCounter,
|
||||
postsGetCounter,
|
||||
getLastResponse: () => lastResponse,
|
||||
setLastResponse: (resp) => { lastResponse = resp },
|
||||
close: async () => {
|
||||
try {
|
||||
await adapters.levelDb.closeDbs()
|
||||
} catch (err) {
|
||||
// ignore close errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function loadFixture (world, name) {
|
||||
if (name !== 'three-top-level-posts-and-one-reply') {
|
||||
throw new Error(`Unknown fixture: ${name}`)
|
||||
}
|
||||
|
||||
const posts = [
|
||||
{ txid: 'post-200-b', addr: 'bitcoincash:qaddr-b', text: 'b', seen: 200, blockHeight: 600200 },
|
||||
{ txid: 'post-200-a', addr: 'bitcoincash:qaddr-a', text: 'a', seen: 100, blockHeight: 600200 },
|
||||
{ txid: 'post-100', addr: 'bitcoincash:qaddr-a', text: 'c', seen: 50, blockHeight: 600100 }
|
||||
]
|
||||
const reply = { txid: 'reply-1', parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600050 }
|
||||
|
||||
for (const post of posts) {
|
||||
await world.adapters.level.postsDb.put(post.txid, {
|
||||
addr: post.addr,
|
||||
text: post.text,
|
||||
seen: post.seen,
|
||||
blockHeight: post.blockHeight
|
||||
})
|
||||
await world.adapters.level.postHeightsDb.put(
|
||||
String(post.blockHeight).padStart(12, '0') + ':' + post.txid,
|
||||
{ txid: post.txid, blockHeight: post.blockHeight }
|
||||
)
|
||||
}
|
||||
|
||||
await world.adapters.level.postsDb.put('reply-1', {
|
||||
addr: 'bitcoincash:qaddr-a',
|
||||
text: 'reply body',
|
||||
seen: 10,
|
||||
blockHeight: 600050
|
||||
})
|
||||
await world.adapters.level.postHeightsDb.put(
|
||||
'000000600050:reply-1',
|
||||
{ txid: 'reply-1', blockHeight: 600050 }
|
||||
)
|
||||
await world.adapters.level.postParentsDb.put('reply-1', reply)
|
||||
await world.adapters.level.postChildrenDb.put('post-200-a:reply-1', reply)
|
||||
}
|
||||
|
||||
const handlers = [
|
||||
{
|
||||
name: 'db instance with posts and postHeights stores',
|
||||
pattern: /^a psf-memo-db instance with a posts store and a postHeights secondary index$/,
|
||||
async run () {
|
||||
// World is already created with both stores.
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'load fixture',
|
||||
pattern: /^the fixture "(.+)" is loaded into the posts store$/,
|
||||
async run (m, example, world) {
|
||||
await loadFixture(world, m[1])
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'request recent posts',
|
||||
pattern: /^the client requests \/posts\/recent with limit (<limit>) and offset (<offset>)$/,
|
||||
async run (m, example, world) {
|
||||
const limit = parseInt(resolveParam(m[1], example), 10)
|
||||
const offset = parseInt(resolveParam(m[2], example), 10)
|
||||
const resp = await world.listRecentPosts.execute({ limit, offset })
|
||||
world.setLastResponse(resp)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'request posts by addr',
|
||||
pattern: /^the client requests \/posts\/by\/(<addr>) with limit (<limit>) and offset (<offset>)$/,
|
||||
async run (m, example, world) {
|
||||
const addr = resolveParam(m[1], example)
|
||||
const limit = parseInt(resolveParam(m[2], example), 10)
|
||||
const offset = parseInt(resolveParam(m[3], example), 10)
|
||||
const resp = await world.listPostsByAddr.execute({ addr, limit, offset })
|
||||
world.setLastResponse(resp)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'posts sorted by block height descending',
|
||||
pattern: /^the response posts are sorted by block height descending$/,
|
||||
run (m, example, world) {
|
||||
const posts = world.getLastResponse().posts
|
||||
for (let i = 1; i < posts.length; i++) {
|
||||
if (posts[i].blockHeight > posts[i - 1].blockHeight) {
|
||||
throw new Error(`Posts not sorted by descending block height at index ${i}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'response contains expected txids',
|
||||
pattern: /^the response contains the txids (<expected_txids>)$/,
|
||||
run (m, example, world) {
|
||||
const expected = resolveParam(m[1], example).split(',').map((s) => s.trim())
|
||||
const actual = world.getLastResponse().posts.map((p) => p.txid)
|
||||
if (expected.join(',') !== actual.join(',')) {
|
||||
throw new Error(`Expected txids ${expected.join(',')}, got ${actual.join(',')}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'response contains only posts by addr',
|
||||
pattern: /^the response contains only posts by (<addr>)$/,
|
||||
run (m, example, world) {
|
||||
const addr = resolveParam(m[1], example)
|
||||
const posts = world.getLastResponse().posts
|
||||
for (const post of posts) {
|
||||
if (post.addr !== addr) {
|
||||
throw new Error(`Expected post by ${addr}, got ${post.addr}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'response pagination metadata',
|
||||
pattern: /^the response pagination shows total (<total>) and hasMore (<hasMore>)$/,
|
||||
run (m, example, world) {
|
||||
const expectedTotal = parseInt(resolveParam(m[1], example), 10)
|
||||
const expectedHasMore = resolveParam(m[2], example) === 'true'
|
||||
const pagination = world.getLastResponse().pagination
|
||||
if (pagination.total !== expectedTotal) {
|
||||
throw new Error(`Expected total ${expectedTotal}, got ${pagination.total}`)
|
||||
}
|
||||
if (pagination.hasMore !== expectedHasMore) {
|
||||
throw new Error(`Expected hasMore ${expectedHasMore}, got ${pagination.hasMore}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'bounded postHeights reads',
|
||||
pattern: /^no more than (<limit>) postHeights entries are read after applying the offset$/,
|
||||
run (m, example, world) {
|
||||
const limit = parseInt(resolveParam(m[1], example), 10)
|
||||
const reads = world.postHeightsIteratorCounter.calls
|
||||
if (reads > limit) {
|
||||
throw new Error(`Read ${reads} postHeights entries, expected at most ${limit}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'bounded posts loaded by txid',
|
||||
pattern: /^no more than (<limit>) posts are loaded by txid$/,
|
||||
run (m, example, world) {
|
||||
const limit = parseInt(resolveParam(m[1], example), 10)
|
||||
const reads = world.postsGetCounter.calls
|
||||
if (reads > limit) {
|
||||
throw new Error(`Loaded ${reads} posts by txid, expected at most ${limit}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'post has replyCount',
|
||||
pattern: /^the response post with txid (<txid>) has replyCount (<replyCount>)$/,
|
||||
run (m, example, world) {
|
||||
const txid = resolveParam(m[1], example)
|
||||
const expected = parseInt(resolveParam(m[2], example), 10)
|
||||
const post = world.getLastResponse().posts.find((p) => p.txid === txid)
|
||||
if (!post) {
|
||||
throw new Error(`Post ${txid} not found in response`)
|
||||
}
|
||||
if (post.replyCount !== expected) {
|
||||
throw new Error(`Expected replyCount ${expected} for ${txid}, got ${post.replyCount}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'single postChildren scan',
|
||||
pattern: /^the postChildren store was iterated exactly once$/,
|
||||
run (m, example, world) {
|
||||
const calls = world.postChildrenIteratorCounter.calls
|
||||
if (calls !== 1) {
|
||||
throw new Error(`Expected exactly one postChildren scan, got ${calls}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
async function handleStep (step, example, world) {
|
||||
for (const handler of handlers) {
|
||||
const match = handler.pattern.exec(step.text)
|
||||
if (match) {
|
||||
await handler.run(match, example, world, step)
|
||||
return
|
||||
}
|
||||
}
|
||||
throw new Error(`Unsupported step: ${step.keyword} ${step.text}`)
|
||||
}
|
||||
|
||||
export { createWorld, handleStep }
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
Acceptance runtime for psf-memo-db.
|
||||
|
||||
Expands each scenario (and each example row) from parser JSON IR into
|
||||
scenario executions, prepends background steps, and dispatches every step to
|
||||
the project step handlers.
|
||||
*/
|
||||
|
||||
import { createWorld, handleStep } from './handlers.js'
|
||||
|
||||
// Expand the IR scenarios into concrete executions.
|
||||
function expandScenarios (ir) {
|
||||
const executions = []
|
||||
const background = ir.background || []
|
||||
|
||||
for (const scenario of ir.scenarios) {
|
||||
const examples = (scenario.examples && scenario.examples.length > 0)
|
||||
? scenario.examples.map((example, i) => ({ example, suffix: `example_${i + 1}` }))
|
||||
: [{ example: {}, suffix: 'example_1' }]
|
||||
|
||||
for (const { example, suffix } of examples) {
|
||||
executions.push({
|
||||
name: `${scenario.name}/${suffix}`,
|
||||
steps: [...background, ...scenario.steps],
|
||||
example
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return executions
|
||||
}
|
||||
|
||||
async function runFeature (ir) {
|
||||
const executions = expandScenarios(ir)
|
||||
const results = []
|
||||
let failures = 0
|
||||
|
||||
for (const ex of executions) {
|
||||
const world = await createWorld()
|
||||
let failure = null
|
||||
|
||||
for (const step of ex.steps) {
|
||||
try {
|
||||
await handleStep(step, ex.example, world)
|
||||
} catch (err) {
|
||||
failure = err.message
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
await world.close()
|
||||
|
||||
if (failure) {
|
||||
failures++
|
||||
results.push({ name: ex.name, status: 'failed', detail: failure })
|
||||
} else {
|
||||
results.push({ name: ex.name, status: 'passed' })
|
||||
}
|
||||
}
|
||||
|
||||
return { feature: ir.name, results, failures, total: results.length }
|
||||
}
|
||||
|
||||
export { expandScenarios, runFeature }
|
||||
@@ -8,6 +8,7 @@
|
||||
"prestart": "npm run docs",
|
||||
"start": "node index.js",
|
||||
"test": "export SVC_ENV=test && c8 --reporter=text mocha --exit --timeout 15000 --recursive test/unit/",
|
||||
"acceptance": "node acceptance/acceptance.js",
|
||||
"lint": "standard --env mocha --fix",
|
||||
"docs": "./node_modules/.bin/apidoc -i src/ -o docs"
|
||||
},
|
||||
|
||||
@@ -24,6 +24,7 @@ class Adapters {
|
||||
})
|
||||
this.postQuery = new PostQuery({
|
||||
postsDb: level.postsDb,
|
||||
postHeightsDb: level.postHeightsDb,
|
||||
postParentsDb: level.postParentsDb,
|
||||
postChildrenDb: level.postChildrenDb
|
||||
})
|
||||
|
||||
@@ -12,6 +12,7 @@ const dbDir = `${__dirname}/../../leveldb`
|
||||
const DB_NAMES = [
|
||||
'status',
|
||||
'posts',
|
||||
'postHeights',
|
||||
'postParents',
|
||||
'postChildren',
|
||||
'likes',
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
/*
|
||||
Adapter for scanning posts with stored block height.
|
||||
Excludes reply posts (txids present in postParentsDb).
|
||||
Adapter for efficient post queries using a postHeights secondary index.
|
||||
*/
|
||||
|
||||
const HEIGHT_PAD = 12
|
||||
|
||||
class PostQuery {
|
||||
constructor (localConfig = {}) {
|
||||
const { postsDb, postParentsDb, postChildrenDb } = localConfig
|
||||
const { postsDb, postHeightsDb, postParentsDb, postChildrenDb } = localConfig
|
||||
if (!postsDb) {
|
||||
throw new Error('postsDb required when instantiating PostQuery adapter.')
|
||||
}
|
||||
if (!postHeightsDb) {
|
||||
throw new Error('postHeightsDb required when instantiating PostQuery adapter.')
|
||||
}
|
||||
if (!postParentsDb) {
|
||||
throw new Error('postParentsDb required when instantiating PostQuery adapter.')
|
||||
}
|
||||
@@ -16,12 +20,32 @@ class PostQuery {
|
||||
throw new Error('postChildrenDb required when instantiating PostQuery adapter.')
|
||||
}
|
||||
this.postsDb = postsDb
|
||||
this.postHeightsDb = postHeightsDb
|
||||
this.postParentsDb = postParentsDb
|
||||
this.postChildrenDb = postChildrenDb
|
||||
this.scanPostsWithBlockHeight = this.scanPostsWithBlockHeight.bind(this)
|
||||
this.scanPostsByAddr = this.scanPostsByAddr.bind(this)
|
||||
|
||||
this.scanRecentPostTxids = this.scanRecentPostTxids.bind(this)
|
||||
this.scanPostsByAddrTxids = this.scanPostsByAddrTxids.bind(this)
|
||||
this.loadPostsByTxids = this.loadPostsByTxids.bind(this)
|
||||
this.countTopLevelPosts = this.countTopLevelPosts.bind(this)
|
||||
this.countTopLevelPostsByAddr = this.countTopLevelPostsByAddr.bind(this)
|
||||
this.loadReplyTxids = this.loadReplyTxids.bind(this)
|
||||
this.buildReplyCountMap = this.buildReplyCountMap.bind(this)
|
||||
this.txidFromPostHeight = this.txidFromPostHeight.bind(this)
|
||||
}
|
||||
|
||||
static padHeight (height) {
|
||||
return String(height).padStart(HEIGHT_PAD, '0')
|
||||
}
|
||||
|
||||
static postHeightKey (blockHeight, txid) {
|
||||
return `${PostQuery.padHeight(blockHeight)}:${txid}`
|
||||
}
|
||||
|
||||
txidFromPostHeight (key, value) {
|
||||
if (value && typeof value.txid === 'string') return value.txid
|
||||
const parts = String(key).split(':')
|
||||
return parts[parts.length - 1]
|
||||
}
|
||||
|
||||
async loadReplyTxids () {
|
||||
@@ -38,82 +62,120 @@ class PostQuery {
|
||||
const counts = new Map()
|
||||
|
||||
for await (const [, child] of this.postChildrenDb.iterator()) {
|
||||
const parentTxid = child.parentTxid
|
||||
if (!parentTxid) continue
|
||||
counts.set(parentTxid, (counts.get(parentTxid) || 0) + 1)
|
||||
}
|
||||
|
||||
return counts
|
||||
}
|
||||
|
||||
async scanPostsWithBlockHeight () {
|
||||
const [replyTxids, replyCounts] = await Promise.all([
|
||||
this.loadReplyTxids(),
|
||||
this.buildReplyCountMap()
|
||||
])
|
||||
const posts = []
|
||||
|
||||
for await (const [txid, post] of this.postsDb.iterator()) {
|
||||
if (replyTxids.has(txid)) continue
|
||||
posts.push({
|
||||
txid,
|
||||
addr: post.addr,
|
||||
text: post.text,
|
||||
seen: post.seen,
|
||||
blockHeight: post.blockHeight ?? 0,
|
||||
replyCount: replyCounts.get(txid) ?? 0
|
||||
})
|
||||
}
|
||||
|
||||
return posts
|
||||
}
|
||||
|
||||
async scanPostsByAddr (addr) {
|
||||
const [replyTxids, replyCounts] = await Promise.all([
|
||||
this.loadReplyTxids(),
|
||||
this.buildReplyCountMap()
|
||||
])
|
||||
const posts = []
|
||||
|
||||
for await (const [txid, post] of this.postsDb.iterator()) {
|
||||
if (post.addr !== addr) continue
|
||||
if (replyTxids.has(txid)) continue
|
||||
posts.push({
|
||||
txid,
|
||||
addr: post.addr,
|
||||
text: post.text,
|
||||
seen: post.seen,
|
||||
blockHeight: post.blockHeight ?? 0,
|
||||
replyCount: replyCounts.get(txid) ?? 0
|
||||
})
|
||||
}
|
||||
|
||||
return posts
|
||||
}
|
||||
|
||||
async buildReplyCountMap () {
|
||||
const counts = new Map()
|
||||
let total = 0
|
||||
|
||||
for await (const [childTxid, child] of this.postChildrenDb.iterator()) {
|
||||
total++
|
||||
|
||||
console.log('Indexed reply:', {
|
||||
childTxid,
|
||||
child,
|
||||
parentTxid: child?.parentTxid
|
||||
})
|
||||
|
||||
const parentTxid = child?.parentTxid
|
||||
if (!parentTxid) continue
|
||||
|
||||
counts.set(parentTxid, (counts.get(parentTxid) || 0) + 1)
|
||||
}
|
||||
|
||||
console.log(`Total postChildrenDb records: ${total}`)
|
||||
|
||||
return counts
|
||||
}
|
||||
}
|
||||
|
||||
async scanRecentPostTxids ({ limit, offset }) {
|
||||
const replyTxids = await this.loadReplyTxids()
|
||||
const txids = []
|
||||
let skipped = 0
|
||||
|
||||
for await (const [key, value] of this.postHeightsDb.iterator({ reverse: true })) {
|
||||
const txid = this.txidFromPostHeight(key, value)
|
||||
if (replyTxids.has(txid)) continue
|
||||
|
||||
if (skipped < offset) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
txids.push(txid)
|
||||
if (txids.length >= limit) break
|
||||
}
|
||||
|
||||
return txids
|
||||
}
|
||||
|
||||
async scanPostsByAddrTxids (addr, { limit, offset }) {
|
||||
const replyTxids = await this.loadReplyTxids()
|
||||
const txids = []
|
||||
let skipped = 0
|
||||
|
||||
for await (const [key, value] of this.postHeightsDb.iterator({ reverse: true })) {
|
||||
const txid = this.txidFromPostHeight(key, value)
|
||||
if (replyTxids.has(txid)) continue
|
||||
|
||||
let post
|
||||
try {
|
||||
post = await this.postsDb.get(txid)
|
||||
} catch (err) {
|
||||
if (err.notFound || err.code === 'LEVEL_NOT_FOUND') continue
|
||||
throw err
|
||||
}
|
||||
|
||||
if (post.addr !== addr) continue
|
||||
|
||||
if (skipped < offset) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
txids.push(txid)
|
||||
if (txids.length >= limit) break
|
||||
}
|
||||
|
||||
return txids
|
||||
}
|
||||
|
||||
async loadPostsByTxids (txids) {
|
||||
const posts = []
|
||||
|
||||
for (const txid of txids) {
|
||||
try {
|
||||
const post = await this.postsDb.get(txid)
|
||||
posts.push({
|
||||
txid,
|
||||
addr: post.addr,
|
||||
text: post.text,
|
||||
seen: post.seen,
|
||||
blockHeight: post.blockHeight ?? 0
|
||||
})
|
||||
} catch (err) {
|
||||
if (err.notFound || err.code === 'LEVEL_NOT_FOUND') continue
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
return posts
|
||||
}
|
||||
|
||||
async countTopLevelPosts () {
|
||||
const replyTxids = await this.loadReplyTxids()
|
||||
let count = 0
|
||||
|
||||
for await (const [key, value] of this.postHeightsDb.iterator()) {
|
||||
const txid = this.txidFromPostHeight(key, value)
|
||||
if (replyTxids.has(txid)) continue
|
||||
count++
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
async countTopLevelPostsByAddr (addr) {
|
||||
const replyTxids = await this.loadReplyTxids()
|
||||
let count = 0
|
||||
|
||||
for await (const [key, value] of this.postHeightsDb.iterator()) {
|
||||
const txid = this.txidFromPostHeight(key, value)
|
||||
if (replyTxids.has(txid)) continue
|
||||
|
||||
try {
|
||||
const post = await this.postsDb.get(txid)
|
||||
if (post.addr === addr) count++
|
||||
} catch (err) {
|
||||
if (err.notFound || err.code === 'LEVEL_NOT_FOUND') continue
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
}
|
||||
|
||||
export default PostQuery
|
||||
|
||||
@@ -36,6 +36,7 @@ export function makeCrudHandlers ({ dbProp, keyParam, bodyIdField, bodyDataField
|
||||
|
||||
export const ENTITY_CONFIG = [
|
||||
{ route: 'post', dbProp: 'postsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'postData' },
|
||||
{ route: 'postheight', dbProp: 'postHeightsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'postHeightData' },
|
||||
{ route: 'postparent', dbProp: 'postParentsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'parentData' },
|
||||
{ route: 'postchild', dbProp: 'postChildrenDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'childData' },
|
||||
{ route: 'like', dbProp: 'likesDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'likeData' },
|
||||
|
||||
@@ -26,7 +26,7 @@ class PostsRouter {
|
||||
attach (app) {
|
||||
this.router.get('/recent', this.postsRESTController.getRecentPosts)
|
||||
this.router.get('/by/:addr', this.postsRESTController.getPostsByAddr)
|
||||
this.router.get('/:txid/thread',this.postsRESTController.getPostThread)
|
||||
this.router.get('/:txid/thread', this.postsRESTController.getPostThread)
|
||||
app.use(this.router.routes())
|
||||
app.use(this.router.allowedMethods())
|
||||
}
|
||||
|
||||
@@ -47,4 +47,3 @@ class UseCases {
|
||||
}
|
||||
|
||||
export default UseCases
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/*
|
||||
Use case: list posts for an address ordered by block height (most recent first), paginated.
|
||||
Uses the postHeights secondary index for efficient sorting and pagination.
|
||||
*/
|
||||
|
||||
const DEFAULT_LIMIT = 100
|
||||
@@ -15,6 +16,7 @@ class ListPostsByAddr {
|
||||
throw new Error('postQuery adapter required for ListPostsByAddr use case.')
|
||||
}
|
||||
this.execute = this.execute.bind(this)
|
||||
this.attachReplyCounts = this.attachReplyCounts.bind(this)
|
||||
}
|
||||
|
||||
parseLimit (limit) {
|
||||
@@ -57,13 +59,11 @@ class ListPostsByAddr {
|
||||
return addr
|
||||
}
|
||||
|
||||
sortPosts (posts) {
|
||||
return posts.sort((a, b) => {
|
||||
if (b.blockHeight !== a.blockHeight) {
|
||||
return b.blockHeight - a.blockHeight
|
||||
}
|
||||
return (b.seen || 0) - (a.seen || 0)
|
||||
})
|
||||
attachReplyCounts (posts, replyCounts) {
|
||||
return posts.map((post) => ({
|
||||
...post,
|
||||
replyCount: replyCounts.get(post.txid) ?? 0
|
||||
}))
|
||||
}
|
||||
|
||||
async execute (inObj = {}) {
|
||||
@@ -71,13 +71,15 @@ class ListPostsByAddr {
|
||||
const limit = this.parseLimit(inObj.limit)
|
||||
const offset = this.parseOffset(inObj.offset)
|
||||
|
||||
const allPosts = await this.adapters.postQuery.scanPostsByAddr(addr)
|
||||
const sorted = this.sortPosts(allPosts)
|
||||
const total = sorted.length
|
||||
const posts = sorted.slice(offset, offset + limit)
|
||||
const txids = await this.adapters.postQuery.scanPostsByAddrTxids(addr, { limit, offset })
|
||||
const [posts, replyCounts, total] = await Promise.all([
|
||||
this.adapters.postQuery.loadPostsByTxids(txids),
|
||||
this.adapters.postQuery.buildReplyCountMap(),
|
||||
this.adapters.postQuery.countTopLevelPostsByAddr(addr)
|
||||
])
|
||||
|
||||
return {
|
||||
posts,
|
||||
posts: this.attachReplyCounts(posts, replyCounts),
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/*
|
||||
Use case: list posts ordered by block height (most recent first), paginated.
|
||||
Uses the postHeights secondary index for efficient sorting and pagination.
|
||||
*/
|
||||
|
||||
const DEFAULT_LIMIT = 100
|
||||
@@ -15,6 +16,7 @@ class ListRecentPosts {
|
||||
throw new Error('postQuery adapter required for ListRecentPosts use case.')
|
||||
}
|
||||
this.execute = this.execute.bind(this)
|
||||
this.attachReplyCounts = this.attachReplyCounts.bind(this)
|
||||
}
|
||||
|
||||
parseLimit (limit) {
|
||||
@@ -48,26 +50,26 @@ class ListRecentPosts {
|
||||
return parsed
|
||||
}
|
||||
|
||||
sortPosts (posts) {
|
||||
return posts.sort((a, b) => {
|
||||
if (b.blockHeight !== a.blockHeight) {
|
||||
return b.blockHeight - a.blockHeight
|
||||
}
|
||||
return (b.seen || 0) - (a.seen || 0)
|
||||
})
|
||||
attachReplyCounts (posts, replyCounts) {
|
||||
return posts.map((post) => ({
|
||||
...post,
|
||||
replyCount: replyCounts.get(post.txid) ?? 0
|
||||
}))
|
||||
}
|
||||
|
||||
async execute (inObj = {}) {
|
||||
const limit = this.parseLimit(inObj.limit)
|
||||
const offset = this.parseOffset(inObj.offset)
|
||||
|
||||
const allPosts = await this.adapters.postQuery.scanPostsWithBlockHeight()
|
||||
const sorted = this.sortPosts(allPosts)
|
||||
const total = sorted.length
|
||||
const posts = sorted.slice(offset, offset + limit)
|
||||
const txids = await this.adapters.postQuery.scanRecentPostTxids({ limit, offset })
|
||||
const [posts, replyCounts, total] = await Promise.all([
|
||||
this.adapters.postQuery.loadPostsByTxids(txids),
|
||||
this.adapters.postQuery.buildReplyCountMap(),
|
||||
this.adapters.postQuery.countTopLevelPosts()
|
||||
])
|
||||
|
||||
return {
|
||||
posts,
|
||||
posts: this.attachReplyCounts(posts, replyCounts),
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
|
||||
@@ -8,11 +8,13 @@ describe('#PostQuery', () => {
|
||||
let postsDb
|
||||
let postParentsDb
|
||||
let postChildrenDb
|
||||
let postHeightsDb
|
||||
|
||||
beforeEach(() => {
|
||||
sandbox = sinon.createSandbox()
|
||||
postsDb = {
|
||||
iterator: sandbox.stub()
|
||||
iterator: sandbox.stub(),
|
||||
get: sandbox.stub()
|
||||
}
|
||||
postParentsDb = {
|
||||
iterator: sandbox.stub()
|
||||
@@ -20,125 +22,217 @@ describe('#PostQuery', () => {
|
||||
postChildrenDb = {
|
||||
iterator: sandbox.stub()
|
||||
}
|
||||
postHeightsDb = {
|
||||
iterator: sandbox.stub()
|
||||
}
|
||||
|
||||
async function * emptyParents () {}
|
||||
async function * emptyChildren () {}
|
||||
async function * emptyHeights () {}
|
||||
postParentsDb.iterator.returns(emptyParents())
|
||||
postChildrenDb.iterator.returns(emptyChildren())
|
||||
uut = new PostQuery({ postsDb, postParentsDb, postChildrenDb })
|
||||
postHeightsDb.iterator.returns(emptyHeights())
|
||||
|
||||
uut = new PostQuery({ postsDb, postParentsDb, postChildrenDb, postHeightsDb })
|
||||
})
|
||||
|
||||
afterEach(() => sandbox.restore())
|
||||
|
||||
it('should scan posts and read block height from stored document', async () => {
|
||||
async function * mockIterator () {
|
||||
yield ['tx1', { addr: 'addr1', text: 'hello', seen: 1000, blockHeight: 600100 }]
|
||||
yield ['tx2', { addr: 'addr2', text: 'world', seen: 2000, blockHeight: 600200 }]
|
||||
it('should throw when postHeightsDb is missing', () => {
|
||||
try {
|
||||
// eslint-disable-next-line no-new
|
||||
new PostQuery({ postsDb, postParentsDb, postChildrenDb })
|
||||
assert.fail('Expected error')
|
||||
} catch (err) {
|
||||
assert.include(err.message, 'postHeightsDb required')
|
||||
}
|
||||
postsDb.iterator.returns(mockIterator())
|
||||
})
|
||||
|
||||
const result = await uut.scanPostsWithBlockHeight()
|
||||
describe('#scanRecentPostTxids', () => {
|
||||
it('should return top-level post txids sorted by block height descending', async () => {
|
||||
async function * mockHeights () {
|
||||
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
|
||||
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
|
||||
yield ['000000600100:post-100', { txid: 'post-100' }]
|
||||
}
|
||||
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
|
||||
|
||||
const result = await uut.scanRecentPostTxids({ limit: 2, offset: 0 })
|
||||
|
||||
assert.deepEqual(result, ['post-200-b', 'post-200-a'])
|
||||
})
|
||||
|
||||
it('should skip replies when selecting recent posts', async () => {
|
||||
async function * mockParents () {
|
||||
yield ['reply-1', { parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 }]
|
||||
}
|
||||
async function * mockHeights () {
|
||||
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
|
||||
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
|
||||
yield ['000000600150:reply-1', { txid: 'reply-1' }]
|
||||
yield ['000000600100:post-100', { txid: 'post-100' }]
|
||||
}
|
||||
postParentsDb.iterator.returns(mockParents())
|
||||
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
|
||||
|
||||
const result = await uut.scanRecentPostTxids({ limit: 2, offset: 0 })
|
||||
|
||||
assert.deepEqual(result, ['post-200-b', 'post-200-a'])
|
||||
})
|
||||
|
||||
it('should apply offset after skipping replies', async () => {
|
||||
async function * mockHeights () {
|
||||
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
|
||||
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
|
||||
yield ['000000600100:post-100', { txid: 'post-100' }]
|
||||
}
|
||||
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
|
||||
|
||||
const result = await uut.scanRecentPostTxids({ limit: 2, offset: 1 })
|
||||
|
||||
assert.deepEqual(result, ['post-200-a', 'post-100'])
|
||||
})
|
||||
|
||||
it('should stop reading after collecting limit top-level posts', async () => {
|
||||
async function * mockHeights () {
|
||||
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
|
||||
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
|
||||
yield ['000000600100:post-100', { txid: 'post-100' }]
|
||||
}
|
||||
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
|
||||
|
||||
await uut.scanRecentPostTxids({ limit: 2, offset: 0 })
|
||||
|
||||
assert.isTrue(postHeightsDb.iterator.calledOnce)
|
||||
})
|
||||
})
|
||||
|
||||
describe('#scanPostsByAddrTxids', () => {
|
||||
it('should return txids for the address sorted by block height descending', async () => {
|
||||
async function * mockHeights () {
|
||||
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
|
||||
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
|
||||
yield ['000000600100:post-100', { txid: 'post-100' }]
|
||||
}
|
||||
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
|
||||
postsDb.get.callsFake(async (txid) => {
|
||||
const posts = {
|
||||
'post-200-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
|
||||
'post-200-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600200 },
|
||||
'post-100': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600100 }
|
||||
}
|
||||
return posts[txid]
|
||||
})
|
||||
|
||||
const result = await uut.scanPostsByAddrTxids('addr-a', { limit: 2, offset: 0 })
|
||||
|
||||
assert.deepEqual(result, ['post-200-a', 'post-100'])
|
||||
})
|
||||
|
||||
it('should apply offset and limit for the address', async () => {
|
||||
async function * mockHeights () {
|
||||
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
|
||||
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
|
||||
yield ['000000600100:post-100', { txid: 'post-100' }]
|
||||
}
|
||||
postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
|
||||
postsDb.get.callsFake(async (txid) => {
|
||||
const posts = {
|
||||
'post-200-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
|
||||
'post-200-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600200 },
|
||||
'post-100': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600100 }
|
||||
}
|
||||
return posts[txid]
|
||||
})
|
||||
|
||||
const result = await uut.scanPostsByAddrTxids('addr-a', { limit: 1, offset: 1 })
|
||||
|
||||
assert.deepEqual(result, ['post-100'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('#loadPostsByTxids', () => {
|
||||
it('should load posts by txid', async () => {
|
||||
postsDb.get.withArgs('tx1').resolves({ addr: 'a1', text: 'hello', seen: 1000, blockHeight: 600100 })
|
||||
postsDb.get.withArgs('tx2').resolves({ addr: 'a2', text: 'world', seen: 2000, blockHeight: 600200 })
|
||||
|
||||
const result = await uut.loadPostsByTxids(['tx1', 'tx2'])
|
||||
|
||||
assert.equal(result.length, 2)
|
||||
assert.equal(result[0].txid, 'tx1')
|
||||
assert.equal(result[0].blockHeight, 600100)
|
||||
assert.equal(result[0].replyCount, 0)
|
||||
assert.equal(result[1].txid, 'tx2')
|
||||
assert.equal(result[1].blockHeight, 600200)
|
||||
assert.equal(result[1].replyCount, 0)
|
||||
})
|
||||
|
||||
it('should use block height 0 when field is missing', async () => {
|
||||
async function * mockIterator () {
|
||||
yield ['tx-missing', { addr: 'addr1', text: 'hi', seen: 1000 }]
|
||||
}
|
||||
postsDb.iterator.returns(mockIterator())
|
||||
|
||||
const result = await uut.scanPostsWithBlockHeight()
|
||||
|
||||
assert.equal(result[0].blockHeight, 0)
|
||||
assert.equal(result[0].replyCount, 0)
|
||||
})
|
||||
|
||||
it('should scan posts for a single address', async () => {
|
||||
async function * mockIterator () {
|
||||
yield ['tx1', { addr: 'addr-a', text: 'hello', seen: 1000, blockHeight: 600100 }]
|
||||
yield ['tx2', { addr: 'addr-b', text: 'world', seen: 2000, blockHeight: 600200 }]
|
||||
yield ['tx3', { addr: 'addr-a', text: 'again', seen: 3000, blockHeight: 600300 }]
|
||||
}
|
||||
postsDb.iterator.returns(mockIterator())
|
||||
|
||||
const result = await uut.scanPostsByAddr('addr-a')
|
||||
|
||||
assert.equal(result.length, 2)
|
||||
assert.equal(result[0].txid, 'tx1')
|
||||
assert.equal(result[1].txid, 'tx3')
|
||||
})
|
||||
|
||||
it('should exclude reply posts from recent scan', async () => {
|
||||
async function * mockParents () {
|
||||
yield ['tx-reply', { parentTxid: 'tx1', childTxid: 'tx-reply', blockHeight: 600150 }]
|
||||
}
|
||||
async function * mockPosts () {
|
||||
yield ['tx1', { addr: 'addr1', text: 'top', seen: 1000, blockHeight: 600100 }]
|
||||
yield ['tx-reply', { addr: 'addr1', text: 'reply', seen: 1500, blockHeight: 600150 }]
|
||||
yield ['tx2', { addr: 'addr2', text: 'other', seen: 2000, blockHeight: 600200 }]
|
||||
}
|
||||
postParentsDb.iterator.returns(mockParents())
|
||||
postsDb.iterator.returns(mockPosts())
|
||||
|
||||
const result = await uut.scanPostsWithBlockHeight()
|
||||
|
||||
assert.equal(result.length, 2)
|
||||
assert.equal(result[0].txid, 'tx1')
|
||||
assert.equal(result[1].txid, 'tx2')
|
||||
})
|
||||
|
||||
it('should exclude reply posts from address scan', async () => {
|
||||
async function * mockParents () {
|
||||
yield ['tx-reply', { parentTxid: 'tx1', childTxid: 'tx-reply', blockHeight: 600150 }]
|
||||
}
|
||||
async function * mockPosts () {
|
||||
yield ['tx1', { addr: 'addr-a', text: 'top', seen: 1000, blockHeight: 600100 }]
|
||||
yield ['tx-reply', { addr: 'addr-a', text: 'reply', seen: 1500, blockHeight: 600150 }]
|
||||
}
|
||||
postParentsDb.iterator.returns(mockParents())
|
||||
postsDb.iterator.returns(mockPosts())
|
||||
it('should skip missing posts', async () => {
|
||||
const err = new Error('not found')
|
||||
err.notFound = true
|
||||
postsDb.get.withArgs('tx1').rejects(err)
|
||||
postsDb.get.withArgs('tx2').resolves({ addr: 'a2', text: 'world', seen: 2000, blockHeight: 600200 })
|
||||
|
||||
const result = await uut.scanPostsByAddr('addr-a')
|
||||
const result = await uut.loadPostsByTxids(['tx1', 'tx2'])
|
||||
|
||||
assert.equal(result.length, 1)
|
||||
assert.equal(result[0].txid, 'tx1')
|
||||
assert.equal(result[0].txid, 'tx2')
|
||||
})
|
||||
})
|
||||
|
||||
it('should include replyCount from postChildren scan', async () => {
|
||||
describe('#countTopLevelPosts', () => {
|
||||
it('should count top-level posts excluding replies', async () => {
|
||||
async function * mockParents () {
|
||||
yield ['reply-1', { parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 }]
|
||||
}
|
||||
async function * mockHeights () {
|
||||
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
|
||||
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
|
||||
yield ['000000600150:reply-1', { txid: 'reply-1' }]
|
||||
yield ['000000600100:post-100', { txid: 'post-100' }]
|
||||
}
|
||||
postParentsDb.iterator.returns(mockParents())
|
||||
postHeightsDb.iterator.returns(mockHeights())
|
||||
|
||||
const result = await uut.countTopLevelPosts()
|
||||
|
||||
assert.equal(result, 3)
|
||||
})
|
||||
})
|
||||
|
||||
describe('#countTopLevelPostsByAddr', () => {
|
||||
it('should count top-level posts for an address', async () => {
|
||||
async function * mockHeights () {
|
||||
yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
|
||||
yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
|
||||
yield ['000000600100:post-100', { txid: 'post-100' }]
|
||||
}
|
||||
postHeightsDb.iterator.returns(mockHeights())
|
||||
postsDb.get.callsFake(async (txid) => {
|
||||
const posts = {
|
||||
'post-200-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
|
||||
'post-200-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600200 },
|
||||
'post-100': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600100 }
|
||||
}
|
||||
return posts[txid]
|
||||
})
|
||||
|
||||
const result = await uut.countTopLevelPostsByAddr('addr-a')
|
||||
|
||||
assert.equal(result, 2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('#buildReplyCountMap', () => {
|
||||
it('should count replies per parent from postChildren', async () => {
|
||||
async function * mockChildren () {
|
||||
yield ['tx1:reply-a', { parentTxid: 'tx1', childTxid: 'reply-a', blockHeight: 600150 }]
|
||||
yield ['tx1:reply-b', { parentTxid: 'tx1', childTxid: 'reply-b', blockHeight: 600160 }]
|
||||
yield ['tx2:reply-c', { parentTxid: 'tx2', childTxid: 'reply-c', blockHeight: 600170 }]
|
||||
}
|
||||
async function * mockPosts () {
|
||||
yield ['tx1', { addr: 'addr1', text: 'top', seen: 1000, blockHeight: 600100 }]
|
||||
yield ['tx2', { addr: 'addr2', text: 'other', seen: 2000, blockHeight: 600200 }]
|
||||
}
|
||||
postChildrenDb.iterator.returns(mockChildren())
|
||||
postsDb.iterator.returns(mockPosts())
|
||||
|
||||
const result = await uut.scanPostsWithBlockHeight()
|
||||
const result = await uut.buildReplyCountMap()
|
||||
|
||||
assert.equal(result.length, 2)
|
||||
assert.equal(result.find((p) => p.txid === 'tx1').replyCount, 2)
|
||||
assert.equal(result.find((p) => p.txid === 'tx2').replyCount, 1)
|
||||
assert.equal(result.get('tx1'), 2)
|
||||
assert.equal(result.get('tx2'), 1)
|
||||
})
|
||||
|
||||
it('should default replyCount to 0 when post has no replies', async () => {
|
||||
async function * mockPosts () {
|
||||
yield ['tx1', { addr: 'addr1', text: 'solo', seen: 1000, blockHeight: 600100 }]
|
||||
}
|
||||
postsDb.iterator.returns(mockPosts())
|
||||
|
||||
const result = await uut.scanPostsByAddr('addr1')
|
||||
|
||||
assert.equal(result.length, 1)
|
||||
assert.equal(result[0].replyCount, 0)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -19,7 +19,7 @@ describe('#LevelRESTController', () => {
|
||||
sandbox = sinon.createSandbox()
|
||||
uut = new LevelRESTControllerLib({
|
||||
adapters: {
|
||||
level: { postsDb: mockDb, statusDb: mockDb },
|
||||
level: { postsDb: mockDb, postHeightsDb: mockDb, statusDb: mockDb },
|
||||
dbBackup: { zipDb: sandbox.stub().resolves(true) }
|
||||
},
|
||||
useCases: {}
|
||||
@@ -44,4 +44,15 @@ describe('#LevelRESTController', () => {
|
||||
assert.equal(ctx.body.success, true)
|
||||
assert.equal(ctx.body.txid, 'abc')
|
||||
})
|
||||
|
||||
it('should expose a postheight entity handler', async () => {
|
||||
const ctx = {
|
||||
params: {},
|
||||
request: { body: { key: '600000:abc', postHeightData: { txid: 'abc', blockHeight: 600000 } } },
|
||||
body: null
|
||||
}
|
||||
await uut.entityHandlers.postheight.create(ctx)
|
||||
assert.equal(ctx.body.success, true)
|
||||
assert.equal(ctx.body.key, '600000:abc')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,23 +5,32 @@ import ListPostsByAddr from '../../../src/use-cases/list-posts-by-addr.js'
|
||||
describe('#ListPostsByAddr', () => {
|
||||
let uut
|
||||
let sandbox
|
||||
let postQuery
|
||||
|
||||
const mockPosts = [
|
||||
{ txid: 'tx-a', addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
|
||||
{ txid: 'tx-b', addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
|
||||
{ txid: 'tx-c', addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600200 }
|
||||
]
|
||||
const mockPosts = {
|
||||
'tx-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
|
||||
'tx-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
|
||||
'tx-c': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600200 }
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
sandbox = sinon.createSandbox()
|
||||
postQuery = {
|
||||
scanPostsByAddrTxids: sandbox.stub().callsFake(async (addr, { limit, offset }) => {
|
||||
const all = Object.entries(mockPosts)
|
||||
.filter(([txid, post]) => post.addr === addr)
|
||||
.sort((a, b) => b[1].blockHeight - a[1].blockHeight)
|
||||
.map(([txid]) => txid)
|
||||
return all.slice(offset, offset + limit)
|
||||
}),
|
||||
loadPostsByTxids: sandbox.stub().callsFake(async (txids) => {
|
||||
return txids.map((txid) => ({ txid, ...mockPosts[txid] }))
|
||||
}),
|
||||
buildReplyCountMap: sandbox.stub().resolves(new Map()),
|
||||
countTopLevelPostsByAddr: sandbox.stub().resolves(2)
|
||||
}
|
||||
uut = new ListPostsByAddr({
|
||||
adapters: {
|
||||
postQuery: {
|
||||
scanPostsByAddr: sandbox.stub().callsFake(async (addr) => {
|
||||
return mockPosts.filter((post) => post.addr === addr)
|
||||
})
|
||||
}
|
||||
}
|
||||
adapters: { postQuery }
|
||||
})
|
||||
})
|
||||
|
||||
@@ -45,4 +54,12 @@ describe('#ListPostsByAddr', () => {
|
||||
assert.include(err.message, 'addr is required')
|
||||
}
|
||||
})
|
||||
|
||||
it('should pass addr, limit, and offset to postQuery', async () => {
|
||||
await uut.execute({ addr: 'addr-a', limit: 5, offset: 10 })
|
||||
|
||||
assert.equal(postQuery.scanPostsByAddrTxids.calledOnce, true)
|
||||
assert.equal(postQuery.scanPostsByAddrTxids.firstCall.args[0], 'addr-a')
|
||||
assert.deepEqual(postQuery.scanPostsByAddrTxids.firstCall.args[1], { limit: 5, offset: 10 })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,21 +5,26 @@ import ListRecentPosts from '../../../src/use-cases/list-recent-posts.js'
|
||||
describe('#ListRecentPosts', () => {
|
||||
let uut
|
||||
let sandbox
|
||||
let postQuery
|
||||
|
||||
const mockPosts = [
|
||||
{ txid: 'tx-a', addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
|
||||
{ txid: 'tx-b', addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
|
||||
{ txid: 'tx-c', addr: 'addr-c', text: 'c', seen: 50, blockHeight: 600200 }
|
||||
]
|
||||
const mockPosts = {
|
||||
'tx-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
|
||||
'tx-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
|
||||
'tx-c': { addr: 'addr-c', text: 'c', seen: 50, blockHeight: 600200 }
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
sandbox = sinon.createSandbox()
|
||||
postQuery = {
|
||||
scanRecentPostTxids: sandbox.stub().resolves(['tx-b', 'tx-c', 'tx-a']),
|
||||
loadPostsByTxids: sandbox.stub().callsFake(async (txids) => {
|
||||
return txids.map((txid) => ({ txid, ...mockPosts[txid] }))
|
||||
}),
|
||||
buildReplyCountMap: sandbox.stub().resolves(new Map([['tx-b', 1]])),
|
||||
countTopLevelPosts: sandbox.stub().resolves(3)
|
||||
}
|
||||
uut = new ListRecentPosts({
|
||||
adapters: {
|
||||
postQuery: {
|
||||
scanPostsWithBlockHeight: sandbox.stub().resolves([...mockPosts])
|
||||
}
|
||||
}
|
||||
adapters: { postQuery }
|
||||
})
|
||||
})
|
||||
|
||||
@@ -32,11 +37,14 @@ describe('#ListRecentPosts', () => {
|
||||
assert.equal(result.posts[0].txid, 'tx-b')
|
||||
assert.equal(result.posts[1].txid, 'tx-c')
|
||||
assert.equal(result.posts[2].txid, 'tx-a')
|
||||
assert.equal(result.posts[0].replyCount, 1)
|
||||
assert.equal(result.posts[1].replyCount, 0)
|
||||
assert.equal(result.pagination.total, 3)
|
||||
assert.equal(result.pagination.hasMore, false)
|
||||
})
|
||||
|
||||
it('should paginate with limit and offset', async () => {
|
||||
postQuery.scanRecentPostTxids.resolves(['tx-c'])
|
||||
const result = await uut.execute({ limit: 1, offset: 1 })
|
||||
|
||||
assert.equal(result.posts.length, 1)
|
||||
@@ -62,4 +70,11 @@ describe('#ListRecentPosts', () => {
|
||||
assert.include(err.message, 'limit cannot exceed')
|
||||
}
|
||||
})
|
||||
|
||||
it('should pass limit and offset to postQuery', async () => {
|
||||
await uut.execute({ limit: 5, offset: 10 })
|
||||
|
||||
assert.equal(postQuery.scanRecentPostTxids.calledOnce, true)
|
||||
assert.deepEqual(postQuery.scanRecentPostTxids.firstCall.args[0], { limit: 5, offset: 10 })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
Normal acceptance runner for psf-memo-indexer.
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const root = path.resolve(__dirname, '..')
|
||||
const specsDir = path.join(root, 'specs')
|
||||
const buildDir = path.join(root, 'build', 'acceptance')
|
||||
const irDir = path.join(buildDir, 'ir')
|
||||
const genDir = path.join(buildDir, 'generated')
|
||||
const apsDir = path.join(root, '..', 'tmp', 'aps-spec')
|
||||
|
||||
function sh (cmd, args, opts = {}) {
|
||||
return execFileSync(cmd, args, {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
...opts
|
||||
}).toString()
|
||||
}
|
||||
|
||||
function ensureAps () {
|
||||
if (fs.existsSync(apsDir)) return
|
||||
fs.mkdirSync(path.dirname(apsDir), { recursive: true })
|
||||
sh('git', ['clone', '--depth', '1',
|
||||
'https://github.com/unclebob/Acceptance-Pipeline-Specification.git', apsDir])
|
||||
}
|
||||
|
||||
function main () {
|
||||
ensureAps()
|
||||
|
||||
const features = fs
|
||||
.readdirSync(specsDir)
|
||||
.filter((f) => f.endsWith('.feature'))
|
||||
.sort()
|
||||
|
||||
if (features.length === 0) {
|
||||
console.log('No feature files found under specs/.')
|
||||
return
|
||||
}
|
||||
|
||||
fs.mkdirSync(irDir, { recursive: true })
|
||||
fs.mkdirSync(genDir, { recursive: true })
|
||||
|
||||
for (const featureFile of features) {
|
||||
const base = featureFile.replace(/\.feature$/i, '')
|
||||
const featurePath = path.join(specsDir, featureFile)
|
||||
const irPath = path.join(irDir, `${base}.json`)
|
||||
|
||||
sh('bb', ['gherkin-parser', featurePath, irPath], { cwd: apsDir })
|
||||
sh('node', [path.join(__dirname, 'lib', 'generate.js'), irPath, genDir])
|
||||
}
|
||||
|
||||
const tests = fs
|
||||
.readdirSync(genDir)
|
||||
.filter((f) => f.endsWith('.acceptance.test.js'))
|
||||
.sort()
|
||||
|
||||
let failures = 0
|
||||
for (const testFile of tests) {
|
||||
try {
|
||||
const out = sh('node', [path.join(genDir, testFile)])
|
||||
process.stdout.write(out)
|
||||
console.log(`ACCEPTANCE PASS: ${testFile}`)
|
||||
} catch (err) {
|
||||
failures++
|
||||
process.stdout.write(err.stdout || '')
|
||||
process.stderr.write(err.stderr || '')
|
||||
console.error(`ACCEPTANCE FAIL: ${testFile}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (failures > 0) {
|
||||
console.error(`ACCEPTANCE: ${failures} failing test file(s)`)
|
||||
process.exit(1)
|
||||
} else {
|
||||
console.log(`ACCEPTANCE: all ${tests.length} generated test file(s) passed`)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,96 @@
|
||||
/*
|
||||
Project-specific acceptance entrypoint generator for psf-memo-indexer.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import crypto from 'node:crypto'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
function metadataName (featureName) {
|
||||
const slug = featureName
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
return `${slug || 'feature'}.json`
|
||||
}
|
||||
|
||||
function relativeImport (fromDir, targetFile) {
|
||||
let rel = path.relative(fromDir, targetFile).replace(/\\/g, '/')
|
||||
if (!rel.startsWith('.')) rel = `./${rel}`
|
||||
return rel
|
||||
}
|
||||
|
||||
function main () {
|
||||
const irArg = process.argv[2]
|
||||
const outArg = process.argv[3]
|
||||
|
||||
if (!irArg || !outArg) {
|
||||
console.error('usage: acceptance-entrypoint-generator <json-ir> <generated-test-output-dir>')
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
let ir
|
||||
try {
|
||||
ir = JSON.parse(fs.readFileSync(irArg, 'utf8'))
|
||||
} catch (err) {
|
||||
console.error(`Failed to read JSON IR "${irArg}": ${err.message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const genDir = outArg
|
||||
fs.mkdirSync(genDir, { recursive: true })
|
||||
|
||||
const featureKey = path.basename(irArg).replace(/\.json$/i, '')
|
||||
const testFile = path.join(genDir, `${featureKey}.acceptance.test.js`)
|
||||
const relRuntime = relativeImport(genDir, path.join(__dirname, 'runtime.js'))
|
||||
|
||||
const body = `import { runFeature } from '${relRuntime}'
|
||||
const ir = ${JSON.stringify(ir, null, 2)}
|
||||
async function main () {
|
||||
const report = await runFeature(ir)
|
||||
for (const r of report.results) {
|
||||
console.log((r.status === 'passed' ? 'PASS ' : 'FAIL ') + r.name)
|
||||
if (r.detail) console.log(' ' + r.detail)
|
||||
}
|
||||
if (report.failures > 0) {
|
||||
console.error('ACCEPTANCE FAILURES: ' + report.failures + ' of ' + report.total)
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
main().catch((err) => { console.error(err); process.exit(1) })
|
||||
`
|
||||
|
||||
try {
|
||||
fs.writeFileSync(testFile, body)
|
||||
} catch (err) {
|
||||
console.error(`Failed to write generated test "${testFile}": ${err.message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const metaDir = path.join(genDir, 'metadata')
|
||||
fs.mkdirSync(metaDir, { recursive: true })
|
||||
const hash = crypto
|
||||
.createHash('sha256')
|
||||
.update(fs.readFileSync(testFile))
|
||||
.digest('hex')
|
||||
|
||||
const metadata = {
|
||||
schema_version: 1,
|
||||
feature_path: `${featureKey}.feature`,
|
||||
ir_path: path.resolve(irArg),
|
||||
implementation_hash: `sha256:${hash}`,
|
||||
hash_scope: 'generated_files',
|
||||
generated_files: [testFile]
|
||||
}
|
||||
fs.writeFileSync(
|
||||
path.join(metaDir, metadataName(featureKey)),
|
||||
JSON.stringify(metadata, null, 2)
|
||||
)
|
||||
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,268 @@
|
||||
/*
|
||||
Project step handlers for the psf-memo-indexer acceptance pipeline.
|
||||
|
||||
These handlers exercise the real Memo action handlers (handlePost, handleReply)
|
||||
against an in-memory database that exposes the same CRUD surface as the
|
||||
psf-memo-db entity routes used by the indexer.
|
||||
*/
|
||||
|
||||
import crypto from 'node:crypto'
|
||||
import { handlePost } from '../../src/use-cases/action-types/post.js'
|
||||
import { handleReply } from '../../src/use-cases/action-types/reply.js'
|
||||
|
||||
function makeInMemoryDb () {
|
||||
const store = new Map()
|
||||
return {
|
||||
async get (key) {
|
||||
if (!store.has(key)) {
|
||||
const err = new Error('not found')
|
||||
err.notFound = true
|
||||
throw err
|
||||
}
|
||||
return store.get(key)
|
||||
},
|
||||
async create (key, data) {
|
||||
store.set(key, data)
|
||||
return { success: true }
|
||||
},
|
||||
async update (key, data) {
|
||||
store.set(key, data)
|
||||
return { success: true }
|
||||
},
|
||||
async delete (key) {
|
||||
store.delete(key)
|
||||
return { success: true }
|
||||
},
|
||||
entries () {
|
||||
return Array.from(store.entries())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resolveParam (value, example) {
|
||||
const match = /^\u003c([A-Za-z0-9_]+)\u003e$/.exec(String(value).trim())
|
||||
if (match) {
|
||||
const param = match[1]
|
||||
if (!(param in example)) {
|
||||
throw new Error(`Missing example value for "${param}"`)
|
||||
}
|
||||
return example[param]
|
||||
}
|
||||
return String(value).trim()
|
||||
}
|
||||
|
||||
function deriveTxid (symbolic) {
|
||||
// Produce a deterministic 32-byte buffer from a symbolic test txid and return
|
||||
// its big-endian hex representation, which is what the indexer stores.
|
||||
return crypto.createHash('sha256').update(symbolic).digest().toString('hex')
|
||||
}
|
||||
|
||||
function resolveTxid (value, example, world) {
|
||||
const resolved = resolveParam(value, example)
|
||||
if (!world.txidMap) world.txidMap = new Map()
|
||||
if (!world.txidMap.has(resolved)) {
|
||||
world.txidMap.set(resolved, deriveTxid(resolved))
|
||||
}
|
||||
return world.txidMap.get(resolved)
|
||||
}
|
||||
|
||||
async function createWorld () {
|
||||
const postsDb = makeInMemoryDb()
|
||||
const postHeightsDb = makeInMemoryDb()
|
||||
const postParentsDb = makeInMemoryDb()
|
||||
const postChildrenDb = makeInMemoryDb()
|
||||
|
||||
const adapters = {
|
||||
postDb: postsDb,
|
||||
postHeightDb: postHeightsDb,
|
||||
postParentDb: postParentsDb,
|
||||
postChildDb: postChildrenDb,
|
||||
processErrorDb: makeInMemoryDb()
|
||||
}
|
||||
|
||||
return {
|
||||
adapters,
|
||||
postsDb,
|
||||
postHeightsDb,
|
||||
postParentsDb,
|
||||
postChildrenDb,
|
||||
txidMap: new Map(),
|
||||
lastTxid: null,
|
||||
lastHeight: null,
|
||||
lastAddr: null
|
||||
}
|
||||
}
|
||||
|
||||
const handlers = [
|
||||
{
|
||||
name: 'db instance with posts and postHeights stores',
|
||||
pattern: /^a psf-memo-db instance with posts and postHeights stores$/,
|
||||
async run () {
|
||||
// World is already created with both stores.
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'indexer configured to write to db',
|
||||
pattern: /^a psf-memo-indexer configured to write to that database$/,
|
||||
async run () {
|
||||
// Adapters object is already configured.
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'process a Memo post transaction',
|
||||
pattern: /^the indexer processes a Memo post transaction (.+) from (.+) at block height (.+) with text "(.+)"$/,
|
||||
async run (m, example, world) {
|
||||
const txid = resolveTxid(m[1], example, world)
|
||||
const addr = resolveParam(m[2], example)
|
||||
const height = parseInt(resolveParam(m[3], example), 10)
|
||||
const text = resolveParam(m[4], example)
|
||||
|
||||
world.lastTxid = txid
|
||||
world.lastHeight = height
|
||||
world.lastAddr = addr
|
||||
|
||||
const prefix = Buffer.from('6d02', 'hex')
|
||||
const message = Buffer.from(text, 'utf8')
|
||||
|
||||
await handlePost({
|
||||
adapters: world.adapters,
|
||||
txid,
|
||||
signerAddr: addr,
|
||||
seen: Date.now(),
|
||||
blockHeight: height,
|
||||
decoded: {
|
||||
action: 'post',
|
||||
prefix,
|
||||
pushDatas: [prefix, message]
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'process a Memo reply transaction',
|
||||
pattern: /^the indexer processes a Memo reply transaction (.+) to parent (.+) from (.+) at block height (.+) with text "(.+)"$/,
|
||||
async run (m, example, world) {
|
||||
const txid = resolveTxid(m[1], example, world)
|
||||
const parentTxid = resolveTxid(m[2], example, world)
|
||||
const addr = resolveParam(m[3], example)
|
||||
const height = parseInt(resolveParam(m[4], example), 10)
|
||||
const text = resolveParam(m[5], example)
|
||||
|
||||
world.lastTxid = txid
|
||||
world.lastHeight = height
|
||||
world.lastAddr = addr
|
||||
|
||||
const prefix = Buffer.from('6d03', 'hex')
|
||||
// handleReply expects the parent tx hash as a 32-byte buffer in the
|
||||
// little-endian wire format; txHashFromPush reverses it to big-endian hex.
|
||||
const parentHash = Buffer.from(parentTxid, 'hex').reverse()
|
||||
const message = Buffer.from(text, 'utf8')
|
||||
|
||||
await handleReply({
|
||||
adapters: world.adapters,
|
||||
txid,
|
||||
signerAddr: addr,
|
||||
seen: Date.now(),
|
||||
blockHeight: height,
|
||||
decoded: {
|
||||
action: 'reply',
|
||||
prefix,
|
||||
pushDatas: [prefix, parentHash, message]
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'process the same Memo post transaction again',
|
||||
pattern: /^the indexer processes the same Memo post transaction (.+) again$/,
|
||||
async run (m, example, world) {
|
||||
const txid = resolveTxid(m[1], example, world)
|
||||
const post = await world.postsDb.get(txid)
|
||||
|
||||
const prefix = Buffer.from('6d02', 'hex')
|
||||
const message = Buffer.from(post.text, 'utf8')
|
||||
|
||||
await handlePost({
|
||||
adapters: world.adapters,
|
||||
txid,
|
||||
signerAddr: post.addr,
|
||||
seen: post.seen,
|
||||
blockHeight: post.blockHeight,
|
||||
decoded: {
|
||||
action: 'post',
|
||||
prefix,
|
||||
pushDatas: [prefix, message]
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'posts store contains post document',
|
||||
pattern: /^the posts store contains (.+) post document for (.+)$/,
|
||||
run (m, example, world) {
|
||||
const expectedCount = parseInt(resolveParam(m[1], example), 10)
|
||||
const txid = resolveTxid(m[2], example, world)
|
||||
const matching = world.postsDb.entries().filter(([key]) => key === txid)
|
||||
if (matching.length !== expectedCount) {
|
||||
throw new Error(`Expected ${expectedCount} post document(s) for ${txid}, got ${matching.length}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'postHeights store contains entry',
|
||||
pattern: /^the postHeights store contains (.+) entry whose key starts with the block height (.+) and ends with (.+)$/,
|
||||
run (m, example, world) {
|
||||
const expectedCount = parseInt(resolveParam(m[1], example), 10)
|
||||
const height = resolveParam(m[2], example)
|
||||
const txid = resolveTxid(m[3], example, world)
|
||||
const prefix = String(height).padStart(12, '0')
|
||||
const matching = world.postHeightsDb.entries().filter(([key, value]) => {
|
||||
return key.startsWith(prefix) && (key.endsWith(`:${txid}`) || value?.txid === txid)
|
||||
})
|
||||
if (matching.length !== expectedCount) {
|
||||
throw new Error(`Expected ${expectedCount} postHeights entry/entries for height ${height} txid ${txid}, got ${matching.length}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'postParents store contains link',
|
||||
pattern: /^the postParents store contains a link from (.+) to (.+)$/,
|
||||
run (m, example, world) {
|
||||
const childTxid = resolveTxid(m[1], example, world)
|
||||
const parentTxid = resolveTxid(m[2], example, world)
|
||||
const link = world.postParentsDb.entries().find(([key, value]) => {
|
||||
return key === childTxid && value?.parentTxid === parentTxid
|
||||
})
|
||||
if (!link) {
|
||||
throw new Error(`Expected postParents link from ${childTxid} to ${parentTxid}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'postChildren store contains link',
|
||||
pattern: /^the postChildren store contains a link from (.+) to (.+)$/,
|
||||
run (m, example, world) {
|
||||
const parentTxid = resolveTxid(m[1], example, world)
|
||||
const childTxid = resolveTxid(m[2], example, world)
|
||||
const link = world.postChildrenDb.entries().find(([key, value]) => {
|
||||
return value?.parentTxid === parentTxid && value?.childTxid === childTxid
|
||||
})
|
||||
if (!link) {
|
||||
throw new Error(`Expected postChildren link from ${parentTxid} to ${childTxid}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
async function handleStep (step, example, world) {
|
||||
for (const handler of handlers) {
|
||||
const match = handler.pattern.exec(step.text)
|
||||
if (match) {
|
||||
await handler.run(match, example, world, step)
|
||||
return
|
||||
}
|
||||
}
|
||||
throw new Error(`Unsupported step: ${step.keyword} ${step.text}`)
|
||||
}
|
||||
|
||||
export { createWorld, handleStep }
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
Acceptance runtime for psf-memo-indexer.
|
||||
*/
|
||||
|
||||
import { createWorld, handleStep } from './handlers.js'
|
||||
|
||||
function expandScenarios (ir) {
|
||||
const executions = []
|
||||
const background = ir.background || []
|
||||
|
||||
for (const scenario of ir.scenarios) {
|
||||
const examples = (scenario.examples && scenario.examples.length > 0)
|
||||
? scenario.examples.map((example, i) => ({ example, suffix: `example_${i + 1}` }))
|
||||
: [{ example: {}, suffix: 'example_1' }]
|
||||
|
||||
for (const { example, suffix } of examples) {
|
||||
executions.push({
|
||||
name: `${scenario.name}/${suffix}`,
|
||||
steps: [...background, ...scenario.steps],
|
||||
example
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return executions
|
||||
}
|
||||
|
||||
async function runFeature (ir) {
|
||||
const executions = expandScenarios(ir)
|
||||
const results = []
|
||||
let failures = 0
|
||||
|
||||
for (const ex of executions) {
|
||||
const world = await createWorld()
|
||||
let failure = null
|
||||
|
||||
for (const step of ex.steps) {
|
||||
try {
|
||||
await handleStep(step, ex.example, world)
|
||||
} catch (err) {
|
||||
failure = err.message
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (failure) {
|
||||
failures++
|
||||
results.push({ name: ex.name, status: 'failed', detail: failure })
|
||||
} else {
|
||||
results.push({ name: ex.name, status: 'passed' })
|
||||
}
|
||||
}
|
||||
|
||||
return { feature: ir.name, results, failures, total: results.length }
|
||||
}
|
||||
|
||||
export { expandScenarios, runFeature }
|
||||
@@ -7,6 +7,7 @@
|
||||
"block-indexer": "node --max-old-space-size=8192 psf-memo-block-indexer.js",
|
||||
"tx-indexer": "node --max-old-space-size=4096 psf-memo-tx-indexer.js",
|
||||
"test": "c8 --reporter=text mocha --exit --recursive test/unit/",
|
||||
"acceptance": "node acceptance/acceptance.js",
|
||||
"lint": "standard --env mocha --fix"
|
||||
},
|
||||
"author": "Chris Troutner",
|
||||
|
||||
@@ -20,6 +20,7 @@ class Adapters {
|
||||
this.dbCtrl = new DbCtrl()
|
||||
|
||||
this.postDb = createEntityDb('post', 'txid', 'postData')
|
||||
this.postHeightDb = createEntityDb('postheight', 'key', 'postHeightData')
|
||||
this.postParentDb = createEntityDb('postparent', 'txid', 'parentData')
|
||||
this.postChildDb = createEntityDb('postchild', 'key', 'childData')
|
||||
this.likeDb = createEntityDb('like', 'txid', 'likeData')
|
||||
|
||||
@@ -62,6 +62,11 @@ export function roomKey (roomName, txid) {
|
||||
return `${roomName}:${txid}`
|
||||
}
|
||||
|
||||
export function postHeightKey (blockHeight, txid) {
|
||||
const padded = String(blockHeight).padStart(12, '0')
|
||||
return `${padded}:${txid}`
|
||||
}
|
||||
|
||||
export function postChildKey (parentTxid, childTxid) {
|
||||
return `${parentTxid}:${childTxid}`
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js'
|
||||
import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas, postHeightKey } from './helpers.js'
|
||||
import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
|
||||
|
||||
export async function handlePost (ctx) {
|
||||
@@ -21,9 +21,16 @@ export async function handlePost (ctx) {
|
||||
}
|
||||
|
||||
const postData = { addr: signerAddr, text, seen, blockHeight }
|
||||
const heightKey = postHeightKey(blockHeight, txid)
|
||||
try {
|
||||
await adapters.postDb.get(txid)
|
||||
} catch (err) {
|
||||
await adapters.postDb.create(txid, postData)
|
||||
}
|
||||
|
||||
try {
|
||||
await adapters.postHeightDb.get(heightKey)
|
||||
} catch (err) {
|
||||
await adapters.postHeightDb.create(heightKey, { txid, blockHeight })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,9 +7,12 @@ describe('#handlePost', () => {
|
||||
it('should save a post to the database', async () => {
|
||||
const create = sinon.stub().resolves({ success: true })
|
||||
const get = sinon.stub().rejects(new Error('not found'))
|
||||
const postHeightCreate = sinon.stub().resolves({ success: true })
|
||||
const postHeightGet = sinon.stub().rejects(new Error('not found'))
|
||||
|
||||
const adapters = {
|
||||
postDb: { create, get },
|
||||
postHeightDb: { create: postHeightCreate, get: postHeightGet },
|
||||
processErrorDb: { create: sinon.stub() }
|
||||
}
|
||||
|
||||
@@ -31,5 +34,40 @@ describe('#handlePost', () => {
|
||||
assert.equal(create.firstCall.args[0], 'abc123')
|
||||
assert.equal(create.firstCall.args[1].text, 'hello memo')
|
||||
assert.equal(create.firstCall.args[1].blockHeight, 600100)
|
||||
|
||||
assert.equal(postHeightCreate.callCount, 1)
|
||||
assert.equal(postHeightCreate.firstCall.args[0], '000000600100:abc123')
|
||||
assert.equal(postHeightCreate.firstCall.args[1].txid, 'abc123')
|
||||
assert.equal(postHeightCreate.firstCall.args[1].blockHeight, 600100)
|
||||
})
|
||||
|
||||
it('should not duplicate postHeight entries when reprocessing', async () => {
|
||||
const create = sinon.stub().resolves({ success: true })
|
||||
const get = sinon.stub().resolves({ addr: 'bitcoincash:qptest', text: 'hello memo', seen: 1000, blockHeight: 600100 })
|
||||
const postHeightCreate = sinon.stub().resolves({ success: true })
|
||||
const postHeightGet = sinon.stub().resolves({ txid: 'abc123', blockHeight: 600100 })
|
||||
|
||||
const adapters = {
|
||||
postDb: { create, get },
|
||||
postHeightDb: { create: postHeightCreate, get: postHeightGet },
|
||||
processErrorDb: { create: sinon.stub() }
|
||||
}
|
||||
|
||||
const message = Buffer.from('hello memo')
|
||||
await handlePost({
|
||||
adapters,
|
||||
txid: 'abc123',
|
||||
signerAddr: 'bitcoincash:qptest',
|
||||
seen: 1000,
|
||||
blockHeight: 600100,
|
||||
decoded: {
|
||||
action: 'post',
|
||||
prefix: PREFIX_POST,
|
||||
pushDatas: [PREFIX_POST, message]
|
||||
}
|
||||
})
|
||||
|
||||
assert.equal(create.callCount, 0)
|
||||
assert.equal(postHeightCreate.callCount, 0)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { assert } from 'chai'
|
||||
import sinon from 'sinon'
|
||||
import { handleReply } from '../../../../src/use-cases/action-types/reply.js'
|
||||
import { PREFIX_REPLY } from '../../../../src/lib/memo-codes.js'
|
||||
|
||||
describe('#handleReply', () => {
|
||||
it('should save a reply and its postHeight index entry', async () => {
|
||||
const parentTxid = Buffer.alloc(32, 0xab)
|
||||
const message = Buffer.from('hi there')
|
||||
|
||||
const postParentCreate = sinon.stub().resolves({ success: true })
|
||||
const postChildCreate = sinon.stub().resolves({ success: true })
|
||||
const postDbGet = sinon.stub().rejects(new Error('not found'))
|
||||
const postDbCreate = sinon.stub().resolves({ success: true })
|
||||
const postHeightGet = sinon.stub().rejects(new Error('not found'))
|
||||
const postHeightCreate = sinon.stub().resolves({ success: true })
|
||||
|
||||
const adapters = {
|
||||
postParentDb: { create: postParentCreate },
|
||||
postChildDb: { create: postChildCreate },
|
||||
postDb: { get: postDbGet, create: postDbCreate },
|
||||
postHeightDb: { get: postHeightGet, create: postHeightCreate },
|
||||
processErrorDb: { create: sinon.stub() }
|
||||
}
|
||||
|
||||
await handleReply({
|
||||
adapters,
|
||||
txid: 'reply-abc',
|
||||
signerAddr: 'bitcoincash:qptest',
|
||||
seen: 1000,
|
||||
blockHeight: 600150,
|
||||
decoded: {
|
||||
action: 'reply',
|
||||
prefix: PREFIX_REPLY,
|
||||
pushDatas: [PREFIX_REPLY, parentTxid, message]
|
||||
}
|
||||
})
|
||||
|
||||
assert.equal(postParentCreate.callCount, 1)
|
||||
assert.equal(postParentCreate.firstCall.args[0], 'reply-abc')
|
||||
assert.equal(postParentCreate.firstCall.args[1].parentTxid, parentTxid.toString('hex'))
|
||||
|
||||
assert.equal(postChildCreate.callCount, 1)
|
||||
assert.include(postChildCreate.firstCall.args[0], 'reply-abc')
|
||||
|
||||
assert.equal(postDbCreate.callCount, 1)
|
||||
assert.equal(postDbCreate.firstCall.args[0], 'reply-abc')
|
||||
assert.equal(postDbCreate.firstCall.args[1].text, 'hi there')
|
||||
|
||||
assert.equal(postHeightCreate.callCount, 1)
|
||||
assert.equal(postHeightCreate.firstCall.args[0], '000000600150:reply-abc')
|
||||
assert.equal(postHeightCreate.firstCall.args[1].txid, 'reply-abc')
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user