From 65e85c9aef2a719b29597aeb0567519dbd46b2a3 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Fri, 4 Sep 2026 17:55:35 -0700 Subject: [PATCH] Implement capped-scan feed query performance for /posts/recent - Replace full-postHeights and full-postChildren scans in ListRecentPosts with a single capped scan and per-page reply counting. - Add PostQuery.scanRecentPostTxidsAndCount with TOTAL_SCAN_CAP=10. - Update acceptance handlers and fixture for the new feed-query-performance spec, and add entry-count wrappers for bounded-read assertions. By coder. --- psf-memo-db/acceptance/lib/handlers.js | 87 ++++++++++++++++++- psf-memo-db/src/adapters/post-query.js | 33 +++++-- .../src/use-cases/list-recent-posts.js | 9 +- .../test/unit/adapters/post-query.unit.js | 71 ++++++++++++++- .../unit/use-cases/list-recent-posts.unit.js | 13 ++- 5 files changed, 190 insertions(+), 23 deletions(-) diff --git a/psf-memo-db/acceptance/lib/handlers.js b/psf-memo-db/acceptance/lib/handlers.js index 14006e1..5fcb71c 100644 --- a/psf-memo-db/acceptance/lib/handlers.js +++ b/psf-memo-db/acceptance/lib/handlers.js @@ -44,10 +44,18 @@ function resolveParam (value, example) { } function wrapIterator (db, counter) { - const original = db.iterator.bind(db) + const originalIterator = db.iterator.bind(db) db.iterator = function (...args) { counter.calls++ - return original(...args) + const iter = originalIterator(...args) + const originalAsyncIterator = iter[Symbol.asyncIterator].bind(iter) + iter[Symbol.asyncIterator] = async function * () { + for await (const entry of originalAsyncIterator()) { + counter.entries = (counter.entries || 0) + 1 + yield entry + } + } + return iter } } @@ -189,6 +197,11 @@ async function loadFixture (world, name) { return } + if (name === 'many-posts-with-replies') { + await loadManyPostsWithReplies(world) + return + } + if (name !== 'three-top-level-posts-and-one-reply') { throw new Error(`Unknown fixture: ${name}`) } @@ -308,6 +321,54 @@ async function loadPostsWithLikes (world) { } } +async function loadManyPostsWithReplies (world) { + const posts = [] + for (let i = 0; i <= 20; i++) { + const id = String(i).padStart(3, '0') + const txid = `post-${id}` + const blockHeight = 600000 + i + posts.push({ txid, addr: 'bitcoincash:qaddr', text: `post ${id}`, seen: i, blockHeight }) + } + + for (const post of posts) { + await world.adapters.level.postsDb.put(post.txid, { + addr: post.addr, + text: post.text, + seen: post.seen, + blockHeight: post.blockHeight + }) + await world.adapters.level.postHeightsDb.put( + String(post.blockHeight).padStart(12, '0') + ':' + post.txid, + { txid: post.txid, blockHeight: post.blockHeight } + ) + await world.adapters.level.addrPostHeightsDb.put( + `${post.addr}:${String(post.blockHeight).padStart(12, '0')}:${post.txid}`, + { txid: post.txid, addr: post.addr, blockHeight: post.blockHeight } + ) + } + + const replies = [ + { txid: 'reply-020-a', parentTxid: 'post-020', childTxid: 'reply-020-a', blockHeight: 599900 }, + { txid: 'reply-020-b', parentTxid: 'post-020', childTxid: 'reply-020-b', blockHeight: 599901 }, + { txid: 'reply-019-a', parentTxid: 'post-019', childTxid: 'reply-019-a', blockHeight: 599902 } + ] + + for (const reply of replies) { + await world.adapters.level.postsDb.put(reply.txid, { + addr: 'bitcoincash:qaddr', + text: `reply to ${reply.parentTxid}`, + seen: reply.blockHeight, + blockHeight: reply.blockHeight + }) + await world.adapters.level.postHeightsDb.put( + String(reply.blockHeight).padStart(12, '0') + ':' + reply.txid, + { txid: reply.txid, blockHeight: reply.blockHeight } + ) + await world.adapters.level.postParentsDb.put(reply.txid, reply) + await world.adapters.level.postChildrenDb.put(`${reply.parentTxid}:${reply.txid}`, reply) + } +} + async function backfillIndexes (world) { const posts = [] for await (const [txid, post] of world.adapters.level.postsDb.iterator()) { @@ -594,6 +655,28 @@ const handlers = [ } } }, + { + name: 'bounded postChildren entries read', + pattern: /^the postChildren store was read at most () entries$/, + run (m, example, world) { + const max = parseInt(resolveParam(m[1], example), 10) + const reads = world.postChildrenIteratorCounter.entries || 0 + if (reads > max) { + throw new Error(`Read ${reads} postChildren entries, expected at most ${max}`) + } + } + }, + { + name: 'bounded postHeights entries read', + pattern: /^the postHeights store was read at most () entries$/, + run (m, example, world) { + const max = parseInt(resolveParam(m[1], example), 10) + const reads = world.postHeightsIteratorCounter.entries || 0 + if (reads > max) { + throw new Error(`Read ${reads} postHeights entries, expected at most ${max}`) + } + } + }, { name: 'bounded postHeights reads', pattern: /^no more than () postHeights entries are read after applying the offset$/, diff --git a/psf-memo-db/src/adapters/post-query.js b/psf-memo-db/src/adapters/post-query.js index 87c9831..06170dc 100644 --- a/psf-memo-db/src/adapters/post-query.js +++ b/psf-memo-db/src/adapters/post-query.js @@ -11,6 +11,7 @@ import { getPostOrNull as getPostOrNullShared } from './lib/get-post-or-null.js' import { loadMutedAddrs, isMutedPost } from './lib/muted-posts.js' const HEIGHT_PAD = 12 +const TOTAL_SCAN_CAP = 10 class PostQuery { constructor (localConfig = {}) { @@ -46,6 +47,7 @@ class PostQuery { this.muteQuery = muteQuery || null this.scanRecentPostTxids = this.scanRecentPostTxids.bind(this) + this.scanRecentPostTxidsAndCount = this.scanRecentPostTxidsAndCount.bind(this) this.scanPostsByAddrTxids = this.scanPostsByAddrTxids.bind(this) this.loadPostsByTxids = this.loadPostsByTxids.bind(this) this.countTopLevelPosts = this.countTopLevelPosts.bind(this) @@ -201,24 +203,45 @@ class PostQuery { } } - async scanRecentPostTxids ({ limit, offset, viewerAddr = null }) { + async scanRecentPostTxids (args) { + const { txids } = await this.scanRecentPostTxidsAndCount(args) + return txids + } + + // Scan the global postHeights index newest first, returning the page txids + // plus a capped total count. The raw scan is limited to offset + limit + + // TOTAL_SCAN_CAP postHeights entries so the first pages avoid walking the + // entire index; the returned total is capped to TOTAL_SCAN_CAP and drives + // hasMore via assemblePostPage. + async scanRecentPostTxidsAndCount ({ limit, offset, viewerAddr = null, totalScanCap = TOTAL_SCAN_CAP }) { const mutedAddrs = await loadMutedAddrs(this.muteQuery, viewerAddr) const txids = [] let skipped = 0 + let eligibleCount = 0 + let rawCount = 0 + const maxRaw = offset + limit + totalScanCap - for await (const txid of this.topLevelPostTxids({ reverse: true })) { + for await (const [key, value] of this.postHeightsDb.iterator({ reverse: true })) { + rawCount++ + const txid = this.txidFromPostHeight(key, value) + if (await this.isReply(txid)) continue if (await isMutedPost((t) => this.getPostOrNull(t), txid, mutedAddrs)) continue + eligibleCount++ + if (skipped < offset) { skipped++ continue } - txids.push(txid) - if (txids.length >= limit) break + if (txids.length < limit) { + txids.push(txid) + } + + if (rawCount >= maxRaw) break } - return txids + return { txids, total: Math.min(eligibleCount, totalScanCap) } } // Iterate posts for a single address using the addrPostHeights index, diff --git a/psf-memo-db/src/use-cases/list-recent-posts.js b/psf-memo-db/src/use-cases/list-recent-posts.js index e54da13..e6059b8 100644 --- a/psf-memo-db/src/use-cases/list-recent-posts.js +++ b/psf-memo-db/src/use-cases/list-recent-posts.js @@ -18,12 +18,11 @@ class ListRecentPosts extends ListUseCase { const scanArgs = { limit, offset } if (viewerAddr) scanArgs.viewerAddr = viewerAddr - const txids = await this.adapters.postQuery.scanRecentPostTxids(scanArgs) - const [posts, replyCounts, likeCounts, total] = await Promise.all([ + const { txids, total } = await this.adapters.postQuery.scanRecentPostTxidsAndCount(scanArgs) + const [posts, replyCounts, likeCounts] = await Promise.all([ this.adapters.postQuery.loadPostsByTxids(txids), - this.adapters.postQuery.buildReplyCountMap(), - this.adapters.postQuery.countLikesForTxids(txids), - this.adapters.postQuery.countTopLevelPosts(viewerAddr) + this.adapters.postQuery.countRepliesForTxids(txids), + this.adapters.postQuery.countLikesForTxids(txids) ]) return assemblePostPage({ posts, replyCounts, likeCounts, total, limit, offset }) diff --git a/psf-memo-db/test/unit/adapters/post-query.unit.js b/psf-memo-db/test/unit/adapters/post-query.unit.js index 5e3a15b..a523ee3 100644 --- a/psf-memo-db/test/unit/adapters/post-query.unit.js +++ b/psf-memo-db/test/unit/adapters/post-query.unit.js @@ -47,6 +47,10 @@ describe('#PostQuery', () => { likesDb.iterator.returns(empty()) postLikesDb.iterator.returns(empty()) + const notFoundErr = new Error('not found') + notFoundErr.notFound = true + postParentsDb.get.rejects(notFoundErr) + uut = new PostQuery({ postsDb, postHeightsDb, @@ -131,16 +135,13 @@ describe('#PostQuery', () => { }) it('should skip replies when selecting recent posts', async () => { - async function * mockParents () { - yield ['reply-1', { parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 }] - } + postParentsDb.get.withArgs('reply-1').resolves({ parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 }) async function * mockHeights () { yield ['000000600200:post-200-b', { txid: 'post-200-b' }] yield ['000000600200:post-200-a', { txid: 'post-200-a' }] yield ['000000600150:reply-1', { txid: 'reply-1' }] yield ['000000600100:post-100', { txid: 'post-100' }] } - postParentsDb.iterator.returns(mockParents()) postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) const result = await uut.scanRecentPostTxids({ limit: 2, offset: 0 }) @@ -175,6 +176,68 @@ describe('#PostQuery', () => { }) }) + describe('#scanRecentPostTxidsAndCount', () => { + it('should return txids and a capped total count', async () => { + async function * mockHeights () { + for (let i = 20; i >= 0; i--) { + const id = String(i).padStart(3, '0') + yield [`000000${600000 + i}:post-${id}`, { txid: `post-${id}` }] + } + } + postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) + + const result = await uut.scanRecentPostTxidsAndCount({ limit: 3, offset: 0 }) + + assert.deepEqual(result.txids, ['post-020', 'post-019', 'post-018']) + assert.equal(result.total, 10) + }) + + it('should cap the raw postHeights scan to limit + offset + cap', async () => { + async function * mockHeights () { + for (let i = 20; i >= 0; i--) { + const id = String(i).padStart(3, '0') + yield [`000000${600000 + i}:post-${id}`, { txid: `post-${id}` }] + } + } + postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) + + const result = await uut.scanRecentPostTxidsAndCount({ limit: 5, offset: 0, totalScanCap: 10 }) + + assert.deepEqual(result.txids.length, 5) + assert.equal(result.total, 10) + }) + + it('should return actual total when the index exhausts before the cap', async () => { + async function * mockHeights () { + yield ['000000600200:post-200-b', { txid: 'post-200-b' }] + yield ['000000600200:post-200-a', { txid: 'post-200-a' }] + yield ['000000600100:post-100', { txid: 'post-100' }] + } + postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) + + const result = await uut.scanRecentPostTxidsAndCount({ limit: 10, offset: 0, totalScanCap: 10 }) + + assert.deepEqual(result.txids, ['post-200-b', 'post-200-a', 'post-100']) + assert.equal(result.total, 3) + }) + + it('should skip replies when counting and selecting recent posts', async () => { + postParentsDb.get.withArgs('reply-1').resolves({ parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 }) + async function * mockHeights () { + yield ['000000600200:post-200-b', { txid: 'post-200-b' }] + yield ['000000600200:post-200-a', { txid: 'post-200-a' }] + yield ['000000600150:reply-1', { txid: 'reply-1' }] + yield ['000000600100:post-100', { txid: 'post-100' }] + } + postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) + + const result = await uut.scanRecentPostTxidsAndCount({ limit: 2, offset: 0, totalScanCap: 10 }) + + assert.deepEqual(result.txids, ['post-200-b', 'post-200-a']) + assert.equal(result.total, 3) + }) + }) + describe('#scanPostsByAddrTxids', () => { beforeEach(() => { const err = new Error('not found') diff --git a/psf-memo-db/test/unit/use-cases/list-recent-posts.unit.js b/psf-memo-db/test/unit/use-cases/list-recent-posts.unit.js index e3dbe11..b19193b 100644 --- a/psf-memo-db/test/unit/use-cases/list-recent-posts.unit.js +++ b/psf-memo-db/test/unit/use-cases/list-recent-posts.unit.js @@ -16,13 +16,12 @@ describe('#ListRecentPosts', () => { beforeEach(() => { sandbox = sinon.createSandbox() postQuery = { - scanRecentPostTxids: sandbox.stub().resolves(['tx-b', 'tx-c', 'tx-a']), + scanRecentPostTxidsAndCount: sandbox.stub().resolves({ txids: ['tx-b', 'tx-c', 'tx-a'], total: 3 }), loadPostsByTxids: sandbox.stub().callsFake(async (txids) => { return txids.map((txid) => ({ txid, ...mockPosts[txid] })) }), - buildReplyCountMap: sandbox.stub().resolves(new Map([['tx-b', 1]])), - countLikesForTxids: sandbox.stub().resolves(new Map([['tx-b', 3], ['tx-a', 5]])), - countTopLevelPosts: sandbox.stub().resolves(3) + countRepliesForTxids: sandbox.stub().resolves(new Map([['tx-b', 1]])), + countLikesForTxids: sandbox.stub().resolves(new Map([['tx-b', 3], ['tx-a', 5]])) } uut = new ListRecentPosts({ adapters: { postQuery } @@ -48,7 +47,7 @@ describe('#ListRecentPosts', () => { }) it('should paginate with limit and offset', async () => { - postQuery.scanRecentPostTxids.resolves(['tx-c']) + postQuery.scanRecentPostTxidsAndCount.resolves({ txids: ['tx-c'], total: 3 }) const result = await uut.execute({ limit: 1, offset: 1 }) assert.equal(result.posts.length, 1) @@ -78,7 +77,7 @@ describe('#ListRecentPosts', () => { it('should pass limit and offset to postQuery', async () => { await uut.execute({ limit: 5, offset: 10 }) - assert.equal(postQuery.scanRecentPostTxids.calledOnce, true) - assert.deepEqual(postQuery.scanRecentPostTxids.firstCall.args[0], { limit: 5, offset: 10 }) + assert.equal(postQuery.scanRecentPostTxidsAndCount.calledOnce, true) + assert.deepEqual(postQuery.scanRecentPostTxidsAndCount.firstCall.args[0], { limit: 5, offset: 10 }) }) })