Implement capped-scan feed query performance for /posts/recent

- Replace full-postHeights and full-postChildren scans in ListRecentPosts with
  a single capped scan and per-page reply counting.
- Add PostQuery.scanRecentPostTxidsAndCount with TOTAL_SCAN_CAP=10.
- Update acceptance handlers and fixture for the new feed-query-performance
  spec, and add entry-count wrappers for bounded-read assertions.

By coder.

psf-memo-db/acceptance/lib/handlers.js

@@ -44,10 +44,18 @@ function resolveParam (value, example) {
 }
 
 function wrapIterator (db, counter) {
-  const original = db.iterator.bind(db)
+  const originalIterator = db.iterator.bind(db)
   db.iterator = function (...args) {
     counter.calls++
-    return original(...args)
+    const iter = originalIterator(...args)
+    const originalAsyncIterator = iter[Symbol.asyncIterator].bind(iter)
+    iter[Symbol.asyncIterator] = async function * () {
+      for await (const entry of originalAsyncIterator()) {
+        counter.entries = (counter.entries || 0) + 1
+        yield entry
+      }
+    }
+    return iter
   }
 }
 
@@ -189,6 +197,11 @@ async function loadFixture (world, name) {
     return
   }
 
+  if (name === 'many-posts-with-replies') {
+    await loadManyPostsWithReplies(world)
+    return
+  }
+
   if (name !== 'three-top-level-posts-and-one-reply') {
     throw new Error(`Unknown fixture: ${name}`)
   }
@@ -308,6 +321,54 @@ async function loadPostsWithLikes (world) {
   }
 }
 
+async function loadManyPostsWithReplies (world) {
+  const posts = []
+  for (let i = 0; i <= 20; i++) {
+    const id = String(i).padStart(3, '0')
+    const txid = `post-${id}`
+    const blockHeight = 600000 + i
+    posts.push({ txid, addr: 'bitcoincash:qaddr', text: `post ${id}`, seen: i, blockHeight })
+  }
+
+  for (const post of posts) {
+    await world.adapters.level.postsDb.put(post.txid, {
+      addr: post.addr,
+      text: post.text,
+      seen: post.seen,
+      blockHeight: post.blockHeight
+    })
+    await world.adapters.level.postHeightsDb.put(
+      String(post.blockHeight).padStart(12, '0') + ':' + post.txid,
+      { txid: post.txid, blockHeight: post.blockHeight }
+    )
+    await world.adapters.level.addrPostHeightsDb.put(
+      `${post.addr}:${String(post.blockHeight).padStart(12, '0')}:${post.txid}`,
+      { txid: post.txid, addr: post.addr, blockHeight: post.blockHeight }
+    )
+  }
+
+  const replies = [
+    { txid: 'reply-020-a', parentTxid: 'post-020', childTxid: 'reply-020-a', blockHeight: 599900 },
+    { txid: 'reply-020-b', parentTxid: 'post-020', childTxid: 'reply-020-b', blockHeight: 599901 },
+    { txid: 'reply-019-a', parentTxid: 'post-019', childTxid: 'reply-019-a', blockHeight: 599902 }
+  ]
+
+  for (const reply of replies) {
+    await world.adapters.level.postsDb.put(reply.txid, {
+      addr: 'bitcoincash:qaddr',
+      text: `reply to ${reply.parentTxid}`,
+      seen: reply.blockHeight,
+      blockHeight: reply.blockHeight
+    })
+    await world.adapters.level.postHeightsDb.put(
+      String(reply.blockHeight).padStart(12, '0') + ':' + reply.txid,
+      { txid: reply.txid, blockHeight: reply.blockHeight }
+    )
+    await world.adapters.level.postParentsDb.put(reply.txid, reply)
+    await world.adapters.level.postChildrenDb.put(`${reply.parentTxid}:${reply.txid}`, reply)
+  }
+}
+
 async function backfillIndexes (world) {
   const posts = []
   for await (const [txid, post] of world.adapters.level.postsDb.iterator()) {
@@ -594,6 +655,28 @@ const handlers = [
       }
     }
   },
+  {
+    name: 'bounded postChildren entries read',
+    pattern: /^the postChildren store was read at most (<max_entries>) entries$/,
+    run (m, example, world) {
+      const max = parseInt(resolveParam(m[1], example), 10)
+      const reads = world.postChildrenIteratorCounter.entries || 0
+      if (reads > max) {
+        throw new Error(`Read ${reads} postChildren entries, expected at most ${max}`)
+      }
+    }
+  },
+  {
+    name: 'bounded postHeights entries read',
+    pattern: /^the postHeights store was read at most (<max_entries>) entries$/,
+    run (m, example, world) {
+      const max = parseInt(resolveParam(m[1], example), 10)
+      const reads = world.postHeightsIteratorCounter.entries || 0
+      if (reads > max) {
+        throw new Error(`Read ${reads} postHeights entries, expected at most ${max}`)
+      }
+    }
+  },
   {
     name: 'bounded postHeights reads',
     pattern: /^no more than (<limit>) postHeights entries are read after applying the offset$/,

psf-memo-db/src/adapters/post-query.js

@@ -11,6 +11,7 @@ import { getPostOrNull as getPostOrNullShared } from './lib/get-post-or-null.js'
 import { loadMutedAddrs, isMutedPost } from './lib/muted-posts.js'
 
 const HEIGHT_PAD = 12
+const TOTAL_SCAN_CAP = 10
 
 class PostQuery {
   constructor (localConfig = {}) {
@@ -46,6 +47,7 @@ class PostQuery {
     this.muteQuery = muteQuery || null
 
     this.scanRecentPostTxids = this.scanRecentPostTxids.bind(this)
+    this.scanRecentPostTxidsAndCount = this.scanRecentPostTxidsAndCount.bind(this)
     this.scanPostsByAddrTxids = this.scanPostsByAddrTxids.bind(this)
     this.loadPostsByTxids = this.loadPostsByTxids.bind(this)
     this.countTopLevelPosts = this.countTopLevelPosts.bind(this)
@@ -201,24 +203,45 @@ class PostQuery {
     }
   }
 
-  async scanRecentPostTxids ({ limit, offset, viewerAddr = null }) {
+  async scanRecentPostTxids (args) {
+    const { txids } = await this.scanRecentPostTxidsAndCount(args)
+    return txids
+  }
+
+  // Scan the global postHeights index newest first, returning the page txids
+  // plus a capped total count. The raw scan is limited to offset + limit +
+  // TOTAL_SCAN_CAP postHeights entries so the first pages avoid walking the
+  // entire index; the returned total is capped to TOTAL_SCAN_CAP and drives
+  // hasMore via assemblePostPage.
+  async scanRecentPostTxidsAndCount ({ limit, offset, viewerAddr = null, totalScanCap = TOTAL_SCAN_CAP }) {
     const mutedAddrs = await loadMutedAddrs(this.muteQuery, viewerAddr)
     const txids = []
     let skipped = 0
+    let eligibleCount = 0
+    let rawCount = 0
+    const maxRaw = offset + limit + totalScanCap
 
-    for await (const txid of this.topLevelPostTxids({ reverse: true })) {
+    for await (const [key, value] of this.postHeightsDb.iterator({ reverse: true })) {
+      rawCount++
+      const txid = this.txidFromPostHeight(key, value)
+      if (await this.isReply(txid)) continue
       if (await isMutedPost((t) => this.getPostOrNull(t), txid, mutedAddrs)) continue
 
+      eligibleCount++
+
       if (skipped < offset) {
         skipped++
         continue
       }
 
-      txids.push(txid)
-      if (txids.length >= limit) break
+      if (txids.length < limit) {
+        txids.push(txid)
+      }
+
+      if (rawCount >= maxRaw) break
     }
 
-    return txids
+    return { txids, total: Math.min(eligibleCount, totalScanCap) }
   }
 
   // Iterate posts for a single address using the addrPostHeights index,

psf-memo-db/src/use-cases/list-recent-posts.js

@@ -18,12 +18,11 @@ class ListRecentPosts extends ListUseCase {
 
     const scanArgs = { limit, offset }
     if (viewerAddr) scanArgs.viewerAddr = viewerAddr
-    const txids = await this.adapters.postQuery.scanRecentPostTxids(scanArgs)
-    const [posts, replyCounts, likeCounts, total] = await Promise.all([
+    const { txids, total } = await this.adapters.postQuery.scanRecentPostTxidsAndCount(scanArgs)
+    const [posts, replyCounts, likeCounts] = await Promise.all([
       this.adapters.postQuery.loadPostsByTxids(txids),
-      this.adapters.postQuery.buildReplyCountMap(),
-      this.adapters.postQuery.countLikesForTxids(txids),
-      this.adapters.postQuery.countTopLevelPosts(viewerAddr)
+      this.adapters.postQuery.countRepliesForTxids(txids),
+      this.adapters.postQuery.countLikesForTxids(txids)
     ])
 
     return assemblePostPage({ posts, replyCounts, likeCounts, total, limit, offset })

psf-memo-db/test/unit/adapters/post-query.unit.js

@@ -47,6 +47,10 @@ describe('#PostQuery', () => {
     likesDb.iterator.returns(empty())
     postLikesDb.iterator.returns(empty())
 
+    const notFoundErr = new Error('not found')
+    notFoundErr.notFound = true
+    postParentsDb.get.rejects(notFoundErr)
+
     uut = new PostQuery({
       postsDb,
       postHeightsDb,
@@ -131,16 +135,13 @@ describe('#PostQuery', () => {
     })
 
     it('should skip replies when selecting recent posts', async () => {
-      async function * mockParents () {
-        yield ['reply-1', { parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 }]
-      }
+      postParentsDb.get.withArgs('reply-1').resolves({ parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 })
       async function * mockHeights () {
         yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
         yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
         yield ['000000600150:reply-1', { txid: 'reply-1' }]
         yield ['000000600100:post-100', { txid: 'post-100' }]
       }
-      postParentsDb.iterator.returns(mockParents())
       postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
 
       const result = await uut.scanRecentPostTxids({ limit: 2, offset: 0 })
@@ -175,6 +176,68 @@ describe('#PostQuery', () => {
     })
   })
 
+  describe('#scanRecentPostTxidsAndCount', () => {
+    it('should return txids and a capped total count', async () => {
+      async function * mockHeights () {
+        for (let i = 20; i >= 0; i--) {
+          const id = String(i).padStart(3, '0')
+          yield [`000000${600000 + i}:post-${id}`, { txid: `post-${id}` }]
+        }
+      }
+      postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
+
+      const result = await uut.scanRecentPostTxidsAndCount({ limit: 3, offset: 0 })
+
+      assert.deepEqual(result.txids, ['post-020', 'post-019', 'post-018'])
+      assert.equal(result.total, 10)
+    })
+
+    it('should cap the raw postHeights scan to limit + offset + cap', async () => {
+      async function * mockHeights () {
+        for (let i = 20; i >= 0; i--) {
+          const id = String(i).padStart(3, '0')
+          yield [`000000${600000 + i}:post-${id}`, { txid: `post-${id}` }]
+        }
+      }
+      postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
+
+      const result = await uut.scanRecentPostTxidsAndCount({ limit: 5, offset: 0, totalScanCap: 10 })
+
+      assert.deepEqual(result.txids.length, 5)
+      assert.equal(result.total, 10)
+    })
+
+    it('should return actual total when the index exhausts before the cap', async () => {
+      async function * mockHeights () {
+        yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
+        yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
+        yield ['000000600100:post-100', { txid: 'post-100' }]
+      }
+      postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
+
+      const result = await uut.scanRecentPostTxidsAndCount({ limit: 10, offset: 0, totalScanCap: 10 })
+
+      assert.deepEqual(result.txids, ['post-200-b', 'post-200-a', 'post-100'])
+      assert.equal(result.total, 3)
+    })
+
+    it('should skip replies when counting and selecting recent posts', async () => {
+      postParentsDb.get.withArgs('reply-1').resolves({ parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600150 })
+      async function * mockHeights () {
+        yield ['000000600200:post-200-b', { txid: 'post-200-b' }]
+        yield ['000000600200:post-200-a', { txid: 'post-200-a' }]
+        yield ['000000600150:reply-1', { txid: 'reply-1' }]
+        yield ['000000600100:post-100', { txid: 'post-100' }]
+      }
+      postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
+
+      const result = await uut.scanRecentPostTxidsAndCount({ limit: 2, offset: 0, totalScanCap: 10 })
+
+      assert.deepEqual(result.txids, ['post-200-b', 'post-200-a'])
+      assert.equal(result.total, 3)
+    })
+  })
+
   describe('#scanPostsByAddrTxids', () => {
     beforeEach(() => {
       const err = new Error('not found')

psf-memo-db/test/unit/use-cases/list-recent-posts.unit.js

@@ -16,13 +16,12 @@ describe('#ListRecentPosts', () => {
   beforeEach(() => {
     sandbox = sinon.createSandbox()
     postQuery = {
-      scanRecentPostTxids: sandbox.stub().resolves(['tx-b', 'tx-c', 'tx-a']),
+      scanRecentPostTxidsAndCount: sandbox.stub().resolves({ txids: ['tx-b', 'tx-c', 'tx-a'], total: 3 }),
       loadPostsByTxids: sandbox.stub().callsFake(async (txids) => {
         return txids.map((txid) => ({ txid, ...mockPosts[txid] }))
       }),
-      buildReplyCountMap: sandbox.stub().resolves(new Map([['tx-b', 1]])),
-      countLikesForTxids: sandbox.stub().resolves(new Map([['tx-b', 3], ['tx-a', 5]])),
-      countTopLevelPosts: sandbox.stub().resolves(3)
+      countRepliesForTxids: sandbox.stub().resolves(new Map([['tx-b', 1]])),
+      countLikesForTxids: sandbox.stub().resolves(new Map([['tx-b', 3], ['tx-a', 5]]))
     }
     uut = new ListRecentPosts({
       adapters: { postQuery }
@@ -48,7 +47,7 @@ describe('#ListRecentPosts', () => {
   })
 
   it('should paginate with limit and offset', async () => {
-    postQuery.scanRecentPostTxids.resolves(['tx-c'])
+    postQuery.scanRecentPostTxidsAndCount.resolves({ txids: ['tx-c'], total: 3 })
     const result = await uut.execute({ limit: 1, offset: 1 })
 
     assert.equal(result.posts.length, 1)
@@ -78,7 +77,7 @@ describe('#ListRecentPosts', () => {
   it('should pass limit and offset to postQuery', async () => {
     await uut.execute({ limit: 5, offset: 10 })
 
-    assert.equal(postQuery.scanRecentPostTxids.calledOnce, true)
-    assert.deepEqual(postQuery.scanRecentPostTxids.firstCall.args[0], { limit: 5, offset: 10 })
+    assert.equal(postQuery.scanRecentPostTxidsAndCount.calledOnce, true)
+    assert.deepEqual(postQuery.scanRecentPostTxidsAndCount.firstCall.args[0], { limit: 5, offset: 10 })
   })
 })