Implement following feed capped scan

By coder.

psf-memo-db/acceptance/lib/handlers.js

@@ -17,6 +17,7 @@ import ListPostsByAddr from '../../src/use-cases/list-posts-by-addr.js'
 import GetPostThread from '../../src/use-cases/get-post-thread.js'
 import FollowState from '../../src/use-cases/follow-state.js'
 import ListFollowing from '../../src/use-cases/list-following.js'
+import ListFollowingFeed from '../../src/use-cases/list-following-feed.js'
 import ListFollowers from '../../src/use-cases/list-followers.js'
 import ListTopics from '../../src/use-cases/list-topics.js'
 import ListTopicPosts from '../../src/use-cases/list-topic-posts.js'
@@ -112,6 +113,7 @@ async function createWorld () {
   const likesIteratorCounter = { calls: 0 }
   const postLikesIteratorCounter = { calls: 0 }
   const followsIteratorCounter = { calls: 0 }
+  const postParentsIteratorCounter = { calls: 0 }
   const followeeHeightsIteratorCounter = { calls: 0, entries: 0 }
   const roomsIteratorCounter = { calls: 0, entries: 0 }
   const topicRecencyIteratorCounter = { calls: 0, entries: 0 }
@@ -122,6 +124,7 @@ async function createWorld () {
   wrapIterator(adapters.level.likesDb, likesIteratorCounter)
   wrapIterator(adapters.level.postLikesDb, postLikesIteratorCounter)
   wrapIterator(adapters.level.followsDb, followsIteratorCounter)
+  wrapIterator(adapters.level.postParentsDb, postParentsIteratorCounter)
   wrapIterator(adapters.level.followeeHeightsDb, followeeHeightsIteratorCounter)
   wrapIterator(adapters.level.roomsDb, roomsIteratorCounter)
   wrapIterator(adapters.level.topicRecencyDb, topicRecencyIteratorCounter)
@@ -131,6 +134,7 @@ async function createWorld () {
   const getPostThread = new GetPostThread({ adapters })
   const followState = new FollowState({ adapters })
   const listFollowing = new ListFollowing({ adapters })
+  const listFollowingFeed = new ListFollowingFeed({ adapters })
   const listFollowers = new ListFollowers({ adapters })
   const listTopics = new ListTopics({ adapters })
   const listTopicPosts = new ListTopicPosts({ adapters })
@@ -152,6 +156,7 @@ async function createWorld () {
     getPostThread,
     followState,
     listFollowing,
+    listFollowingFeed,
     listFollowers,
     listTopics,
     listTopicPosts,
@@ -170,6 +175,7 @@ async function createWorld () {
     likesIteratorCounter,
     postLikesIteratorCounter,
     followsIteratorCounter,
+    postParentsIteratorCounter,
     followeeHeightsIteratorCounter,
     roomsIteratorCounter,
     topicRecencyIteratorCounter,
@@ -278,6 +284,16 @@ async function loadFixture (world, name) {
     return
   }
 
+  if (name === 'following-feed-capped') {
+    await loadFollowingFeedCapped(world)
+    return
+  }
+
+  if (name === 'following-feed-mixed') {
+    await loadFollowingFeedMixed(world)
+    return
+  }
+
   if (name !== 'three-top-level-posts-and-one-reply') {
     throw new Error(`Unknown fixture: ${name}`)
   }
@@ -470,6 +486,81 @@ async function loadManyTopLevelPosts (world) {
   }
 }
 
+async function loadFollowingFeedCapped (world) {
+  const viewer = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d'
+  const followee = 'bitcoincash:qr95sy3j9xwd2ap32xkykttr4cvcu7as4y0qverfuy'
+  world.fixtureViewer = viewer
+
+  await world.adapters.level.followsDb.put(`${viewer}:${hash160(followee)}`, {
+    followerAddr: viewer,
+    followeePkHash: hash160(followee),
+    unfollow: false,
+    txid: 'follow-capped',
+    seen: 1,
+    blockHeight: 600000
+  })
+
+  // 510 eligible top-level posts, larger than the 500 total-scan cap.
+  for (let i = 0; i < 510; i++) {
+    const id = String(i).padStart(3, '0')
+    const txid = `post-${id}`
+    const blockHeight = 600000 + i
+    await world.adapters.level.postsDb.put(txid, {
+      addr: followee,
+      text: `post ${id}`,
+      seen: i,
+      blockHeight
+    })
+    await world.adapters.level.postHeightsDb.put(
+      `${padHeight(blockHeight)}:${txid}`,
+      { txid, blockHeight }
+    )
+  }
+}
+
+// Fixture "following-feed-mixed" from following-feed-performance.feature: the
+// viewer follows one author while the viewer and another author also have
+// top-level posts, and the followed author has a reply that must be excluded.
+async function loadFollowingFeedMixed (world) {
+  const viewer = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d'
+  const followee = 'bitcoincash:qr95sy3j9xwd2ap32xkykttr4cvcu7as4y0qverfuy'
+  const other = 'bitcoincash:qpm2qsznhks23z7629mms6s4cwef74vcwvy22gdx6a'
+  world.fixtureViewer = viewer
+
+  await world.adapters.level.followsDb.put(`${viewer}:${hash160(followee)}`, {
+    followerAddr: viewer,
+    followeePkHash: hash160(followee),
+    unfollow: false,
+    txid: 'follow-mixed',
+    seen: 1,
+    blockHeight: 600000
+  })
+
+  const posts = [
+    { txid: 'post-A1', addr: followee, blockHeight: 600100 },
+    { txid: 'post-A2', addr: followee, blockHeight: 600200 },
+    { txid: 'post-viewer', addr: viewer, blockHeight: 600150 },
+    { txid: 'post-other', addr: other, blockHeight: 600250 },
+    { txid: 'reply-A2', addr: followee, blockHeight: 600300 }
+  ]
+  for (const post of posts) {
+    await world.adapters.level.postsDb.put(post.txid, {
+      addr: post.addr,
+      text: post.txid,
+      seen: post.blockHeight,
+      blockHeight: post.blockHeight
+    })
+    await world.adapters.level.postHeightsDb.put(
+      `${padHeight(post.blockHeight)}:${post.txid}`,
+      { txid: post.txid, blockHeight: post.blockHeight }
+    )
+  }
+
+  const reply = { txid: 'reply-A2', parentTxid: 'post-A2', childTxid: 'reply-A2', blockHeight: 600300 }
+  await world.adapters.level.postParentsDb.put('reply-A2', reply)
+  await world.adapters.level.postChildrenDb.put('post-A2:reply-A2', reply)
+}
+
 async function backfillIndexes (world) {
   const posts = []
   for await (const [txid, post] of world.adapters.level.postsDb.iterator()) {
@@ -1013,6 +1104,17 @@ const handlers = [
       world.setLastResponse(resp)
     }
   },
+  {
+    name: 'request following feed',
+    pattern: /^the client requests \/posts\/following\/(<[A-Za-z0-9_]+>) with limit (<[A-Za-z0-9_]+>) and offset (<[A-Za-z0-9_]+>)$/,
+    async run (m, example, world) {
+      const addr = m[1] === '<viewer>' ? world.fixtureViewer : resolveParam(m[1], example)
+      const limit = parseInt(resolveParam(m[2], example), 10)
+      const offset = parseInt(resolveParam(m[3], example), 10)
+      const resp = await world.listFollowingFeed.execute({ addr, limit, offset })
+      world.setLastResponse(resp)
+    }
+  },
   {
     name: 'posts sorted by block height descending',
     pattern: /^the response posts are sorted by block height descending$/,
@@ -2076,6 +2178,15 @@ const handlers = [
         throw new Error(`Expected follows store not to be iterated, got ${world.followsIteratorCounter.calls} call(s)`)
       }
     }
+  },
+  {
+    name: 'postParents store was not iterated',
+    pattern: /^the postParents store was not iterated$/,
+    run (m, example, world) {
+      if (world.postParentsIteratorCounter.calls !== 0) {
+        throw new Error(`Expected postParents store not to be iterated, got ${world.postParentsIteratorCounter.calls} call(s)`)
+      }
+    }
   }
 
 ]

psf-memo-db/src/adapters/post-query.js

@@ -282,38 +282,41 @@ class PostQuery {
 
   // Iterate the global postHeights index newest first, returning only top-level
   // posts (replies excluded) authored by addresses the viewer follows, excluding
-  // the viewer's own posts. Returns both the page txids and total matching count.
-  async scanFollowingFeedTxidsAndCount (viewerAddr, followingAddrs, { limit, offset }) {
+  // the viewer's own posts. Reply detection uses per-candidate point lookups so
+  // the postParents store is never fully iterated. The scan stops after
+  // offset + limit + totalScanCap eligible followed posts, and the returned
+  // total is capped to totalScanCap so the first pages stay bounded while
+  // hasMore still works.
+  async scanFollowingFeedTxidsAndCount (viewerAddr, followingAddrs, { limit, offset, totalScanCap = TOTAL_SCAN_CAP }) {
     const followeeSet = new Set(followingAddrs.filter((addr) => addr !== viewerAddr))
-    const replyTxids = await this.loadReplyTxids()
     const txids = []
     let skipped = 0
-    let total = 0
+    let eligibleCount = 0
+    const maxEligible = offset + limit + totalScanCap
 
     for await (const [key, value] of this.postHeightsDb.iterator({ reverse: true })) {
       const txid = this.txidFromPostHeight(key, value)
-      if (!(await this.isFolloweePost(txid, replyTxids, followeeSet))) continue
+      if (!(await this.isFolloweePost(txid, followeeSet))) continue
 
-      total++
+      eligibleCount++
 
       if (skipped < offset) {
         skipped++
-        continue
-      }
-
-      if (txids.length < limit) {
+      } else if (txids.length < limit) {
         txids.push(txid)
       }
+
+      if (eligibleCount >= maxEligible) break
     }
 
-    return { txids, total }
+    return { txids, total: Math.min(eligibleCount, totalScanCap) }
   }
 
   // True when a post is a top-level (non-reply) post authored by a followed
-  // address. Loads the post record to check authorship; missing records are
-  // treated as not matching.
-  async isFolloweePost (txid, replyTxids, followeeSet) {
-    if (replyTxids.has(txid)) return false
+  // address. Reply detection is a point lookup on postParents; missing records
+  // are treated as not matching.
+  async isFolloweePost (txid, followeeSet) {
+    if (await this.isReply(txid)) return false
     const post = await this.getPostOrNull(txid)
     if (!post) return false
     return followeeSet.has(post.addr)

psf-memo-db/test/property/following-feed-query.property.test.js

@@ -57,10 +57,11 @@ function makePostsDb (posts) {
 
 function makeParentsDb (replyTxids) {
   return {
-    async * iterator () {
-      for (const txid of replyTxids) {
-        yield [txid, { parentTxid: 'parent' }]
-      }
+    async get (txid) {
+      if (replyTxids.has(txid)) return { parentTxid: 'parent', childTxid: txid }
+      const err = new Error('not found')
+      err.notFound = true
+      throw err
     }
   }
 }

psf-memo-db/test/unit/adapters/post-query.unit.js

@@ -384,16 +384,18 @@ describe('#PostQuery', () => {
       assert.equal(result.total, 2)
     })
 
-    it('should exclude replies from the following feed', async () => {
-      async function * mockParents () {
-        yield ['reply-a', { parentTxid: 'post-a', childTxid: 'reply-a' }]
-      }
+    it('should exclude replies using point lookups without iterating postParents', async () => {
       async function * mockHeights () {
         yield ['000000600300:reply-a', { txid: 'reply-a' }]
         yield ['000000600200:post-a', { txid: 'post-a' }]
       }
-      postParentsDb.iterator.returns(mockParents())
       postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
+      postParentsDb.get.callsFake(async (txid) => {
+        if (txid === 'reply-a') return { parentTxid: 'post-a', childTxid: 'reply-a' }
+        const err = new Error('not found')
+        err.notFound = true
+        throw err
+      })
       postsDb.get.callsFake(async (txid) => {
         return { addr: followeeA, text: txid }
       })
@@ -406,6 +408,77 @@ describe('#PostQuery', () => {
 
       assert.deepEqual(result.txids, ['post-a'])
       assert.equal(result.total, 1)
+      assert.equal(postParentsDb.iterator.called, false)
+    })
+
+    it('should cap the total at 500 and stop scanning after offset + limit + cap eligible posts', async () => {
+      let reads = 0
+      async function * mockHeights () {
+        for (let i = 599; i >= 0; i--) {
+          reads++
+          const id = String(i).padStart(3, '0')
+          yield [`000000${600000 + i}:post-${id}`, { txid: `post-${id}` }]
+        }
+      }
+      postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
+      postsDb.get.callsFake(async (txid) => ({ addr: followeeA, text: txid }))
+
+      const result = await uut.scanFollowingFeedTxidsAndCount(
+        viewerAddr,
+        [followeeA],
+        { limit: 3, offset: 0 }
+      )
+
+      assert.deepEqual(result.txids, ['post-599', 'post-598', 'post-597'])
+      assert.equal(result.total, 500)
+      assert.equal(reads, 503) // offset + limit + default cap eligible posts
+    })
+
+    it('should report the exact total when eligible posts are below the cap', async () => {
+      async function * mockHeights () {
+        yield ['000000600200:post-a', { txid: 'post-a' }]
+        yield ['000000600100:post-b', { txid: 'post-b' }]
+      }
+      postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
+      postsDb.get.callsFake(async (txid) => ({ addr: followeeA, text: txid }))
+
+      const result = await uut.scanFollowingFeedTxidsAndCount(
+        viewerAddr,
+        [followeeA],
+        { limit: 10, offset: 0 }
+      )
+
+      assert.deepEqual(result.txids, ['post-a', 'post-b'])
+      assert.equal(result.total, 2)
+    })
+
+    it('should count only eligible followed posts toward the scan cap', async () => {
+      let reads = 0
+      async function * mockHeights () {
+        // Interleave eligible followed posts with non-followed top-level posts.
+        for (let i = 1199; i >= 0; i--) {
+          reads++
+          const txid = i % 2 === 0 ? `followed-${i}` : `other-${i}`
+          yield [`000000${600000 + i}:${txid}`, { txid }]
+        }
+      }
+      postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights())
+      postsDb.get.callsFake(async (txid) => ({
+        addr: txid.startsWith('followed') ? followeeA : 'bitcoincash:other',
+        text: txid
+      }))
+
+      const result = await uut.scanFollowingFeedTxidsAndCount(
+        viewerAddr,
+        [followeeA],
+        { limit: 3, offset: 0 }
+      )
+
+      assert.deepEqual(result.txids, ['followed-1198', 'followed-1196', 'followed-1194'])
+      assert.equal(result.total, 500)
+      // Non-eligible raw entries do not count toward the cap, so the scan keeps
+      // going past 503 raw entries until it has seen 503 eligible posts.
+      assert.isAbove(reads, 503)
     })
 
     it('should apply limit and offset', async () => {