diff --git a/psf-memo-db/acceptance/lib/handlers.js b/psf-memo-db/acceptance/lib/handlers.js index 4166423..c463c9c 100644 --- a/psf-memo-db/acceptance/lib/handlers.js +++ b/psf-memo-db/acceptance/lib/handlers.js @@ -17,6 +17,7 @@ import ListPostsByAddr from '../../src/use-cases/list-posts-by-addr.js' import GetPostThread from '../../src/use-cases/get-post-thread.js' import FollowState from '../../src/use-cases/follow-state.js' import ListFollowing from '../../src/use-cases/list-following.js' +import ListFollowingFeed from '../../src/use-cases/list-following-feed.js' import ListFollowers from '../../src/use-cases/list-followers.js' import ListTopics from '../../src/use-cases/list-topics.js' import ListTopicPosts from '../../src/use-cases/list-topic-posts.js' @@ -112,6 +113,7 @@ async function createWorld () { const likesIteratorCounter = { calls: 0 } const postLikesIteratorCounter = { calls: 0 } const followsIteratorCounter = { calls: 0 } + const postParentsIteratorCounter = { calls: 0 } const followeeHeightsIteratorCounter = { calls: 0, entries: 0 } const roomsIteratorCounter = { calls: 0, entries: 0 } const topicRecencyIteratorCounter = { calls: 0, entries: 0 } @@ -122,6 +124,7 @@ async function createWorld () { wrapIterator(adapters.level.likesDb, likesIteratorCounter) wrapIterator(adapters.level.postLikesDb, postLikesIteratorCounter) wrapIterator(adapters.level.followsDb, followsIteratorCounter) + wrapIterator(adapters.level.postParentsDb, postParentsIteratorCounter) wrapIterator(adapters.level.followeeHeightsDb, followeeHeightsIteratorCounter) wrapIterator(adapters.level.roomsDb, roomsIteratorCounter) wrapIterator(adapters.level.topicRecencyDb, topicRecencyIteratorCounter) @@ -131,6 +134,7 @@ async function createWorld () { const getPostThread = new GetPostThread({ adapters }) const followState = new FollowState({ adapters }) const listFollowing = new ListFollowing({ adapters }) + const listFollowingFeed = new ListFollowingFeed({ adapters }) const listFollowers = new ListFollowers({ adapters }) const listTopics = new ListTopics({ adapters }) const listTopicPosts = new ListTopicPosts({ adapters }) @@ -152,6 +156,7 @@ async function createWorld () { getPostThread, followState, listFollowing, + listFollowingFeed, listFollowers, listTopics, listTopicPosts, @@ -170,6 +175,7 @@ async function createWorld () { likesIteratorCounter, postLikesIteratorCounter, followsIteratorCounter, + postParentsIteratorCounter, followeeHeightsIteratorCounter, roomsIteratorCounter, topicRecencyIteratorCounter, @@ -278,6 +284,16 @@ async function loadFixture (world, name) { return } + if (name === 'following-feed-capped') { + await loadFollowingFeedCapped(world) + return + } + + if (name === 'following-feed-mixed') { + await loadFollowingFeedMixed(world) + return + } + if (name !== 'three-top-level-posts-and-one-reply') { throw new Error(`Unknown fixture: ${name}`) } @@ -470,6 +486,81 @@ async function loadManyTopLevelPosts (world) { } } +async function loadFollowingFeedCapped (world) { + const viewer = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d' + const followee = 'bitcoincash:qr95sy3j9xwd2ap32xkykttr4cvcu7as4y0qverfuy' + world.fixtureViewer = viewer + + await world.adapters.level.followsDb.put(`${viewer}:${hash160(followee)}`, { + followerAddr: viewer, + followeePkHash: hash160(followee), + unfollow: false, + txid: 'follow-capped', + seen: 1, + blockHeight: 600000 + }) + + // 510 eligible top-level posts, larger than the 500 total-scan cap. + for (let i = 0; i < 510; i++) { + const id = String(i).padStart(3, '0') + const txid = `post-${id}` + const blockHeight = 600000 + i + await world.adapters.level.postsDb.put(txid, { + addr: followee, + text: `post ${id}`, + seen: i, + blockHeight + }) + await world.adapters.level.postHeightsDb.put( + `${padHeight(blockHeight)}:${txid}`, + { txid, blockHeight } + ) + } +} + +// Fixture "following-feed-mixed" from following-feed-performance.feature: the +// viewer follows one author while the viewer and another author also have +// top-level posts, and the followed author has a reply that must be excluded. +async function loadFollowingFeedMixed (world) { + const viewer = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d' + const followee = 'bitcoincash:qr95sy3j9xwd2ap32xkykttr4cvcu7as4y0qverfuy' + const other = 'bitcoincash:qpm2qsznhks23z7629mms6s4cwef74vcwvy22gdx6a' + world.fixtureViewer = viewer + + await world.adapters.level.followsDb.put(`${viewer}:${hash160(followee)}`, { + followerAddr: viewer, + followeePkHash: hash160(followee), + unfollow: false, + txid: 'follow-mixed', + seen: 1, + blockHeight: 600000 + }) + + const posts = [ + { txid: 'post-A1', addr: followee, blockHeight: 600100 }, + { txid: 'post-A2', addr: followee, blockHeight: 600200 }, + { txid: 'post-viewer', addr: viewer, blockHeight: 600150 }, + { txid: 'post-other', addr: other, blockHeight: 600250 }, + { txid: 'reply-A2', addr: followee, blockHeight: 600300 } + ] + for (const post of posts) { + await world.adapters.level.postsDb.put(post.txid, { + addr: post.addr, + text: post.txid, + seen: post.blockHeight, + blockHeight: post.blockHeight + }) + await world.adapters.level.postHeightsDb.put( + `${padHeight(post.blockHeight)}:${post.txid}`, + { txid: post.txid, blockHeight: post.blockHeight } + ) + } + + const reply = { txid: 'reply-A2', parentTxid: 'post-A2', childTxid: 'reply-A2', blockHeight: 600300 } + await world.adapters.level.postParentsDb.put('reply-A2', reply) + await world.adapters.level.postChildrenDb.put('post-A2:reply-A2', reply) +} + async function backfillIndexes (world) { const posts = [] for await (const [txid, post] of world.adapters.level.postsDb.iterator()) { @@ -1013,6 +1104,17 @@ const handlers = [ world.setLastResponse(resp) } }, + { + name: 'request following feed', + pattern: /^the client requests \/posts\/following\/(<[A-Za-z0-9_]+>) with limit (<[A-Za-z0-9_]+>) and offset (<[A-Za-z0-9_]+>)$/, + async run (m, example, world) { + const addr = m[1] === '' ? world.fixtureViewer : resolveParam(m[1], example) + const limit = parseInt(resolveParam(m[2], example), 10) + const offset = parseInt(resolveParam(m[3], example), 10) + const resp = await world.listFollowingFeed.execute({ addr, limit, offset }) + world.setLastResponse(resp) + } + }, { name: 'posts sorted by block height descending', pattern: /^the response posts are sorted by block height descending$/, @@ -2076,6 +2178,15 @@ const handlers = [ throw new Error(`Expected follows store not to be iterated, got ${world.followsIteratorCounter.calls} call(s)`) } } + }, + { + name: 'postParents store was not iterated', + pattern: /^the postParents store was not iterated$/, + run (m, example, world) { + if (world.postParentsIteratorCounter.calls !== 0) { + throw new Error(`Expected postParents store not to be iterated, got ${world.postParentsIteratorCounter.calls} call(s)`) + } + } } ] diff --git a/psf-memo-db/src/adapters/post-query.js b/psf-memo-db/src/adapters/post-query.js index 40d8c99..b9c48f9 100644 --- a/psf-memo-db/src/adapters/post-query.js +++ b/psf-memo-db/src/adapters/post-query.js @@ -282,38 +282,41 @@ class PostQuery { // Iterate the global postHeights index newest first, returning only top-level // posts (replies excluded) authored by addresses the viewer follows, excluding - // the viewer's own posts. Returns both the page txids and total matching count. - async scanFollowingFeedTxidsAndCount (viewerAddr, followingAddrs, { limit, offset }) { + // the viewer's own posts. Reply detection uses per-candidate point lookups so + // the postParents store is never fully iterated. The scan stops after + // offset + limit + totalScanCap eligible followed posts, and the returned + // total is capped to totalScanCap so the first pages stay bounded while + // hasMore still works. + async scanFollowingFeedTxidsAndCount (viewerAddr, followingAddrs, { limit, offset, totalScanCap = TOTAL_SCAN_CAP }) { const followeeSet = new Set(followingAddrs.filter((addr) => addr !== viewerAddr)) - const replyTxids = await this.loadReplyTxids() const txids = [] let skipped = 0 - let total = 0 + let eligibleCount = 0 + const maxEligible = offset + limit + totalScanCap for await (const [key, value] of this.postHeightsDb.iterator({ reverse: true })) { const txid = this.txidFromPostHeight(key, value) - if (!(await this.isFolloweePost(txid, replyTxids, followeeSet))) continue + if (!(await this.isFolloweePost(txid, followeeSet))) continue - total++ + eligibleCount++ if (skipped < offset) { skipped++ - continue - } - - if (txids.length < limit) { + } else if (txids.length < limit) { txids.push(txid) } + + if (eligibleCount >= maxEligible) break } - return { txids, total } + return { txids, total: Math.min(eligibleCount, totalScanCap) } } // True when a post is a top-level (non-reply) post authored by a followed - // address. Loads the post record to check authorship; missing records are - // treated as not matching. - async isFolloweePost (txid, replyTxids, followeeSet) { - if (replyTxids.has(txid)) return false + // address. Reply detection is a point lookup on postParents; missing records + // are treated as not matching. + async isFolloweePost (txid, followeeSet) { + if (await this.isReply(txid)) return false const post = await this.getPostOrNull(txid) if (!post) return false return followeeSet.has(post.addr) diff --git a/psf-memo-db/test/property/following-feed-query.property.test.js b/psf-memo-db/test/property/following-feed-query.property.test.js index 34d8ec9..cb36057 100644 --- a/psf-memo-db/test/property/following-feed-query.property.test.js +++ b/psf-memo-db/test/property/following-feed-query.property.test.js @@ -57,10 +57,11 @@ function makePostsDb (posts) { function makeParentsDb (replyTxids) { return { - async * iterator () { - for (const txid of replyTxids) { - yield [txid, { parentTxid: 'parent' }] - } + async get (txid) { + if (replyTxids.has(txid)) return { parentTxid: 'parent', childTxid: txid } + const err = new Error('not found') + err.notFound = true + throw err } } } diff --git a/psf-memo-db/test/unit/adapters/post-query.unit.js b/psf-memo-db/test/unit/adapters/post-query.unit.js index 780a6bc..b870ae0 100644 --- a/psf-memo-db/test/unit/adapters/post-query.unit.js +++ b/psf-memo-db/test/unit/adapters/post-query.unit.js @@ -384,16 +384,18 @@ describe('#PostQuery', () => { assert.equal(result.total, 2) }) - it('should exclude replies from the following feed', async () => { - async function * mockParents () { - yield ['reply-a', { parentTxid: 'post-a', childTxid: 'reply-a' }] - } + it('should exclude replies using point lookups without iterating postParents', async () => { async function * mockHeights () { yield ['000000600300:reply-a', { txid: 'reply-a' }] yield ['000000600200:post-a', { txid: 'post-a' }] } - postParentsDb.iterator.returns(mockParents()) postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) + postParentsDb.get.callsFake(async (txid) => { + if (txid === 'reply-a') return { parentTxid: 'post-a', childTxid: 'reply-a' } + const err = new Error('not found') + err.notFound = true + throw err + }) postsDb.get.callsFake(async (txid) => { return { addr: followeeA, text: txid } }) @@ -406,6 +408,77 @@ describe('#PostQuery', () => { assert.deepEqual(result.txids, ['post-a']) assert.equal(result.total, 1) + assert.equal(postParentsDb.iterator.called, false) + }) + + it('should cap the total at 500 and stop scanning after offset + limit + cap eligible posts', async () => { + let reads = 0 + async function * mockHeights () { + for (let i = 599; i >= 0; i--) { + reads++ + const id = String(i).padStart(3, '0') + yield [`000000${600000 + i}:post-${id}`, { txid: `post-${id}` }] + } + } + postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) + postsDb.get.callsFake(async (txid) => ({ addr: followeeA, text: txid })) + + const result = await uut.scanFollowingFeedTxidsAndCount( + viewerAddr, + [followeeA], + { limit: 3, offset: 0 } + ) + + assert.deepEqual(result.txids, ['post-599', 'post-598', 'post-597']) + assert.equal(result.total, 500) + assert.equal(reads, 503) // offset + limit + default cap eligible posts + }) + + it('should report the exact total when eligible posts are below the cap', async () => { + async function * mockHeights () { + yield ['000000600200:post-a', { txid: 'post-a' }] + yield ['000000600100:post-b', { txid: 'post-b' }] + } + postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) + postsDb.get.callsFake(async (txid) => ({ addr: followeeA, text: txid })) + + const result = await uut.scanFollowingFeedTxidsAndCount( + viewerAddr, + [followeeA], + { limit: 10, offset: 0 } + ) + + assert.deepEqual(result.txids, ['post-a', 'post-b']) + assert.equal(result.total, 2) + }) + + it('should count only eligible followed posts toward the scan cap', async () => { + let reads = 0 + async function * mockHeights () { + // Interleave eligible followed posts with non-followed top-level posts. + for (let i = 1199; i >= 0; i--) { + reads++ + const txid = i % 2 === 0 ? `followed-${i}` : `other-${i}` + yield [`000000${600000 + i}:${txid}`, { txid }] + } + } + postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) + postsDb.get.callsFake(async (txid) => ({ + addr: txid.startsWith('followed') ? followeeA : 'bitcoincash:other', + text: txid + })) + + const result = await uut.scanFollowingFeedTxidsAndCount( + viewerAddr, + [followeeA], + { limit: 3, offset: 0 } + ) + + assert.deepEqual(result.txids, ['followed-1198', 'followed-1196', 'followed-1194']) + assert.equal(result.total, 500) + // Non-eligible raw entries do not count toward the cap, so the scan keeps + // going past 503 raw entries until it has seen 503 eligible posts. + assert.isAbove(reads, 503) }) it('should apply limit and offset', async () => {