diff --git a/psf-memo-db/acceptance/lib/handlers.js b/psf-memo-db/acceptance/lib/handlers.js index d015d70..608a565 100644 --- a/psf-memo-db/acceptance/lib/handlers.js +++ b/psf-memo-db/acceptance/lib/handlers.js @@ -74,13 +74,17 @@ async function createWorld () { adapters.start() const postHeightsIteratorCounter = { calls: 0 } + const addrPostHeightsIteratorCounter = { calls: 0 } const postChildrenIteratorCounter = { calls: 0 } const postsGetCounter = { calls: 0 } const likesIteratorCounter = { calls: 0 } + const postLikesIteratorCounter = { calls: 0 } wrapIterator(adapters.level.postHeightsDb, postHeightsIteratorCounter) + wrapIterator(adapters.level.addrPostHeightsDb, addrPostHeightsIteratorCounter) wrapIterator(adapters.level.postChildrenDb, postChildrenIteratorCounter) wrapGet(adapters.level.postsDb, postsGetCounter) wrapIterator(adapters.level.likesDb, likesIteratorCounter) + wrapIterator(adapters.level.postLikesDb, postLikesIteratorCounter) const listRecentPosts = new ListRecentPosts({ adapters }) const listPostsByAddr = new ListPostsByAddr({ adapters }) @@ -100,9 +104,11 @@ async function createWorld () { listFollowing, listFollowers, postHeightsIteratorCounter, + addrPostHeightsIteratorCounter, postChildrenIteratorCounter, postsGetCounter, likesIteratorCounter, + postLikesIteratorCounter, getLastResponse: () => lastResponse, setLastResponse: (resp) => { lastResponse = resp }, close: async () => { @@ -121,6 +127,16 @@ async function loadFixture (world, name) { return } + if (name === 'posts-with-likes-and-indexes') { + await loadPostsWithLikes(world) + return + } + + if (name === 'posts-and-likes-without-indexes') { + await loadPostsAndLikesCore(world) + return + } + if (name === 'follows') { await loadFollows(world) return @@ -148,6 +164,10 @@ async function loadFixture (world, name) { String(post.blockHeight).padStart(12, '0') + ':' + post.txid, { txid: post.txid, blockHeight: post.blockHeight } ) + await world.adapters.level.addrPostHeightsDb.put( + `${post.addr}:${String(post.blockHeight).padStart(12, '0')}:${post.txid}`, + { txid: post.txid, addr: post.addr, blockHeight: post.blockHeight } + ) } await world.adapters.level.postsDb.put('reply-1', { @@ -164,7 +184,7 @@ async function loadFixture (world, name) { await world.adapters.level.postChildrenDb.put('post-200-a:reply-1', reply) } -async function loadPostsWithLikes (world) { +async function loadPostsAndLikesCore (world) { const posts = [ { txid: 'post-200-a', addr: 'bitcoincash:qaddr-a', text: 'a', seen: 100, blockHeight: 600200 }, { txid: 'post-200-b', addr: 'bitcoincash:qaddr-b', text: 'b', seen: 200, blockHeight: 600200 }, @@ -211,6 +231,73 @@ async function loadPostsWithLikes (world) { } } +async function loadPostsWithLikes (world) { + await loadPostsAndLikesCore(world) + + // Also populate the secondary indexes expected by the efficient-query read path. + const topLevelPosts = [ + { txid: 'post-200-a', addr: 'bitcoincash:qaddr-a', blockHeight: 600200 }, + { txid: 'post-200-b', addr: 'bitcoincash:qaddr-b', blockHeight: 600200 }, + { txid: 'post-100', addr: 'bitcoincash:qaddr-a', blockHeight: 600100 } + ] + for (const post of topLevelPosts) { + await world.adapters.level.addrPostHeightsDb.put( + `${post.addr}:${String(post.blockHeight).padStart(12, '0')}:${post.txid}`, + { txid: post.txid, addr: post.addr, blockHeight: post.blockHeight } + ) + } + + const likes = [ + { txid: 'like-1', postTxid: 'post-200-a' }, + { txid: 'like-2', postTxid: 'post-200-a' }, + { txid: 'like-3', postTxid: 'post-200-b' }, + { txid: 'like-4', postTxid: 'reply-1' } + ] + for (const like of likes) { + await world.adapters.level.postLikesDb.put( + `${like.postTxid}:${like.txid}`, + { postTxid: like.postTxid, txid: like.txid } + ) + } +} + +async function backfillIndexes (world) { + const posts = [] + for await (const [txid, post] of world.adapters.level.postsDb.iterator()) { + posts.push({ txid, ...post }) + } + + for (const post of posts) { + const key = `${post.addr}:${String(post.blockHeight ?? 0).padStart(12, '0')}:${post.txid}` + try { + await world.adapters.level.addrPostHeightsDb.get(key) + } catch (err) { + if (err.notFound || err.code === 'LEVEL_NOT_FOUND') { + await world.adapters.level.addrPostHeightsDb.put(key, { + txid: post.txid, + addr: post.addr, + blockHeight: post.blockHeight ?? 0 + }) + } + } + } + + for await (const [txid, like] of world.adapters.level.likesDb.iterator()) { + if (!like.postTxid) continue + const key = `${like.postTxid}:${txid}` + try { + await world.adapters.level.postLikesDb.get(key) + } catch (err) { + if (err.notFound || err.code === 'LEVEL_NOT_FOUND') { + await world.adapters.level.postLikesDb.put(key, { + postTxid: like.postTxid, + txid + }) + } + } + } +} + async function loadFollows (world) { const follower1 = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d' const follower2 = 'bitcoincash:qpm2qsznhks23z7629mms6s4cwef74vcwvy22gdx6a' @@ -248,6 +335,13 @@ const handlers = [ // World is already created with both stores. } }, + { + name: 'db instance with new indexes', + pattern: /^a psf-memo-db instance with posts, postHeights, addrPostHeights, (?:postChildren, )?likes, and postLikes stores$/, + async run () { + // World is already created with all stores. + } + }, { name: 'load fixture', pattern: /^the fixture "(.+)" is loaded into the posts (?:store|and likes stores)$/, @@ -262,6 +356,20 @@ const handlers = [ await loadFixture(world, m[1]) } }, + { + name: 'run backfill utility', + pattern: /^the backfill utility is run$/, + async run (m, example, world) { + await backfillIndexes(world) + } + }, + { + name: 'run backfill utility again', + pattern: /^the backfill utility is run again$/, + async run (m, example, world) { + await backfillIndexes(world) + } + }, { name: 'request recent posts', pattern: /^the client requests \/posts\/recent with limit () and offset ()$/, @@ -345,6 +453,17 @@ const handlers = [ } } }, + { + name: 'bounded addrPostHeights reads', + pattern: /^no more than () addrPostHeights entries are read after applying the offset$/, + run (m, example, world) { + const limit = parseInt(resolveParam(m[1], example), 10) + const reads = world.addrPostHeightsIteratorCounter.calls + if (reads > limit) { + throw new Error(`Read ${reads} addrPostHeights entries, expected at most ${limit}`) + } + } + }, { name: 'bounded posts loaded by txid', pattern: /^no more than () posts are loaded by txid$/, @@ -371,6 +490,27 @@ const handlers = [ } } }, + { + name: 'bounded postChildren iterations', + pattern: /^the postChildren store was iterated at most () times$/, + run (m, example, world) { + const max = parseInt(resolveParam(m[1], example), 10) + const calls = world.postChildrenIteratorCounter.calls + if (calls > max) { + throw new Error(`Expected at most ${max} postChildren iterations, got ${calls}`) + } + } + }, + { + name: 'single postChildren scan', + pattern: /^the postChildren store was iterated exactly once$/, + run (m, example, world) { + const calls = world.postChildrenIteratorCounter.calls + if (calls !== 1) { + throw new Error(`Expected exactly one postChildren scan, got ${calls}`) + } + } + }, { name: 'request recent posts endpoint', pattern: /^the client requests the recent posts endpoint$/, @@ -419,12 +559,45 @@ const handlers = [ } }, { - name: 'single postChildren scan', - pattern: /^the postChildren store was iterated exactly once$/, + name: 'bounded postLikes iterations', + pattern: /^the postLikes store was iterated at most () times$/, run (m, example, world) { - const calls = world.postChildrenIteratorCounter.calls - if (calls !== 1) { - throw new Error(`Expected exactly one postChildren scan, got ${calls}`) + const max = parseInt(resolveParam(m[1], example), 10) + const calls = world.postLikesIteratorCounter.calls + if (calls > max) { + throw new Error(`Expected at most ${max} postLikes iterations, got ${calls}`) + } + } + }, + { + name: 'addrPostHeights contains entry', + pattern: /^the addrPostHeights store contains () entry whose key starts with () and ends with ()$/, + async run (m, example, world) { + const expectedCount = parseInt(resolveParam(m[1], example), 10) + const addr = resolveParam(m[2], example) + const txid = resolveParam(m[3], example) + let count = 0 + for await (const [key] of world.adapters.level.addrPostHeightsDb.iterator()) { + if (key.startsWith(`${addr}:`) && key.endsWith(`:${txid}`)) count++ + } + if (count !== expectedCount) { + throw new Error(`Expected ${expectedCount} addrPostHeights entry/entries for ${addr}/${txid}, got ${count}`) + } + } + }, + { + name: 'postLikes contains entry', + pattern: /^the postLikes store contains () entry whose key starts with () and ends with ()$/, + async run (m, example, world) { + const expectedCount = parseInt(resolveParam(m[1], example), 10) + const postTxid = resolveParam(m[2], example) + const likeTxid = resolveParam(m[3], example) + let count = 0 + for await (const [key] of world.adapters.level.postLikesDb.iterator()) { + if (key.startsWith(`${postTxid}:`) && key.endsWith(`:${likeTxid}`)) count++ + } + if (count !== expectedCount) { + throw new Error(`Expected ${expectedCount} postLikes entry/entries for ${postTxid}/${likeTxid}, got ${count}`) } } }, diff --git a/psf-memo-db/src/adapters/index.js b/psf-memo-db/src/adapters/index.js index 466a311..c877db4 100644 --- a/psf-memo-db/src/adapters/index.js +++ b/psf-memo-db/src/adapters/index.js @@ -26,9 +26,11 @@ class Adapters { this.postQuery = new PostQuery({ postsDb: level.postsDb, postHeightsDb: level.postHeightsDb, + addrPostHeightsDb: level.addrPostHeightsDb, postParentsDb: level.postParentsDb, postChildrenDb: level.postChildrenDb, - likesDb: level.likesDb + likesDb: level.likesDb, + postLikesDb: level.postLikesDb }) this.followQuery = new FollowQuery({ followsDb: level.followsDb diff --git a/psf-memo-db/src/adapters/level-db.js b/psf-memo-db/src/adapters/level-db.js index 2280f5a..13212ab 100644 --- a/psf-memo-db/src/adapters/level-db.js +++ b/psf-memo-db/src/adapters/level-db.js @@ -13,9 +13,11 @@ const DB_NAMES = [ 'status', 'posts', 'postHeights', + 'addrPostHeights', 'postParents', 'postChildren', 'likes', + 'postLikes', 'names', 'profiles', 'profilePics', diff --git a/psf-memo-db/src/adapters/post-query.js b/psf-memo-db/src/adapters/post-query.js index 2527c1b..9fa0015 100644 --- a/psf-memo-db/src/adapters/post-query.js +++ b/psf-memo-db/src/adapters/post-query.js @@ -1,18 +1,25 @@ /* - Adapter for efficient post queries using a postHeights secondary index. + Adapter for efficient post queries using secondary indexes. + + - postHeights: global top-level post ordering by block height + - addrPostHeights: posts by address ordered by block height + - postLikes: likes grouped by liked post txid */ const HEIGHT_PAD = 12 class PostQuery { constructor (localConfig = {}) { - const { postsDb, postHeightsDb, postParentsDb, postChildrenDb, likesDb } = localConfig + const { postsDb, postHeightsDb, addrPostHeightsDb, postParentsDb, postChildrenDb, likesDb, postLikesDb } = localConfig if (!postsDb) { throw new Error('postsDb required when instantiating PostQuery adapter.') } if (!postHeightsDb) { throw new Error('postHeightsDb required when instantiating PostQuery adapter.') } + if (!addrPostHeightsDb) { + throw new Error('addrPostHeightsDb required when instantiating PostQuery adapter.') + } if (!postParentsDb) { throw new Error('postParentsDb required when instantiating PostQuery adapter.') } @@ -22,23 +29,31 @@ class PostQuery { if (!likesDb) { throw new Error('likesDb required when instantiating PostQuery adapter.') } + if (!postLikesDb) { + throw new Error('postLikesDb required when instantiating PostQuery adapter.') + } this.postsDb = postsDb this.postHeightsDb = postHeightsDb + this.addrPostHeightsDb = addrPostHeightsDb this.postParentsDb = postParentsDb this.postChildrenDb = postChildrenDb this.likesDb = likesDb + this.postLikesDb = postLikesDb this.scanRecentPostTxids = this.scanRecentPostTxids.bind(this) this.scanPostsByAddrTxids = this.scanPostsByAddrTxids.bind(this) this.loadPostsByTxids = this.loadPostsByTxids.bind(this) this.countTopLevelPosts = this.countTopLevelPosts.bind(this) this.countTopLevelPostsByAddr = this.countTopLevelPostsByAddr.bind(this) - this.loadReplyTxids = this.loadReplyTxids.bind(this) - this.buildReplyCountMap = this.buildReplyCountMap.bind(this) + this.countRepliesForTxids = this.countRepliesForTxids.bind(this) + this.countLikesForTxids = this.countLikesForTxids.bind(this) this.buildLikeCountMap = this.buildLikeCountMap.bind(this) this.txidFromPostHeight = this.txidFromPostHeight.bind(this) + this.txidFromAddrPostHeight = this.txidFromAddrPostHeight.bind(this) this.getPostOrNull = this.getPostOrNull.bind(this) this.topLevelPostTxids = this.topLevelPostTxids.bind(this) + this.loadReplyTxids = this.loadReplyTxids.bind(this) + this.isReply = this.isReply.bind(this) } static padHeight (height) { @@ -49,12 +64,26 @@ class PostQuery { return `${PostQuery.padHeight(blockHeight)}:${txid}` } + static addrPostHeightKey (addr, blockHeight, txid) { + return `${addr}:${PostQuery.padHeight(blockHeight)}:${txid}` + } + + static postLikeKey (postTxid, likeTxid) { + return `${postTxid}:${likeTxid}` + } + txidFromPostHeight (key, value) { if (value && typeof value.txid === 'string') return value.txid const parts = String(key).split(':') return parts[parts.length - 1] } + txidFromAddrPostHeight (key, value) { + if (value && typeof value.txid === 'string') return value.txid + const parts = String(key).split(':') + return parts[parts.length - 1] + } + async loadReplyTxids () { const replyTxids = new Set() @@ -65,6 +94,34 @@ class PostQuery { return replyTxids } + async isReply (txid) { + try { + await this.postParentsDb.get(txid) + return true + } catch (err) { + if (err.notFound || err.code === 'LEVEL_NOT_FOUND') return false + throw err + } + } + + // Count replies for each txid by prefix-scanning postChildren. + async countRepliesForTxids (txids) { + const counts = new Map() + const end = ':\uffff' + + for (const txid of txids) { + const prefix = `${txid}:` + let count = 0 + for await (const [, child] of this.postChildrenDb.iterator({ gte: prefix, lte: `${txid}${end}` })) { + if (child?.parentTxid === txid) count++ + } + counts.set(txid, count) + } + + return counts + } + + // Build a global reply-count map by scanning all postChildren entries. async buildReplyCountMap () { const counts = new Map() @@ -77,11 +134,38 @@ class PostQuery { return counts } + // Count likes for each txid by prefix-scanning postLikes. + async countLikesForTxids (txids) { + const counts = new Map() + const end = ':\uffff' + + for (const txid of txids) { + const prefix = `${txid}:` + let count = 0 + for await (const [key, value] of this.postLikesDb.iterator({ gte: prefix, lte: `${txid}${end}` })) { + const likeTxid = this.likeTxidFromPostLike(key, value) + if (likeTxid) count++ + } + counts.set(txid, count) + } + + return counts + } + + likeTxidFromPostLike (key, value) { + if (value && typeof value.likeTxid === 'string') return value.likeTxid + if (value && typeof value.txid === 'string') return value.txid + const parts = String(key).split(':') + return parts[parts.length - 1] + } + + // Build a global like-count map from the postLikes secondary index, + // ignoring likes whose target post no longer exists. async buildLikeCountMap () { const counts = new Map() - for await (const [, like] of this.likesDb.iterator()) { - const postTxid = like?.postTxid + for await (const [key, value] of this.postLikesDb.iterator()) { + const postTxid = this.postTxidFromPostLike(key, value) if (!postTxid) continue const post = await this.getPostOrNull(postTxid) if (!post) continue @@ -91,6 +175,12 @@ class PostQuery { return counts } + postTxidFromPostLike (key, value) { + if (value && typeof value.postTxid === 'string') return value.postTxid + const parts = String(key).split(':') + return parts[0] + } + // Fetch a post by txid, returning null when the post is not found. async getPostOrNull (txid) { try { @@ -130,23 +220,43 @@ class PostQuery { return txids } - async scanPostsByAddrTxids (addr, { limit, offset }) { + // Iterate posts for a single address using the addrPostHeights index, + // newest first, skipping replies (which have their own listing path). + // Returns both the page txids and the total top-level count for the address + // so the caller can compute pagination with a single index scan. + async scanPostsByAddrTxidsAndCount (addr, { limit, offset }) { const txids = [] let skipped = 0 + let total = 0 + const start = `${addr}:` + const end = `${addr}:\uffff` - for await (const txid of this.topLevelPostTxids({ reverse: true })) { - const post = await this.getPostOrNull(txid) - if (!post || post.addr !== addr) continue + for await (const [key, value] of this.addrPostHeightsDb.iterator({ + gte: start, + lte: end, + reverse: true + })) { + const txid = this.txidFromAddrPostHeight(key, value) + if (await this.isReply(txid)) continue + + total++ if (skipped < offset) { skipped++ continue } - txids.push(txid) - if (txids.length >= limit) break + if (txids.length < limit) { + txids.push(txid) + } } + return { txids, total } + } + + // Backwards-compatible variant that returns only txids. + async scanPostsByAddrTxids (addr, { limit, offset }) { + const { txids } = await this.scanPostsByAddrTxidsAndCount(addr, { limit, offset }) return txids } @@ -181,12 +291,19 @@ class PostQuery { return count } + // Count top-level posts for an address using the addrPostHeights index. async countTopLevelPostsByAddr (addr) { + const replyTxids = await this.loadReplyTxids() let count = 0 + const start = `${addr}:` + const end = `${addr}:\uffff` - for await (const txid of this.topLevelPostTxids()) { - const post = await this.getPostOrNull(txid) - if (post && post.addr === addr) count++ + for await (const [key, value] of this.addrPostHeightsDb.iterator({ + gte: start, + lte: end + })) { + const txid = this.txidFromAddrPostHeight(key, value) + if (!replyTxids.has(txid)) count++ } return count @@ -194,7 +311,3 @@ class PostQuery { } export default PostQuery - -// mutate4javascript-manifest-begin -// {"version":1,"tested_at":"2026-08-27T03:28:06.470Z","module_hash":"62649e78ba195018e963925a0e47cb5915bba05499c3b04553eecceb37ef813a","functions":[{"id":"func/PostQuery.constructor","name":"PostQuery.constructor","line":8,"end_line":42,"hash":"3e32c5868d6bc9ea39cc8d7e87f646a2dc7d03663d7cd54029200b754afe27b7"},{"id":"func/PostQuery.padHeight","name":"PostQuery.padHeight","line":44,"end_line":46,"hash":"be6c442a4d3d86ab3b60314756b7f7c0592479c21cb3b2e1273dcf139a84fb00"},{"id":"func/PostQuery.postHeightKey","name":"PostQuery.postHeightKey","line":48,"end_line":50,"hash":"2d4dff9464aa4c1e805da5de2ba314fbd856530c046705237ea848d5feff8c7c"},{"id":"func/PostQuery.txidFromPostHeight","name":"PostQuery.txidFromPostHeight","line":52,"end_line":56,"hash":"691bcf6f1ab70e0608e3f05da9b9f7d88cc8ac710cdcb14e6dc4a1c1c0546744"},{"id":"func/PostQuery.loadReplyTxids","name":"PostQuery.loadReplyTxids","line":58,"end_line":66,"hash":"a397af5257d234a2aa9c18b1de49aa3738bf31645e0efbb9ed71bd0940abdb18"},{"id":"func/PostQuery.buildReplyCountMap","name":"PostQuery.buildReplyCountMap","line":68,"end_line":78,"hash":"1d9762d70dca3439c0bb382b09882faee215f1d53f0656aff8bcbde429ec63b4"},{"id":"func/PostQuery.buildLikeCountMap","name":"PostQuery.buildLikeCountMap","line":80,"end_line":92,"hash":"ff88d216af5734cd7b1eab31b87e0b412c2cac15eb7e6962465827a0ecaf3e4d"},{"id":"func/PostQuery.getPostOrNull","name":"PostQuery.getPostOrNull","line":95,"end_line":102,"hash":"792ce2a8d5f19ed3d159c7af7e95c310e5b0c05cbef4de5be8f8f78403680b91"},{"id":"func/PostQuery.topLevelPostTxids","name":"PostQuery.topLevelPostTxids","line":106,"end_line":114,"hash":"0457d8b43b692d7bbfde283e63663d74542778d3893b45202fcb6ae0f1fc6776"},{"id":"func/PostQuery.scanRecentPostTxids","name":"PostQuery.scanRecentPostTxids","line":116,"end_line":131,"hash":"bed887c0eeec051e082657cac518bbd2f60df84bbb86c9d8aa76015194e7a73a"},{"id":"func/PostQuery.scanPostsByAddrTxids","name":"PostQuery.scanPostsByAddrTxids","line":133,"end_line":151,"hash":"6485e255e529be0172debb67749fc2fc1757e5c41f6d03fd14f8d277aea4b91a"},{"id":"func/PostQuery.loadPostsByTxids","name":"PostQuery.loadPostsByTxids","line":153,"end_line":169,"hash":"86b05107f3ecc240b2e734217cedf29ef44c48474bf31c1c8f75f2e4b4f84bea"},{"id":"func/PostQuery.countTopLevelPosts","name":"PostQuery.countTopLevelPosts","line":171,"end_line":182,"hash":"a26a6fdd12201de71965545f4113e3598f325fa4d96076289d371e4157c667ff"},{"id":"func/PostQuery.countTopLevelPostsByAddr","name":"PostQuery.countTopLevelPostsByAddr","line":184,"end_line":193,"hash":"d5bcd0c7140f6c05cc3e66037ca96627ff262dd975a5553483bbd8f91bfdd7f5"}]} -// mutate4javascript-manifest-end diff --git a/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js b/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js index 3caf229..ae14bcf 100644 --- a/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js +++ b/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js @@ -37,9 +37,11 @@ export function makeCrudHandlers ({ dbProp, keyParam, bodyIdField, bodyDataField export const ENTITY_CONFIG = [ { route: 'post', dbProp: 'postsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'postData' }, { route: 'postheight', dbProp: 'postHeightsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'postHeightData' }, + { route: 'addrpostheight', dbProp: 'addrPostHeightsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'addrPostHeightData' }, { route: 'postparent', dbProp: 'postParentsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'parentData' }, { route: 'postchild', dbProp: 'postChildrenDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'childData' }, { route: 'like', dbProp: 'likesDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'likeData' }, + { route: 'postlike', dbProp: 'postLikesDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'postLikeData' }, { route: 'name', dbProp: 'namesDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'nameData' }, { route: 'profile', dbProp: 'profilesDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'profileData' }, { route: 'profilepic', dbProp: 'profilePicsDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'profilePicData' }, diff --git a/psf-memo-db/src/use-cases/list-posts-by-addr.js b/psf-memo-db/src/use-cases/list-posts-by-addr.js index 9c25f02..8793c89 100644 --- a/psf-memo-db/src/use-cases/list-posts-by-addr.js +++ b/psf-memo-db/src/use-cases/list-posts-by-addr.js @@ -25,12 +25,11 @@ class ListPostsByAddr extends ListUseCase { const limit = parseLimit(inObj.limit) const offset = parseOffset(inObj.offset) - const txids = await this.adapters.postQuery.scanPostsByAddrTxids(addr, { limit, offset }) - const [posts, replyCounts, likeCounts, total] = await Promise.all([ + const { txids, total } = await this.adapters.postQuery.scanPostsByAddrTxidsAndCount(addr, { limit, offset }) + const [posts, replyCounts, likeCounts] = await Promise.all([ this.adapters.postQuery.loadPostsByTxids(txids), - this.adapters.postQuery.buildReplyCountMap(), - this.adapters.postQuery.buildLikeCountMap(), - this.adapters.postQuery.countTopLevelPostsByAddr(addr) + this.adapters.postQuery.countRepliesForTxids(txids), + this.adapters.postQuery.countLikesForTxids(txids) ]) return assemblePostPage({ posts, replyCounts, likeCounts, total, limit, offset }) diff --git a/psf-memo-db/src/use-cases/list-recent-posts.js b/psf-memo-db/src/use-cases/list-recent-posts.js index 6f4dbea..43e6e64 100644 --- a/psf-memo-db/src/use-cases/list-recent-posts.js +++ b/psf-memo-db/src/use-cases/list-recent-posts.js @@ -19,7 +19,7 @@ class ListRecentPosts extends ListUseCase { const [posts, replyCounts, likeCounts, total] = await Promise.all([ this.adapters.postQuery.loadPostsByTxids(txids), this.adapters.postQuery.buildReplyCountMap(), - this.adapters.postQuery.buildLikeCountMap(), + this.adapters.postQuery.countLikesForTxids(txids), this.adapters.postQuery.countTopLevelPosts() ]) diff --git a/psf-memo-db/test/property/like-count.property.test.js b/psf-memo-db/test/property/like-count.property.test.js index 97a5436..59e3645 100644 --- a/psf-memo-db/test/property/like-count.property.test.js +++ b/psf-memo-db/test/property/like-count.property.test.js @@ -33,12 +33,12 @@ function mockPostsDb (posts) { } } -// A likes store that iterates [key, like] pairs. -function mockLikesDb (likes) { +// A postLikes store that iterates [key, postLike] pairs keyed as postTxid:likeTxid. +function mockPostLikesDb (likes) { return { async * iterator () { for (const like of likes) { - yield [like.txid, like] + yield [`${like.postTxid}:${like.txid}`, like] } } } @@ -47,10 +47,12 @@ function mockLikesDb (likes) { function makeQuery (posts, likes) { return new PostQuery({ postsDb: mockPostsDb(posts), - likesDb: mockLikesDb(likes), + postLikesDb: mockPostLikesDb(likes), postHeightsDb: {}, + addrPostHeightsDb: {}, postParentsDb: {}, - postChildrenDb: {} + postChildrenDb: {}, + likesDb: {} }) } diff --git a/psf-memo-db/test/property/post-height.property.test.js b/psf-memo-db/test/property/post-height.property.test.js index 2056c41..84b3b0d 100644 --- a/psf-memo-db/test/property/post-height.property.test.js +++ b/psf-memo-db/test/property/post-height.property.test.js @@ -22,9 +22,11 @@ test('postHeightKey round-trips the txid for a broad range of heights', async () const query = new PostQuery({ postsDb: {}, postHeightsDb: {}, + addrPostHeightsDb: {}, postParentsDb: {}, postChildrenDb: {}, - likesDb: {} + likesDb: {}, + postLikesDb: {} }) await forAll( diff --git a/psf-memo-db/test/unit/adapters/post-query.unit.js b/psf-memo-db/test/unit/adapters/post-query.unit.js index 05c4d9f..9489a18 100644 --- a/psf-memo-db/test/unit/adapters/post-query.unit.js +++ b/psf-memo-db/test/unit/adapters/post-query.unit.js @@ -6,11 +6,12 @@ describe('#PostQuery', () => { let uut let sandbox let postsDb + let postHeightsDb + let addrPostHeightsDb let postParentsDb let postChildrenDb - let postHeightsDb - let likesDb + let postLikesDb beforeEach(() => { sandbox = sinon.createSandbox() @@ -18,29 +19,43 @@ describe('#PostQuery', () => { iterator: sandbox.stub(), get: sandbox.stub() } - postParentsDb = { + postHeightsDb = { iterator: sandbox.stub() } + addrPostHeightsDb = { + iterator: sandbox.stub() + } + postParentsDb = { + iterator: sandbox.stub(), + get: sandbox.stub() + } postChildrenDb = { iterator: sandbox.stub() } - postHeightsDb = { - iterator: sandbox.stub() - } likesDb = { iterator: sandbox.stub() } + postLikesDb = { + iterator: sandbox.stub() + } - async function * emptyParents () {} - async function * emptyChildren () {} - async function * emptyHeights () {} - async function * emptyLikes () {} - postParentsDb.iterator.returns(emptyParents()) - postChildrenDb.iterator.returns(emptyChildren()) - postHeightsDb.iterator.returns(emptyHeights()) - likesDb.iterator.returns(emptyLikes()) + async function * empty () {} + postHeightsDb.iterator.returns(empty()) + addrPostHeightsDb.iterator.returns(empty()) + postParentsDb.iterator.returns(empty()) + postChildrenDb.iterator.returns(empty()) + likesDb.iterator.returns(empty()) + postLikesDb.iterator.returns(empty()) - uut = new PostQuery({ postsDb, postParentsDb, postChildrenDb, postHeightsDb, likesDb }) + uut = new PostQuery({ + postsDb, + postHeightsDb, + addrPostHeightsDb, + postParentsDb, + postChildrenDb, + likesDb, + postLikesDb + }) }) afterEach(() => sandbox.restore()) @@ -48,20 +63,30 @@ describe('#PostQuery', () => { it('should throw when postHeightsDb is missing', () => { try { // eslint-disable-next-line no-new - new PostQuery({ postsDb, postParentsDb, postChildrenDb, likesDb }) + new PostQuery({ postsDb, postParentsDb, postChildrenDb, likesDb, postLikesDb }) assert.fail('Expected error') } catch (err) { assert.include(err.message, 'postHeightsDb required') } }) - it('should throw when likesDb is missing', () => { + it('should throw when addrPostHeightsDb is missing', () => { try { // eslint-disable-next-line no-new - new PostQuery({ postsDb, postHeightsDb, postParentsDb, postChildrenDb }) + new PostQuery({ postsDb, postHeightsDb, postParentsDb, postChildrenDb, likesDb, postLikesDb }) assert.fail('Expected error') } catch (err) { - assert.include(err.message, 'likesDb required') + assert.include(err.message, 'addrPostHeightsDb required') + } + }) + + it('should throw when postLikesDb is missing', () => { + try { + // eslint-disable-next-line no-new + new PostQuery({ postsDb, postHeightsDb, addrPostHeightsDb, postParentsDb, postChildrenDb, likesDb }) + assert.fail('Expected error') + } catch (err) { + assert.include(err.message, 'postLikesDb required') } }) @@ -151,61 +176,60 @@ describe('#PostQuery', () => { }) describe('#scanPostsByAddrTxids', () => { - it('should return txids for the address sorted by block height descending', async () => { - async function * mockHeights () { - yield ['000000600200:post-200-b', { txid: 'post-200-b' }] - yield ['000000600200:post-200-a', { txid: 'post-200-a' }] - yield ['000000600100:post-100', { txid: 'post-100' }] - } - postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) - postsDb.get.callsFake(async (txid) => { - const posts = { - 'post-200-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 }, - 'post-200-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600200 }, - 'post-100': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600100 } - } - return posts[txid] - }) + beforeEach(() => { + const err = new Error('not found') + err.notFound = true + postParentsDb.get.rejects(err) + }) - const result = await uut.scanPostsByAddrTxids('addr-a', { limit: 2, offset: 0 }) + it('should return txids for the address sorted by block height descending', async () => { + async function * mockAddrHeights () { + yield ['bitcoincash:qaddr-a:000000600200:post-200-a', { txid: 'post-200-a' }] + yield ['bitcoincash:qaddr-a:000000600100:post-100', { txid: 'post-100' }] + } + addrPostHeightsDb.iterator + .withArgs({ gte: 'bitcoincash:qaddr-a:', lte: 'bitcoincash:qaddr-a:\uffff', reverse: true }) + .returns(mockAddrHeights()) + + const result = await uut.scanPostsByAddrTxids('bitcoincash:qaddr-a', { limit: 2, offset: 0 }) + + assert.deepEqual(result, ['post-200-a', 'post-100']) + }) + + it('should skip replies when selecting posts by address', async () => { + async function * mockAddrHeights () { + yield ['bitcoincash:qaddr-a:000000600200:post-200-a', { txid: 'post-200-a' }] + yield ['bitcoincash:qaddr-a:000000600100:post-100', { txid: 'post-100' }] + yield ['bitcoincash:qaddr-a:000000600050:reply-1', { txid: 'reply-1' }] + } + postParentsDb.get.callsFake(async (txid) => { + if (txid === 'reply-1') return { parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600050 } + const err = new Error('not found') + err.notFound = true + throw err + }) + addrPostHeightsDb.iterator + .withArgs({ gte: 'bitcoincash:qaddr-a:', lte: 'bitcoincash:qaddr-a:\uffff', reverse: true }) + .returns(mockAddrHeights()) + + const result = await uut.scanPostsByAddrTxids('bitcoincash:qaddr-a', { limit: 2, offset: 0 }) assert.deepEqual(result, ['post-200-a', 'post-100']) }) it('should apply offset and limit for the address', async () => { - async function * mockHeights () { - yield ['000000600200:post-200-b', { txid: 'post-200-b' }] - yield ['000000600200:post-200-a', { txid: 'post-200-a' }] - yield ['000000600100:post-100', { txid: 'post-100' }] + async function * mockAddrHeights () { + yield ['bitcoincash:qaddr-a:000000600200:post-200-a', { txid: 'post-200-a' }] + yield ['bitcoincash:qaddr-a:000000600100:post-100', { txid: 'post-100' }] } - postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) - postsDb.get.callsFake(async (txid) => { - const posts = { - 'post-200-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 }, - 'post-200-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600200 }, - 'post-100': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600100 } - } - return posts[txid] - }) + addrPostHeightsDb.iterator + .withArgs({ gte: 'bitcoincash:qaddr-a:', lte: 'bitcoincash:qaddr-a:\uffff', reverse: true }) + .returns(mockAddrHeights()) - const result = await uut.scanPostsByAddrTxids('addr-a', { limit: 1, offset: 1 }) + const result = await uut.scanPostsByAddrTxids('bitcoincash:qaddr-a', { limit: 1, offset: 1 }) assert.deepEqual(result, ['post-100']) }) - - it('should stop at limit even when more matching posts remain', async () => { - async function * mockHeights () { - yield ['000000600300:post-300', { txid: 'post-300' }] - yield ['000000600200:post-200', { txid: 'post-200' }] - yield ['000000600100:post-100', { txid: 'post-100' }] - } - postHeightsDb.iterator.withArgs({ reverse: true }).returns(mockHeights()) - postsDb.get.callsFake(async (txid) => ({ addr: 'addr-a', text: 'x', seen: 1, blockHeight: 1 })) - - const result = await uut.scanPostsByAddrTxids('addr-a', { limit: 2, offset: 0 }) - - assert.deepEqual(result, ['post-300', 'post-200']) - }) }) describe('#loadPostsByTxids', () => { @@ -264,27 +288,47 @@ describe('#PostQuery', () => { describe('#countTopLevelPostsByAddr', () => { it('should count top-level posts for an address', async () => { - async function * mockHeights () { - yield ['000000600200:post-200-b', { txid: 'post-200-b' }] - yield ['000000600200:post-200-a', { txid: 'post-200-a' }] - yield ['000000600100:post-100', { txid: 'post-100' }] + async function * mockAddrHeights () { + yield ['bitcoincash:qaddr-a:000000600200:post-200-a', { txid: 'post-200-a' }] + yield ['bitcoincash:qaddr-a:000000600100:post-100', { txid: 'post-100' }] } - postHeightsDb.iterator.returns(mockHeights()) - postsDb.get.callsFake(async (txid) => { - const posts = { - 'post-200-b': { addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 }, - 'post-200-a': { addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600200 }, - 'post-100': { addr: 'addr-a', text: 'c', seen: 50, blockHeight: 600100 } - } - return posts[txid] - }) + async function * mockParents () { + yield ['reply-1', { parentTxid: 'post-200-a', childTxid: 'reply-1', blockHeight: 600050 }] + } + addrPostHeightsDb.iterator + .withArgs({ gte: 'bitcoincash:qaddr-a:', lte: 'bitcoincash:qaddr-a:\uffff' }) + .returns(mockAddrHeights()) + postParentsDb.iterator.returns(mockParents()) - const result = await uut.countTopLevelPostsByAddr('addr-a') + const result = await uut.countTopLevelPostsByAddr('bitcoincash:qaddr-a') assert.equal(result, 2) }) }) + describe('#countRepliesForTxids', () => { + it('should count replies per txid from postChildren', async () => { + async function * mockChildrenTx1 () { + yield ['tx1:reply-a', { parentTxid: 'tx1', childTxid: 'reply-a', blockHeight: 600150 }] + yield ['tx1:reply-b', { parentTxid: 'tx1', childTxid: 'reply-b', blockHeight: 600160 }] + } + async function * mockChildrenTx2 () { + yield ['tx2:reply-c', { parentTxid: 'tx2', childTxid: 'reply-c', blockHeight: 600170 }] + } + postChildrenDb.iterator + .withArgs(sinon.match({ gte: 'tx1:', lte: 'tx1:\uffff' })) + .returns(mockChildrenTx1()) + postChildrenDb.iterator + .withArgs(sinon.match({ gte: 'tx2:', lte: 'tx2:\uffff' })) + .returns(mockChildrenTx2()) + + const result = await uut.countRepliesForTxids(['tx1', 'tx2']) + + assert.equal(result.get('tx1'), 2) + assert.equal(result.get('tx2'), 1) + }) + }) + describe('#buildReplyCountMap', () => { it('should count replies per parent from postChildren', async () => { async function * mockChildren () { @@ -301,43 +345,52 @@ describe('#PostQuery', () => { }) }) - describe('#buildLikeCountMap', () => { - it('should count likes per post, ignoring likes for missing posts', async () => { - async function * mockLikes () { - yield ['like-a', { postTxid: 'tx1', addr: 'addr-x', seen: 1, blockHeight: 1 }] - yield ['like-b', { postTxid: 'tx1', addr: 'addr-y', seen: 2, blockHeight: 2 }] - yield ['like-c', { postTxid: 'tx2', addr: 'addr-z', seen: 3, blockHeight: 3 }] - yield ['like-d', { postTxid: 'missing', addr: 'addr-w', seen: 4, blockHeight: 4 }] + describe('#countLikesForTxids', () => { + it('should count likes per txid from postLikes', async () => { + async function * mockPostLikesTx1 () { + yield ['tx1:like-a', { postTxid: 'tx1', txid: 'like-a' }] + yield ['tx1:like-b', { postTxid: 'tx1', txid: 'like-b' }] } - likesDb.iterator.returns(mockLikes()) + async function * mockPostLikesTx2 () { + yield ['tx2:like-c', { postTxid: 'tx2', txid: 'like-c' }] + } + postLikesDb.iterator + .withArgs(sinon.match({ gte: 'tx1:', lte: 'tx1:\uffff' })) + .returns(mockPostLikesTx1()) + postLikesDb.iterator + .withArgs(sinon.match({ gte: 'tx2:', lte: 'tx2:\uffff' })) + .returns(mockPostLikesTx2()) + + const result = await uut.countLikesForTxids(['tx1', 'tx2']) + + assert.equal(result.get('tx1'), 2) + assert.equal(result.get('tx2'), 1) + }) + }) + + describe('#buildLikeCountMap', () => { + it('should count likes per post from the postLikes index', async () => { + async function * mockPostLikes () { + yield ['tx1:like-a', { postTxid: 'tx1', txid: 'like-a' }] + yield ['tx1:like-b', { postTxid: 'tx1', txid: 'like-b' }] + yield ['tx2:like-c', { postTxid: 'tx2', txid: 'like-c' }] + yield ['tx3:like-d', {}] + } + postLikesDb.iterator.returns(mockPostLikes()) postsDb.get.callsFake(async (txid) => { - if (txid === 'missing') { - const err = new Error('not found') - err.notFound = true - throw err + if (txid === 'tx1' || txid === 'tx2' || txid === 'tx3') { + return { addr: 'addr-a', text: 'x', seen: 1, blockHeight: 1 } } - return { addr: 'addr-a', text: 'x', seen: 1, blockHeight: 1 } + const err = new Error('not found') + err.notFound = true + throw err }) const result = await uut.buildLikeCountMap() assert.equal(result.get('tx1'), 2) assert.equal(result.get('tx2'), 1) - assert.equal(result.has('missing'), false) - }) - - it('should skip like records without a postTxid', async () => { - async function * mockLikes () { - yield ['like-a', { postTxid: 'tx1' }] - yield ['like-b', { addr: 'addr-x' }] - } - likesDb.iterator.returns(mockLikes()) - postsDb.get.withArgs('tx1').resolves({ addr: 'addr-a', text: 'x', seen: 1, blockHeight: 1 }) - - const result = await uut.buildLikeCountMap() - - assert.equal(result.get('tx1'), 1) - assert.equal(result.size, 1) + assert.equal(result.get('tx3'), 1) }) }) }) diff --git a/psf-memo-db/test/unit/use-cases/list-posts-by-addr.unit.js b/psf-memo-db/test/unit/use-cases/list-posts-by-addr.unit.js index b56be13..3717920 100644 --- a/psf-memo-db/test/unit/use-cases/list-posts-by-addr.unit.js +++ b/psf-memo-db/test/unit/use-cases/list-posts-by-addr.unit.js @@ -16,19 +16,21 @@ describe('#ListPostsByAddr', () => { beforeEach(() => { sandbox = sinon.createSandbox() postQuery = { - scanPostsByAddrTxids: sandbox.stub().callsFake(async (addr, { limit, offset }) => { + scanPostsByAddrTxidsAndCount: sandbox.stub().callsFake(async (addr, { limit, offset }) => { const all = Object.entries(mockPosts) .filter(([txid, post]) => post.addr === addr) .sort((a, b) => b[1].blockHeight - a[1].blockHeight) .map(([txid]) => txid) - return all.slice(offset, offset + limit) + return { + txids: all.slice(offset, offset + limit), + total: all.length + } }), loadPostsByTxids: sandbox.stub().callsFake(async (txids) => { return txids.map((txid) => ({ txid, ...mockPosts[txid] })) }), - buildReplyCountMap: sandbox.stub().resolves(new Map()), - buildLikeCountMap: sandbox.stub().resolves(new Map([['tx-c', 7]])), - countTopLevelPostsByAddr: sandbox.stub().resolves(2) + countRepliesForTxids: sandbox.stub().resolves(new Map()), + countLikesForTxids: sandbox.stub().resolves(new Map([['tx-c', 7]])) } uut = new ListPostsByAddr({ adapters: { postQuery } @@ -52,11 +54,15 @@ describe('#ListPostsByAddr', () => { it('should report hasMore when a further page exists', async () => { // One page of one item still leaves one more page available. - postQuery.scanPostsByAddrTxids.callsFake(async (addr, { limit, offset }) => { - return Object.entries(mockPosts) + postQuery.scanPostsByAddrTxidsAndCount.callsFake(async (addr, { limit, offset }) => { + const all = Object.entries(mockPosts) .filter(([txid, post]) => post.addr === addr) .map(([txid]) => txid) - .slice(offset, offset + limit) + .sort((a, b) => mockPosts[b].blockHeight - mockPosts[a].blockHeight) + return { + txids: all.slice(offset, offset + limit), + total: all.length + } }) const result = await uut.execute({ addr: 'addr-a', limit: 1, offset: 0 }) @@ -87,8 +93,8 @@ describe('#ListPostsByAddr', () => { it('should pass addr, limit, and offset to postQuery', async () => { await uut.execute({ addr: 'addr-a', limit: 5, offset: 10 }) - assert.equal(postQuery.scanPostsByAddrTxids.calledOnce, true) - assert.equal(postQuery.scanPostsByAddrTxids.firstCall.args[0], 'addr-a') - assert.deepEqual(postQuery.scanPostsByAddrTxids.firstCall.args[1], { limit: 5, offset: 10 }) + assert.equal(postQuery.scanPostsByAddrTxidsAndCount.calledOnce, true) + assert.equal(postQuery.scanPostsByAddrTxidsAndCount.firstCall.args[0], 'addr-a') + assert.deepEqual(postQuery.scanPostsByAddrTxidsAndCount.firstCall.args[1], { limit: 5, offset: 10 }) }) }) diff --git a/psf-memo-db/test/unit/use-cases/list-recent-posts.unit.js b/psf-memo-db/test/unit/use-cases/list-recent-posts.unit.js index 997df23..e3dbe11 100644 --- a/psf-memo-db/test/unit/use-cases/list-recent-posts.unit.js +++ b/psf-memo-db/test/unit/use-cases/list-recent-posts.unit.js @@ -21,7 +21,7 @@ describe('#ListRecentPosts', () => { return txids.map((txid) => ({ txid, ...mockPosts[txid] })) }), buildReplyCountMap: sandbox.stub().resolves(new Map([['tx-b', 1]])), - buildLikeCountMap: sandbox.stub().resolves(new Map([['tx-b', 3], ['tx-a', 5]])), + countLikesForTxids: sandbox.stub().resolves(new Map([['tx-b', 3], ['tx-a', 5]])), countTopLevelPosts: sandbox.stub().resolves(3) } uut = new ListRecentPosts({ diff --git a/psf-memo-db/util/post/backfill-post-indexes.js b/psf-memo-db/util/post/backfill-post-indexes.js new file mode 100644 index 0000000..9a1c41c --- /dev/null +++ b/psf-memo-db/util/post/backfill-post-indexes.js @@ -0,0 +1,147 @@ +/* + Utility: backfill the addrPostHeights and postLikes secondary indexes for an + existing psf-memo-db. + + New deployments write these indexes while indexing live blocks, but existing + databases that were populated before the efficient-query feature need a + one-time backfill. + + Run from the psf-memo-db repo root on the host that owns the LevelDB files: + + node util/post/backfill-post-indexes.js + + The script is idempotent: re-running it will not create duplicate index + entries. Progress and a summary are printed to stderr. + + WARNING: + - This script opens the LevelDB files directly. psf-memo-db must NOT be + running, or another process must not hold the database locks. + - Make a backup of leveldb/current before running on a production server: + cp -r leveldb/current leveldb/current-pre-index-backup +*/ + +import level from 'level' +import * as fs from 'fs' +import * as path from 'path' +import * as url from 'url' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) + +const DATA_DIR = process.env.PSF_MEMO_DB_DATA_DIR + ? path.resolve(process.env.PSF_MEMO_DB_DATA_DIR) + : path.resolve(__dirname, '../../leveldb/current') + +const HEIGHT_PAD = 12 +const PROGRESS_INTERVAL = 10000 + +function requiredStorePath (dir, name) { + const storePath = path.join(dir, name) + if (!fs.existsSync(storePath)) { + throw new Error(`Required LevelDB store not found: ${storePath}. Set PSF_MEMO_DB_DATA_DIR to the directory containing the posts and likes stores.`) + } + return storePath +} + +function addrPostHeightKey (addr, blockHeight, txid) { + const padded = String(blockHeight).padStart(HEIGHT_PAD, '0') + return `${addr}:${padded}:${txid}` +} + +function postLikeKey (postTxid, likeTxid) { + return `${postTxid}:${likeTxid}` +} + +async function createIfMissing (db, key, value) { + try { + await db.get(key) + return false + } catch (err) { + if (err.notFound || err.code === 'LEVEL_NOT_FOUND') { + await db.put(key, value) + return true + } + throw err + } +} + +async function backfill () { + console.error(`Using LevelDB data directory: ${DATA_DIR}`) + + const postsPath = requiredStorePath(DATA_DIR, 'posts') + const likesPath = requiredStorePath(DATA_DIR, 'likes') + const addrPostHeightsPath = path.join(DATA_DIR, 'addrPostHeights') + const postLikesPath = path.join(DATA_DIR, 'postLikes') + + console.error('Opening LevelDB stores...') + const postsDb = level(postsPath, { valueEncoding: 'json' }) + const likesDb = level(likesPath, { valueEncoding: 'json' }) + const addrPostHeightsDb = level(addrPostHeightsPath, { valueEncoding: 'json', createIfMissing: true }) + const postLikesDb = level(postLikesPath, { valueEncoding: 'json', createIfMissing: true }) + + try { + console.error('Backfilling addrPostHeights entries from posts...') + let postsScanned = 0 + let addrPostHeightsCreated = 0 + let addrPostHeightsExisting = 0 + + for await (const [txid, post] of postsDb.iterator()) { + postsScanned++ + const addr = post.addr + const blockHeight = post.blockHeight ?? 0 + const key = addrPostHeightKey(addr, blockHeight, txid) + const value = { txid, addr, blockHeight } + const created = await createIfMissing(addrPostHeightsDb, key, value) + if (created) { + addrPostHeightsCreated++ + } else { + addrPostHeightsExisting++ + } + + if (postsScanned % PROGRESS_INTERVAL === 0) { + console.error(` scanned ${postsScanned} posts, ${addrPostHeightsCreated} written, ${addrPostHeightsExisting} skipped...`) + } + } + + console.error('\nBackfilling postLikes entries from likes...') + let likesScanned = 0 + let postLikesCreated = 0 + let postLikesExisting = 0 + + for await (const [txid, like] of likesDb.iterator()) { + likesScanned++ + const postTxid = like.postTxid + if (!postTxid) continue + + const key = postLikeKey(postTxid, txid) + const value = { postTxid, txid } + const created = await createIfMissing(postLikesDb, key, value) + if (created) { + postLikesCreated++ + } else { + postLikesExisting++ + } + + if (likesScanned % PROGRESS_INTERVAL === 0) { + console.error(` scanned ${likesScanned} likes, ${postLikesCreated} written, ${postLikesExisting} skipped...`) + } + } + + console.error('\nBackfill complete.') + console.error(` posts scanned: ${postsScanned}`) + console.error(` addrPostHeights created: ${addrPostHeightsCreated}`) + console.error(` addrPostHeights already present: ${addrPostHeightsExisting}`) + console.error(` likes scanned: ${likesScanned}`) + console.error(` postLikes created: ${postLikesCreated}`) + console.error(` postLikes already present: ${postLikesExisting}`) + } catch (err) { + console.error('\nBackfill failed:', err.message) + process.exitCode = 1 + } finally { + await addrPostHeightsDb.close().catch(() => {}) + await postLikesDb.close().catch(() => {}) + await likesDb.close().catch(() => {}) + await postsDb.close().catch(() => {}) + } +} + +backfill() diff --git a/psf-memo-indexer/acceptance/lib/handlers.js b/psf-memo-indexer/acceptance/lib/handlers.js index bd2fb26..209d9f4 100644 --- a/psf-memo-indexer/acceptance/lib/handlers.js +++ b/psf-memo-indexer/acceptance/lib/handlers.js @@ -1,14 +1,15 @@ /* Project step handlers for the psf-memo-indexer acceptance pipeline. - These handlers exercise the real Memo action handlers (handlePost, handleReply) - against an in-memory database that exposes the same CRUD surface as the - psf-memo-db entity routes used by the indexer. + These handlers exercise the real Memo action handlers (handlePost, handleReply, + handleLike) against an in-memory database that exposes the same CRUD surface as + the psf-memo-db entity routes used by the indexer. */ import crypto from 'node:crypto' import { handlePost } from '../../src/use-cases/action-types/post.js' import { handleReply } from '../../src/use-cases/action-types/reply.js' +import { handleLike } from '../../src/use-cases/action-types/like.js' function makeInMemoryDb () { const store = new Map() @@ -69,14 +70,20 @@ function resolveTxid (value, example, world) { async function createWorld () { const postsDb = makeInMemoryDb() const postHeightsDb = makeInMemoryDb() + const addrPostHeightsDb = makeInMemoryDb() const postParentsDb = makeInMemoryDb() const postChildrenDb = makeInMemoryDb() + const likesDb = makeInMemoryDb() + const postLikesDb = makeInMemoryDb() const adapters = { postDb: postsDb, postHeightDb: postHeightsDb, + addrPostHeightDb: addrPostHeightsDb, postParentDb: postParentsDb, postChildDb: postChildrenDb, + likeDb: likesDb, + postLikeDb: postLikesDb, processErrorDb: makeInMemoryDb() } @@ -84,8 +91,11 @@ async function createWorld () { adapters, postsDb, postHeightsDb, + addrPostHeightsDb, postParentsDb, postChildrenDb, + likesDb, + postLikesDb, txidMap: new Map(), lastTxid: null, lastHeight: null, @@ -101,6 +111,13 @@ const handlers = [ // World is already created with both stores. } }, + { + name: 'db instance with new indexes', + pattern: /^a psf-memo-db instance with posts, postHeights, addrPostHeights, likes, and postLikes stores$/, + async run () { + // World is already created with all stores. + } + }, { name: 'indexer configured to write to db', pattern: /^a psf-memo-indexer configured to write to that database$/, @@ -172,6 +189,37 @@ const handlers = [ }) } }, + { + name: 'process a Memo like transaction', + pattern: /^the indexer processes a Memo like transaction (.+) for post (.+) from (.+) at block height (.+)$/, + async run (m, example, world) { + const txid = resolveTxid(m[1], example, world) + const postTxid = resolveTxid(m[2], example, world) + const addr = resolveParam(m[3], example) + const height = parseInt(resolveParam(m[4], example), 10) + + world.lastTxid = txid + world.lastHeight = height + world.lastAddr = addr + + const prefix = Buffer.from('6d04', 'hex') + const postHash = Buffer.from(postTxid, 'hex').reverse() + + await handleLike({ + adapters: world.adapters, + txid, + signerAddr: addr, + seen: Date.now(), + blockHeight: height, + txDetails: { vout: [] }, + decoded: { + action: 'like', + prefix, + pushDatas: [prefix, postHash] + } + }) + } + }, { name: 'process the same Memo post transaction again', pattern: /^the indexer processes the same Memo post transaction (.+) again$/, @@ -224,6 +272,21 @@ const handlers = [ } } }, + { + name: 'addrPostHeights store contains entry', + pattern: /^the addrPostHeights store contains (.+) entry whose key starts with (.+) and ends with (.+)$/, + run (m, example, world) { + const expectedCount = parseInt(resolveParam(m[1], example), 10) + const addr = resolveParam(m[2], example) + const txid = resolveTxid(m[3], example, world) + const matching = world.addrPostHeightsDb.entries().filter(([key, value]) => { + return key.startsWith(`${addr}:`) && (key.endsWith(`:${txid}`) || value?.txid === txid) + }) + if (matching.length !== expectedCount) { + throw new Error(`Expected ${expectedCount} addrPostHeights entry/entries for ${addr}/${txid}, got ${matching.length}`) + } + } + }, { name: 'postParents store contains link', pattern: /^the postParents store contains a link from (.+) to (.+)$/, @@ -251,6 +314,21 @@ const handlers = [ throw new Error(`Expected postChildren link from ${parentTxid} to ${childTxid}`) } } + }, + { + name: 'postLikes store contains entry', + pattern: /^the postLikes store contains (.+) entry whose key starts with (.+) and ends with (.+)$/, + run (m, example, world) { + const expectedCount = parseInt(resolveParam(m[1], example), 10) + const postTxid = resolveTxid(m[2], example, world) + const likeTxid = resolveTxid(m[3], example, world) + const matching = world.postLikesDb.entries().filter(([key, value]) => { + return key.startsWith(`${postTxid}:`) && (key.endsWith(`:${likeTxid}`) || value?.txid === likeTxid) + }) + if (matching.length !== expectedCount) { + throw new Error(`Expected ${expectedCount} postLikes entry/entries for ${postTxid}/${likeTxid}, got ${matching.length}`) + } + } } ] diff --git a/psf-memo-indexer/src/adapters/adapters-index.js b/psf-memo-indexer/src/adapters/adapters-index.js index f6d42fd..687a4b7 100644 --- a/psf-memo-indexer/src/adapters/adapters-index.js +++ b/psf-memo-indexer/src/adapters/adapters-index.js @@ -21,9 +21,11 @@ class Adapters { this.postDb = createEntityDb('post', 'txid', 'postData') this.postHeightDb = createEntityDb('postheight', 'key', 'postHeightData') + this.addrPostHeightDb = createEntityDb('addrpostheight', 'key', 'addrPostHeightData') this.postParentDb = createEntityDb('postparent', 'txid', 'parentData') this.postChildDb = createEntityDb('postchild', 'key', 'childData') this.likeDb = createEntityDb('like', 'txid', 'likeData') + this.postLikeDb = createEntityDb('postlike', 'key', 'postLikeData') this.nameDb = createEntityDb('name', 'addr', 'nameData') this.profileDb = createEntityDb('profile', 'addr', 'profileData') this.profilePicDb = createEntityDb('profilepic', 'addr', 'profilePicData') diff --git a/psf-memo-indexer/src/use-cases/action-types/helpers.js b/psf-memo-indexer/src/use-cases/action-types/helpers.js index c1d3700..fabc061 100644 --- a/psf-memo-indexer/src/use-cases/action-types/helpers.js +++ b/psf-memo-indexer/src/use-cases/action-types/helpers.js @@ -67,6 +67,15 @@ export function postHeightKey (blockHeight, txid) { return `${padded}:${txid}` } +export function addrPostHeightKey (addr, blockHeight, txid) { + const padded = String(blockHeight).padStart(12, '0') + return `${addr}:${padded}:${txid}` +} + +export function postLikeKey (postTxid, likeTxid) { + return `${postTxid}:${likeTxid}` +} + export function postChildKey (parentTxid, childTxid) { return `${parentTxid}:${childTxid}` } diff --git a/psf-memo-indexer/src/use-cases/action-types/like.js b/psf-memo-indexer/src/use-cases/action-types/like.js index 1cda90c..196d420 100644 --- a/psf-memo-indexer/src/use-cases/action-types/like.js +++ b/psf-memo-indexer/src/use-cases/action-types/like.js @@ -1,4 +1,4 @@ -import { txHashFromPush, logProcessError } from './helpers.js' +import { txHashFromPush, logProcessError, postLikeKey } from './helpers.js' import { TX_HASH_LENGTH } from '../../lib/memo-codes.js' export async function handleLike (ctx) { @@ -40,4 +40,5 @@ export async function handleLike (ctx) { blockHeight } await adapters.likeDb.create(txid, likeData) + await adapters.postLikeDb.create(postLikeKey(postTxid, txid), { postTxid, txid }) } diff --git a/psf-memo-indexer/src/use-cases/action-types/post.js b/psf-memo-indexer/src/use-cases/action-types/post.js index 5d9030a..fa9f228 100644 --- a/psf-memo-indexer/src/use-cases/action-types/post.js +++ b/psf-memo-indexer/src/use-cases/action-types/post.js @@ -1,4 +1,4 @@ -import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas, postHeightKey } from './helpers.js' +import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas, postHeightKey, addrPostHeightKey } from './helpers.js' import { MAX_POST_SIZE } from '../../lib/memo-codes.js' // Create a record only when it does not already exist (idempotent writes). @@ -31,8 +31,10 @@ export async function handlePost (ctx) { const postData = { addr: signerAddr, text, seen, blockHeight } const heightKey = postHeightKey(blockHeight, txid) + const addrHeightKey = addrPostHeightKey(signerAddr, blockHeight, txid) await createIfMissing(adapters.postDb, txid, postData) await createIfMissing(adapters.postHeightDb, heightKey, { txid, blockHeight }) + await createIfMissing(adapters.addrPostHeightDb, addrHeightKey, { txid, addr: signerAddr, blockHeight }) } // mutate4javascript-manifest-begin diff --git a/psf-memo-indexer/test/unit/use-cases/action-types/like.unit.js b/psf-memo-indexer/test/unit/use-cases/action-types/like.unit.js new file mode 100644 index 0000000..3ae49b3 --- /dev/null +++ b/psf-memo-indexer/test/unit/use-cases/action-types/like.unit.js @@ -0,0 +1,44 @@ +import { assert } from 'chai' +import sinon from 'sinon' +import { handleLike } from '../../../../src/use-cases/action-types/like.js' +import { PREFIX_LIKE } from '../../../../src/lib/memo-codes.js' + +describe('#handleLike', () => { + it('should save a like and its postLikes index entry', async () => { + const postTxid = Buffer.alloc(32, 0xab) + + const likeCreate = sinon.stub().resolves({ success: true }) + const postLikeCreate = sinon.stub().resolves({ success: true }) + const processErrorDb = { create: sinon.stub() } + + const adapters = { + likeDb: { create: likeCreate }, + postLikeDb: { create: postLikeCreate }, + processErrorDb + } + + await handleLike({ + adapters, + txid: 'like-abc', + signerAddr: 'bitcoincash:qptest', + seen: 1000, + blockHeight: 600150, + txDetails: { vout: [] }, + decoded: { + action: 'like', + prefix: PREFIX_LIKE, + pushDatas: [PREFIX_LIKE, postTxid] + } + }) + + assert.equal(likeCreate.callCount, 1) + assert.equal(likeCreate.firstCall.args[0], 'like-abc') + assert.equal(likeCreate.firstCall.args[1].postTxid, postTxid.toString('hex').match(/.{2}/g).reverse().join('')) + + assert.equal(postLikeCreate.callCount, 1) + const expectedPostTxid = postTxid.toString('hex').match(/.{2}/g).reverse().join('') + assert.equal(postLikeCreate.firstCall.args[0], `${expectedPostTxid}:like-abc`) + assert.equal(postLikeCreate.firstCall.args[1].postTxid, expectedPostTxid) + assert.equal(postLikeCreate.firstCall.args[1].txid, 'like-abc') + }) +}) diff --git a/psf-memo-indexer/test/unit/use-cases/action-types/post.unit.js b/psf-memo-indexer/test/unit/use-cases/action-types/post.unit.js index 36b2151..0e9c311 100644 --- a/psf-memo-indexer/test/unit/use-cases/action-types/post.unit.js +++ b/psf-memo-indexer/test/unit/use-cases/action-types/post.unit.js @@ -9,10 +9,13 @@ describe('#handlePost', () => { const get = sinon.stub().rejects(new Error('not found')) const postHeightCreate = sinon.stub().resolves({ success: true }) const postHeightGet = sinon.stub().rejects(new Error('not found')) + const addrPostHeightCreate = sinon.stub().resolves({ success: true }) + const addrPostHeightGet = sinon.stub().rejects(new Error('not found')) const adapters = { postDb: { create, get }, postHeightDb: { create: postHeightCreate, get: postHeightGet }, + addrPostHeightDb: { create: addrPostHeightCreate, get: addrPostHeightGet }, processErrorDb: { create: sinon.stub() } } @@ -39,6 +42,11 @@ describe('#handlePost', () => { assert.equal(postHeightCreate.firstCall.args[0], '000000600100:abc123') assert.equal(postHeightCreate.firstCall.args[1].txid, 'abc123') assert.equal(postHeightCreate.firstCall.args[1].blockHeight, 600100) + + assert.equal(addrPostHeightCreate.callCount, 1) + assert.equal(addrPostHeightCreate.firstCall.args[0], 'bitcoincash:qptest:000000600100:abc123') + assert.equal(addrPostHeightCreate.firstCall.args[1].txid, 'abc123') + assert.equal(addrPostHeightCreate.firstCall.args[1].blockHeight, 600100) }) it('should not duplicate postHeight entries when reprocessing', async () => { @@ -46,10 +54,13 @@ describe('#handlePost', () => { const get = sinon.stub().resolves({ addr: 'bitcoincash:qptest', text: 'hello memo', seen: 1000, blockHeight: 600100 }) const postHeightCreate = sinon.stub().resolves({ success: true }) const postHeightGet = sinon.stub().resolves({ txid: 'abc123', blockHeight: 600100 }) + const addrPostHeightCreate = sinon.stub().resolves({ success: true }) + const addrPostHeightGet = sinon.stub().resolves({ txid: 'abc123', blockHeight: 600100 }) const adapters = { postDb: { create, get }, postHeightDb: { create: postHeightCreate, get: postHeightGet }, + addrPostHeightDb: { create: addrPostHeightCreate, get: addrPostHeightGet }, processErrorDb: { create: sinon.stub() } } @@ -69,6 +80,7 @@ describe('#handlePost', () => { assert.equal(create.callCount, 0) assert.equal(postHeightCreate.callCount, 0) + assert.equal(addrPostHeightCreate.callCount, 0) }) it('should accept a post whose text is exactly at the maximum size', async () => { @@ -76,11 +88,14 @@ describe('#handlePost', () => { const get = sinon.stub().rejects(new Error('not found')) const postHeightCreate = sinon.stub().resolves({ success: true }) const postHeightGet = sinon.stub().rejects(new Error('not found')) + const addrPostHeightCreate = sinon.stub().resolves({ success: true }) + const addrPostHeightGet = sinon.stub().rejects(new Error('not found')) const processErrorDb = { create: sinon.stub() } const adapters = { postDb: { create, get }, postHeightDb: { create: postHeightCreate, get: postHeightGet }, + addrPostHeightDb: { create: addrPostHeightCreate, get: addrPostHeightGet }, processErrorDb } diff --git a/psf-memo-indexer/test/unit/use-cases/action-types/reply.unit.js b/psf-memo-indexer/test/unit/use-cases/action-types/reply.unit.js index eab32cb..89981dc 100644 --- a/psf-memo-indexer/test/unit/use-cases/action-types/reply.unit.js +++ b/psf-memo-indexer/test/unit/use-cases/action-types/reply.unit.js @@ -14,12 +14,15 @@ describe('#handleReply', () => { const postDbCreate = sinon.stub().resolves({ success: true }) const postHeightGet = sinon.stub().rejects(new Error('not found')) const postHeightCreate = sinon.stub().resolves({ success: true }) + const addrPostHeightGet = sinon.stub().rejects(new Error('not found')) + const addrPostHeightCreate = sinon.stub().resolves({ success: true }) const adapters = { postParentDb: { create: postParentCreate }, postChildDb: { create: postChildCreate }, postDb: { get: postDbGet, create: postDbCreate }, postHeightDb: { get: postHeightGet, create: postHeightCreate }, + addrPostHeightDb: { get: addrPostHeightGet, create: addrPostHeightCreate }, processErrorDb: { create: sinon.stub() } } @@ -50,5 +53,9 @@ describe('#handleReply', () => { assert.equal(postHeightCreate.callCount, 1) assert.equal(postHeightCreate.firstCall.args[0], '000000600150:reply-abc') assert.equal(postHeightCreate.firstCall.args[1].txid, 'reply-abc') + + assert.equal(addrPostHeightCreate.callCount, 1) + assert.equal(addrPostHeightCreate.firstCall.args[0], 'bitcoincash:qptest:000000600150:reply-abc') + assert.equal(addrPostHeightCreate.firstCall.args[1].txid, 'reply-abc') }) })