diff --git a/psf-memo-db/.env-example b/psf-memo-db/.env-example index aac8fc1..52ea345 100644 --- a/psf-memo-db/.env-example +++ b/psf-memo-db/.env-example @@ -2,3 +2,4 @@ PORT=5021 SVC_ENV=development BACKUP_QTY=3 EXIT_ON_MISSING_BACKUP=false +NOTIFICATION_BLOCK_WINDOW=25000 diff --git a/psf-memo-db/acceptance/lib/handlers.js b/psf-memo-db/acceptance/lib/handlers.js index 63b2559..4166423 100644 --- a/psf-memo-db/acceptance/lib/handlers.js +++ b/psf-memo-db/acceptance/lib/handlers.js @@ -24,11 +24,24 @@ import TopicFollowState from '../../src/use-cases/topic-follow-state.js' import ListTopicFollowers from '../../src/use-cases/list-topic-followers.js' import MuteState from '../../src/use-cases/mute-state.js' import ListMuted from '../../src/use-cases/list-muted.js' +import ListNotifications from '../../src/use-cases/list-notifications.js' import GetPoll from '../../src/use-cases/get-poll.js' import GetPollOptions from '../../src/use-cases/get-poll-options.js' import GetPollVotes from '../../src/use-cases/get-poll-votes.js' import { repairTxidEncoding } from '../../src/lib/repair-txid-encoding.js' import { backfillTopicIndexes, topicRecencyKey } from '../../src/lib/backfill-topic-indexes.js' +import { backfillFolloweeIndex, followeeHeightKey } from '../../src/lib/backfill-followee-index.js' +import BCHJS from '@psf/bch-js' + +const bchjs = new BCHJS({ restURL: process.env.RESTURL || 'https://api.fullstack.cash/v5/' }) + +function hash160 (addr) { + return bchjs.Address.toHash160(addr) +} + +function padHeight (blockHeight) { + return String(blockHeight ?? 0).padStart(12, '0') +} const __dirname = path.dirname(fileURLToPath(import.meta.url)) const tmpDir = path.resolve(__dirname, '..', '..', 'tmp', 'acceptance') @@ -98,6 +111,8 @@ async function createWorld () { const postsGetCounter = { calls: 0 } const likesIteratorCounter = { calls: 0 } const postLikesIteratorCounter = { calls: 0 } + const followsIteratorCounter = { calls: 0 } + const followeeHeightsIteratorCounter = { calls: 0, entries: 0 } const roomsIteratorCounter = { calls: 0, entries: 0 } const topicRecencyIteratorCounter = { calls: 0, entries: 0 } wrapIterator(adapters.level.postHeightsDb, postHeightsIteratorCounter) @@ -106,6 +121,8 @@ async function createWorld () { wrapGet(adapters.level.postsDb, postsGetCounter) wrapIterator(adapters.level.likesDb, likesIteratorCounter) wrapIterator(adapters.level.postLikesDb, postLikesIteratorCounter) + wrapIterator(adapters.level.followsDb, followsIteratorCounter) + wrapIterator(adapters.level.followeeHeightsDb, followeeHeightsIteratorCounter) wrapIterator(adapters.level.roomsDb, roomsIteratorCounter) wrapIterator(adapters.level.topicRecencyDb, topicRecencyIteratorCounter) @@ -121,6 +138,7 @@ async function createWorld () { const listTopicFollowers = new ListTopicFollowers({ adapters }) const muteState = new MuteState({ adapters }) const listMuted = new ListMuted({ adapters }) + const listNotifications = new ListNotifications({ adapters }) const getPoll = new GetPoll({ adapters }) const getPollOptions = new GetPollOptions({ adapters }) const getPollVotes = new GetPollVotes({ adapters }) @@ -141,6 +159,7 @@ async function createWorld () { listTopicFollowers, muteState, listMuted, + listNotifications, getPoll, getPollOptions, getPollVotes, @@ -150,6 +169,8 @@ async function createWorld () { postsGetCounter, likesIteratorCounter, postLikesIteratorCounter, + followsIteratorCounter, + followeeHeightsIteratorCounter, roomsIteratorCounter, topicRecencyIteratorCounter, getLastResponse: () => lastResponse, @@ -237,6 +258,16 @@ async function loadFixture (world, name) { return } + if (name === 'follows-by-followee') { + await loadFollowsByFollowee(world) + return + } + + if (name === 'notifications-window') { + await loadNotificationsWindow(world) + return + } + if (name === 'many-posts-with-replies') { await loadManyPostsWithReplies(world) return @@ -498,6 +529,169 @@ async function loadFollows (world) { } } +// Fixture "follows-by-followee" from backfill-followee-index.feature: raw +// follows records the backfill projects into followeeHeights. +async function loadFollowsByFollowee (world) { + const viewer = 'bitcoincash:qqg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zye3kwllue' + const followerA = 'bitcoincash:qq3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygrg4dtdzf' + const followerB = 'bitcoincash:qqenxvenxvenxvenxvenxvenxvenxvenxvn254yg3p' + const followerC = 'bitcoincash:qpzyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs7fn3s6pt' + const other = 'bitcoincash:qp24242424242424242424242424242425wtjflljr' + + const records = [ + { follower: followerA, followee: viewer, unfollow: false, height: 690400 }, + { follower: followerB, followee: viewer, unfollow: true, height: 690600 }, + // The Examples table lists followee qpzyg... and follower qp2424... for this + // record, so store it that orientation to match the asserted index key. + { follower: other, followee: followerC, unfollow: false, height: 690200 } + ] + + for (const record of records) { + const followeePkHash = hash160(record.followee) + await world.adapters.level.followsDb.put(`${record.follower}:${followeePkHash}`, { + followerAddr: record.follower, + followeePkHash, + unfollow: record.unfollow, + txid: `follow-${record.follower.slice(-8)}`, + seen: 1, + blockHeight: record.height + }) + } +} + +// Fixture "notifications-window" from notifications-query-performance.feature. +async function loadNotificationsWindow (world) { + const viewer = 'bitcoincash:qqg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zye3kwllue' + const followerA = 'bitcoincash:qq3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygrg4dtdzf' + const followerB = 'bitcoincash:qqenxvenxvenxvenxvenxvenxvenxvenxvn254yg3p' + const oldActor = 'bitcoincash:qpzyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs7fn3s6pt' + const liker = 'bitcoincash:qp24242424242424242424242424242425wtjflljr' + const replier = 'bitcoincash:qpnxvenxvenxvenxvenxvenxvenxvenxvc5j32tdvn' + + // The feature identifies the viewer by the fixture name rather than an + // example column, so expose it for step resolution. + world.fixtureViewer = viewer + + await world.adapters.level.statusDb.put('status', { + startBlockHeight: 0, + syncedBlockHeight: 700000, + chainBlockHeight: 700000 + }) + + const posts = [ + { txid: 'post-recent', addr: viewer, blockHeight: 690000, text: 'recent' }, + { txid: 'post-old', addr: viewer, blockHeight: 600000, text: 'old' }, + { txid: 'post-other', addr: followerA, blockHeight: 690000, text: 'other' } + ] + for (const post of posts) { + await world.adapters.level.postsDb.put(post.txid, { + addr: post.addr, + text: post.text, + seen: post.blockHeight, + blockHeight: post.blockHeight + }) + await world.adapters.level.postHeightsDb.put(`${padHeight(post.blockHeight)}:${post.txid}`, { + txid: post.txid, + blockHeight: post.blockHeight + }) + await world.adapters.level.addrPostHeightsDb.put( + `${post.addr}:${padHeight(post.blockHeight)}:${post.txid}`, + { txid: post.txid, addr: post.addr, blockHeight: post.blockHeight } + ) + } + + const likes = [ + { txid: 'like-recent', postTxid: 'post-recent', addr: liker, blockHeight: 690100 }, + { txid: 'like-old', postTxid: 'post-old', addr: oldActor, blockHeight: 690200 }, + { txid: 'like-other', postTxid: 'post-other', addr: followerA, blockHeight: 690300 } + ] + for (const like of likes) { + await world.adapters.level.likesDb.put(like.txid, { + addr: like.addr, + postTxid: like.postTxid, + seen: like.blockHeight, + tip: 0, + blockHeight: like.blockHeight + }) + await world.adapters.level.postLikesDb.put( + `${like.postTxid}:${like.txid}`, + { postTxid: like.postTxid, txid: like.txid } + ) + } + + const replies = [ + { parent: 'post-recent', child: 'reply-recent', addr: replier, blockHeight: 690150 }, + { parent: 'post-old', child: 'reply-old', addr: oldActor, blockHeight: 690250 }, + { parent: 'post-other', child: 'reply-other', addr: followerA, blockHeight: 690300 } + ] + for (const reply of replies) { + await world.adapters.level.postsDb.put(reply.child, { + addr: reply.addr, + text: 'reply', + seen: reply.blockHeight, + blockHeight: reply.blockHeight + }) + await world.adapters.level.postChildrenDb.put( + `${reply.parent}:${reply.child}`, + { parentTxid: reply.parent, childTxid: reply.child, blockHeight: reply.blockHeight } + ) + } + + const viewerHash = hash160(viewer) + const followEvents = [ + { follower: followerA, height: 690400, unfollow: false }, + { follower: followerB, height: 690550, unfollow: false }, + { follower: followerB, height: 690600, unfollow: true }, + { follower: oldActor, height: 600000, unfollow: false } + ] + for (const event of followEvents) { + const record = { + followerAddr: event.follower, + followeePkHash: viewerHash, + unfollow: event.unfollow, + txid: `follow-${event.follower.slice(-6)}-${event.height}`, + seen: event.height, + blockHeight: event.height + } + await world.adapters.level.followeeHeightsDb.put( + followeeHeightKey(viewerHash, event.height, event.follower), + record + ) + // Mirror the event in the follows store; the query must ignore it. + await world.adapters.level.followsDb.put(`${event.follower}:${viewerHash}`, record) + } + + // Filler so an accidental full-store scan is observable. + for (let i = 0; i < 100; i++) { + await world.adapters.level.likesDb.put(`filler-like-${i}`, { + addr: 'bitcoincash:filler', + postTxid: `filler-post-${i}`, + seen: i, + tip: 0, + blockHeight: 1 + }) + await world.adapters.level.postLikesDb.put( + `filler-post-${i}:filler-like-${i}`, + { postTxid: `filler-post-${i}`, txid: `filler-like-${i}` } + ) + await world.adapters.level.postChildrenDb.put( + `filler-parent-${i}:filler-child-${i}`, + { parentTxid: `filler-parent-${i}`, childTxid: `filler-child-${i}`, blockHeight: 1 } + ) + await world.adapters.level.followeeHeightsDb.put( + followeeHeightKey(`fillerhash-${i}`, 690000, `filler-follower-${i}`), + { + followerAddr: `filler-follower-${i}`, + followeePkHash: `fillerhash-${i}`, + unfollow: false, + txid: `filler-${i}`, + seen: i, + blockHeight: 690000 + } + ) + } +} + async function loadTopicsWithPosts (world) { const roomEntries = [ { key: 'bitcoin:post-300', room: 'bitcoin', txid: 'post-300', type: 'post', blockHeight: 300 }, @@ -872,7 +1066,7 @@ const handlers = [ }, { name: 'bounded postChildren entries read', - pattern: /^the postChildren store was read at most () entries$/, + pattern: /^the postChildren store was read at most (<[A-Za-z0-9_]+>) entries$/, run (m, example, world) { const max = parseInt(resolveParam(m[1], example), 10) const reads = world.postChildrenIteratorCounter.entries || 0 @@ -1732,6 +1926,156 @@ const handlers = [ throw new Error(`Expected muted ${expected.join(',')}, got ${actual.join(',')}`) } } + }, + { + name: 'db instance with follows and followeeHeights stores', + pattern: /^a psf-memo-db instance with follows and followeeHeights stores$/, + async run () { + // World is already created with both stores. + } + }, + { + name: 'run followee index backfill utility', + pattern: /^the followee index backfill utility is run$/, + async run (m, example, world) { + await backfillFolloweeIndex({ + followsDb: world.adapters.level.followsDb, + followeeHeightsDb: world.adapters.level.followeeHeightsDb + }) + } + }, + { + name: 'run followee index backfill utility again', + pattern: /^the followee index backfill utility is run again$/, + async run (m, example, world) { + await backfillFolloweeIndex({ + followsDb: world.adapters.level.followsDb, + followeeHeightsDb: world.adapters.level.followeeHeightsDb + }) + } + }, + { + name: 'followeeHeights store contains entry', + pattern: /^the followeeHeights store contains (<[A-Za-z0-9_]+>) entr(?:y|ies) for followee (<[A-Za-z0-9_]+>) from (<[A-Za-z0-9_]+>) at block height (<[A-Za-z0-9_]+>) marked unfollow (<[A-Za-z0-9_]+>|true|false)$/, + async run (m, example, world) { + const expectedCount = parseInt(resolveParam(m[1], example), 10) + const followee = resolveParam(m[2], example) + const follower = resolveParam(m[3], example) + const height = parseInt(resolveParam(m[4], example), 10) + const expectedUnfollow = resolveParam(m[5], example) === 'true' + const key = followeeHeightKey(hash160(followee), height, follower) + let count = 0 + for await (const [k, value] of world.adapters.level.followeeHeightsDb.iterator()) { + if (k === key && value?.unfollow === expectedUnfollow) count++ + } + if (count !== expectedCount) { + throw new Error(`Expected ${expectedCount} followeeHeights entry/entries for ${key} marked unfollow ${expectedUnfollow}, got ${count}`) + } + } + }, + { + name: 'db instance with notification stores', + pattern: /^a psf-memo-db instance with posts, postHeights, addrPostHeights, postChildren, likes, postLikes, follows, followeeHeights, and status stores$/, + async run () { + // World is already created with all stores. + } + }, + { + name: 'set notification window', + pattern: /^a notification window of (\S+) blocks$/, + run (m, example, world) { + const window = parseInt(resolveParam(m[1], example), 10) + world.adapters.notificationsQuery.notificationBlockWindow = window + } + }, + { + name: 'request notifications', + pattern: /^the client requests \/posts\/notifications\/(<[A-Za-z0-9_]+>) with limit (<[A-Za-z0-9_]+>) and offset (<[A-Za-z0-9_]+>)$/, + async run (m, example, world) { + const addr = m[1] === '' ? world.fixtureViewer : resolveParam(m[1], example) + const limit = parseInt(resolveParam(m[2], example), 10) + const offset = parseInt(resolveParam(m[3], example), 10) + const resp = await world.listNotifications.execute({ addr, limit, offset }) + world.setLastResponse(resp) + } + }, + { + name: 'response contains N notifications', + pattern: /^the response contains (<[A-Za-z0-9_]+>) notifications$/, + run (m, example, world) { + const expected = parseInt(resolveParam(m[1], example), 10) + const notifications = world.getLastResponse().notifications + if (notifications.length !== expected) { + throw new Error(`Expected ${expected} notifications, got ${notifications.length}`) + } + } + }, + { + name: 'notification at index', + pattern: /^the response notification at index (<[A-Za-z0-9_]+>) has type (<[A-Za-z0-9_]+>) from (<[A-Za-z0-9_]+>)$/, + run (m, example, world) { + const index = parseInt(resolveParam(m[1], example), 10) + const type = resolveParam(m[2], example) + const actor = resolveParam(m[3], example) + const notification = world.getLastResponse().notifications[index] + if (!notification) { + throw new Error(`No notification at index ${index}`) + } + if (notification.type !== type || notification.addr !== actor) { + throw new Error(`Expected ${type} from ${actor} at index ${index}, got ${notification.type} from ${notification.addr}`) + } + } + }, + { + name: 'contains no notification from actor', + pattern: /^the response contains no notification from (<[A-Za-z0-9_]+>)$/, + run (m, example, world) { + const actor = resolveParam(m[1], example) + const found = world.getLastResponse().notifications.find((n) => n.addr === actor) + if (found) { + throw new Error(`Expected no notification from ${actor}, got ${JSON.stringify(found)}`) + } + } + }, + { + name: 'bounded postLikes entries read', + pattern: /^the postLikes store was read at most (<[A-Za-z0-9_]+>) entries$/, + run (m, example, world) { + const max = parseInt(resolveParam(m[1], example), 10) + const reads = world.postLikesIteratorCounter.entries || 0 + if (reads > max) { + throw new Error(`Read ${reads} postLikes entries, expected at most ${max}`) + } + } + }, + { + name: 'bounded followeeHeights entries read', + pattern: /^the followeeHeights store was read at most (<[A-Za-z0-9_]+>) entries$/, + run (m, example, world) { + const max = parseInt(resolveParam(m[1], example), 10) + const reads = world.followeeHeightsIteratorCounter.entries || 0 + if (reads > max) { + throw new Error(`Read ${reads} followeeHeights entries, expected at most ${max}`) + } + } + }, + { + name: 'likes store was not iterated', + pattern: /^the likes store was not iterated$/, + run (m, example, world) { + if (world.likesIteratorCounter.calls !== 0) { + throw new Error(`Expected likes store not to be iterated, got ${world.likesIteratorCounter.calls} call(s)`) + } + } + }, + { + name: 'follows store was not iterated', + pattern: /^the follows store was not iterated$/, + run (m, example, world) { + if (world.followsIteratorCounter.calls !== 0) { + throw new Error(`Expected follows store not to be iterated, got ${world.followsIteratorCounter.calls} call(s)`) + } + } } ] diff --git a/psf-memo-db/config/env/common.js b/psf-memo-db/config/env/common.js index e124b82..7afe7e3 100644 --- a/psf-memo-db/config/env/common.js +++ b/psf-memo-db/config/env/common.js @@ -12,5 +12,10 @@ export default { useIpfs: false, version: pkgInfo.version, backupQty: process.env.BACKUP_QTY ? parseInt(process.env.BACKUP_QTY) : 3, - exitOnMissingBackup: process.env.EXIT_ON_MISSING_BACKUP === 'true' + exitOnMissingBackup: process.env.EXIT_ON_MISSING_BACKUP === 'true', + // Number of blocks before the chain tip that still count as a notification. + // GET /posts/notifications/:addr uses cutoff = chainBlockHeight - window. + notificationBlockWindow: process.env.NOTIFICATION_BLOCK_WINDOW + ? parseInt(process.env.NOTIFICATION_BLOCK_WINDOW, 10) + : 25000 } diff --git a/psf-memo-db/src/adapters/index.js b/psf-memo-db/src/adapters/index.js index 5d686e2..88032ac 100644 --- a/psf-memo-db/src/adapters/index.js +++ b/psf-memo-db/src/adapters/index.js @@ -14,7 +14,8 @@ import SearchQuery from './search-query.js' import NotificationsQuery from './notifications-query.js' class Adapters { - constructor () { + constructor (localConfig = {}) { + this.notificationBlockWindow = localConfig.notificationBlockWindow this.levelDb = new LevelDb() this.openDatabases = this.openDatabases.bind(this) this.start = this.start.bind(this) @@ -68,12 +69,14 @@ class Adapters { }) this.notificationsQuery = new NotificationsQuery({ postsDb: level.postsDb, - postParentsDb: level.postParentsDb, + addrPostHeightsDb: level.addrPostHeightsDb, postChildrenDb: level.postChildrenDb, - likesDb: level.likesDb, postLikesDb: level.postLikesDb, - followsDb: level.followsDb, - muteQuery: this.muteQuery + likesDb: level.likesDb, + followeeHeightsDb: level.followeeHeightsDb, + statusDb: level.statusDb, + muteQuery: this.muteQuery, + notificationBlockWindow: this.notificationBlockWindow }) return true } diff --git a/psf-memo-db/src/adapters/level-db.js b/psf-memo-db/src/adapters/level-db.js index 41ad848..64bbcae 100644 --- a/psf-memo-db/src/adapters/level-db.js +++ b/psf-memo-db/src/adapters/level-db.js @@ -22,6 +22,7 @@ const DB_NAMES = [ 'profiles', 'profilePics', 'follows', + 'followeeHeights', 'mutes', 'rooms', 'topicSummaries', diff --git a/psf-memo-db/src/adapters/notifications-query.js b/psf-memo-db/src/adapters/notifications-query.js index 6eacd53..9ad837d 100644 --- a/psf-memo-db/src/adapters/notifications-query.js +++ b/psf-memo-db/src/adapters/notifications-query.js @@ -1,74 +1,150 @@ /* Adapter for aggregating the viewer's notifications. - Notifications are read-only: the DB collects replies to the viewer's posts, - likes on the viewer's posts, and new follows of the viewer, then returns them - sorted newest-first with limit/offset pagination. + Notifications are read-only: the DB collects likes on the viewer's posts, + replies to the viewer's posts, and new follows of the viewer, then returns + them sorted newest-first with limit/offset pagination. + + The work is bounded to the viewer's activity inside a configurable block + window (notificationBlockWindow, default 25000; cutoff = + status.chainBlockHeight - window): + + - the viewer's own posts come from the addrPostHeights index, range-limited + to the window; + - likes and replies come from prefix-scanning postLikes and postChildren + for those posts only; the global likes store is never iterated and + postChildren is only scanned per viewer post; + - follows come from the followeeHeights index, range-limited to the window; + the follows store is never iterated. + + A notification is drawn from the viewer's content inside the window, so an + interaction with a post older than the window is not returned even when the + interaction itself is recent. */ import BCHJS from '@psf/bch-js' import { getPostOrNull } from './lib/get-post-or-null.js' import { loadMutedAddrs } from './lib/muted-posts.js' +const HEIGHT_PAD = 12 +const DEFAULT_NOTIFICATION_BLOCK_WINDOW = 25000 + +function padHeight (blockHeight) { + return String(blockHeight ?? 0).padStart(HEIGHT_PAD, '0') +} + +// Read a record by key, returning null when it does not exist. +async function getRecordOrNull (db, key) { + try { + return await db.get(key) + } catch (err) { + if (err.notFound || err.code === 'LEVEL_NOT_FOUND' || err.response?.status === 404) return null + throw err + } +} + class NotificationsQuery { constructor (localConfig = {}) { const { postsDb, - postParentsDb, + addrPostHeightsDb, postChildrenDb, - likesDb, postLikesDb, - followsDb, + likesDb, + followeeHeightsDb, + statusDb = null, muteQuery, + notificationBlockWindow, bchjs = new BCHJS({ restURL: process.env.RESTURL || 'https://api.fullstack.cash/v5/' }) } = localConfig if (!postsDb) { throw new Error('postsDb required when instantiating NotificationsQuery adapter.') } - if (!postParentsDb) { - throw new Error('postParentsDb required when instantiating NotificationsQuery adapter.') + if (!addrPostHeightsDb) { + throw new Error('addrPostHeightsDb required when instantiating NotificationsQuery adapter.') } if (!postChildrenDb) { throw new Error('postChildrenDb required when instantiating NotificationsQuery adapter.') } - if (!likesDb) { - throw new Error('likesDb required when instantiating NotificationsQuery adapter.') - } if (!postLikesDb) { throw new Error('postLikesDb required when instantiating NotificationsQuery adapter.') } - if (!followsDb) { - throw new Error('followsDb required when instantiating NotificationsQuery adapter.') + if (!likesDb) { + throw new Error('likesDb required when instantiating NotificationsQuery adapter.') + } + if (!followeeHeightsDb) { + throw new Error('followeeHeightsDb required when instantiating NotificationsQuery adapter.') } this.postsDb = postsDb - this.postParentsDb = postParentsDb + this.addrPostHeightsDb = addrPostHeightsDb this.postChildrenDb = postChildrenDb - this.likesDb = likesDb this.postLikesDb = postLikesDb - this.followsDb = followsDb + this.likesDb = likesDb + this.followeeHeightsDb = followeeHeightsDb + this.statusDb = statusDb this.muteQuery = muteQuery || null + this.notificationBlockWindow = notificationBlockWindow ?? DEFAULT_NOTIFICATION_BLOCK_WINDOW this.bchjs = bchjs this.listNotifications = this.listNotifications.bind(this) + this._windowCutoff = this._windowCutoff.bind(this) + this._scanViewerPosts = this._scanViewerPosts.bind(this) this._collectFollowNotifications = this._collectFollowNotifications.bind(this) this._collectLikeNotifications = this._collectLikeNotifications.bind(this) this._collectReplyNotifications = this._collectReplyNotifications.bind(this) - this._replyNotificationChild = this._replyNotificationChild.bind(this) - this._followNotificationAddr = this._followNotificationAddr.bind(this) - this._likeNotificationPost = this._likeNotificationPost.bind(this) + this._followerFromKey = this._followerFromKey.bind(this) this._sortNotifications = this._sortNotifications.bind(this) } - // Collect active follows where this address is the followee. - async _collectFollowNotifications (addr, mutedAddrs) { - const myHash160 = this.bchjs.Address.toHash160(addr) - const notifications = [] + // The lowest block height that counts as inside the notification window, or + // null when no status is available (treat the whole history as in-window). + async _windowCutoff () { + if (!this.statusDb) return null + const status = await getRecordOrNull(this.statusDb, 'status') + if (!status) return null + const chainBlockHeight = status.chainBlockHeight ?? 0 + return chainBlockHeight - this.notificationBlockWindow + } - for await (const [key, record] of this.followsDb.iterator()) { - const followerAddr = this._followNotificationAddr(key, record, addr, myHash160, mutedAddrs) - if (followerAddr === null) continue + // Prefix-scan the viewer's posts within the window from addrPostHeights, + // newest-first is not required because likes/replies are scanned per txid. + async _scanViewerPosts (addr, cutoff) { + const prefix = `${addr}:` + const start = cutoff === null ? prefix : `${addr}:${padHeight(cutoff)}` + const end = `${addr}:\uffff` + const posts = [] + + for await (const [key, value] of this.addrPostHeightsDb.iterator({ gte: start, lte: end })) { + const txid = value?.txid || key.slice(key.lastIndexOf(':') + 1) + if (!txid) continue + posts.push({ txid, blockHeight: value?.blockHeight ?? 0 }) + } + + return posts + } + + // Collect the viewer's follows from the followeeHeights index within the + // window, keeping the newest entry per follower and ignoring unfollows. + async _collectFollowNotifications (addr, cutoff, mutedAddrs) { + const myHash160 = this.bchjs.Address.toHash160(addr) + const start = cutoff === null ? `${myHash160}:` : `${myHash160}:${padHeight(cutoff)}` + const end = `${myHash160}:\uffff` + const newestByFollower = new Map() + + for await (const [key, record] of this.followeeHeightsDb.iterator({ gte: start, lte: end })) { + const followerAddr = record?.followerAddr || this._followerFromKey(key) + if (!followerAddr) continue + // Ascending height order means the last entry for a follower wins. + newestByFollower.set(followerAddr, record) + } + + const notifications = [] + for (const [followerAddr, record] of newestByFollower) { + if (record.unfollow === true) continue + if (followerAddr === addr) continue + if (mutedAddrs.has(followerAddr)) continue notifications.push({ type: 'follow', @@ -82,89 +158,75 @@ class NotificationsQuery { return notifications } - // Return the follower address when a follow record is a valid follow - // notification for addr, else null. Filters out unfollow records, follows of - // other addresses, self-follows, and follows from muted addresses. - _followNotificationAddr (key, record, addr, myHash160, mutedAddrs) { - if (record.unfollow === true) return null - if (record.followeePkHash !== myHash160) return null - const followerAddr = record.followerAddr || key.slice(0, key.lastIndexOf(':')) - if (followerAddr === addr) return null - if (mutedAddrs.has(followerAddr)) return null - return followerAddr - } - - // Collect likes on posts authored by this address, excluding self-likes. - async _collectLikeNotifications (addr, mutedAddrs) { + // Prefix-scan postLikes for each viewer post and load the like record to get + // the actor and height. The global likes store is never iterated. + async _collectLikeNotifications (posts, addr, mutedAddrs) { const notifications = [] - for await (const [likeTxid, like] of this.likesDb.iterator()) { - const post = await this._likeNotificationPost(like, addr, mutedAddrs) - if (!post) continue + for (const post of posts) { + const prefix = `${post.txid}:` + for await (const [key, value] of this.postLikesDb.iterator({ gte: prefix, lte: `${post.txid}:\uffff` })) { + const likeTxid = value?.txid || key.slice(key.lastIndexOf(':') + 1) + if (!likeTxid) continue - notifications.push({ - type: 'like', - txid: likeTxid, - addr: like.addr, - postTxid: like.postTxid, - blockHeight: like.blockHeight ?? 0, - seen: like.seen ?? 0 - }) + const like = await getRecordOrNull(this.likesDb, likeTxid) + if (!like) continue + if (like.addr === addr) continue + if (mutedAddrs.has(like.addr)) continue + + notifications.push({ + type: 'like', + txid: likeTxid, + addr: like.addr, + postTxid: post.txid, + blockHeight: like.blockHeight ?? post.blockHeight ?? 0, + seen: like.seen ?? 0 + }) + } } return notifications } - // Return the liked post when a like is a valid like notification for addr, - // else null. Filters out missing likes, self-likes, likes from muted - // addresses, and likes on posts not authored by addr. - async _likeNotificationPost (like, addr, mutedAddrs) { - if (!like || like.addr === addr || mutedAddrs.has(like.addr)) return null - const post = await getPostOrNull(this.postsDb, like.postTxid) - if (!post || post.addr !== addr) return null - return post - } - - // Collect replies to posts authored by this address, excluding own replies. - async _collectReplyNotifications (addr, mutedAddrs) { + // Prefix-scan postChildren for each viewer post and load the child post for + // the actor and text. postChildren is only read per viewer post, never + // full-scanned. + async _collectReplyNotifications (posts, addr, mutedAddrs) { const notifications = [] - for await (const [, child] of this.postChildrenDb.iterator()) { - const parentTxid = child?.parentTxid - const childTxid = child?.childTxid - if (!parentTxid || !childTxid) continue + for (const post of posts) { + const prefix = `${post.txid}:` + for await (const [key, child] of this.postChildrenDb.iterator({ gte: prefix, lte: `${post.txid}:\uffff` })) { + const childTxid = child?.childTxid || key.slice(key.lastIndexOf(':') + 1) + if (!childTxid) continue + if (child?.parentTxid && child.parentTxid !== post.txid) continue - const childPost = await this._replyNotificationChild(child, addr) - if (!childPost) continue - if (mutedAddrs.has(childPost.addr)) continue + const childPost = await getPostOrNull(this.postsDb, childTxid) + if (!childPost) continue + if (childPost.addr === addr) continue + if (mutedAddrs.has(childPost.addr)) continue - notifications.push({ - type: 'reply', - txid: childTxid, - addr: childPost.addr, - postTxid: parentTxid, - text: childPost.text, - blockHeight: child.blockHeight ?? childPost.blockHeight ?? 0, - seen: childPost.seen ?? 0 - }) + notifications.push({ + type: 'reply', + txid: childTxid, + addr: childPost.addr, + postTxid: post.txid, + text: childPost.text, + blockHeight: child?.blockHeight ?? childPost.blockHeight ?? post.blockHeight ?? 0, + seen: childPost.seen ?? 0 + }) + } } return notifications } - // Return the child post when a child record is a valid reply notification - // for addr: it links a parent authored by addr to a child authored by someone - // else. Returns null when the parent is missing or not authored by addr, or - // the child is missing or authored by addr. Callers must have already - // verified the record carries parentTxid and childTxid. - async _replyNotificationChild (child, addr) { - const parent = await getPostOrNull(this.postsDb, child.parentTxid) - if (!parent || parent.addr !== addr) return null - - const childPost = await getPostOrNull(this.postsDb, child.childTxid) - if (!childPost || childPost.addr === addr) return null - - return childPost + // Recover the follower address from a followeeHeights key of the form + // `${followeePkHash}:${paddedHeight}:${followerAddr}`. Cash addresses contain + // a colon, so the follower is everything after the second colon. + _followerFromKey (key) { + const parts = String(key).split(':') + return parts.slice(2).join(':') } _sortNotifications (notifications) { @@ -178,9 +240,11 @@ class NotificationsQuery { async listNotifications (addr, { limit, offset } = {}) { const mutedAddrs = await loadMutedAddrs(this.muteQuery, addr) - const follows = await this._collectFollowNotifications(addr, mutedAddrs) - const likes = await this._collectLikeNotifications(addr, mutedAddrs) - const replies = await this._collectReplyNotifications(addr, mutedAddrs) + const cutoff = await this._windowCutoff() + const viewerPosts = await this._scanViewerPosts(addr, cutoff) + const follows = await this._collectFollowNotifications(addr, cutoff, mutedAddrs) + const likes = await this._collectLikeNotifications(viewerPosts, addr, mutedAddrs) + const replies = await this._collectReplyNotifications(viewerPosts, addr, mutedAddrs) const all = this._sortNotifications(follows.concat(likes).concat(replies)) const total = all.length @@ -191,7 +255,3 @@ class NotificationsQuery { } export default NotificationsQuery - -// mutate4javascript-manifest-begin -// {"version":1,"tested_at":"2026-09-04T20:09:09.555Z","module_hash":"5ba9ad53d1edc8797d7922446cb19e75a4158f8822eea588d9e94bb1fb2fa971","functions":[{"id":"func/NotificationsQuery.constructor","name":"NotificationsQuery.constructor","line":14,"end_line":62,"hash":"87579ac7d3e764ab30e21e73d9780aaa4c6d562c610b5e13b4ede182b694845e"},{"id":"func/NotificationsQuery._collectFollowNotifications","name":"NotificationsQuery._collectFollowNotifications","line":65,"end_line":83,"hash":"a4b9f141ee0de921232efff0aaf73503c0c4313fedac4c37b79894cd74f29877"},{"id":"func/NotificationsQuery._followNotificationAddr","name":"NotificationsQuery._followNotificationAddr","line":88,"end_line":95,"hash":"814bf4e51edac65108806f012ed3f0c12dc5e9dbe09dfb83af95e8292aab1669"},{"id":"func/NotificationsQuery._collectLikeNotifications","name":"NotificationsQuery._collectLikeNotifications","line":98,"end_line":116,"hash":"e3ab4911550211f3bd9f8f85139d69392433bb95d824b0fd8ac628eb9fe755cf"},{"id":"func/NotificationsQuery._likeNotificationPost","name":"NotificationsQuery._likeNotificationPost","line":121,"end_line":126,"hash":"c72690ed0975ca35ec00f1fb22a602c7301a327e8d36b17032e8e9a36a89adcb"},{"id":"func/NotificationsQuery._collectReplyNotifications","name":"NotificationsQuery._collectReplyNotifications","line":129,"end_line":153,"hash":"270ba026b18025bcef2c92776bbf8143f5f56da0c565285708d5196cf777ba83"},{"id":"func/NotificationsQuery._replyNotificationChild","name":"NotificationsQuery._replyNotificationChild","line":160,"end_line":168,"hash":"7ed6bf9d9f5147ba189c99304b76a1a0d77bc9c7909e194f9b74545c84227ed0"},{"id":"func/NotificationsQuery._sortNotifications","name":"NotificationsQuery._sortNotifications","line":170,"end_line":175,"hash":"61b4ae0b6d450e172db3e59240a3c2ccd5fdf0e6ea1e31023df7bb3d52fdbd5c"},{"id":"func/NotificationsQuery.listNotifications","name":"NotificationsQuery.listNotifications","line":178,"end_line":190,"hash":"3507c459f6dc22482aa693a92fe13da16af05d65dd4e37366a9b8db98654ec14"}]} -// mutate4javascript-manifest-end diff --git a/psf-memo-db/src/controllers/index.js b/psf-memo-db/src/controllers/index.js index 0c7a131..b2247f7 100644 --- a/psf-memo-db/src/controllers/index.js +++ b/psf-memo-db/src/controllers/index.js @@ -5,10 +5,11 @@ import Adapters from '../adapters/index.js' import UseCases from '../use-cases/index.js' import RESTControllers from './rest-api/index.js' +import config from '../../config/index.js' class Controllers { constructor () { - this.adapters = new Adapters() + this.adapters = new Adapters({ notificationBlockWindow: config.notificationBlockWindow }) this.useCases = new UseCases({ adapters: this.adapters }) this.initAdapters = this.initAdapters.bind(this) this.initUseCases = this.initUseCases.bind(this) diff --git a/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js b/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js index 7acfbea..64fe85f 100644 --- a/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js +++ b/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js @@ -46,6 +46,7 @@ export const ENTITY_CONFIG = [ { route: 'profile', dbProp: 'profilesDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'profileData' }, { route: 'profilepic', dbProp: 'profilePicsDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'profilePicData' }, { route: 'follow', dbProp: 'followsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'followData' }, + { route: 'followeeheight', dbProp: 'followeeHeightsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'followeeHeightData' }, { route: 'room', dbProp: 'roomsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'roomData' }, { route: 'topicsummary', dbProp: 'topicSummariesDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'topicSummaryData' }, { route: 'topicrecency', dbProp: 'topicRecencyDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'topicRecencyData' }, diff --git a/psf-memo-db/src/lib/backfill-followee-index.js b/psf-memo-db/src/lib/backfill-followee-index.js new file mode 100644 index 0000000..94ffbcb --- /dev/null +++ b/psf-memo-db/src/lib/backfill-followee-index.js @@ -0,0 +1,60 @@ +/* + Library to build the followeeHeights notification read index from an + existing follows store. + + followeeHeights is keyed `${followeePkHash}:${padded blockHeight}:${followerAddr}` + with value `{ followerAddr, followeePkHash, unfollow, txid, seen, blockHeight }`. + The read side range-scans one followee's entries to find its newest follow per + follower, so the index must carry every follow and unfollow event at its block + height. + + The backfill writes one entry per follows record at that record's latest block + height. It rebuilds from the follows store only, is idempotent, and leaves the + follows store unchanged. + + The LevelDB handles are injected so the logic stays testable and free of + file-system concerns; the CLI wrapper in util/follow opens the real stores. +*/ + +const HEIGHT_PAD = 12 + +export function followeeHeightKey (followeePkHash, blockHeight, followerAddr) { + const padded = String(blockHeight ?? 0).padStart(HEIGHT_PAD, '0') + return `${followeePkHash}:${padded}:${followerAddr}` +} + +// Recover the follower and followee from a follows key of the form +// `${followerAddr}:${followeePkHash}`. Cash addresses contain a colon, so the +// followee hash is the segment after the final colon. +function partsFromKey (key) { + const str = String(key) + const idx = str.lastIndexOf(':') + return { + followerAddr: str.slice(0, idx), + followeePkHash: str.slice(idx + 1) + } +} + +export async function backfillFolloweeIndex ({ followsDb, followeeHeightsDb }) { + let follows = 0 + + for await (const [key, record] of followsDb.iterator()) { + const fallback = partsFromKey(key) + const followerAddr = record?.followerAddr || fallback.followerAddr + const followeePkHash = record?.followeePkHash || fallback.followeePkHash + if (!followerAddr || !followeePkHash) continue + + const blockHeight = record?.blockHeight ?? 0 + await followeeHeightsDb.put(followeeHeightKey(followeePkHash, blockHeight, followerAddr), { + followerAddr, + followeePkHash, + unfollow: record?.unfollow ?? false, + txid: record?.txid, + seen: record?.seen, + blockHeight + }) + follows++ + } + + return { follows } +} diff --git a/psf-memo-db/test/property/notifications-query.property.test.js b/psf-memo-db/test/property/notifications-query.property.test.js index 710a539..76fe4f4 100644 --- a/psf-memo-db/test/property/notifications-query.property.test.js +++ b/psf-memo-db/test/property/notifications-query.property.test.js @@ -3,180 +3,196 @@ The unit suite probes listNotifications at a handful of fixed fixtures. These properties pin down invariants that should hold over broad random - notification records: + notification records when every interaction is inside the window: + - membership: every returned notification originated from an active follow, + a like on one of the viewer's posts, or a reply to one of the viewer's + posts; - newest-first ordering: notifications are returned sorted by blockHeight - descending. - - tie-break ordering: when blockHeights tie, notifications are ordered by - seen descending. + descending, with seen descending as the tie-break; - pagination conservation: applying offset/limit returns exactly the full matching set sliced to the page, and reports an exact total. - - membership: every returned notification originated from an active follow, - a like on one of my posts, or a reply to one of my posts. - The reference implementation below mirrors the adapter's aggregation and - sort logic so the two can be cross-checked. + The reference implementation below mirrors the adapter's window-free + aggregation and sort logic so the two can be cross-checked. */ import test from 'node:test' import { seededRandom, forAll, intGen } from './harness.js' import NotificationsQuery from '../../src/adapters/notifications-query.js' -const rng = seededRandom(20260903) +const rng = seededRandom(20260918) const VIEWER = 'bitcoincash:viewer' const MY_HASH = 'hash-viewer' const OTHERS = ['bitcoincash:a', 'bitcoincash:b', 'bitcoincash:c'] const VIEWER_POSTS = ['vp1', 'vp2', 'vp3', 'vp4'] -function makeIterator (items) { - return (async function * () { - for (const item of items) yield item - }()) -} +const PAD = 12 +const pad = (h) => String(h).padStart(PAD, '0') -function buildQuery ({ posts, follows, likes, children }) { - const postsDb = { - async get (txid) { - const post = posts.get(txid) - if (!post) { +function makeDb (entries = []) { + const map = new Map(entries) + return { + map, + async get (key) { + if (!map.has(key)) { const err = new Error('not found') err.notFound = true throw err } - return post - } - } - const bchjs = { - Address: { - toHash160: (addr) => (addr === VIEWER ? MY_HASH : 'hash-' + addr) + return map.get(key) + }, + iterator (opts = {}) { + let keys = [...map.keys()].sort() + if (opts.gte !== undefined) keys = keys.filter((key) => key >= opts.gte) + if (opts.lte !== undefined) keys = keys.filter((key) => key <= opts.lte) + return (async function * () { + for (const key of keys) yield [key, map.get(key)] + }()) } } +} + +function buildQuery () { + // statusDb null => no window; every generated interaction is in scope. return new NotificationsQuery({ - postsDb, - postParentsDb: {}, - postChildrenDb: { iterator: () => makeIterator(children) }, - likesDb: { iterator: () => makeIterator(likes) }, - postLikesDb: {}, - followsDb: { iterator: () => makeIterator(follows) }, - bchjs + postsDb: makeDb(), + addrPostHeightsDb: makeDb(), + postChildrenDb: makeDb(), + postLikesDb: makeDb(), + likesDb: makeDb(), + followeeHeightsDb: makeDb(), + statusDb: null, + bchjs: { Address: { toHash160: (addr) => (addr === VIEWER ? MY_HASH : `hash-${addr}`) } } }) } +function randomNotification () { + return { + blockHeight: intGen(rng, 0, 5000)(), + seen: intGen(rng, 0, 1000)() + } +} + function fixtureGen () { return () => { - const posts = new Map(VIEWER_POSTS.map((txid) => [txid, { addr: VIEWER }])) - const follows = [] - const likeKeys = [] - const danglingLikes = [] - const children = [] - const childPosts = [] + const postsDbEntries = [] + const addrPostHeightEntries = [] + const postLikesEntries = [] + const likesEntries = [] + const postChildrenEntries = [] + const followeeHeightEntries = [] - // Active follow records of me by other addresses. - const nFollows = intGen(rng, 0, 6)() + for (const txid of VIEWER_POSTS) { + postsDbEntries.push([txid, { addr: VIEWER }]) + addrPostHeightEntries.push([`${VIEWER}:${pad(0)}:${txid}`, { txid, addr: VIEWER, blockHeight: 0 }]) + } + + // Follows: random events, including repeated followers so the newest-wins + // rule is exercised. Recorded in the expected order. + const events = [] + const nFollows = intGen(rng, 0, 8)() for (let i = 0; i < nFollows; i++) { const follower = OTHERS[Math.floor(rng() * OTHERS.length)] - const toMe = rng() < 0.7 - follows.push([ - `${follower}:${toMe ? MY_HASH : 'hash-other'}`, - { - followerAddr: follower, - followeePkHash: toMe ? MY_HASH : 'hash-other', - unfollow: rng() < 0.2, - txid: 'follow' + i, - // a fraction omit blockHeight/seen to exercise the ?? 0 defaults - blockHeight: rng() < 0.2 ? undefined : intGen(rng, 0, 5000)(), - seen: rng() < 0.2 ? undefined : intGen(rng, 0, 1000)() - } + const n = randomNotification() + const unfollow = rng() < 0.25 + events.push({ follower, ...n, unfollow }) + followeeHeightEntries.push([ + `${MY_HASH}:${pad(n.blockHeight)}:${follower}`, + { followerAddr: follower, followeePkHash: MY_HASH, unfollow, txid: `follow${i}`, seen: n.seen, blockHeight: n.blockHeight } ]) } - // Likes on my posts by other addresses. - const nLikes = intGen(rng, 0, 6)() + // Likes on the viewer's posts by other addresses. + const likeRecords = [] + const nLikes = intGen(rng, 0, 8)() for (let i = 0; i < nLikes; i++) { const postTxid = VIEWER_POSTS[Math.floor(rng() * VIEWER_POSTS.length)] - likeKeys.push(['like' + i, { - addr: OTHERS[Math.floor(rng() * OTHERS.length)], - postTxid, - blockHeight: intGen(rng, 0, 5000)(), - seen: intGen(rng, 0, 1000)() - }]) + const actor = OTHERS[Math.floor(rng() * OTHERS.length)] + const n = randomNotification() + const likeTxid = `like${i}` + likeRecords.push({ likeTxid, addr: actor, ...n }) + postLikesEntries.push([`${postTxid}:${likeTxid}`, { postTxid, txid: likeTxid }]) + likesEntries.push([likeTxid, { addr: actor, postTxid, blockHeight: n.blockHeight, seen: n.seen }]) } - // Likes whose target post is missing, to exercise the exclusion path. - const nDangling = intGen(rng, 0, 3)() - for (let i = 0; i < nDangling; i++) { - danglingLikes.push(['dl' + i, { addr: OTHERS[0], postTxid: 'missing-post', blockHeight: 1, seen: 1 }]) - } - const allLikes = likeKeys.concat(danglingLikes) - - // Replies to my posts by other addresses. - const nReplies = intGen(rng, 0, 6)() + // Replies to the viewer's posts by other addresses. + const replyRecords = [] + const nReplies = intGen(rng, 0, 8)() for (let i = 0; i < nReplies; i++) { const parentTxid = VIEWER_POSTS[Math.floor(rng() * VIEWER_POSTS.length)] - const childTxid = 'child' + i - const childPost = { - addr: OTHERS[Math.floor(rng() * OTHERS.length)], - blockHeight: intGen(rng, 0, 5000)(), - seen: intGen(rng, 0, 1000)() - } - childPosts.push(childTxid) - posts.set(childTxid, childPost) - children.push([`${parentTxid}:${childTxid}`, { parentTxid, childTxid }]) + const childTxid = `child${i}` + const actor = OTHERS[Math.floor(rng() * OTHERS.length)] + const n = randomNotification() + replyRecords.push({ childTxid, addr: actor, ...n }) + postChildrenEntries.push([`${parentTxid}:${childTxid}`, { parentTxid, childTxid, blockHeight: n.blockHeight }]) + postsDbEntries.push([childTxid, { addr: actor, text: 'reply', blockHeight: n.blockHeight, seen: n.seen }]) } return { - query: buildQuery({ posts, follows, likes: allLikes, children }), - follows, - likeKeys, - childPosts, - posts, + events, + likeRecords, + replyRecords, + postsDbEntries, + addrPostHeightEntries, + postLikesEntries, + likesEntries, + postChildrenEntries, + followeeHeightEntries, limit: intGen(rng, 1, 8)(), offset: intGen(rng, 0, 10)() } } } -function buildExpected ({ follows, likeKeys, childPosts, posts, limit, offset }) { +// Build the adapter with the generated stores populated. +function populate (input) { + const query = buildQuery() + input.postsDbEntries.forEach(([key, value]) => query.postsDb.map.set(key, value)) + input.addrPostHeightEntries.forEach(([key, value]) => query.addrPostHeightsDb.map.set(key, value)) + input.postLikesEntries.forEach(([key, value]) => query.postLikesDb.map.set(key, value)) + input.likesEntries.forEach(([key, value]) => query.likesDb.map.set(key, value)) + input.postChildrenEntries.forEach(([key, value]) => query.postChildrenDb.map.set(key, value)) + input.followeeHeightEntries.forEach(([key, value]) => query.followeeHeightsDb.map.set(key, value)) + return query +} + +// Mirror the adapter's window-free aggregation without using the adapter. +function buildExpected ({ events, likeRecords, replyRecords, limit, offset }) { const out = [] - for (const [, record] of follows) { - if (record.unfollow === true) continue - if (record.followeePkHash !== MY_HASH) continue - if (record.followerAddr === VIEWER) continue - out.push({ blockHeight: record.blockHeight ?? 0, seen: record.seen ?? 0 }) + // Newest follow event per follower wins. + const newestByFollower = new Map() + for (const event of [...events].sort((a, b) => a.blockHeight - b.blockHeight)) { + newestByFollower.set(event.follower, event) + } + for (const event of newestByFollower.values()) { + if (event.unfollow) continue + out.push({ blockHeight: event.blockHeight, seen: event.seen }) } - for (const [, like] of likeKeys) { - if (!like || like.addr === VIEWER) continue - const post = posts.get(like.postTxid) - if (!post || post.addr !== VIEWER) continue - out.push({ blockHeight: like.blockHeight ?? 0, seen: like.seen ?? 0 }) - } - - for (const childTxid of childPosts) { - // Parent and child both exist in the fixture; child is always authored by - // a non-viewer, parent by the viewer, so every child here qualifies. - out.push({ blockHeight: posts.get(childTxid).blockHeight ?? 0, seen: posts.get(childTxid).seen ?? 0 }) - } + for (const like of likeRecords) out.push({ blockHeight: like.blockHeight, seen: like.seen }) + for (const reply of replyRecords) out.push({ blockHeight: reply.blockHeight, seen: reply.seen }) out.sort((a, b) => { if (b.blockHeight !== a.blockHeight) return b.blockHeight - a.blockHeight return (b.seen ?? 0) - (a.seen ?? 0) }) - return { - total: out.length, - page: out.slice(offset, offset + limit) - } + return { total: out.length, page: out.slice(offset, offset + limit) } } test('notifications are sorted newest-first with seen tie-break and exact pagination', async () => { await forAll( fixtureGen(), - async ({ query, follows, likeKeys, childPosts, posts, limit, offset }) => { - const { notifications, total } = await query.listNotifications(VIEWER, { limit, offset }) - const expected = buildExpected({ follows, likeKeys, childPosts, posts, limit, offset }) + async (input) => { + const query = populate(input) + const { notifications, total } = await query.listNotifications(VIEWER, { + limit: input.limit, + offset: input.offset + }) + const expected = buildExpected(input) if (total !== expected.total) return false if (notifications.length !== expected.page.length) return false @@ -193,8 +209,12 @@ test('notifications are sorted newest-first with seen tie-break and exact pagina test('the returned page is globally ordered by blockHeight then seen descending', async () => { await forAll( fixtureGen(), - async ({ query, limit, offset }) => { - const { notifications } = await query.listNotifications(VIEWER, { limit, offset }) + async (input) => { + const query = populate(input) + const { notifications } = await query.listNotifications(VIEWER, { + limit: input.limit, + offset: input.offset + }) for (let i = 1; i < notifications.length; i++) { const prev = notifications[i - 1] const cur = notifications[i] diff --git a/psf-memo-db/test/unit/adapters/notifications-query.unit.js b/psf-memo-db/test/unit/adapters/notifications-query.unit.js index 4ed71d4..23992f6 100644 --- a/psf-memo-db/test/unit/adapters/notifications-query.unit.js +++ b/psf-memo-db/test/unit/adapters/notifications-query.unit.js @@ -1,526 +1,350 @@ import { assert } from 'chai' import sinon from 'sinon' import NotificationsQuery from '../../../src/adapters/notifications-query.js' -import { getPostOrNull } from '../../../src/adapters/lib/get-post-or-null.js' describe('#NotificationsQuery', () => { let sandbox - let postsDb - let postChildrenDb - let likesDb - let followsDb - let bchjs - let uut - const MY_ADDR = 'bitcoincash:qqlrzp23w08434twtmvr4fxw672whkjy0py26r63g3d' - const THEIR_ADDR = 'bitcoincash:qr95sy3j9xwd2ap32xkykttr4cvcu7as4y0qverfuy' - const MY_HASH160 = 'myhash160' - const THEIR_HASH160 = 'theirhash160' + const VIEWER = 'bitcoincash:viewer' + const VIEWER_HASH = 'hash-viewer' + const FOLLOWER = 'bitcoincash:follower' + const OTHER = 'bitcoincash:other' + const LIKER = 'bitcoincash:liker' + const REPLIER = 'bitcoincash:replier' - function makeIterator (items) { - return (async function * () { - for (const item of items) yield item - }()) + const HEIGHT_PAD = 12 + const pad = (h) => String(h).padStart(HEIGHT_PAD, '0') + + // A minimal LevelDB double that honors gte/lte range options and records + // iteration calls so tests can assert that a store was never scanned. + function makeDb (entries = []) { + const map = new Map(entries) + const iteratorCalls = [] + return { + map, + iteratorCalls, + async get (key) { + if (!map.has(key)) { + const err = new Error('not found') + err.notFound = true + throw err + } + return map.get(key) + }, + iterator (opts = {}) { + iteratorCalls.push(opts) + let keys = [...map.keys()].sort() + if (opts.gte !== undefined) keys = keys.filter((key) => key >= opts.gte) + if (opts.lte !== undefined) keys = keys.filter((key) => key <= opts.lte) + return (async function * () { + for (const key of keys) yield [key, map.get(key)] + }()) + } + } + } + + function addrPostHeight (addr, height, txid) { + return [`${addr}:${pad(height)}:${txid}`, { txid, addr, blockHeight: height }] + } + + function postLike (postTxid, likeTxid) { + return [`${postTxid}:${likeTxid}`, { postTxid, txid: likeTxid }] + } + + function child (parentTxid, childTxid, blockHeight) { + return [`${parentTxid}:${childTxid}`, { parentTxid, childTxid, blockHeight }] + } + + function followeeHeight (followeeHash, height, follower, unfollow = false) { + return [ + `${followeeHash}:${pad(height)}:${follower}`, + { followerAddr: follower, followeePkHash: followeeHash, unfollow, txid: `follow-${height}`, seen: height, blockHeight: height } + ] + } + + function buildQuery (overrides = {}) { + const postsDb = overrides.postsDb || makeDb() + const addrPostHeightsDb = overrides.addrPostHeightsDb || makeDb() + const postChildrenDb = overrides.postChildrenDb || makeDb() + const postLikesDb = overrides.postLikesDb || makeDb() + const likesDb = overrides.likesDb || makeDb() + const followeeHeightsDb = overrides.followeeHeightsDb || makeDb() + const statusDb = 'statusDb' in overrides + ? overrides.statusDb + : makeDb([['status', { chainBlockHeight: 700000 }]]) + + const bchjs = { + Address: { + toHash160: (addr) => (addr === VIEWER ? VIEWER_HASH : `hash-${addr}`) + } + } + + return new NotificationsQuery({ + postsDb, + addrPostHeightsDb, + postChildrenDb, + postLikesDb, + likesDb, + followeeHeightsDb, + statusDb, + notificationBlockWindow: overrides.notificationBlockWindow, + muteQuery: overrides.muteQuery, + bchjs + }) } beforeEach(() => { sandbox = sinon.createSandbox() - - postsDb = { get: sandbox.stub() } - postChildrenDb = { iterator: sandbox.stub() } - likesDb = { iterator: sandbox.stub() } - followsDb = { iterator: sandbox.stub() } - - bchjs = { - Address: { - toHash160: sandbox.stub() - } - } - - bchjs.Address.toHash160.withArgs(MY_ADDR).returns(MY_HASH160) - bchjs.Address.toHash160.withArgs(THEIR_ADDR).returns(THEIR_HASH160) - - uut = new NotificationsQuery({ - postsDb, - postParentsDb: {}, - postChildrenDb, - likesDb, - postLikesDb: {}, - followsDb, - bchjs - }) }) afterEach(() => sandbox.restore()) - it('should throw when postsDb is missing', () => { - try { - // eslint-disable-next-line no-new - new NotificationsQuery({ postChildrenDb, likesDb, followsDb, bchjs }) - assert.fail('Expected error') - } catch (err) { - assert.include(err.message, 'postsDb required') + it('should throw when required stores are missing', () => { + const cases = [ + ['postsDb', /postsDb required/], + ['addrPostHeightsDb', /addrPostHeightsDb required/], + ['postChildrenDb', /postChildrenDb required/], + ['postLikesDb', /postLikesDb required/], + ['likesDb', /likesDb required/], + ['followeeHeightsDb', /followeeHeightsDb required/] + ] + + for (const [omit, expected] of cases) { + const config = { + postsDb: makeDb(), + addrPostHeightsDb: makeDb(), + postChildrenDb: makeDb(), + postLikesDb: makeDb(), + likesDb: makeDb(), + followeeHeightsDb: makeDb() + } + delete config[omit] + + try { + // eslint-disable-next-line no-new + new NotificationsQuery(config) + assert.fail(`Expected error for missing ${omit}`) + } catch (err) { + assert.match(err.message, expected) + } } }) - it('should throw when postChildrenDb is missing', () => { - try { - // eslint-disable-next-line no-new - new NotificationsQuery({ postsDb, postParentsDb: {}, likesDb, postLikesDb: {}, followsDb, bchjs }) - assert.fail('Expected error') - } catch (err) { - assert.include(err.message, 'postChildrenDb required') - } - }) + it('should include an in-window follow, reply, and like sorted newest-first', async () => { + const postsDb = makeDb([ + ['post-recent', { addr: VIEWER, text: 'hi' }], + ['reply-recent', { addr: REPLIER, text: 'reply', blockHeight: 690150, seen: 1 }] + ]) + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 690000, 'post-recent')]) + const postLikesDb = makeDb([postLike('post-recent', 'like-recent')]) + const likesDb = makeDb([['like-recent', { addr: LIKER, postTxid: 'post-recent', blockHeight: 690100, seen: 2 }]]) + const postChildrenDb = makeDb([child('post-recent', 'reply-recent', 690150)]) + const followeeHeightsDb = makeDb([followeeHeight(VIEWER_HASH, 690400, FOLLOWER)]) - it('should throw when likesDb is missing', () => { - try { - // eslint-disable-next-line no-new - new NotificationsQuery({ postsDb, postParentsDb: {}, postChildrenDb, postLikesDb: {}, followsDb, bchjs }) - assert.fail('Expected error') - } catch (err) { - assert.include(err.message, 'likesDb required') - } - }) - - it('should throw when followsDb is missing', () => { - try { - // eslint-disable-next-line no-new - new NotificationsQuery({ postsDb, postParentsDb: {}, postChildrenDb, likesDb, postLikesDb: {}, bchjs }) - assert.fail('Expected error') - } catch (err) { - assert.include(err.message, 'followsDb required') - } - }) - - it('should throw when postParentsDb is missing', () => { - try { - // eslint-disable-next-line no-new - new NotificationsQuery({ postsDb, postChildrenDb, likesDb, postLikesDb: {}, followsDb, bchjs }) - assert.fail('Expected error') - } catch (err) { - assert.include(err.message, 'postParentsDb required') - } - }) - - it('should throw when postLikesDb is missing', () => { - try { - // eslint-disable-next-line no-new - new NotificationsQuery({ postsDb, postParentsDb: {}, postChildrenDb, likesDb, followsDb, bchjs }) - assert.fail('Expected error') - } catch (err) { - assert.include(err.message, 'postLikesDb required') - } - }) - - it('should exclude notifications from muted addresses when a mute query is provided', async () => { - const myPostTxid = 'a'.repeat(64) - const replyTxid = 'b'.repeat(64) - const likeTxid = 'c'.repeat(64) - - postChildrenDb.iterator.returns(makeIterator([ - [`${myPostTxid}:${replyTxid}`, { parentTxid: myPostTxid, childTxid: replyTxid, blockHeight: 100 }] - ])) - likesDb.iterator.returns(makeIterator([ - [likeTxid, { addr: THEIR_ADDR, postTxid: myPostTxid, blockHeight: 300 }] - ])) - followsDb.iterator.returns(makeIterator([])) - - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - postsDb.get.withArgs(replyTxid).resolves({ addr: THEIR_ADDR, text: 'reply' }) - - const muteQuery = { - listMuted: sandbox.stub().resolves([THEIR_ADDR]) - } - uut = new NotificationsQuery({ - postsDb, - postParentsDb: {}, - postChildrenDb, - likesDb, - postLikesDb: {}, - followsDb, - muteQuery, - bchjs - }) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 0) - assert.isTrue(muteQuery.listMuted.calledOnceWith(MY_ADDR)) - }) - - it('should return null from getPostOrNull when the post is not found', async () => { - const missingTxid = 'c'.repeat(64) - const notFound = new Error('not found') - notFound.notFound = true - postsDb.get.withArgs(missingTxid).rejects(notFound) - - const result = await getPostOrNull(postsDb, missingTxid) - assert.equal(result, null) - }) - - it('should rethrow non-notFound errors from getPostOrNull', async () => { - const txid = 'd'.repeat(64) - const boom = new Error('boom') - postsDb.get.withArgs(txid).rejects(boom) - - try { - await getPostOrNull(postsDb, txid) - assert.fail('Expected error') - } catch (err) { - assert.equal(err, boom) - } - }) - - it('should include a reply to my post', async () => { - const myPostTxid = 'a'.repeat(64) - const replyTxid = 'b'.repeat(64) - - postChildrenDb.iterator.returns(makeIterator([ - [`${myPostTxid}:${replyTxid}`, { parentTxid: myPostTxid, childTxid: replyTxid, blockHeight: 200 }] - ])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - postsDb.get.withArgs(replyTxid).resolves({ addr: THEIR_ADDR, text: 'nice post', blockHeight: 200 }) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 1) - assert.equal(result.notifications.length, 1) - const n = result.notifications[0] - assert.equal(n.type, 'reply') - assert.equal(n.txid, replyTxid) - assert.equal(n.addr, THEIR_ADDR) - assert.equal(n.postTxid, myPostTxid) - assert.equal(n.text, 'nice post') - }) - - it('should include a like on my post', async () => { - const myPostTxid = 'a'.repeat(64) - const likeTxid = 'b'.repeat(64) - - postChildrenDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([ - [likeTxid, { addr: THEIR_ADDR, postTxid: myPostTxid, blockHeight: 300 }] - ])) - - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 1) - const n = result.notifications[0] - assert.equal(n.type, 'like') - assert.equal(n.txid, likeTxid) - assert.equal(n.addr, THEIR_ADDR) - assert.equal(n.postTxid, myPostTxid) - }) - - it('should include a follow of me', async () => { - postChildrenDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([ - [`${THEIR_ADDR}:${MY_HASH160}`, { followerAddr: THEIR_ADDR, followeePkHash: MY_HASH160, unfollow: false, txid: 'c'.repeat(64), blockHeight: 150 }] - ])) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 1) - const n = result.notifications[0] - assert.equal(n.type, 'follow') - assert.equal(n.txid, 'c'.repeat(64)) - assert.equal(n.addr, THEIR_ADDR) - }) - - it('should fall back to the follow key first component when followerAddr is missing', async () => { - postChildrenDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([])) - // A follow record can omit followerAddr; the address must be derived from - // the first component of the `addr:` key. - followsDb.iterator.returns(makeIterator([ - [`${THEIR_ADDR}:${MY_HASH160}`, { followeePkHash: MY_HASH160, unfollow: false, txid: 'c'.repeat(64), blockHeight: 150 }] - ])) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 1) - const n = result.notifications[0] - assert.equal(n.type, 'follow') - assert.equal(n.addr, THEIR_ADDR) - }) - - it('should exclude my own replies', async () => { - const myPostTxid = 'a'.repeat(64) - const replyTxid = 'b'.repeat(64) - - postChildrenDb.iterator.returns(makeIterator([ - [`${myPostTxid}:${replyTxid}`, { parentTxid: myPostTxid, childTxid: replyTxid, blockHeight: 100 }] - ])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - postsDb.get.withArgs(replyTxid).resolves({ addr: MY_ADDR, text: 'my own reply' }) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 0) - }) - - it('should exclude replies to posts by other people', async () => { - const theirPostTxid = 'a'.repeat(64) - const replyTxid = 'b'.repeat(64) - - postChildrenDb.iterator.returns(makeIterator([ - [`${theirPostTxid}:${replyTxid}`, { parentTxid: theirPostTxid, childTxid: replyTxid, blockHeight: 100 }] - ])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - - postsDb.get.withArgs(theirPostTxid).resolves({ addr: THEIR_ADDR, text: 'alice post' }) - postsDb.get.withArgs(replyTxid).resolves({ addr: 'bitcoincash:other', text: 'reply' }) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 0) - }) - - it('should exclude follows of other people', async () => { - postChildrenDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([ - [`${THEIR_ADDR}:${'otherhash'}`, { followerAddr: THEIR_ADDR, followeePkHash: 'otherhash', unfollow: false, txid: 'c'.repeat(64), blockHeight: 150 }] - ])) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 0) - }) - - it('should exclude unfollows', async () => { - postChildrenDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([ - [`${THEIR_ADDR}:${MY_HASH160}`, { followerAddr: THEIR_ADDR, followeePkHash: MY_HASH160, unfollow: true, txid: 'c'.repeat(64), blockHeight: 150 }] - ])) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 0) - }) - - it('should sort notifications by block height descending', async () => { - const myPostTxid = 'a'.repeat(64) - const replyTxid = 'b'.repeat(64) - const likeTxid = 'c'.repeat(64) - - postChildrenDb.iterator.returns(makeIterator([ - [`${myPostTxid}:${replyTxid}`, { parentTxid: myPostTxid, childTxid: replyTxid, blockHeight: 100 }] - ])) - likesDb.iterator.returns(makeIterator([ - [likeTxid, { addr: THEIR_ADDR, postTxid: myPostTxid, blockHeight: 300 }] - ])) - followsDb.iterator.returns(makeIterator([])) - - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - postsDb.get.withArgs(replyTxid).resolves({ addr: THEIR_ADDR, text: 'reply' }) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 2) - assert.equal(result.notifications[0].type, 'like') - assert.equal(result.notifications[1].type, 'reply') - }) - - it('should sort three notifications by block height descending', async () => { - const myPostTxid = 'a'.repeat(64) - const replyTxid = 'b'.repeat(64) - const likeTxid = 'c'.repeat(64) - - postChildrenDb.iterator.returns(makeIterator([ - [`${myPostTxid}:${replyTxid}`, { parentTxid: myPostTxid, childTxid: replyTxid, blockHeight: 200 }] - ])) - likesDb.iterator.returns(makeIterator([ - [likeTxid, { addr: THEIR_ADDR, postTxid: myPostTxid, blockHeight: 300 }] - ])) - followsDb.iterator.returns(makeIterator([ - [`${THEIR_ADDR}:${MY_HASH160}`, { followerAddr: THEIR_ADDR, followeePkHash: MY_HASH160, unfollow: false, txid: 'd'.repeat(64), blockHeight: 100 }] - ])) - - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - postsDb.get.withArgs(replyTxid).resolves({ addr: THEIR_ADDR, text: 'reply' }) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) + const uut = buildQuery({ postsDb, addrPostHeightsDb, postLikesDb, likesDb, postChildrenDb, followeeHeightsDb }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) assert.equal(result.total, 3) - assert.equal(result.notifications[0].type, 'like') - assert.equal(result.notifications[0].blockHeight, 300) - assert.equal(result.notifications[1].type, 'reply') - assert.equal(result.notifications[1].blockHeight, 200) - assert.equal(result.notifications[2].type, 'follow') - assert.equal(result.notifications[2].blockHeight, 100) + assert.deepEqual(result.notifications.map((n) => n.type), ['follow', 'reply', 'like']) + assert.equal(result.notifications[0].addr, FOLLOWER) + assert.equal(result.notifications[0].blockHeight, 690400) + assert.equal(result.notifications[1].addr, REPLIER) + assert.equal(result.notifications[2].addr, LIKER) }) - it('should paginate notifications', async () => { - const myPostTxid = 'a'.repeat(64) - const replyTxid = 'b'.repeat(64) - const likeTxid = 'c'.repeat(64) + it('should exclude interactions with posts older than the window', async () => { + const postsDb = makeDb([ + ['post-old', { addr: VIEWER, text: 'old' }], + ['reply-old', { addr: REPLIER, text: 'reply', blockHeight: 690250, seen: 1 }] + ]) + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 600000, 'post-old')]) + const postLikesDb = makeDb([postLike('post-old', 'like-old')]) + const likesDb = makeDb([['like-old', { addr: LIKER, postTxid: 'post-old', blockHeight: 690200, seen: 1 }]]) + const postChildrenDb = makeDb([child('post-old', 'reply-old', 690250)]) + const followeeHeightsDb = makeDb([followeeHeight(VIEWER_HASH, 690400, FOLLOWER)]) - postChildrenDb.iterator.returns(makeIterator([ - [`${myPostTxid}:${replyTxid}`, { parentTxid: myPostTxid, childTxid: replyTxid, blockHeight: 100 }] - ])) - likesDb.iterator.returns(makeIterator([ - [likeTxid, { addr: THEIR_ADDR, postTxid: myPostTxid, blockHeight: 300 }] - ])) - followsDb.iterator.returns(makeIterator([])) + const uut = buildQuery({ postsDb, addrPostHeightsDb, postLikesDb, likesDb, postChildrenDb, followeeHeightsDb }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - postsDb.get.withArgs(replyTxid).resolves({ addr: THEIR_ADDR, text: 'reply' }) + // The old post's like and reply are excluded; only the in-window follow remains. + assert.equal(result.total, 1) + assert.deepEqual(result.notifications.map((n) => n.type), ['follow']) + }) - const result = await uut.listNotifications(MY_ADDR, { limit: 1, offset: 0 }) + it('should include an out-of-window post when the window reaches it', async () => { + const postsDb = makeDb([['post-old', { addr: VIEWER, text: 'old' }]]) + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 600000, 'post-old')]) + const postLikesDb = makeDb([postLike('post-old', 'like-old')]) + const likesDb = makeDb([['like-old', { addr: LIKER, postTxid: 'post-old', blockHeight: 690200, seen: 1 }]]) + + const uut = buildQuery({ + postsDb, + addrPostHeightsDb, + postLikesDb, + likesDb, + postChildrenDb: makeDb(), + followeeHeightsDb: makeDb(), + notificationBlockWindow: 100000 + }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) + + assert.equal(result.total, 1) + assert.equal(result.notifications[0].blockHeight, 690200) + }) + + it('should drop a follower whose newest entry is an unfollow', async () => { + const followeeHeightsDb = makeDb([ + followeeHeight(VIEWER_HASH, 690550, FOLLOWER, false), + followeeHeight(VIEWER_HASH, 690600, FOLLOWER, true) + ]) + + const uut = buildQuery({ followeeHeightsDb }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) + + assert.equal(result.total, 0) + }) + + it('should keep a follower whose newest entry is a follow after an older unfollow', async () => { + const followeeHeightsDb = makeDb([ + followeeHeight(VIEWER_HASH, 690550, FOLLOWER, true), + followeeHeight(VIEWER_HASH, 690600, FOLLOWER, false) + ]) + + const uut = buildQuery({ followeeHeightsDb }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) + + assert.equal(result.total, 1) + assert.equal(result.notifications[0].addr, FOLLOWER) + }) + + it('should exclude self-likes, self-replies, and self-follows', async () => { + const postsDb = makeDb([ + ['post-recent', { addr: VIEWER, text: 'hi' }], + ['reply-self', { addr: VIEWER, text: 'mine', blockHeight: 690150 }] + ]) + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 690000, 'post-recent')]) + const postLikesDb = makeDb([postLike('post-recent', 'like-self')]) + const likesDb = makeDb([['like-self', { addr: VIEWER, postTxid: 'post-recent', blockHeight: 690100 }]]) + const postChildrenDb = makeDb([child('post-recent', 'reply-self', 690150)]) + const followeeHeightsDb = makeDb([followeeHeight(VIEWER_HASH, 690400, VIEWER)]) + + const uut = buildQuery({ postsDb, addrPostHeightsDb, postLikesDb, likesDb, postChildrenDb, followeeHeightsDb }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) + + assert.equal(result.total, 0) + }) + + it('should exclude notifications from muted addresses', async () => { + const postsDb = makeDb([['reply-recent', { addr: OTHER, text: 'reply', blockHeight: 690150 }]]) + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 690000, 'post-recent')]) + const postChildrenDb = makeDb([child('post-recent', 'reply-recent', 690150)]) + const muteQuery = { listMuted: sandbox.stub().resolves([OTHER]) } + + const uut = buildQuery({ postsDb, addrPostHeightsDb, postChildrenDb, muteQuery }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) + + assert.equal(result.total, 0) + assert.isTrue(muteQuery.listMuted.calledOnceWith(VIEWER)) + }) + + it('should break sort ties by seen descending', async () => { + const postsDb = makeDb([ + ['reply-a', { addr: REPLIER, text: 'a', blockHeight: 690150, seen: 1 }], + ['reply-b', { addr: OTHER, text: 'b', blockHeight: 690150, seen: 2 }] + ]) + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 690000, 'post-recent')]) + const postChildrenDb = makeDb([ + child('post-recent', 'reply-a', 690150), + child('post-recent', 'reply-b', 690150) + ]) + + const uut = buildQuery({ postsDb, addrPostHeightsDb, postChildrenDb }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) + + assert.equal(result.total, 2) + assert.equal(result.notifications[0].seen, 2) + assert.equal(result.notifications[1].seen, 1) + }) + + it('should paginate and report the exact in-window total', async () => { + const followeeHeightsDb = makeDb([ + followeeHeight(VIEWER_HASH, 690400, FOLLOWER), + followeeHeight(VIEWER_HASH, 690300, OTHER) + ]) + + const uut = buildQuery({ followeeHeightsDb }) + const result = await uut.listNotifications(VIEWER, { limit: 1, offset: 1 }) assert.equal(result.total, 2) assert.equal(result.notifications.length, 1) - assert.equal(result.notifications[0].type, 'like') + assert.equal(result.notifications[0].blockHeight, 690300) }) - it('should default follow blockHeight and seen to 0 when missing', async () => { - postChildrenDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([ - [`${THEIR_ADDR}:${MY_HASH160}`, { followerAddr: THEIR_ADDR, followeePkHash: MY_HASH160, unfollow: false, txid: 'c'.repeat(64) }] - ])) + it('should never iterate the likes store', async () => { + const postsDb = makeDb([['post-recent', { addr: VIEWER, text: 'hi' }]]) + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 690000, 'post-recent')]) + const postLikesDb = makeDb([postLike('post-recent', 'like-recent')]) + const likesDb = makeDb([['like-recent', { addr: LIKER, postTxid: 'post-recent', blockHeight: 690100 }]]) + const followeeHeightsDb = makeDb([followeeHeight(VIEWER_HASH, 690400, FOLLOWER)]) + likesDb.iterator = sandbox.stub().throws(new Error('likes store must not be iterated')) - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) + const uut = buildQuery({ postsDb, addrPostHeightsDb, postLikesDb, likesDb, postChildrenDb: makeDb(), followeeHeightsDb }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) - assert.equal(result.notifications.length, 1) - const n = result.notifications[0] - assert.equal(n.type, 'follow') - assert.equal(n.blockHeight, 0) - assert.equal(n.seen, 0) + assert.equal(result.total, 2) + assert.equal(likesDb.iterator.callCount, 0) }) - it('should default like blockHeight and seen to 0 when missing', async () => { - const myPostTxid = 'a'.repeat(64) - const likeTxid = 'b'.repeat(64) + it('should treat the whole history as in-window when status is missing', async () => { + const postsDb = makeDb([['post-old', { addr: VIEWER, text: 'old' }]]) + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 1, 'post-old')]) + const postLikesDb = makeDb([postLike('post-old', 'like-old')]) + const likesDb = makeDb([['like-old', { addr: LIKER, postTxid: 'post-old', blockHeight: 2 }]]) - postChildrenDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([ - [likeTxid, { addr: THEIR_ADDR, postTxid: myPostTxid }] - ])) - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) + const uut = buildQuery({ + postsDb, + addrPostHeightsDb, + postLikesDb, + likesDb, + postChildrenDb: makeDb(), + followeeHeightsDb: makeDb(), + statusDb: null + }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.notifications.length, 1) - const n = result.notifications[0] - assert.equal(n.type, 'like') - assert.equal(n.blockHeight, 0) - assert.equal(n.seen, 0) + assert.equal(result.total, 1) }) - it('should default reply blockHeight and seen to 0 when missing', async () => { - const myPostTxid = 'a'.repeat(64) - const replyTxid = 'b'.repeat(64) + it('should recover the follower from the followeeHeights key when the record lacks it', async () => { + const followeeHeightsDb = makeDb([ + [`${VIEWER_HASH}:${pad(690400)}:${FOLLOWER}`, { unfollow: false, txid: 't', blockHeight: 690400 }] + ]) - postChildrenDb.iterator.returns(makeIterator([ - [`${myPostTxid}:${replyTxid}`, { parentTxid: myPostTxid, childTxid: replyTxid }] - ])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - postsDb.get.withArgs(replyTxid).resolves({ addr: THEIR_ADDR, text: 'reply' }) + const uut = buildQuery({ followeeHeightsDb }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.notifications.length, 1) - const n = result.notifications[0] - assert.equal(n.type, 'reply') - assert.equal(n.blockHeight, 0) - assert.equal(n.seen, 0) + assert.equal(result.total, 1) + assert.equal(result.notifications[0].addr, FOLLOWER) }) - it('should use the child post blockHeight when the child record lacks one', async () => { - const myPostTxid = 'a'.repeat(64) - const replyTxid = 'b'.repeat(64) + it('should skip a missing like record', async () => { + const postsDb = makeDb([['post-recent', { addr: VIEWER, text: 'hi' }]]) + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 690000, 'post-recent')]) + const postLikesDb = makeDb([postLike('post-recent', 'like-missing')]) - postChildrenDb.iterator.returns(makeIterator([ - [`${myPostTxid}:${replyTxid}`, { parentTxid: myPostTxid, childTxid: replyTxid }] - ])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - postsDb.get.withArgs(replyTxid).resolves({ addr: THEIR_ADDR, text: 'reply', blockHeight: 500, seen: 7 }) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.notifications.length, 1) - const n = result.notifications[0] - assert.equal(n.blockHeight, 500) - assert.equal(n.seen, 7) - }) - - it('should break sort ties by seen descending when blockHeight is equal', async () => { - const myPostTxid = 'a'.repeat(64) - - postChildrenDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([ - ['b1'.repeat(32), { addr: THEIR_ADDR, postTxid: myPostTxid, blockHeight: 300, seen: 100 }], - ['b2'.repeat(32), { addr: 'bitcoincash:qother2', postTxid: myPostTxid, blockHeight: 300, seen: 200 }] - ])) - postsDb.get.withArgs(myPostTxid).resolves({ addr: MY_ADDR, text: 'hello' }) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.notifications.length, 2) - assert.equal(result.notifications[0].seen, 200) - assert.equal(result.notifications[1].seen, 100) - }) - - it('should skip a like whose post is missing', async () => { - const myPostTxid = 'a'.repeat(64) - const likeTxid = 'b'.repeat(64) - const missingPost = new Error('not found') - missingPost.notFound = true - - postChildrenDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([ - [likeTxid, { addr: THEIR_ADDR, postTxid: myPostTxid, blockHeight: 300 }] - ])) - postsDb.get.withArgs(myPostTxid).rejects(missingPost) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) + const uut = buildQuery({ postsDb, addrPostHeightsDb, postLikesDb }) + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) assert.equal(result.total, 0) }) - it('should skip a null like record without throwing', async () => { - postChildrenDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - likesDb.iterator.returns(makeIterator([ - [null, null] - ])) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) - - assert.equal(result.total, 0) - }) - - it('should skip a reply child record missing parentTxid or childTxid', async () => { - postChildrenDb.iterator.returns(makeIterator([ - ['missing:child', { parentTxid: undefined, childTxid: 'b'.repeat(64), blockHeight: 100 }] - ])) - likesDb.iterator.returns(makeIterator([])) - followsDb.iterator.returns(makeIterator([])) - - const result = await uut.listNotifications(MY_ADDR, { limit: 100, offset: 0 }) + it('should default notificationBlockWindow to 25000', async () => { + const addrPostHeightsDb = makeDb([addrPostHeight(VIEWER, 1, 'post-old')]) + const uut = buildQuery({ addrPostHeightsDb, statusDb: makeDb([['status', { chainBlockHeight: 700000 }]]) }) + // cutoff 675000: a post at height 1 is out of window. + const result = await uut.listNotifications(VIEWER, { limit: 100, offset: 0 }) assert.equal(result.total, 0) + assert.equal(uut.notificationBlockWindow, 25000) }) }) diff --git a/psf-memo-db/test/unit/lib/backfill-followee-index.unit.js b/psf-memo-db/test/unit/lib/backfill-followee-index.unit.js new file mode 100644 index 0000000..6e1c523 --- /dev/null +++ b/psf-memo-db/test/unit/lib/backfill-followee-index.unit.js @@ -0,0 +1,96 @@ +import { assert } from 'chai' +import { backfillFolloweeIndex, followeeHeightKey } from '../../../src/lib/backfill-followee-index.js' +import { FakeDb } from '../../support/level-double.js' + +describe('#backfillFolloweeIndex', () => { + // followeePkHash is a 20-byte hash160 hex; cash addresses are the followers. + const VIEWER_HASH = 'a1'.repeat(20) + const OTHER_HASH = 'b2'.repeat(20) + const FOLLOWER_A = 'bitcoincash:qq3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygrg4dtdzf' + const FOLLOWER_B = 'bitcoincash:qqenxvenxvenxvenxvenxvenxvenxvenxvn254yg3p' + const FOLLOWER_C = 'bitcoincash:qpzyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs7fn3s6pt' + + function followsDb () { + return new FakeDb([ + [`${FOLLOWER_A}:${VIEWER_HASH}`, { + followerAddr: FOLLOWER_A, + followeePkHash: VIEWER_HASH, + unfollow: false, + txid: 'follow-a', + seen: 1, + blockHeight: 690400 + }], + [`${FOLLOWER_B}:${VIEWER_HASH}`, { + followerAddr: FOLLOWER_B, + followeePkHash: VIEWER_HASH, + unfollow: true, + txid: 'unfollow-b', + seen: 2, + blockHeight: 690600 + }], + [`${FOLLOWER_C}:${OTHER_HASH}`, { + followerAddr: FOLLOWER_C, + followeePkHash: OTHER_HASH, + unfollow: false, + txid: 'follow-c', + seen: 3, + blockHeight: 690200 + }] + ]) + } + + it('should index every follow record at its height, preserving unfollow', async () => { + const db = followsDb() + const followeeHeightsDb = new FakeDb() + + const result = await backfillFolloweeIndex({ followsDb: db, followeeHeightsDb }) + + assert.equal(result.follows, 3) + assert.deepEqual(followeeHeightsDb.store.get(followeeHeightKey(VIEWER_HASH, 690400, FOLLOWER_A)), { + followerAddr: FOLLOWER_A, + followeePkHash: VIEWER_HASH, + unfollow: false, + txid: 'follow-a', + seen: 1, + blockHeight: 690400 + }) + assert.equal(followeeHeightsDb.store.get(followeeHeightKey(VIEWER_HASH, 690600, FOLLOWER_B)).unfollow, true) + assert.equal(followeeHeightsDb.store.get(followeeHeightKey(OTHER_HASH, 690200, FOLLOWER_C)).unfollow, false) + }) + + it('should be idempotent across repeated runs', async () => { + const db = followsDb() + const followeeHeightsDb = new FakeDb() + + await backfillFolloweeIndex({ followsDb: db, followeeHeightsDb }) + const sizeAfterFirst = followeeHeightsDb.store.size + await backfillFolloweeIndex({ followsDb: db, followeeHeightsDb }) + + assert.equal(followeeHeightsDb.store.size, sizeAfterFirst) + assert.equal(sizeAfterFirst, 3) + }) + + it('should leave the follows store unchanged', async () => { + const db = followsDb() + const before = new Map(db.store) + const followeeHeightsDb = new FakeDb() + + await backfillFolloweeIndex({ followsDb: db, followeeHeightsDb }) + + assert.deepEqual(db.store, before) + }) + + it('should recover the follower and followee from the key when fields are missing', async () => { + const db = new FakeDb([ + [`${FOLLOWER_A}:${VIEWER_HASH}`, { unfollow: false, blockHeight: 690400 }] + ]) + const followeeHeightsDb = new FakeDb() + + await backfillFolloweeIndex({ followsDb: db, followeeHeightsDb }) + + const record = followeeHeightsDb.store.get(followeeHeightKey(VIEWER_HASH, 690400, FOLLOWER_A)) + assert.equal(record.followerAddr, FOLLOWER_A) + assert.equal(record.followeePkHash, VIEWER_HASH) + assert.equal(record.unfollow, false) + }) +}) diff --git a/psf-memo-db/util/follow/backfill-followee-index.js b/psf-memo-db/util/follow/backfill-followee-index.js new file mode 100644 index 0000000..c5641f7 --- /dev/null +++ b/psf-memo-db/util/follow/backfill-followee-index.js @@ -0,0 +1,68 @@ +/* + Utility: build the followeeHeights notification index for an existing + psf-memo-db. + + New deployments write followeeHeights while indexing live follow actions, but + existing databases that were populated before the notifications query + performance feature need a one-time backfill. + + Run from the psf-memo-db repo root on the host that owns the LevelDB files: + + node util/follow/backfill-followee-index.js + + The script is idempotent: re-running it produces the same index. Progress and + a summary are printed to stderr. + + WARNING: + - This script opens the LevelDB files directly. psf-memo-db must NOT be + running, or another process must not hold the database locks. + - Make a backup of leveldb/current before running on a production server: + cp -r leveldb/current leveldb/current-pre-followee-index-backup +*/ + +import level from 'level' +import * as fs from 'fs' +import * as path from 'path' +import * as url from 'url' +import { backfillFolloweeIndex } from '../../src/lib/backfill-followee-index.js' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) + +const DATA_DIR = process.env.PSF_MEMO_DB_DATA_DIR + ? path.resolve(process.env.PSF_MEMO_DB_DATA_DIR) + : path.resolve(__dirname, '../../leveldb/current') + +function requiredStorePath (dir, name) { + const storePath = path.join(dir, name) + if (!fs.existsSync(storePath)) { + throw new Error(`Required LevelDB store not found: ${storePath}. Set PSF_MEMO_DB_DATA_DIR to the directory containing the follows store.`) + } + return storePath +} + +async function main () { + console.error(`Using LevelDB data directory: ${DATA_DIR}`) + + const followsPath = requiredStorePath(DATA_DIR, 'follows') + const followeeHeightsPath = path.join(DATA_DIR, 'followeeHeights') + + console.error('Opening LevelDB stores...') + const followsDb = level(followsPath, { valueEncoding: 'json' }) + const followeeHeightsDb = level(followeeHeightsPath, { valueEncoding: 'json', createIfMissing: true }) + + try { + console.error('Backfilling followeeHeights from follows...') + const summary = await backfillFolloweeIndex({ followsDb, followeeHeightsDb }) + + console.error('\nFollowee index backfill complete.') + console.error(` follows indexed: ${summary.follows}`) + } catch (err) { + console.error('\nFollowee index backfill failed:', err.message) + process.exitCode = 1 + } finally { + await followeeHeightsDb.close().catch(() => {}) + await followsDb.close().catch(() => {}) + } +} + +main() diff --git a/psf-memo-indexer/acceptance/lib/handlers.js b/psf-memo-indexer/acceptance/lib/handlers.js index fa08392..9fc5db4 100644 --- a/psf-memo-indexer/acceptance/lib/handlers.js +++ b/psf-memo-indexer/acceptance/lib/handlers.js @@ -14,6 +14,7 @@ import { handleCreatePoll } from '../../src/use-cases/action-types/poll-create.j import { handleAddPollOption } from '../../src/use-cases/action-types/poll-option.js' import { handlePollVote } from '../../src/use-cases/action-types/poll-vote.js' import { handleMute } from '../../src/use-cases/action-types/mute.js' +import { handleFollow } from '../../src/use-cases/action-types/follow.js' import { handleTopicMessage } from '../../src/use-cases/action-types/topic-message.js' import { handleTopicFollow } from '../../src/use-cases/action-types/topic-follow.js' import { topicRecencyKey } from '../../src/use-cases/action-types/helpers.js' @@ -66,6 +67,13 @@ function deriveTxid (symbolic) { return crypto.createHash('sha256').update(symbolic).digest().toString('hex') } +// Derive a deterministic 20-byte pkHash hex for a symbolic test address. The +// follow action payload carries a pkHash, not an address, so the acceptance +// step and its assertion must derive the same value from the address. +function derivePkHash (addr) { + return crypto.createHash('sha256').update(addr).digest().slice(0, 20).toString('hex') +} + function resolveTxid (value, example, world) { const resolved = resolveParam(value, example) if (!world.txidMap) world.txidMap = new Map() @@ -88,6 +96,8 @@ async function createWorld () { const pollOptionDb = makeInMemoryDb() const pollVoteDb = makeInMemoryDb() const muteDb = makeInMemoryDb() + const followDb = makeInMemoryDb() + const followeeHeightsDb = makeInMemoryDb() const roomDb = makeInMemoryDb() const topicSummaryDb = makeInMemoryDb() const topicRecencyDb = makeInMemoryDb() @@ -104,6 +114,8 @@ async function createWorld () { pollOptionDb, pollVoteDb, muteDb, + followDb, + followeeHeightDb: followeeHeightsDb, roomDb, topicSummaryDb, topicRecencyDb, @@ -130,6 +142,8 @@ async function createWorld () { pollOptionsDb: pollOptionDb, pollVotesDb: pollVoteDb, mutesDb: muteDb, + followsDb: followDb, + followeeHeightsDb, roomsDb: roomDb, topicSummariesDb: topicSummaryDb, topicRecencyDb, @@ -588,6 +602,90 @@ const handlers = [ throw new Error(`Expected ${expectedCount} postLikes entry/entries for ${postTxid}/${likeTxid}, got ${matching.length}`) } } + }, + { + name: 'db instance with follows and followeeHeights stores', + pattern: /^a psf-memo-db instance with follows and followeeHeights stores$/, + async run () { + // World is already created with both stores. + } + }, + { + name: 'process a Memo follow', + pattern: /^the indexer processes a Memo follow of (.+) from (.+) at block height (.+)$/, + async run (m, example, world) { + const followee = resolveParam(m[1], example) + const follower = resolveParam(m[2], example) + const height = parseInt(resolveParam(m[3], example), 10) + const txid = deriveTxid(`follow-${follower}-${followee}-${height}`) + + const prefix = Buffer.from('6d06', 'hex') + const hashBuf = Buffer.from(derivePkHash(followee), 'hex') + const ctx = { + adapters: world.adapters, + txid, + signerAddr: follower, + seen: Date.now(), + blockHeight: height, + decoded: { action: 'follow', prefix, pushDatas: [prefix, hashBuf] } + } + world.lastFollow = { followee, follower, height, ctx } + + await handleFollow(ctx) + } + }, + { + name: 'process a Memo unfollow', + pattern: /^the indexer processes a Memo unfollow of (.+) from (.+) at block height (.+)$/, + async run (m, example, world) { + const followee = resolveParam(m[1], example) + const follower = resolveParam(m[2], example) + const height = parseInt(resolveParam(m[3], example), 10) + const txid = deriveTxid(`unfollow-${follower}-${followee}-${height}`) + + const prefix = Buffer.from('6d07', 'hex') + const hashBuf = Buffer.from(derivePkHash(followee), 'hex') + const ctx = { + adapters: world.adapters, + txid, + signerAddr: follower, + seen: Date.now(), + blockHeight: height, + decoded: { action: 'unfollow', prefix, pushDatas: [prefix, hashBuf] } + } + world.lastFollow = { followee, follower, height, ctx } + + await handleFollow(ctx) + } + }, + { + name: 'process the same Memo follow again', + pattern: /^the indexer processes the same Memo follow of (.+) from (.+) again$/, + async run (m, example, world) { + if (!world.lastFollow) { + throw new Error('No previous follow to reprocess') + } + await handleFollow(world.lastFollow.ctx) + } + }, + { + name: 'followeeHeights store contains entry', + pattern: /^the followeeHeights store contains (.+) entr(?:y|ies) for followee (.+) from (.+) at block height (.+) marked unfollow (true|false)$/, + run (m, example, world) { + const expectedCount = parseInt(resolveParam(m[1], example), 10) + const followee = resolveParam(m[2], example) + const follower = resolveParam(m[3], example) + const height = parseInt(resolveParam(m[4], example), 10) + const expectedUnfollow = m[5] === 'true' + const pkHash = derivePkHash(followee) + const key = `${pkHash}:${String(height).padStart(12, '0')}:${follower}` + const matching = world.followeeHeightsDb.entries().filter(([k, value]) => { + return k === key && value?.unfollow === expectedUnfollow + }) + if (matching.length !== expectedCount) { + throw new Error(`Expected ${expectedCount} followeeHeights entry/entries for ${key} unfollow ${expectedUnfollow}, got ${matching.length}`) + } + } } ] diff --git a/psf-memo-indexer/dev-docs/design-decisions-and-tradeoffs.md b/psf-memo-indexer/dev-docs/design-decisions-and-tradeoffs.md index 2311c25..a0ef096 100644 --- a/psf-memo-indexer/dev-docs/design-decisions-and-tradeoffs.md +++ b/psf-memo-indexer/dev-docs/design-decisions-and-tradeoffs.md @@ -166,6 +166,37 @@ This document records **why** the Memo indexer stack looks the way it does, incl **Recommended e2e smoke:** Index block ≥ 525000 containing a known Memo post from [memo-protocol.md](../../memo-protocol.md) explorer links; verify `GET /level/post/:txid`. +## 16. Bounded notification reads (per-object scans + block window) + +**Decision:** `GET /posts/notifications/:addr` no longer full-scans the global +`likes`, `postChildren`, and `follows` stores. Instead it bounds the work to the +viewer's activity inside a configurable block window (`NOTIFICATION_BLOCK_WINDOW`, +default 25000; cutoff = `status.chainBlockHeight - window`): + +- the viewer's posts come from the `addrPostHeights` index, range-limited to the + window; +- likes and replies are prefix-scanned from `postLikes` and `postChildren` for + those posts only; +- follows come from a new followee-keyed, height-ordered `followeeHeights` index + maintained by the indexer, range-limited to the window. + +**Reason:** The original request cost `O(all likes + all replies + all follows)` +and loaded a post for every candidate, so the `/notifications` page blocked on a +scan proportional to the whole database rather than the viewer's activity. + +**Tradeoff:** A notification is drawn from the viewer's content inside the +window, so an interaction with a post older than the window is not returned even +when the interaction itself is recent. This is acceptable for a social inbox: +the window bounds staleness and keeps the read proportional to recent activity. +The `followeeHeights` index is an append-only event log per followee; the read +side takes the newest entry per follower and ignores unfollows, so an unfollow +removes the notification without deleting event history. The `follows` store +remains the source of truth for the current follow graph. + +**Backfill:** Databases populated before this feature need a one-time +`util/follow/backfill-followee-index.js` run, which projects every `follows` +record into a `followeeHeights` entry at its block height and is idempotent. + ## Decision log (quick reference) | Topic | Choice | diff --git a/psf-memo-indexer/dev-docs/psf-memo-db.md b/psf-memo-indexer/dev-docs/psf-memo-db.md index 6b28126..39699c2 100644 --- a/psf-memo-indexer/dev-docs/psf-memo-db.md +++ b/psf-memo-indexer/dev-docs/psf-memo-db.md @@ -48,6 +48,9 @@ leveldb/current/{name} // valueEncoding: 'json' | `profiles` | 64 MB | Profile text | | `profilePics` | 64 MB | Avatar URLs | | `follows` | 64 MB | Follow graph edges | +| `addrPostHeights` | 64 MB | Posts by address ordered by height | +| `postLikes` | 64 MB | Like txids grouped by liked post | +| `followeeHeights` | 64 MB | Follow/unfollow events keyed by followee and height | | `rooms` | 64 MB | Topic posts and follows | | `processErrors` | 64 MB | Skipped / invalid txs | | `ptxs` | 64 MB | Processed tx markers | @@ -77,6 +80,7 @@ All routes are under `/level` with a consistent CRUD pattern generated from `ENT | `profile` | `addr` | `profileData` | cash address | | `profilepic` | `addr` | `profilePicData` | cash address | | `follow` | `key` | `followData` | `follower:followeePkHash` | +| `followeeheight` | `key` | `followeeHeightData` | `followeePkHash:paddedHeight:followerAddr` | | `room` | `key` | `roomData` | composite | | `processerror` | `txid` | `errorData` | txid | | `ptx` | `txid` | `ptxData` | txid | @@ -87,6 +91,7 @@ All routes are under `/level` with a consistent CRUD pattern generated from `ENT |--------|------|----------------| | `GET` | `/profile/recent` | `limit` (default 100, max 100), `offset` (default 0) | | `GET` | `/posts/recent` | `limit` (default 100, max 100), `offset` (default 0) | +| `GET` | `/posts/notifications/:addr` | `limit` (default 100, max 100), `offset` (default 0) | Returns profiles or posts sorted by **block height** (newest first), using the `blockHeight` field stored on each entity document at indexing time. Tie-breaker: `seen` timestamp descending. @@ -130,6 +135,29 @@ Implementation: `profile-query` / `post-query` adapter (LevelDB scan) → `list- **Tradeoff:** Full scan of `profiles` on each request; suitable for moderate corpus sizes. A height-indexed store would be needed for very large archives. +#### Notifications + +`GET /posts/notifications/:addr` returns likes on the viewer's posts, replies to +the viewer's posts, and follows of the viewer, newest first. Work is bounded to +the viewer's activity inside a configurable block window +(`NOTIFICATION_BLOCK_WINDOW`, default `25000`; cutoff = +`status.chainBlockHeight - window`): + +- the viewer's posts are read from `addrPostHeights`, range-limited to the + window; +- likes and replies are prefix-scanned from `postLikes` and `postChildren` for + those posts only (the global `likes` and `postChildren` stores are never + iterated); +- follows are read from `followeeHeights`, range-limited to the window, keeping + the newest entry per follower and ignoring unfollows (the `follows` store is + never iterated). + +`pagination.total` counts only in-window notifications. Because a notification +is drawn from the viewer's content inside the window, an interaction with a +post older than the window is not returned even when the interaction itself is +recent. The indexer writes `followeeHeights` on every follow/unfollow; existing +databases need the one-time `util/follow/backfill-followee-index.js` backfill. + ### Status (special case) Matches SLP status semantics: @@ -187,6 +215,7 @@ Common fields on indexed documents: | profilePic | addr | `url`, `txid`, `seen`, `addr`, `blockHeight` | | like | txid | `addr`, `postTxid`, `seen`, `tip`, `blockHeight` | | follow | composite key | `followerAddr`, `followeePkHash`, `unfollow`, `txid`, `seen`, `blockHeight` | +| followeeHeights | `followeePkHash:paddedHeight:followerAddr` | `followerAddr`, `followeePkHash`, `unfollow`, `txid`, `seen`, `blockHeight` | | postParent / postChild | txid / `parentTxid:childTxid` | `parentTxid`, `childTxid`, `blockHeight` | | room | composite key | `room`, `txid`, `seen`, `type`, `blockHeight` (+ `addr` for follows) | | processError | txid | `error`, `ts`, `blockHeight` | @@ -203,6 +232,7 @@ Common fields on indexed documents: | `SVC_ENV` | `development` | Config profile | | `BACKUP_QTY` | `3` | Retained zip backups | | `EXIT_ON_MISSING_BACKUP` | `false` | Fail restore if zip missing | +| `NOTIFICATION_BLOCK_WINDOW` | `25000` | Blocks before the chain tip that still count as a notification | ## Testing diff --git a/psf-memo-indexer/src/adapters/adapters-index.js b/psf-memo-indexer/src/adapters/adapters-index.js index f7360ca..2cbdc2f 100644 --- a/psf-memo-indexer/src/adapters/adapters-index.js +++ b/psf-memo-indexer/src/adapters/adapters-index.js @@ -30,6 +30,7 @@ class Adapters { this.profileDb = createEntityDb('profile', 'addr', 'profileData') this.profilePicDb = createEntityDb('profilepic', 'addr', 'profilePicData') this.followDb = createEntityDb('follow', 'key', 'followData') + this.followeeHeightDb = createEntityDb('followeeheight', 'key', 'followeeHeightData') this.muteDb = createEntityDb('mute', 'key', 'muteData') this.roomDb = createEntityDb('room', 'key', 'roomData') this.topicSummaryDb = createEntityDb('topicsummary', 'key', 'topicSummaryData') diff --git a/psf-memo-indexer/src/use-cases/action-types/follow.js b/psf-memo-indexer/src/use-cases/action-types/follow.js index eca1d02..77d621b 100644 --- a/psf-memo-indexer/src/use-cases/action-types/follow.js +++ b/psf-memo-indexer/src/use-cases/action-types/follow.js @@ -1,4 +1,4 @@ -import { logProcessError } from './helpers.js' +import { logProcessError, followeeHeightKey } from './helpers.js' import { PK_HASH_LENGTH, PREFIX_UNFOLLOW } from '../../lib/memo-codes.js' export async function handleFollow (ctx) { @@ -19,12 +19,20 @@ export async function handleFollow (ctx) { const followeePkHash = pushDatas[1].toString('hex') const key = `${signerAddr}:${followeePkHash}` - await adapters.followDb.create(key, { + const record = { followerAddr: signerAddr, followeePkHash, unfollow, txid, seen, blockHeight - }) + } + await adapters.followDb.create(key, record) + + // Mirror the event into the followeeHeights notification index so the read + // side can find the viewer's follows without scanning the follows store. + await adapters.followeeHeightDb.create( + followeeHeightKey(followeePkHash, blockHeight, signerAddr), + record + ) } diff --git a/psf-memo-indexer/src/use-cases/action-types/helpers.js b/psf-memo-indexer/src/use-cases/action-types/helpers.js index c3f579b..8c1e40b 100644 --- a/psf-memo-indexer/src/use-cases/action-types/helpers.js +++ b/psf-memo-indexer/src/use-cases/action-types/helpers.js @@ -104,6 +104,14 @@ export function postChildKey (parentTxid, childTxid) { return `${parentTxid}:${childTxid}` } +// followeeHeights is a followee-keyed, height-ordered notification read index: +// `${followeePkHash}:${paddedBlockHeight}:${followerAddr}`. The read side +// range-scans one followee's suffix to find its newest follow per follower. +export function followeeHeightKey (followeePkHash, blockHeight, followerAddr) { + const padded = String(blockHeight ?? 0).padStart(12, '0') + return `${followeePkHash}:${padded}:${followerAddr}` +} + // mutate4javascript-manifest-begin // {"version":1,"tested_at":"2026-09-17T16:25:37.308Z","module_hash":"00fa2f64726246ec4e403d93b7cabaf0fb5e2753614533c0b3651b23d6147749","functions":[{"id":"func/logProcessError","name":"logProcessError","line":7,"end_line":13,"hash":"78f39be271917cad97072ed5e2d1f531a30a37bc9c10169516e0457951c438f4"},{"id":"func/utf8FromPush","name":"utf8FromPush","line":15,"end_line":17,"hash":"356fc665a6389e392ec83522210dd95b2af6be341d00dc5edc0bbabfbd511d9e"},{"id":"func/stripLeadingEmptyPushes","name":"stripLeadingEmptyPushes","line":22,"end_line":28,"hash":"be7c401e07e4ef3670428afe7211860038c69da17abcd03508ea5cb640b46163"},{"id":"func/normalizeTwoPushMemoDatas","name":"normalizeTwoPushMemoDatas","line":34,"end_line":46,"hash":"61d050380c2cef7eed8870a66e347f1d01c56246d733d95d692b436cfb4a20ed"},{"id":"func/txHashFromPush","name":"txHashFromPush","line":48,"end_line":55,"hash":"704934308365cf0fbbb72aa437faf5db20a69bd57a16ba37b19c905ddc922ea9"},{"id":"func/followKey","name":"followKey","line":57,"end_line":59,"hash":"e1c5cfdd78afd3945e18e4af06e443efb306c27b07e3b3534c918ef000da9696"},{"id":"func/roomKey","name":"roomKey","line":61,"end_line":63,"hash":"d4f146c7a938bb6b30b13f071609a3ac4e96a26f116e321b804c16114884cd15"},{"id":"func/topicRecencyKey","name":"topicRecencyKey","line":68,"end_line":71,"hash":"ce7146b8dfca80309058c1512818fe2edc9f1b3658cbe1e06aabf78f65136b90"},{"id":"func/isNotFound","name":"isNotFound","line":75,"end_line":77,"hash":"662978032673c9bac6d64c519899ebfed72e477ea39b9fde19039ffa01de183d"},{"id":"func/getIfPresent","name":"getIfPresent","line":80,"end_line":87,"hash":"b012b3ef1d48f45a65fcbd4ff9574443cc98d2c2f7d12a504af74fc3e0d3f5af"},{"id":"func/postHeightKey","name":"postHeightKey","line":89,"end_line":92,"hash":"25bdb7995aeb2823d0f808812251c3fd8c163089ea32b4f39ddf55520e03cbb1"},{"id":"func/addrPostHeightKey","name":"addrPostHeightKey","line":94,"end_line":97,"hash":"b9b6b08e4c5892905dc6577ad22dd48c13598ca205b1b8c13d7cc9622962879f"},{"id":"func/postLikeKey","name":"postLikeKey","line":99,"end_line":101,"hash":"627ce4b3ca6a9d28fbaa50bc1f51973509e7d8b52eda9208343dac1d936d7f51"},{"id":"func/postChildKey","name":"postChildKey","line":103,"end_line":105,"hash":"4d30329db33e34b0461bd1ba70b078fbef285a338f17f7e4d1166ca5bd107893"}]} // mutate4javascript-manifest-end diff --git a/psf-memo-indexer/test/unit/use-cases/action-types/follow.unit.js b/psf-memo-indexer/test/unit/use-cases/action-types/follow.unit.js new file mode 100644 index 0000000..521afb9 --- /dev/null +++ b/psf-memo-indexer/test/unit/use-cases/action-types/follow.unit.js @@ -0,0 +1,121 @@ +import { assert } from 'chai' +import sinon from 'sinon' +import { handleFollow } from '../../../../src/use-cases/action-types/follow.js' +import { PREFIX_UNFOLLOW } from '../../../../src/lib/memo-codes.js' + +describe('#handleFollow', () => { + let adapters + let followCreate + let followeeHeightCreate + let processErrorCreate + + const FOLLOWER = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d' + const FOLLOWER2 = 'bitcoincash:qpm2qsznhks23z7629mms6s4cwef74vcwvy22gdx6a' + const FOLLOWEE_HASH = 'cb481232299cd5743151ac4b2d63ae198e7bb0a9' + + function baseCtx (overrides = {}) { + const hashBuf = Buffer.from(FOLLOWEE_HASH, 'hex') + return { + adapters, + txid: 'follow-tx', + signerAddr: FOLLOWER, + seen: 1000, + blockHeight: 690100, + decoded: { + action: 'follow', + prefix: Buffer.from('6d06', 'hex'), + pushDatas: [Buffer.from('6d06', 'hex'), hashBuf] + }, + ...overrides + } + } + + beforeEach(() => { + followCreate = sinon.stub().resolves({ success: true }) + followeeHeightCreate = sinon.stub().resolves({ success: true }) + processErrorCreate = sinon.stub().resolves({ success: true }) + adapters = { + followDb: { create: followCreate }, + followeeHeightDb: { create: followeeHeightCreate }, + processErrorDb: { create: processErrorCreate } + } + }) + + it('should save the follows record and mirror it into followeeHeights', async () => { + await handleFollow(baseCtx()) + + assert.equal(followCreate.callCount, 1) + assert.equal(followCreate.firstCall.args[0], `${FOLLOWER}:${FOLLOWEE_HASH}`) + assert.equal(followCreate.firstCall.args[1].followerAddr, FOLLOWER) + assert.equal(followCreate.firstCall.args[1].followeePkHash, FOLLOWEE_HASH) + assert.equal(followCreate.firstCall.args[1].unfollow, false) + + assert.equal(followeeHeightCreate.callCount, 1) + assert.equal( + followeeHeightCreate.firstCall.args[0], + `${FOLLOWEE_HASH}:000000690100:${FOLLOWER}` + ) + const indexed = followeeHeightCreate.firstCall.args[1] + assert.equal(indexed.followerAddr, FOLLOWER) + assert.equal(indexed.followeePkHash, FOLLOWEE_HASH) + assert.equal(indexed.unfollow, false) + assert.equal(indexed.txid, 'follow-tx') + assert.equal(indexed.blockHeight, 690100) + }) + + it('should mark an unfollow entry as unfollow true', async () => { + await handleFollow(baseCtx({ + decoded: { + action: 'unfollow', + prefix: PREFIX_UNFOLLOW, + pushDatas: [PREFIX_UNFOLLOW, Buffer.from(FOLLOWEE_HASH, 'hex')] + } + })) + + assert.equal(followCreate.firstCall.args[1].unfollow, true) + assert.equal(followeeHeightCreate.callCount, 1) + assert.equal(followeeHeightCreate.firstCall.args[1].unfollow, true) + }) + + it('should use the same followeeHeights key when a follow is reprocessed', async () => { + await handleFollow(baseCtx()) + await handleFollow(baseCtx()) + + assert.equal(followeeHeightCreate.callCount, 2) + assert.equal( + followeeHeightCreate.firstCall.args[0], + followeeHeightCreate.secondCall.args[0] + ) + }) + + it('should scope the followeeHeights key to the follower and height', async () => { + await handleFollow(baseCtx({ signerAddr: FOLLOWER2, blockHeight: 690200 })) + + assert.equal( + followeeHeightCreate.firstCall.args[0], + `${FOLLOWEE_HASH}:000000690200:${FOLLOWER2}` + ) + }) + + it('should log a process error when push data count is not 2', async () => { + await handleFollow(baseCtx({ + decoded: { pushDatas: [Buffer.from('6d06', 'hex'), Buffer.alloc(20, 1), Buffer.alloc(20, 2)] } + })) + + assert.equal(processErrorCreate.callCount, 1) + assert.include(processErrorCreate.firstCall.args[1].error, 'invalid follow push data count') + assert.equal(followCreate.callCount, 0) + assert.equal(followeeHeightCreate.callCount, 0) + }) + + it('should log a process error when the followee hash has the wrong size', async () => { + await handleFollow(baseCtx({ + decoded: { pushDatas: [Buffer.from('6d06', 'hex'), Buffer.alloc(32, 1)] } + })) + + assert.equal(processErrorCreate.callCount, 1) + assert.include(processErrorCreate.firstCall.args[1].error, 'follow pk hash wrong size') + assert.equal(followCreate.callCount, 0) + assert.equal(followeeHeightCreate.callCount, 0) + }) +})