diff --git a/psf-memo-client/test/unit/topic-discovery-page.test.js b/psf-memo-client/test/unit/topic-discovery-page.test.js index 0cb4201..bd4682d 100644 --- a/psf-memo-client/test/unit/topic-discovery-page.test.js +++ b/psf-memo-client/test/unit/topic-discovery-page.test.js @@ -110,6 +110,29 @@ test('getTopic returns null when the topic is not loaded', async () => { assert.equal(page.getTopic('bitcoin'), null) }) +test('openTopic navigates to the encoded topic feed path', () => { + const calls = [] + const page = new TopicDiscoveryPage({ + memoDb: makeMemoDb({ topics: [] }), + navigate: (path) => calls.push(path) + }) + + const result = page.openTopic('space room') + + assert.deepEqual(result, { path: '/topics/space%20room' }) + assert.deepEqual(calls, ['/topics/space%20room']) +}) + +test('topicFeedPath percent-encodes the room name', () => { + assert.equal(TopicDiscoveryPage.topicFeedPath('a/b'), '/topics/a%2Fb') +}) + +test('openTopic uses a no-op navigate by default', () => { + const page = new TopicDiscoveryPage({ memoDb: makeMemoDb({ topics: [] }) }) + + assert.deepEqual(page.openTopic('bitcoin'), { path: '/topics/bitcoin' }) +}) + test('exposes the topics page path', () => { assert.equal(TopicDiscoveryPage.TOPICS_PATH, '/topics') }) diff --git a/psf-memo-db/src/adapters/topic-query.js b/psf-memo-db/src/adapters/topic-query.js index 5cf9f6c..73983ca 100644 --- a/psf-memo-db/src/adapters/topic-query.js +++ b/psf-memo-db/src/adapters/topic-query.js @@ -47,6 +47,7 @@ class TopicQuery { this.listRoomFollowers = this.listRoomFollowers.bind(this) this.roomFromKey = this.roomFromKey.bind(this) this.txidFromKey = this.txidFromKey.bind(this) + this.roomRange = this.roomRange.bind(this) this.followAddrFromValue = this.followAddrFromValue.bind(this) } @@ -60,6 +61,13 @@ class TopicQuery { return parts[parts.length - 1] } + // Key range bounding every record for a room. The trailing \uffff sorts + // after any address or txid segment, so the range is exclusive of other + // rooms regardless of their name. + roomRange (room) { + return { gte: `${room}:`, lte: `${room}:\uffff` } + } + // The topicSummaries key is the room name; fall back to the key when the // stored value omits it. summaryRoom (key, value) { @@ -110,11 +118,9 @@ class TopicQuery { async getTopicPostTxids (room, { limit, offset, viewerAddr = null }) { const mutedAddrs = await loadMutedAddrs(this.muteQuery, viewerAddr) - const start = `${room}:` - const end = `${room}:\uffff` const entries = [] - for await (const [key, value] of this.roomsDb.iterator({ gte: start, lte: end })) { + for await (const [key, value] of this.roomsDb.iterator(this.roomRange(room))) { if (value?.type !== 'post') continue const txid = (value && typeof value.txid === 'string') ? value.txid : this.txidFromKey(key) if (await isMutedPost((t) => this.postsDb.get(t).catch(() => null), txid, mutedAddrs)) continue @@ -144,10 +150,8 @@ class TopicQuery { // Return the cash addresses that currently follow the room. async listRoomFollowers (room) { - const start = `${room}:` - const end = `${room}:\uffff` const followers = [] - for await (const [key, value] of this.roomsDb.iterator({ gte: start, lte: end })) { + for await (const [key, value] of this.roomsDb.iterator(this.roomRange(room))) { if (value?.type !== 'follow') continue if (value?.unfollow === true) continue const addr = this.followAddrFromValue(value, key) diff --git a/psf-memo-db/src/lib/backfill-topic-indexes.js b/psf-memo-db/src/lib/backfill-topic-indexes.js index dba4ca4..ffdc253 100644 --- a/psf-memo-db/src/lib/backfill-topic-indexes.js +++ b/psf-memo-db/src/lib/backfill-topic-indexes.js @@ -28,20 +28,31 @@ export function topicRecencyKey (blockHeight, room) { return `${String(inverted).padStart(HEIGHT_PAD, '0')}:${room}` } +// The room name is stored on the record; fall back to the first key segment so +// a record written without it is still attributed to a room. +function roomFromEntry (key, value) { + if (value && typeof value.room === 'string') return value.room + return String(key).split(':')[0] +} + +// Fold one post entry into its room summary, tracking the newest height. +function applyPost (summary, value) { + summary.postCount++ + const height = value.blockHeight ?? 0 + if (height > summary.lastHeight) summary.lastHeight = height +} + async function collectSummaries (roomsDb) { const summaries = new Map() for await (const [key, value] of roomsDb.iterator()) { - const room = (value && typeof value.room === 'string') ? value.room : String(key).split(':')[0] + const room = roomFromEntry(key, value) if (!summaries.has(room)) { summaries.set(room, { room, postCount: 0, lastHeight: 0 }) } if (value?.type === 'post') { - const summary = summaries.get(room) - summary.postCount++ - const height = value.blockHeight ?? 0 - if (height > summary.lastHeight) summary.lastHeight = height + applyPost(summaries.get(room), value) } } diff --git a/psf-memo-db/test/property/backfill-topic-indexes.property.test.js b/psf-memo-db/test/property/backfill-topic-indexes.property.test.js new file mode 100644 index 0000000..555c16a --- /dev/null +++ b/psf-memo-db/test/property/backfill-topic-indexes.property.test.js @@ -0,0 +1,193 @@ +/* + Property tests for the topic index backfill. + + The unit tests probe backfillTopicIndexes at a few fixed fixtures. These + properties pin down the invariants that must hold over arbitrary rooms-store + contents: + + - Conservation: every room in the rooms store gets a summary whose + postCount equals its number of post entries and whose lastHeight is the + newest post height (0 for follow-only rooms). + - Recency shape: topicRecency holds exactly one record per summarized room, + keyed at that room's lastHeight, with the matching value. + - Idempotence: running the backfill twice produces identical indexes, and + stale records from earlier runs are removed. + - Key ordering: the inverted topicRecency key sorts newest-height-first and + breaks ties by room name ascending, which is the read side's contract. +*/ + +import test from 'node:test' + +import { seededRandom, forAll, intGen } from './harness.js' +import { backfillTopicIndexes, topicRecencyKey } from '../../src/lib/backfill-topic-indexes.js' + +const rng = seededRandom(20260917) + +const ROOMS = ['room-0', 'room-1', 'room-2', 'room-3'] +const FOLLOW_ADDRS = ['addr-0', 'addr-1', 'addr-2'] + +// In-memory LevelDB-shaped store supporting the iterator/get/put/del surface +// the backfill uses. +function makeDb (records = []) { + const store = new Map(records) + return { + store, + async get (key) { + if (!store.has(key)) { + const err = new Error('not found') + err.notFound = true + throw err + } + return store.get(key) + }, + async put (key, value) { + store.set(key, value) + }, + async del (key) { + store.delete(key) + }, + async * iterator () { + for (const key of Array.from(store.keys()).sort()) { + yield [key, store.get(key)] + } + } + } +} + +// Random rooms-store contents: each room gets a random number of posts and +// follow records, including unfollows and follows with no posts. +function roomsGen () { + return () => { + const entries = [] + let seq = 0 + for (const room of ROOMS) { + const postCount = intGen(rng, 0, 6)() + for (let i = 0; i < postCount; i++) { + entries.push([ + `${room}:post-${seq}`, + { room, txid: `post-${seq}`, type: 'post', blockHeight: intGen(rng, 0, 9000000)() } + ]) + seq++ + } + + const followCount = intGen(rng, 0, 3)() + for (let i = 0; i < followCount; i++) { + const addr = FOLLOW_ADDRS[Math.floor(rng() * FOLLOW_ADDRS.length)] + entries.push([ + `${room}:${addr}-${seq}`, + { room, addr, type: 'follow', unfollow: rng() < 0.4 } + ]) + seq++ + } + } + return entries + } +} + +function expectedSummaries (entries) { + const summaries = new Map() + for (const [, value] of entries) { + const room = value.room + if (!summaries.has(room)) summaries.set(room, { room, postCount: 0, lastHeight: 0 }) + if (value.type === 'post') { + const summary = summaries.get(room) + summary.postCount++ + const height = value.blockHeight ?? 0 + if (height > summary.lastHeight) summary.lastHeight = height + } + } + return summaries +} + +function snapshot (db) { + return JSON.stringify(Array.from(db.store.entries()).sort()) +} + +test('backfill conserves every room summary and builds the recency index', async () => { + await forAll( + roomsGen(), + async (entries) => { + const roomsDb = makeDb(entries) + const topicSummariesDb = makeDb() + const topicRecencyDb = makeDb() + + const result = await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb }) + + const expected = expectedSummaries(entries) + if (result.rooms !== expected.size) return false + if (topicSummariesDb.store.size !== expected.size) return false + + for (const [room, summary] of expected) { + const stored = topicSummariesDb.store.get(room) + if (!stored) return false + if (stored.postCount !== summary.postCount) return false + if (stored.lastHeight !== summary.lastHeight) return false + + const key = topicRecencyKey(summary.lastHeight, room) + const recency = topicRecencyDb.store.get(key) + if (!recency) return false + if (recency.room !== room) return false + if (recency.blockHeight !== summary.lastHeight) return false + } + + // Exactly one recency record per room. + if (topicRecencyDb.store.size !== expected.size) return false + + // Idempotence: a second run leaves the indexes byte-for-byte identical. + const beforeSummaries = snapshot(topicSummariesDb) + const beforeRecency = snapshot(topicRecencyDb) + await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb }) + if (snapshot(topicSummariesDb) !== beforeSummaries) return false + if (snapshot(topicRecencyDb) !== beforeRecency) return false + + return true + }, + { label: 'backfill conservation, recency shape, and idempotence' } + ) +}) + +test('backfill drops stale recency and summary records from earlier runs', async () => { + await forAll( + roomsGen(), + async (entries) => { + const roomsDb = makeDb(entries) + const topicSummariesDb = makeDb([ + ['stale-room', { room: 'stale-room', postCount: 9, lastHeight: 999999 }] + ]) + const topicRecencyDb = makeDb([ + [topicRecencyKey(999999, 'stale-room'), { room: 'stale-room', blockHeight: 999999 }], + [topicRecencyKey(123456, 'another-stale'), { room: 'another-stale', blockHeight: 123456 }] + ]) + + await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb }) + + // No record may survive for a room absent from the rooms store. + for (const [room] of topicSummariesDb.store) { + if (room === 'stale-room') return false + } + for (const [, value] of topicRecencyDb.store) { + if (value.room === 'stale-room' || value.room === 'another-stale') return false + } + + return true + }, + { label: 'backfill removes stale records' } + ) +}) + +test('topicRecencyKey sorts newest height first and ties by room ascending', async () => { + await forAll( + () => ({ + h1: intGen(rng, 0, 5000000)(), + h2: intGen(rng, 0, 5000000)(), + r1: ROOMS[Math.floor(rng() * ROOMS.length)], + r2: ROOMS[Math.floor(rng() * ROOMS.length)] + }), + ({ h1, h2, r1, r2 }) => { + const lower = topicRecencyKey(h1, r1) < topicRecencyKey(h2, r2) + const expected = h1 > h2 || (h1 === h2 && r1 < r2) + return lower === expected + }, + { label: 'topicRecencyKey ordering' } + ) +}) diff --git a/psf-memo-db/test/unit/adapters/topic-query.unit.js b/psf-memo-db/test/unit/adapters/topic-query.unit.js index e4ac0de..22ed58c 100644 --- a/psf-memo-db/test/unit/adapters/topic-query.unit.js +++ b/psf-memo-db/test/unit/adapters/topic-query.unit.js @@ -144,6 +144,26 @@ describe('#TopicQuery', () => { }) }) + describe('#summaryRoom', () => { + it('should return the room from the value when present', () => { + assert.equal(uut.summaryRoom('ignored', { room: 'bitcoin' }), 'bitcoin') + }) + + it('should fall back to the key when the value omits the room', () => { + assert.equal(uut.summaryRoom('cash', {}), 'cash') + }) + }) + + describe('#recencyRoom', () => { + it('should return the room from the value when present', () => { + assert.equal(uut.recencyRoom('ignored', { room: 'bitcoin' }), 'bitcoin') + }) + + it('should parse the room out of the inverted-height key when the value omits it', () => { + assert.equal(uut.recencyRoom(topicRecencyKey(600100, 'bitcoin'), {}), 'bitcoin') + }) + }) + describe('#listTopics', () => { const summaries = [ ['memo', { room: 'memo', postCount: 5, lastHeight: 600500 }], diff --git a/psf-memo-db/test/unit/lib/backfill-topic-indexes.unit.js b/psf-memo-db/test/unit/lib/backfill-topic-indexes.unit.js index cabfc28..3a7701f 100644 --- a/psf-memo-db/test/unit/lib/backfill-topic-indexes.unit.js +++ b/psf-memo-db/test/unit/lib/backfill-topic-indexes.unit.js @@ -86,4 +86,17 @@ describe('#backfillTopicIndexes', () => { assert.isFalse(topicRecencyDb.store.has(topicRecencyKey(0, 'stale'))) assert.isTrue(topicRecencyDb.store.has(topicRecencyKey(600200, 'bitcoin'))) }) + + it('should fall back to the key room segment when a record omits the room', async () => { + const roomsDb = makeDb([ + ['cash:post-1', { type: 'post', blockHeight: 500 }] + ]) + const topicSummariesDb = makeDb() + const topicRecencyDb = makeDb() + + await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb }) + + assert.deepEqual(topicSummariesDb.store.get('cash'), { room: 'cash', postCount: 1, lastHeight: 500 }) + assert.deepEqual(topicRecencyDb.store.get(topicRecencyKey(500, 'cash')), { room: 'cash', blockHeight: 500 }) + }) }) diff --git a/psf-memo-indexer/test/property/topic-indexing.property.test.js b/psf-memo-indexer/test/property/topic-indexing.property.test.js new file mode 100644 index 0000000..9a1ac76 --- /dev/null +++ b/psf-memo-indexer/test/property/topic-indexing.property.test.js @@ -0,0 +1,227 @@ +/* + Property tests for the topic recency indexes maintained by the indexer. + + The unit tests probe handleTopicMessage, handleTopicFollow, and the + topic-indexing helpers at a few fixed fixtures. These properties pin down + the invariants that must hold over arbitrary interleavings of topic posts, + follows, and unfollows: + + - Conservation: each room's summary postCount equals the number of distinct + topic-message txids seen for that room, and lastHeight equals the newest + post height (0 when the room has no posts). + - Recency shape: topicRecency holds exactly one record per summarized room, + keyed at that room's lastHeight, with the matching value. + - Idempotence: replaying the whole action sequence leaves every index + unchanged, so reprocessing a block never double-counts. + - Key ordering: the inverted topicRecency key sorts newest-height-first and + breaks ties by room name ascending, which is the read side's contract. +*/ + +import test from 'node:test' + +import { seededRandom, forAll, intGen } from './harness.js' +import { handleTopicMessage } from '../../src/use-cases/action-types/topic-message.js' +import { handleTopicFollow } from '../../src/use-cases/action-types/topic-follow.js' +import { topicRecencyKey } from '../../src/use-cases/action-types/helpers.js' +import { PREFIX_TOPIC_FOLLOW, PREFIX_TOPIC_UNFOLLOW } from '../../src/lib/memo-codes.js' + +const rng = seededRandom(20260917) + +const ROOMS = ['room-0', 'room-1', 'room-2', 'room-3'] +const FOLLOW_ADDRS = ['bitcoincash:qaddr-0', 'bitcoincash:qaddr-1', 'bitcoincash:qaddr-2'] + +function makeDb () { + const store = new Map() + return { + store, + async get (key) { + if (!store.has(key)) { + const err = new Error('not found') + err.notFound = true + throw err + } + return store.get(key) + }, + async create (key, value) { + if (!store.has(key)) store.set(key, value) + return { success: true } + }, + async update (key, value) { + store.set(key, value) + return { success: true } + }, + async delete (key) { + store.delete(key) + return { success: true } + } + } +} + +function makeAdapters () { + return { + postDb: makeDb(), + postHeightDb: makeDb(), + addrPostHeightDb: makeDb(), + roomDb: makeDb(), + topicSummaryDb: makeDb(), + topicRecencyDb: makeDb(), + processErrorDb: makeDb() + } +} + +async function processEvent (adapters, event) { + if (event.kind === 'post') { + const prefix = Buffer.from('6d0c', 'hex') + await handleTopicMessage({ + adapters, + txid: event.txid, + signerAddr: 'bitcoincash:qauthor', + seen: 1, + blockHeight: event.height, + decoded: { + action: 'topic-message', + prefix, + pushDatas: [prefix, Buffer.from(event.room, 'utf8'), Buffer.from(event.text, 'utf8')] + } + }) + return + } + + const prefix = event.unfollow ? PREFIX_TOPIC_UNFOLLOW : PREFIX_TOPIC_FOLLOW + await handleTopicFollow({ + adapters, + txid: event.txid, + signerAddr: event.addr, + seen: 1, + blockHeight: event.height, + decoded: { + action: 'topic-follow', + prefix, + pushDatas: [prefix, Buffer.from(event.room, 'utf8')] + } + }) +} + +// Random interleaving of topic posts, follows, and unfollows across a small +// room set. Txids are unique per event so the expected conservation values are +// well defined even when the sequence is replayed. +function eventSequenceGen () { + return () => { + const count = intGen(rng, 0, 40)() + const events = [] + for (let i = 0; i < count; i++) { + const room = ROOMS[Math.floor(rng() * ROOMS.length)] + if (rng() < 0.7) { + events.push({ + kind: 'post', + txid: `txid-${i}`, + room, + height: intGen(rng, 0, 9000000)(), + text: `text-${i}` + }) + } else { + events.push({ + kind: 'follow', + txid: `txid-${i}`, + room, + addr: FOLLOW_ADDRS[Math.floor(rng() * FOLLOW_ADDRS.length)], + height: intGen(rng, 0, 9000000)(), + unfollow: rng() < 0.4 + }) + } + } + return events + } +} + +// Expected summaries derived straight from the event list, independent of the +// implementation under test. +function expectedSummaries (events) { + const summaries = new Map() + const ensure = (room) => { + if (!summaries.has(room)) summaries.set(room, { room, postCount: 0, lastHeight: 0 }) + return summaries.get(room) + } + + for (const event of events) { + if (event.kind === 'post') { + const summary = ensure(event.room) + summary.postCount++ + if (event.height > summary.lastHeight) summary.lastHeight = event.height + } else if (!event.unfollow) { + // Only an active follow creates a zero-post room; unfollows do not. + ensure(event.room) + } + } + + return summaries +} + +function snapshot (adapters) { + return JSON.stringify({ + summaries: Array.from(adapters.topicSummaryDb.store.entries()).sort(), + recency: Array.from(adapters.topicRecencyDb.store.entries()).sort() + }) +} + +test('topic indexes conserve counts and heights over arbitrary action sequences', async () => { + await forAll( + eventSequenceGen(), + async (events) => { + const adapters = makeAdapters() + for (const event of events) { + await processEvent(adapters, event) + } + + const expected = expectedSummaries(events) + + // Conservation: one summary per room with the exact count and height. + if (adapters.topicSummaryDb.store.size !== expected.size) return false + for (const [room, summary] of expected) { + const stored = adapters.topicSummaryDb.store.get(room) + if (!stored) return false + if (stored.postCount !== summary.postCount) return false + if (stored.lastHeight !== summary.lastHeight) return false + } + + // Recency shape: exactly one record per room, at its lastHeight. + if (adapters.topicRecencyDb.store.size !== expected.size) return false + for (const [room, summary] of expected) { + const key = topicRecencyKey(summary.lastHeight, room) + const record = adapters.topicRecencyDb.store.get(key) + if (!record) return false + if (record.room !== room) return false + if (record.blockHeight !== summary.lastHeight) return false + } + + // Idempotence: replaying the sequence changes nothing. + const before = snapshot(adapters) + for (const event of events) { + await processEvent(adapters, event) + } + if (snapshot(adapters) !== before) return false + + return true + }, + { label: 'topic index conservation, shape, and idempotence' } + ) +}) + +test('topicRecencyKey sorts newest height first and ties by room ascending', async () => { + await forAll( + () => { + return { + h1: intGen(rng, 0, 5000000)(), + h2: intGen(rng, 0, 5000000)(), + r1: ROOMS[Math.floor(rng() * ROOMS.length)], + r2: ROOMS[Math.floor(rng() * ROOMS.length)] + } + }, + ({ h1, h2, r1, r2 }) => { + const lower = topicRecencyKey(h1, r1) < topicRecencyKey(h2, r2) + const expected = h1 > h2 || (h1 === h2 && r1 < r2) + return lower === expected + }, + { label: 'topicRecencyKey ordering' } + ) +}) diff --git a/psf-memo-indexer/test/unit/use-cases/action-types/topic-follow.unit.js b/psf-memo-indexer/test/unit/use-cases/action-types/topic-follow.unit.js index 98d4d21..3a254db 100644 --- a/psf-memo-indexer/test/unit/use-cases/action-types/topic-follow.unit.js +++ b/psf-memo-indexer/test/unit/use-cases/action-types/topic-follow.unit.js @@ -103,4 +103,26 @@ describe('#handleTopicFollow topic indexes', () => { assert.equal(adapters.topicSummaryDb.store.size, 0) assert.equal(adapters.topicRecencyDb.store.size, 0) }) + + it('should log a process error and write nothing when the push data count is invalid', async () => { + const adapters = makeAdapters() + + await handleTopicFollow({ + adapters, + txid: 'follow-bad', + signerAddr: 'bitcoincash:qaddr-a', + seen: 1, + blockHeight: 600000, + decoded: { + action: 'topic-follow', + prefix: PREFIX_TOPIC_FOLLOW, + pushDatas: [PREFIX_TOPIC_FOLLOW] + } + }) + + assert.equal(adapters.processErrorDb.store.size, 1) + assert.equal(adapters.roomDb.store.size, 0) + assert.equal(adapters.topicSummaryDb.store.size, 0) + assert.equal(adapters.topicRecencyDb.store.size, 0) + }) }) diff --git a/psf-memo-indexer/test/unit/use-cases/action-types/topic-message.unit.js b/psf-memo-indexer/test/unit/use-cases/action-types/topic-message.unit.js index 18acbb8..b189123 100644 --- a/psf-memo-indexer/test/unit/use-cases/action-types/topic-message.unit.js +++ b/psf-memo-indexer/test/unit/use-cases/action-types/topic-message.unit.js @@ -132,4 +132,48 @@ describe('#handleTopicMessage topic indexes', () => { }) assert.equal(adapters.topicRecencyDb.store.size, 1) }) + + it('should log a process error and index nothing for an invalid push data count', async () => { + const adapters = makeAdapters() + const prefix = Buffer.from('6d0c', 'hex') + + await handleTopicMessage({ + adapters, + txid: 'topic-bad', + signerAddr: 'bitcoincash:qaddr-a', + seen: 1, + blockHeight: 600100, + decoded: { + action: 'topic-message', + prefix, + pushDatas: [prefix, Buffer.from('bitcoin', 'utf8')] + } + }) + + assert.equal(adapters.processErrorDb.store.size, 1) + assert.equal(adapters.topicSummaryDb.store.size, 0) + assert.equal(adapters.topicRecencyDb.store.size, 0) + }) + + it('should log a process error and index nothing for an oversized topic message', async () => { + const adapters = makeAdapters() + const prefix = Buffer.from('6d0c', 'hex') + + await handleTopicMessage({ + adapters, + txid: 'topic-big', + signerAddr: 'bitcoincash:qaddr-a', + seen: 1, + blockHeight: 600100, + decoded: { + action: 'topic-message', + prefix, + pushDatas: [prefix, Buffer.from('bitcoin', 'utf8'), Buffer.from('x'.repeat(65000), 'utf8')] + } + }) + + assert.equal(adapters.processErrorDb.store.size, 1) + assert.equal(adapters.topicSummaryDb.store.size, 0) + assert.equal(adapters.topicRecencyDb.store.size, 0) + }) })