diff --git a/psf-memo-client/acceptance/lib/handlers.js b/psf-memo-client/acceptance/lib/handlers.js index bec92cb..2c2e227 100644 --- a/psf-memo-client/acceptance/lib/handlers.js +++ b/psf-memo-client/acceptance/lib/handlers.js @@ -361,13 +361,18 @@ function makeMemoDb () { async getMuteState (muterAddr, muteeAddr) { return muteState[`${muterAddr}:${muteeAddr}`] || false }, - async getTopics () { + async getTopics ({ limit = 50, offset = 0 } = {}) { const list = [] for (const [room, postCount] of topicCounts.entries()) { list.push({ room, postCount }) } list.sort((a, b) => a.room.localeCompare(b.room)) - return { topics: list } + const total = list.length + const page = list.slice(offset, offset + limit) + return { + topics: page, + pagination: { limit, offset, total, hasMore: offset + page.length < total } + } }, async getTopicPosts (room, { limit = 50, offset = 0, viewer = null } = {}) { let all = topicPosts[room] || [] @@ -2141,6 +2146,54 @@ const handlers = [ world.currentPath = TopicDiscoveryPage.TOPICS_PATH } }, + { + name: 'open topics page at offset', + pattern: /^I open the topics page at offset (<[A-Za-z0-9_]+>)$/, + async run (m, example, world) { + const offset = parseInt(resolveParam(m[1], example), 10) + await world.topicDiscoveryPage.load({ limit: 50, offset }) + world.currentPath = TopicDiscoveryPage.TOPICS_PATH + } + }, + { + name: 'API serves N topics', + pattern: /^the psf-memo-db API serves (<[A-Za-z0-9_]+>) topics$/, + run (m, example, world) { + const count = parseInt(resolveParam(m[1], example), 10) + for (let i = 0; i < count; i++) { + world.memoDb.addTopic(`topic-${String(i + 1).padStart(3, '0')}`, 1) + } + } + }, + { + name: 'topics page shows N topics', + pattern: /^the topics page shows (<[A-Za-z0-9_]+>) topics$/, + run (m, example, world) { + const expected = parseInt(resolveParam(m[1], example), 10) + const actual = world.topicDiscoveryPage.topics.length + if (actual !== expected) { + throw new Error(`Expected ${expected} topics on the topics page, got ${actual}.`) + } + } + }, + { + name: 'topics page can load more topics', + pattern: /^the topics page can load more topics$/, + run (m, example, world) { + if (!world.topicDiscoveryPage.canLoadMore()) { + throw new Error('Expected the topics page to have more topics, but pagination says there are none.') + } + } + }, + { + name: 'topics page has no more topics', + pattern: /^the topics page has no more topics$/, + run (m, example, world) { + if (world.topicDiscoveryPage.canLoadMore()) { + throw new Error('Expected the topics page to have no more topics, but pagination says there are more.') + } + } + }, { name: 'topics page shows topic count', pattern: /^the topics page shows the topic () with () posts$/, diff --git a/psf-memo-client/src/components/app-body/topics/index.js b/psf-memo-client/src/components/app-body/topics/index.js index eb3cbf4..88c8a8d 100644 --- a/psf-memo-client/src/components/app-body/topics/index.js +++ b/psf-memo-client/src/components/app-body/topics/index.js @@ -1,10 +1,10 @@ /* - Display the list of Memo topics served by psf-memo-db. + Display a page of Memo topics served by psf-memo-db. */ // Global npm libraries import React, { useState, useEffect } from 'react' -import { Container, Row, Col, Spinner, ListGroup } from 'react-bootstrap' +import { Container, Row, Col, Spinner, ListGroup, Button } from 'react-bootstrap' import { useNavigate } from 'react-router-dom' // Local libraries @@ -12,11 +12,15 @@ import MemoDb from '../../../services/memo-db' import TopicDiscoveryPage from '../../../services/topic-discovery-page' import '../../../App.css' +const PAGE_SIZE = 50 + function Topics (props) { const navigate = useNavigate() const [loading, setLoading] = useState(true) const [error, setError] = useState(null) const [topics, setTopics] = useState([]) + const [pagination, setPagination] = useState(null) + const [offset, setOffset] = useState(0) useEffect(() => { const loadTopics = async () => { @@ -26,23 +30,36 @@ function Topics (props) { try { const memoDb = new MemoDb() const page = new TopicDiscoveryPage({ memoDb, navigate }) - const result = await page.load() + const result = await page.load({ limit: PAGE_SIZE, offset }) setTopics(result.topics || []) + setPagination(result.pagination || null) } catch (err) { setError(err.message || 'Failed to load topics') setTopics([]) + setPagination(null) } setLoading(false) } loadTopics() - }, [navigate]) + }, [navigate, offset]) const handleClick = (room) => { navigate(TopicDiscoveryPage.topicFeedPath(room)) } + const canGoBack = offset > 0 + const canGoNext = pagination?.hasMore ?? false + + const handlePrevious = () => { + setOffset((prev) => Math.max(0, prev - PAGE_SIZE)) + } + + const handleNext = () => { + setOffset((prev) => prev + PAGE_SIZE) + } + return ( @@ -50,6 +67,13 @@ function Topics (props) {

Topics

Discover Memo conversations organized by topic.

+ + {pagination && topics.length > 0 && ( + + Showing {pagination.offset + 1}– + {pagination.offset + topics.length} of {pagination.total} + + )}
{error && ( @@ -84,6 +108,26 @@ function Topics (props) { ))} )} + + {!loading && !error && (pagination || offset > 0) && ( +
+ + + +
+ )}
diff --git a/psf-memo-client/src/services/memo-db.js b/psf-memo-client/src/services/memo-db.js index 93c7f92..5541281 100644 --- a/psf-memo-client/src/services/memo-db.js +++ b/psf-memo-client/src/services/memo-db.js @@ -44,8 +44,8 @@ class MemoDb { return this._getList(`/mute/muted/${encodeURIComponent(muterAddr)}`, 'getMuted', 'muted') } - async getTopics () { - return this.getRecent('/topics', 'getTopics', {}) + async getTopics ({ limit = 50, offset = 0 } = {}) { + return this.getRecent('/topics', 'getTopics', { limit, offset }) } async getTopicPosts (room, { limit = 50, offset = 0, viewer = null } = {}) { diff --git a/psf-memo-client/src/services/topic-discovery-page.js b/psf-memo-client/src/services/topic-discovery-page.js index 56c953b..a08009d 100644 --- a/psf-memo-client/src/services/topic-discovery-page.js +++ b/psf-memo-client/src/services/topic-discovery-page.js @@ -1,29 +1,23 @@ /* - Topic Discovery Page behavior: load and display the list of Memo topics. + Topic Discovery Page behavior: load and display a page of Memo topics. This is the testable controller behind the React "Topics" page. It wraps - the MemoDb client and exposes the loaded topics so the view can render each - topic name and post count. + the MemoDb client and exposes the loaded topics and pagination so the view + can render each topic name and post count and move between pages. */ +const PaginatedPage = require('./paginated-page') + const TOPICS_PATH = '/topics' -class TopicDiscoveryPage { +class TopicDiscoveryPage extends PaginatedPage { constructor (deps = {}) { - this.memoDb = deps.memoDb || null + super(deps, { + listField: 'topics', + loadMethod: 'getTopics', + errorMessage: 'Topic discovery page requires a memo db client.' + }) this.navigate = deps.navigate || (() => {}) - this.topics = [] - } - - async load () { - if (!this.memoDb) { - throw new Error('Topic discovery page requires a memo db client.') - } - - const data = await this.memoDb.getTopics() - this.topics = data.topics || [] - - return { topics: this.topics } } getTopic (room) { diff --git a/psf-memo-client/test/unit/topic-discovery-page.test.js b/psf-memo-client/test/unit/topic-discovery-page.test.js index 691dadd..0cb4201 100644 --- a/psf-memo-client/test/unit/topic-discovery-page.test.js +++ b/psf-memo-client/test/unit/topic-discovery-page.test.js @@ -8,10 +8,13 @@ const test = require('node:test') const assert = require('node:assert/strict') const TopicDiscoveryPage = require('../../src/services/topic-discovery-page') -function makeMemoDb (topics) { +function makeMemoDb ({ topics = [], pagination = null } = {}) { + const calls = [] return { - async getTopics () { - return { topics } + calls, + async getTopics (opts) { + calls.push(opts) + return { topics, pagination } } } } @@ -21,13 +24,56 @@ test('load returns topics with post counts', async () => { { room: 'bitcoin', postCount: 2 }, { room: 'cash', postCount: 1 } ] - const page = new TopicDiscoveryPage({ memoDb: makeMemoDb(topics) }) + const page = new TopicDiscoveryPage({ memoDb: makeMemoDb({ topics }) }) const result = await page.load() assert.deepEqual(result.topics, topics) }) +test('load defaults to a page of 50 from offset 0', async () => { + const memoDb = makeMemoDb({ topics: [] }) + const page = new TopicDiscoveryPage({ memoDb }) + + await page.load() + + assert.deepEqual(memoDb.calls[0], { limit: 50, offset: 0 }) +}) + +test('load forwards limit and offset and stores pagination', async () => { + const memoDb = makeMemoDb({ + topics: [{ room: 'bitcoin', postCount: 2 }], + pagination: { limit: 50, offset: 50, total: 60, hasMore: false } + }) + const page = new TopicDiscoveryPage({ memoDb }) + + const result = await page.load({ limit: 50, offset: 50 }) + + assert.deepEqual(memoDb.calls[0], { limit: 50, offset: 50 }) + assert.deepEqual(result.pagination, { limit: 50, offset: 50, total: 60, hasMore: false }) + assert.equal(page.pagination.offset, 50) +}) + +test('canLoadMore reflects the pagination hasMore flag', async () => { + const more = new TopicDiscoveryPage({ + memoDb: makeMemoDb({ + topics: [{ room: 'a', postCount: 0 }], + pagination: { limit: 50, offset: 0, total: 60, hasMore: true } + }) + }) + await more.load() + assert.equal(more.canLoadMore(), true) + + const last = new TopicDiscoveryPage({ + memoDb: makeMemoDb({ + topics: [{ room: 'a', postCount: 0 }], + pagination: { limit: 50, offset: 50, total: 60, hasMore: false } + }) + }) + await last.load() + assert.equal(last.canLoadMore(), false) +}) + test('load throws when no memo db client is provided', async () => { const page = new TopicDiscoveryPage({}) @@ -39,7 +85,7 @@ test('load throws when no memo db client is provided', async () => { test('stores the provided navigate function', () => { const navigate = () => {} - const page = new TopicDiscoveryPage({ memoDb: makeMemoDb([]), navigate }) + const page = new TopicDiscoveryPage({ memoDb: makeMemoDb({ topics: [] }), navigate }) assert.equal(page.navigate, navigate) }) @@ -49,7 +95,7 @@ test('getTopic returns the matching topic', async () => { { room: 'bitcoin', postCount: 2 }, { room: 'cash', postCount: 1 } ] - const page = new TopicDiscoveryPage({ memoDb: makeMemoDb(topics) }) + const page = new TopicDiscoveryPage({ memoDb: makeMemoDb({ topics }) }) await page.load() @@ -57,7 +103,7 @@ test('getTopic returns the matching topic', async () => { }) test('getTopic returns null when the topic is not loaded', async () => { - const page = new TopicDiscoveryPage({ memoDb: makeMemoDb([]) }) + const page = new TopicDiscoveryPage({ memoDb: makeMemoDb({ topics: [] }) }) await page.load() diff --git a/psf-memo-db/acceptance/lib/handlers.js b/psf-memo-db/acceptance/lib/handlers.js index c103aa2..5caf9b2 100644 --- a/psf-memo-db/acceptance/lib/handlers.js +++ b/psf-memo-db/acceptance/lib/handlers.js @@ -28,6 +28,7 @@ import GetPoll from '../../src/use-cases/get-poll.js' import GetPollOptions from '../../src/use-cases/get-poll-options.js' import GetPollVotes from '../../src/use-cases/get-poll-votes.js' import { repairTxidEncoding } from '../../src/lib/repair-txid-encoding.js' +import { backfillTopicIndexes, topicRecencyKey } from '../../src/lib/backfill-topic-indexes.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const tmpDir = path.resolve(__dirname, '..', '..', 'tmp', 'acceptance') @@ -97,12 +98,16 @@ async function createWorld () { const postsGetCounter = { calls: 0 } const likesIteratorCounter = { calls: 0 } const postLikesIteratorCounter = { calls: 0 } + const roomsIteratorCounter = { calls: 0, entries: 0 } + const topicRecencyIteratorCounter = { calls: 0, entries: 0 } wrapIterator(adapters.level.postHeightsDb, postHeightsIteratorCounter) wrapIterator(adapters.level.addrPostHeightsDb, addrPostHeightsIteratorCounter) wrapIterator(adapters.level.postChildrenDb, postChildrenIteratorCounter) wrapGet(adapters.level.postsDb, postsGetCounter) wrapIterator(adapters.level.likesDb, likesIteratorCounter) wrapIterator(adapters.level.postLikesDb, postLikesIteratorCounter) + wrapIterator(adapters.level.roomsDb, roomsIteratorCounter) + wrapIterator(adapters.level.topicRecencyDb, topicRecencyIteratorCounter) const listRecentPosts = new ListRecentPosts({ adapters }) const listPostsByAddr = new ListPostsByAddr({ adapters }) @@ -145,6 +150,8 @@ async function createWorld () { postsGetCounter, likesIteratorCounter, postLikesIteratorCounter, + roomsIteratorCounter, + topicRecencyIteratorCounter, getLastResponse: () => lastResponse, setLastResponse: (resp) => { lastResponse = resp }, close: async () => { @@ -195,6 +202,16 @@ async function loadFixture (world, name) { return } + if (name === 'topic-indexes') { + await loadTopicIndexes(world) + return + } + + if (name === 'rooms-with-topics-and-follows') { + await loadRoomsWithTopicsAndFollows(world) + return + } + if (name === 'topic-follows') { await loadTopicFollows(world) return @@ -502,6 +519,56 @@ async function loadTopicsWithPosts (world) { for (const [txid, post] of Object.entries(posts)) { await world.adapters.level.postsDb.put(txid, post) } + + // Derived topic indexes (see topic-read.feature). + const summaries = [ + { room: 'bitcoin', postCount: 2, lastHeight: 300 }, + { room: 'cash', postCount: 1, lastHeight: 250 }, + { room: 'dev', postCount: 1, lastHeight: 400 }, + { room: 'lone', postCount: 0, lastHeight: 0 } + ] + for (const summary of summaries) { + await world.adapters.level.topicSummariesDb.put(summary.room, summary) + await world.adapters.level.topicRecencyDb.put( + topicRecencyKey(summary.lastHeight, summary.room), + { room: summary.room, blockHeight: summary.lastHeight } + ) + } +} + +// Fixture "topic-indexes" from topic-pagination.feature: indexes only. +async function loadTopicIndexes (world) { + const summaries = [ + { room: 'memo', postCount: 5, lastHeight: 600500 }, + { room: 'cash', postCount: 2, lastHeight: 600400 }, + { room: 'dance', postCount: 3, lastHeight: 600400 }, + { room: 'anime', postCount: 1, lastHeight: 600300 }, + { room: 'lone', postCount: 0, lastHeight: 0 }, + { room: 'quiet', postCount: 0, lastHeight: 0 } + ] + + for (const summary of summaries) { + await world.adapters.level.topicSummariesDb.put(summary.room, summary) + await world.adapters.level.topicRecencyDb.put( + topicRecencyKey(summary.lastHeight, summary.room), + { room: summary.room, blockHeight: summary.lastHeight } + ) + } +} + +// Fixture "rooms-with-topics-and-follows" from backfill-topic-indexes.feature. +async function loadRoomsWithTopicsAndFollows (world) { + const entries = [ + { key: 'bitcoin:post-100', room: 'bitcoin', txid: 'post-100', type: 'post', blockHeight: 600100 }, + { key: 'bitcoin:post-200', room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 600200 }, + { key: 'bitcoin:addr-f', room: 'bitcoin', addr: 'addr-f', type: 'follow', unfollow: false }, + { key: 'cash:post-250', room: 'cash', txid: 'post-250', type: 'post', blockHeight: 600250 }, + { key: 'lone:addr-f', room: 'lone', addr: 'addr-f', type: 'follow', unfollow: false } + ] + + for (const entry of entries) { + await world.adapters.level.roomsDb.put(entry.key, entry) + } } async function loadTopicFollows (world) { @@ -1119,7 +1186,7 @@ const handlers = [ }, { name: 'response lists topics in order', - pattern: /^the response lists topics in order ()$/, + pattern: /^the response lists topics in order (<[A-Za-z0-9_]+>)$/, run (m, example, world) { const expected = resolveParam(m[1], example).split(',').map((s) => s.trim()) const actual = world.getLastResponse().topics.map((t) => t.room) @@ -1163,6 +1230,157 @@ const handlers = [ await loadFixture(world, m[1]) } }, + { + name: 'db instance with rooms, posts, and topic index stores', + pattern: /^a psf-memo-db instance with rooms, posts, topicSummaries, and topicRecency stores$/, + async run () { + // World is already created with all stores. + } + }, + { + name: 'load fixture into rooms, posts, and topic index stores', + pattern: /^the fixture "(.+)" is loaded into the rooms, posts, topicSummaries, and topicRecency stores$/, + async run (m, example, world) { + await loadFixture(world, m[1]) + } + }, + { + name: 'db instance with topic index stores', + pattern: /^a psf-memo-db instance with topicSummaries and topicRecency stores$/, + async run () { + // World is already created with both topic index stores. + } + }, + { + name: 'load fixture into topic index stores', + pattern: /^the fixture "(.+)" is loaded into the topic index stores$/, + async run (m, example, world) { + await loadFixture(world, m[1]) + } + }, + { + name: 'db instance with rooms and topic index stores', + pattern: /^a psf-memo-db instance with rooms, topicSummaries, and topicRecency stores$/, + async run () { + // World is already created with all stores. + } + }, + { + name: 'request topics with limit and offset', + pattern: /^the client requests \/topics with limit () and offset ()$/, + async run (m, example, world) { + const limit = parseInt(resolveParam(m[1], example), 10) + const offset = parseInt(resolveParam(m[2], example), 10) + const resp = await world.listTopics.execute({ limit, offset }) + world.setLastResponse(resp) + } + }, + { + name: 'response contains quoted topic with post count', + pattern: /^the response contains the topic "()" with post count ()$/, + run (m, example, world) { + const topic = resolveParam(m[1], example) + const expectedCount = parseInt(resolveParam(m[2], example), 10) + const found = world.getLastResponse().topics.find((t) => t.room === topic) + if (!found) { + throw new Error(`Topic ${topic} not found in response`) + } + if (found.postCount !== expectedCount) { + throw new Error(`Expected post count ${expectedCount} for ${topic}, got ${found.postCount}`) + } + } + }, + { + name: 'response pagination shows total and hasMore', + pattern: /^the response pagination shows total () and hasMore ()$/, + run (m, example, world) { + const expectedTotal = parseInt(resolveParam(m[1], example), 10) + const expectedHasMore = resolveParam(m[2], example) === 'true' + const pagination = world.getLastResponse().pagination + if (!pagination) { + throw new Error('Response has no pagination metadata') + } + if (pagination.total !== expectedTotal) { + throw new Error(`Expected total ${expectedTotal}, got ${pagination.total}`) + } + if (pagination.hasMore !== expectedHasMore) { + throw new Error(`Expected hasMore ${expectedHasMore}, got ${pagination.hasMore}`) + } + } + }, + { + name: 'topicRecency store read count', + pattern: /^the topicRecency store was read exactly () records$/, + run (m, example, world) { + const expected = parseInt(resolveParam(m[1], example), 10) + const actual = world.topicRecencyIteratorCounter.entries || 0 + if (actual !== expected) { + throw new Error(`Expected topicRecency to be read ${expected} times, got ${actual}`) + } + } + }, + { + name: 'rooms store was not iterated', + pattern: /^the rooms store was not iterated$/, + run (m, example, world) { + if (world.roomsIteratorCounter.calls !== 0) { + throw new Error(`Expected rooms store not to be iterated, got ${world.roomsIteratorCounter.calls} call(s)`) + } + } + }, + { + name: 'run topic backfill utility', + pattern: /^the topic backfill utility is run$/, + async run (m, example, world) { + await backfillTopicIndexes({ + roomsDb: world.adapters.level.roomsDb, + topicSummariesDb: world.adapters.level.topicSummariesDb, + topicRecencyDb: world.adapters.level.topicRecencyDb + }) + } + }, + { + name: 'run topic backfill utility again', + pattern: /^the topic backfill utility is run again$/, + async run (m, example, world) { + await backfillTopicIndexes({ + roomsDb: world.adapters.level.roomsDb, + topicSummariesDb: world.adapters.level.topicSummariesDb, + topicRecencyDb: world.adapters.level.topicRecencyDb + }) + } + }, + { + name: 'topicSummaries contains room with postCount and lastHeight', + pattern: /^the topicSummaries store contains the room "()" with postCount () and lastHeight (<[A-Za-z0-9_]+>)$/, + async run (m, example, world) { + const room = resolveParam(m[1], example) + const postCount = parseInt(resolveParam(m[2], example), 10) + const lastHeight = parseInt(resolveParam(m[3], example), 10) + const record = await world.adapters.level.topicSummariesDb.get(room) + if (!record) { + throw new Error(`No topicSummaries record for ${room}`) + } + if (record.room !== room || record.postCount !== postCount || record.lastHeight !== lastHeight) { + throw new Error(`Expected ${room} postCount ${postCount} lastHeight ${lastHeight}, got ${JSON.stringify(record)}`) + } + } + }, + { + name: 'topicRecency records room at height', + pattern: /^the topicRecency store records the room "()" at block height ()$/, + async run (m, example, world) { + const room = resolveParam(m[1], example) + const height = parseInt(resolveParam(m[2], example), 10) + const record = await world.adapters.level.topicRecencyDb.get(topicRecencyKey(height, room)) + if (!record) { + throw new Error(`No topicRecency record for ${room} at ${height}`) + } + if (record.room !== room || record.blockHeight !== height) { + throw new Error(`Expected topicRecency ${room} at ${height}, got ${JSON.stringify(record)}`) + } + } + }, { name: 'request topic follow state', pattern: /^the client requests the topic follow state for room (<[A-Za-z0-9_]+>) and address (<[A-Za-z0-9_]+>)$/, diff --git a/psf-memo-db/src/adapters/index.js b/psf-memo-db/src/adapters/index.js index 76079dc..5d686e2 100644 --- a/psf-memo-db/src/adapters/index.js +++ b/psf-memo-db/src/adapters/index.js @@ -50,6 +50,8 @@ class Adapters { this.topicQuery = new TopicQuery({ roomsDb: level.roomsDb, postsDb: level.postsDb, + topicSummariesDb: level.topicSummariesDb, + topicRecencyDb: level.topicRecencyDb, muteQuery: this.muteQuery }) this.pollQuery = new PollQuery({ diff --git a/psf-memo-db/src/adapters/level-db.js b/psf-memo-db/src/adapters/level-db.js index 3579d4d..41ad848 100644 --- a/psf-memo-db/src/adapters/level-db.js +++ b/psf-memo-db/src/adapters/level-db.js @@ -24,6 +24,8 @@ const DB_NAMES = [ 'follows', 'mutes', 'rooms', + 'topicSummaries', + 'topicRecency', 'processErrors', 'ptxs', 'polls', diff --git a/psf-memo-db/src/adapters/topic-query.js b/psf-memo-db/src/adapters/topic-query.js index 69134ef..5cf9f6c 100644 --- a/psf-memo-db/src/adapters/topic-query.js +++ b/psf-memo-db/src/adapters/topic-query.js @@ -5,8 +5,16 @@ - Topic posts are keyed `${room}:${txid}` with type 'post'. - Topic follows are keyed `${room}:${addr}` with type 'follow'. + Topic listing is served from two indexes so it can order and paginate + without iterating the rooms store: + - topicSummaries: one record per room keyed by room, value + { room, postCount, lastHeight }. + - topicRecency: one record per room keyed `${invertedHeight}:${room}`, value + { room, blockHeight }. Follow-only rooms live at height 0. The height is + inverted so an ascending scan yields newest-first. + This adapter exposes: - - listTopics() - distinct rooms with post counts + - listTopics() - ordered, paginated distinct rooms with counts - getTopicPostTxids() - paginated txids for a room sorted by block height */ @@ -14,15 +22,23 @@ import { loadMutedAddrs, isMutedPost } from './lib/muted-posts.js' class TopicQuery { constructor (localConfig = {}) { - const { roomsDb, postsDb, muteQuery } = localConfig + const { roomsDb, postsDb, topicSummariesDb, topicRecencyDb, muteQuery } = localConfig if (!roomsDb) { throw new Error('roomsDb required when instantiating TopicQuery adapter.') } if (!postsDb) { throw new Error('postsDb required when instantiating TopicQuery adapter.') } + if (!topicSummariesDb) { + throw new Error('topicSummariesDb required when instantiating TopicQuery adapter.') + } + if (!topicRecencyDb) { + throw new Error('topicRecencyDb required when instantiating TopicQuery adapter.') + } this.roomsDb = roomsDb this.postsDb = postsDb + this.topicSummariesDb = topicSummariesDb + this.topicRecencyDb = topicRecencyDb this.muteQuery = muteQuery || null this.listTopics = this.listTopics.bind(this) @@ -44,35 +60,52 @@ class TopicQuery { return parts[parts.length - 1] } - async listTopics () { - const topics = new Map() + // The topicSummaries key is the room name; fall back to the key when the + // stored value omits it. + summaryRoom (key, value) { + if (value && typeof value.room === 'string') return value.room + return String(key) + } - for await (const [key, value] of this.roomsDb.iterator()) { - const room = this.roomFromKey(key, value) - if (!topics.has(room)) { - topics.set(room, { postCount: 0, lastHeight: 0 }) - } - const topic = topics.get(room) - if (value?.type === 'post') { - topic.postCount++ - const height = value?.blockHeight ?? 0 - if (height > topic.lastHeight) { - topic.lastHeight = height - } - } + // The topicRecency key is `${invertedHeight}:${room}`; the stored value + // carries the room name, but fall back to the key for robustness. + recencyRoom (key, value) { + if (value && typeof value.room === 'string') return value.room + const parts = String(key).split(':') + return parts.slice(1).join(':') + } + + // Return a page of topics ordered by their most recent post, descending, + // with rooms at the same height ordered by name ascending. Counts and the + // total come from topicSummaries; ordering and pagination come from + // topicRecency, which is read only through offset + limit records. + async listTopics ({ limit = 100, offset = 0 } = {}) { + const postCounts = new Map() + for await (const [key, value] of this.topicSummariesDb.iterator()) { + postCounts.set(this.summaryRoom(key, value), value?.postCount ?? 0) + } + const total = postCounts.size + + const recencyRooms = [] + for await (const [key, value] of this.topicRecencyDb.iterator({ limit: offset + limit })) { + recencyRooms.push(this.recencyRoom(key, value)) } - return Array.from(topics.entries()) - .map(([room, { postCount, lastHeight }]) => ({ room, postCount, lastHeight })) - .sort((a, b) => { - if (b.lastHeight !== a.lastHeight) { - return b.lastHeight - a.lastHeight - } - return a.room.localeCompare(b.room) - }) - // lastHeight is an internal ordering key; keep it out of the public - // API contract so the response exposes only room and postCount. - .map(({ room, postCount }) => ({ room, postCount })) + const pageRooms = recencyRooms.slice(offset, offset + limit) + const topics = pageRooms.map((room) => ({ + room, + postCount: postCounts.get(room) ?? 0 + })) + + return { + topics, + pagination: { + limit, + offset, + total, + hasMore: offset + topics.length < total + } + } } async getTopicPostTxids (room, { limit, offset, viewerAddr = null }) { diff --git a/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js b/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js index 8d94f3f..7acfbea 100644 --- a/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js +++ b/psf-memo-db/src/controllers/rest-api/level/crud-handlers.js @@ -47,6 +47,8 @@ export const ENTITY_CONFIG = [ { route: 'profilepic', dbProp: 'profilePicsDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'profilePicData' }, { route: 'follow', dbProp: 'followsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'followData' }, { route: 'room', dbProp: 'roomsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'roomData' }, + { route: 'topicsummary', dbProp: 'topicSummariesDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'topicSummaryData' }, + { route: 'topicrecency', dbProp: 'topicRecencyDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'topicRecencyData' }, { route: 'processerror', dbProp: 'processErrorsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'errorData' }, { route: 'ptx', dbProp: 'ptxsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'ptxData' }, { route: 'poll', dbProp: 'pollsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'pollData' }, diff --git a/psf-memo-db/src/controllers/rest-api/topics/controller.js b/psf-memo-db/src/controllers/rest-api/topics/controller.js index 0770cc2..9abf50b 100644 --- a/psf-memo-db/src/controllers/rest-api/topics/controller.js +++ b/psf-memo-db/src/controllers/rest-api/topics/controller.js @@ -32,7 +32,11 @@ class TopicsRESTControllerLib { * @apiName GetTopics * @apiGroup REST Topics * - * @apiDescription Returns all distinct Memo topics with their post counts. + * @apiDescription Returns a page of distinct Memo topics ordered by their + * most recent post, newest first. Supports limit and offset. + * + * @apiQuery {Number} [limit=100] Page size (max 100) + * @apiQuery {Number} [offset=0] Number of topics to skip after sorting * * @apiExample Example usage: * curl -X GET "localhost:5021/topics" @@ -40,10 +44,12 @@ class TopicsRESTControllerLib { * @apiSuccess {Object[]} topics Array of topic objects * @apiSuccess {String} topics.room Topic name * @apiSuccess {Number} topics.postCount Number of posts in the topic + * @apiSuccess {Object} pagination Pagination metadata */ async getTopics (ctx) { try { - ctx.body = await this.useCases.listTopics.execute() + const { limit, offset } = ctx.query + ctx.body = await this.useCases.listTopics.execute({ limit, offset }) } catch (err) { this.handleError(ctx, err) } diff --git a/psf-memo-db/src/lib/backfill-topic-indexes.js b/psf-memo-db/src/lib/backfill-topic-indexes.js new file mode 100644 index 0000000..dba4ca4 --- /dev/null +++ b/psf-memo-db/src/lib/backfill-topic-indexes.js @@ -0,0 +1,76 @@ +/* + Library to build the topicSummaries and topicRecency indexes from an + existing rooms store. + + The read side serves GET /topics from these two indexes: + - topicSummaries: one record per room keyed by room, value + { room, postCount, lastHeight }. + - topicRecency: one record per room keyed `${invertedHeight}:${room}`, + value { room, blockHeight }. Follow-only rooms live at height 0. The + height is inverted so an ascending scan yields newest-first. + + The backfill recomputes both indexes from the rooms store, so running it + more than once is idempotent. Stale index records for rooms that no longer + have any room entry, or that moved to a new height, are removed. + + The LevelDB handles are injected so the logic stays testable and free of + file-system concerns; the CLI wrapper in util/room opens the real stores. +*/ + +const HEIGHT_PAD = 12 +// Heights are inverted in the recency key so a plain ascending iterator yields +// rooms from most recent to least recent. Rooms at the same height still sort +// by room name ascending. +const MAX_HEIGHT = 999999999999 + +export function topicRecencyKey (blockHeight, room) { + const inverted = MAX_HEIGHT - (blockHeight ?? 0) + return `${String(inverted).padStart(HEIGHT_PAD, '0')}:${room}` +} + +async function collectSummaries (roomsDb) { + const summaries = new Map() + + for await (const [key, value] of roomsDb.iterator()) { + const room = (value && typeof value.room === 'string') ? value.room : String(key).split(':')[0] + if (!summaries.has(room)) { + summaries.set(room, { room, postCount: 0, lastHeight: 0 }) + } + + if (value?.type === 'post') { + const summary = summaries.get(room) + summary.postCount++ + const height = value.blockHeight ?? 0 + if (height > summary.lastHeight) summary.lastHeight = height + } + } + + return summaries +} + +// Remove index records that no longer have a desired key. +async function removeStale (db, desiredKeys) { + for await (const [key] of db.iterator()) { + if (!desiredKeys.has(key)) { + await db.del(key) + } + } +} + +export async function backfillTopicIndexes ({ roomsDb, topicSummariesDb, topicRecencyDb }) { + const summaries = await collectSummaries(roomsDb) + const desiredRecencyKeys = new Set() + + for (const summary of summaries.values()) { + await topicSummariesDb.put(summary.room, summary) + + const key = topicRecencyKey(summary.lastHeight, summary.room) + desiredRecencyKeys.add(key) + await topicRecencyDb.put(key, { room: summary.room, blockHeight: summary.lastHeight }) + } + + await removeStale(topicRecencyDb, desiredRecencyKeys) + await removeStale(topicSummariesDb, new Set(summaries.keys())) + + return { rooms: summaries.size } +} diff --git a/psf-memo-db/src/use-cases/list-topics.js b/psf-memo-db/src/use-cases/list-topics.js index cf62bbc..b19214d 100644 --- a/psf-memo-db/src/use-cases/list-topics.js +++ b/psf-memo-db/src/use-cases/list-topics.js @@ -1,8 +1,9 @@ /* - Use case: list all distinct Memo topics with their post counts. + Use case: list a page of distinct Memo topics ordered by most recent post. */ import { ListUseCase } from './lib/use-case.js' +import { parseLimit, parseOffset } from './lib/pagination.js' class ListTopics extends ListUseCase { constructor (localConfig = {}) { @@ -10,8 +11,9 @@ class ListTopics extends ListUseCase { } async execute (inObj = {}) { - const topics = await this.adapters.topicQuery.listTopics() - return { topics } + const limit = parseLimit(inObj.limit) + const offset = parseOffset(inObj.offset) + return this.adapters.topicQuery.listTopics({ limit, offset }) } } diff --git a/psf-memo-db/test/property/topic-follow-query.property.test.js b/psf-memo-db/test/property/topic-follow-query.property.test.js index 3e0b3eb..e900e4b 100644 --- a/psf-memo-db/test/property/topic-follow-query.property.test.js +++ b/psf-memo-db/test/property/topic-follow-query.property.test.js @@ -47,10 +47,19 @@ function makeRoomsDb (entries) { } } +// Empty index store; this suite exercises the follow read side, not listTopics. +function makeEmptyIndexDb () { + return { + async * iterator () {} + } +} + function makeQuery (entries) { return new TopicQuery({ roomsDb: makeRoomsDb(entries), - postsDb: {} + postsDb: {}, + topicSummariesDb: makeEmptyIndexDb(), + topicRecencyDb: makeEmptyIndexDb() }) } diff --git a/psf-memo-db/test/property/topic-query.property.test.js b/psf-memo-db/test/property/topic-query.property.test.js index c3537db..6264574 100644 --- a/psf-memo-db/test/property/topic-query.property.test.js +++ b/psf-memo-db/test/property/topic-query.property.test.js @@ -19,6 +19,7 @@ import test from 'node:test' import { seededRandom, forAll, intGen, txidGen } from './harness.js' import TopicQuery from '../../src/adapters/topic-query.js' +import { topicRecencyKey } from '../../src/lib/backfill-topic-indexes.js' const rng = seededRandom(20260828) @@ -45,10 +46,55 @@ function makeRoomsDb (entries) { function makeQuery (entries) { return new TopicQuery({ roomsDb: makeRoomsDb(entries), - postsDb: {} + postsDb: {}, + topicSummariesDb: makeIndexDb([]), + topicRecencyDb: makeIndexDb([]) }) } +// In-memory index store honoring the LevelDB `limit` option used by listTopics. +function makeIndexDb (records) { + const store = new Map(records) + return { + async * iterator (opts = {}) { + const keys = Array.from(store.keys()).sort() + const limit = opts.limit === undefined ? keys.length : opts.limit + for (const key of keys.slice(0, limit)) { + yield [key, store.get(key)] + } + } + } +} + +// Derive topicSummaries and topicRecency records from rooms-store entries the +// way the indexer and backfill do. +function indexDbsFromEntries (entries) { + const summaries = new Map() + for (const e of entries) { + const room = e.value.room + if (!summaries.has(room)) summaries.set(room, { room, postCount: 0, lastHeight: 0 }) + if (e.value.type === 'post') { + const summary = summaries.get(room) + summary.postCount++ + const height = e.value.blockHeight ?? 0 + if (height > summary.lastHeight) summary.lastHeight = height + } + } + + const recency = [] + for (const summary of summaries.values()) { + recency.push([ + topicRecencyKey(summary.lastHeight, summary.room), + { room: summary.room, blockHeight: summary.lastHeight } + ]) + } + + return { + topicSummariesDb: makeIndexDb(Array.from(summaries.entries())), + topicRecencyDb: makeIndexDb(recency) + } +} + // In-memory posts store exposing the LevelDB `get` contract used by the mute // post lookup. Returns the post record or throws a LEVEL_NOT_FOUND-style error. function makePostsDb (entries) { @@ -112,14 +158,15 @@ function fixtureGen () { test('listTopics conserves post counts and returns rooms sorted by most recent post', async () => { await forAll( fixtureGen(), - async ({ entries, rooms }) => { - const query = makeQuery(entries) - const topics = await query.listTopics() + async ({ entries, limit, offset }) => { + const query = new TopicQuery({ + roomsDb: makeRoomsDb(entries), + postsDb: {}, + ...indexDbsFromEntries(entries) + }) + const { topics, pagination } = await query.listTopics({ limit, offset }) const postEntries = entries.filter((e) => e.value.type === 'post') - const totalPosts = topics.reduce((sum, t) => sum + t.postCount, 0) - if (totalPosts !== postEntries.length) return false - const expectedTopics = [...new Set(entries.map((e) => e.value.room))] .map((room) => { const heights = entries @@ -131,7 +178,12 @@ test('listTopics conserves post counts and returns rooms sorted by most recent p if (b.lastHeight !== a.lastHeight) return b.lastHeight - a.lastHeight return a.room.localeCompare(b.room) }) - if (JSON.stringify(topics.map((t) => t.room)) !== JSON.stringify(expectedTopics.map((t) => t.room))) return false + + if (pagination.total !== expectedTopics.length) return false + + const expectedPage = expectedTopics.slice(offset, offset + limit) + if (JSON.stringify(topics.map((t) => t.room)) !== JSON.stringify(expectedPage.map((t) => t.room))) return false + if (pagination.hasMore !== (offset + topics.length < pagination.total)) return false for (const topic of topics) { const roomPosts = postEntries.filter((e) => e.value.room === topic.room).length @@ -139,7 +191,7 @@ test('listTopics conserves post counts and returns rooms sorted by most recent p } return true }, - { label: 'listTopics conservation and ordering' } + { label: 'listTopics conservation, ordering, and pagination' } ) }) @@ -204,6 +256,8 @@ test('getTopicPostTxids excludes muted addresses and conserves total and paginat const query = new TopicQuery({ roomsDb: makeRoomsDb(entries), postsDb: makePostsDb(entries), + topicSummariesDb: makeIndexDb([]), + topicRecencyDb: makeIndexDb([]), muteQuery }) diff --git a/psf-memo-db/test/unit/adapters/topic-query.unit.js b/psf-memo-db/test/unit/adapters/topic-query.unit.js index 4d6fe69..e4ac0de 100644 --- a/psf-memo-db/test/unit/adapters/topic-query.unit.js +++ b/psf-memo-db/test/unit/adapters/topic-query.unit.js @@ -1,6 +1,7 @@ import { assert } from 'chai' import sinon from 'sinon' import TopicQuery from '../../../src/adapters/topic-query.js' +import { topicRecencyKey } from '../../../src/lib/backfill-topic-indexes.js' function makeRoomsDb (records = {}) { const store = new Map(Object.entries(records)) @@ -37,11 +38,38 @@ function makeRoomsDb (records = {}) { } } +// In-memory index store whose iterator honors the LevelDB `limit` option, so +// the read path's bounded recency reads can be asserted. +function makeIteratorDb (records = []) { + const store = new Map(records) + return { + async get (key) { + if (!store.has(key)) { + const err = new Error('not found') + err.notFound = true + throw err + } + return store.get(key) + }, + async * iterator (opts = {}) { + let keys = Array.from(store.keys()).sort() + if (opts.gte !== undefined) keys = keys.filter((k) => k >= opts.gte) + if (opts.lte !== undefined) keys = keys.filter((k) => k <= opts.lte) + const limit = opts.limit === undefined ? keys.length : opts.limit + for (const key of keys.slice(0, limit)) { + yield [key, store.get(key)] + } + } + } +} + describe('#TopicQuery', () => { let uut let sandbox let roomsDb let postsDb + let topicSummariesDb + let topicRecencyDb beforeEach(() => { sandbox = sinon.createSandbox() @@ -52,8 +80,10 @@ describe('#TopicQuery', () => { postsDb = { get: sandbox.stub() } + topicSummariesDb = makeIteratorDb() + topicRecencyDb = makeIteratorDb() - uut = new TopicQuery({ roomsDb, postsDb }) + uut = new TopicQuery({ roomsDb, postsDb, topicSummariesDb, topicRecencyDb }) }) afterEach(() => sandbox.restore()) @@ -71,13 +101,33 @@ describe('#TopicQuery', () => { it('should throw when postsDb is missing', () => { try { // eslint-disable-next-line no-new - new TopicQuery({ roomsDb }) + new TopicQuery({ roomsDb, topicSummariesDb, topicRecencyDb }) assert.fail('Expected error') } catch (err) { assert.include(err.message, 'postsDb required') } }) + it('should throw when topicSummariesDb is missing', () => { + try { + // eslint-disable-next-line no-new + new TopicQuery({ roomsDb, postsDb, topicRecencyDb }) + assert.fail('Expected error') + } catch (err) { + assert.include(err.message, 'topicSummariesDb required') + } + }) + + it('should throw when topicRecencyDb is missing', () => { + try { + // eslint-disable-next-line no-new + new TopicQuery({ roomsDb, postsDb, topicSummariesDb }) + assert.fail('Expected error') + } catch (err) { + assert.include(err.message, 'topicRecencyDb required') + } + }) + describe('#roomFromKey', () => { it('should return the room from the value when present', () => { assert.equal(uut.roomFromKey('ignored:post-1', { room: 'bitcoin' }), 'bitcoin') @@ -95,60 +145,85 @@ describe('#TopicQuery', () => { }) describe('#listTopics', () => { - it('should return distinct topics with post counts', async () => { - async function * mockRooms () { - yield ['bitcoin:post-300', { room: 'bitcoin', txid: 'post-300', type: 'post', blockHeight: 300 }] - yield ['bitcoin:post-200', { room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 200 }] - yield ['cash:post-250', { room: 'cash', txid: 'post-250', type: 'post', blockHeight: 250 }] - yield ['dev:post-100', { room: 'dev', txid: 'post-100', type: 'post', blockHeight: 100 }] - yield ['lone:addr-f', { room: 'lone', addr: 'addr-f', type: 'follow', unfollow: false }] - } - roomsDb.iterator.returns(mockRooms()) + const summaries = [ + ['memo', { room: 'memo', postCount: 5, lastHeight: 600500 }], + ['cash', { room: 'cash', postCount: 2, lastHeight: 600400 }], + ['dance', { room: 'dance', postCount: 3, lastHeight: 600400 }], + ['anime', { room: 'anime', postCount: 1, lastHeight: 600300 }], + ['lone', { room: 'lone', postCount: 0, lastHeight: 0 }], + ['quiet', { room: 'quiet', postCount: 0, lastHeight: 0 }] + ] + const recency = [ + [topicRecencyKey(600500, 'memo'), { room: 'memo', blockHeight: 600500 }], + [topicRecencyKey(600400, 'cash'), { room: 'cash', blockHeight: 600400 }], + [topicRecencyKey(600400, 'dance'), { room: 'dance', blockHeight: 600400 }], + [topicRecencyKey(600300, 'anime'), { room: 'anime', blockHeight: 600300 }], + [topicRecencyKey(0, 'lone'), { room: 'lone', blockHeight: 0 }], + [topicRecencyKey(0, 'quiet'), { room: 'quiet', blockHeight: 0 }] + ] - const result = await uut.listTopics() + it('should return topics ordered by recency with counts from summaries', async () => { + uut = new TopicQuery({ + roomsDb, + postsDb, + topicSummariesDb: makeIteratorDb(summaries), + topicRecencyDb: makeIteratorDb(recency) + }) - assert.deepEqual(result, [ - { room: 'bitcoin', postCount: 2 }, - { room: 'cash', postCount: 1 }, - { room: 'dev', postCount: 1 }, - { room: 'lone', postCount: 0 } + const result = await uut.listTopics({ limit: 100, offset: 0 }) + + assert.deepEqual(result.topics, [ + { room: 'memo', postCount: 5 }, + { room: 'cash', postCount: 2 }, + { room: 'dance', postCount: 3 }, + { room: 'anime', postCount: 1 }, + { room: 'lone', postCount: 0 }, + { room: 'quiet', postCount: 0 } ]) + assert.deepEqual(result.pagination, { limit: 100, offset: 0, total: 6, hasMore: false }) }) - it('should sort topics by most recent post descending', async () => { - async function * mockRooms () { - yield ['zoo:post-1', { room: 'zoo', txid: 'post-1', type: 'post', blockHeight: 1 }] - yield ['alpha:post-1', { room: 'alpha', txid: 'post-1', type: 'post', blockHeight: 2 }] - } - roomsDb.iterator.returns(mockRooms()) + it('should paginate using recency order and report total and hasMore', async () => { + uut = new TopicQuery({ + roomsDb, + postsDb, + topicSummariesDb: makeIteratorDb(summaries), + topicRecencyDb: makeIteratorDb(recency) + }) - const result = await uut.listTopics() + const result = await uut.listTopics({ limit: 2, offset: 2 }) - assert.deepEqual(result.map((t) => t.room), ['alpha', 'zoo']) + assert.deepEqual(result.topics.map((t) => t.room), ['dance', 'anime']) + assert.equal(result.pagination.total, 6) + assert.equal(result.pagination.hasMore, true) }) - it('should treat a post at block height 0 as the least recent', async () => { - async function * mockRooms () { - yield ['a:post-1', { room: 'a', txid: 'post-1', type: 'post', blockHeight: 0 }] - yield ['b:post-1', { room: 'b', txid: 'post-1', type: 'post', blockHeight: 1 }] - } - roomsDb.iterator.returns(mockRooms()) + it('should report hasMore false on the last page', async () => { + uut = new TopicQuery({ + roomsDb, + postsDb, + topicSummariesDb: makeIteratorDb(summaries), + topicRecencyDb: makeIteratorDb(recency) + }) - const result = await uut.listTopics() + const result = await uut.listTopics({ limit: 2, offset: 4 }) - assert.deepEqual(result.map((t) => t.room), ['b', 'a']) + assert.deepEqual(result.topics.map((t) => t.room), ['lone', 'quiet']) + assert.equal(result.pagination.total, 6) + assert.equal(result.pagination.hasMore, false) }) - it('should treat a post with no block height as height 0', async () => { - async function * mockRooms () { - yield ['a:post-1', { room: 'a', txid: 'post-1', type: 'post' }] - yield ['b:post-1', { room: 'b', txid: 'post-1', type: 'post', blockHeight: 1 }] - } - roomsDb.iterator.returns(mockRooms()) + it('should read the recency index without iterating the rooms store', async () => { + uut = new TopicQuery({ + roomsDb, + postsDb, + topicSummariesDb: makeIteratorDb(summaries), + topicRecencyDb: makeIteratorDb(recency) + }) - const result = await uut.listTopics() + await uut.listTopics({ limit: 2, offset: 2 }) - assert.deepEqual(result.map((t) => t.room), ['b', 'a']) + assert.equal(roomsDb.iterator.callCount, 0) }) }) @@ -258,7 +333,7 @@ describe('#TopicQuery', () => { const muteQuery = { listMuted: sandbox.stub().resolves(['muted-addr']) } - uut = new TopicQuery({ roomsDb, postsDb, muteQuery }) + uut = new TopicQuery({ roomsDb, postsDb, topicSummariesDb, topicRecencyDb, muteQuery }) const result = await uut.getTopicPostTxids('bitcoin', { limit: 100, offset: 0, viewerAddr: 'viewer-addr' }) @@ -320,7 +395,9 @@ describe('#TopicQuery', () => { 'bitcoin:addr-c': { room: 'bitcoin', addr: 'addr-c', type: 'follow', unfollow: true }, 'cash:addr-a': { room: 'cash', addr: 'addr-a', type: 'follow', unfollow: false } }), - postsDb + postsDb, + topicSummariesDb, + topicRecencyDb }) const result = await query.listRoomFollowers('bitcoin') @@ -332,7 +409,9 @@ describe('#TopicQuery', () => { roomsDb: makeRoomsDb({ 'bitcoin:addr-a': { room: 'bitcoin', type: 'follow', unfollow: false } }), - postsDb + postsDb, + topicSummariesDb, + topicRecencyDb }) const result = await query.listRoomFollowers('bitcoin') @@ -342,7 +421,9 @@ describe('#TopicQuery', () => { it('should return an empty array for a room with no followers', async () => { const query = new TopicQuery({ roomsDb: makeRoomsDb({}), - postsDb + postsDb, + topicSummariesDb, + topicRecencyDb }) const result = await query.listRoomFollowers('lone') diff --git a/psf-memo-db/test/unit/controllers/topics.controller.unit.js b/psf-memo-db/test/unit/controllers/topics.controller.unit.js index 3cb34dd..f96a429 100644 --- a/psf-memo-db/test/unit/controllers/topics.controller.unit.js +++ b/psf-memo-db/test/unit/controllers/topics.controller.unit.js @@ -45,10 +45,11 @@ describe('#TopicsRESTController', () => { afterEach(() => sandbox.restore()) it('should return topics from use case', async () => { - const ctx = { body: null, throw: sandbox.stub() } + const ctx = { query: { limit: '2', offset: '4' }, body: null, throw: sandbox.stub() } await uut.getTopics(ctx) assert.equal(uut.useCases.listTopics.execute.callCount, 1) + assert.deepEqual(uut.useCases.listTopics.execute.firstCall.args[0], { limit: '2', offset: '4' }) assert.equal(ctx.body.topics.length, 2) assert.equal(ctx.body.topics[0].room, 'bitcoin') }) @@ -106,7 +107,7 @@ describe('#TopicsRESTController', () => { it('should throw a 500 when the use case fails without a status', async () => { uut.useCases.listTopics.execute = sandbox.stub().rejects(new Error('boom')) - const ctx = { body: null, throw: sandbox.stub() } + const ctx = { query: {}, body: null, throw: sandbox.stub() } await uut.getTopics(ctx) diff --git a/psf-memo-db/test/unit/lib/backfill-topic-indexes.unit.js b/psf-memo-db/test/unit/lib/backfill-topic-indexes.unit.js new file mode 100644 index 0000000..cabfc28 --- /dev/null +++ b/psf-memo-db/test/unit/lib/backfill-topic-indexes.unit.js @@ -0,0 +1,89 @@ +import { assert } from 'chai' +import { backfillTopicIndexes, topicRecencyKey } from '../../../src/lib/backfill-topic-indexes.js' + +function makeDb (records = []) { + const store = new Map(records) + return { + store, + async get (key) { + if (!store.has(key)) { + const err = new Error('not found') + err.notFound = true + throw err + } + return store.get(key) + }, + async put (key, value) { + store.set(key, value) + }, + async del (key) { + store.delete(key) + }, + async * iterator () { + for (const key of Array.from(store.keys()).sort()) { + yield [key, store.get(key)] + } + } + } +} + +describe('#backfillTopicIndexes', () => { + it('should summarize rooms with posts and follow-only rooms', async () => { + const roomsDb = makeDb([ + ['bitcoin:post-100', { room: 'bitcoin', txid: 'post-100', type: 'post', blockHeight: 600100 }], + ['bitcoin:post-200', { room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 600200 }], + ['bitcoin:addr-f', { room: 'bitcoin', addr: 'addr-f', type: 'follow', unfollow: false }], + ['cash:post-250', { room: 'cash', txid: 'post-250', type: 'post', blockHeight: 600250 }], + ['lone:addr-f', { room: 'lone', addr: 'addr-f', type: 'follow', unfollow: false }] + ]) + const topicSummariesDb = makeDb() + const topicRecencyDb = makeDb() + + const result = await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb }) + + assert.equal(result.rooms, 3) + assert.deepEqual(topicSummariesDb.store.get('bitcoin'), { room: 'bitcoin', postCount: 2, lastHeight: 600200 }) + assert.deepEqual(topicSummariesDb.store.get('cash'), { room: 'cash', postCount: 1, lastHeight: 600250 }) + assert.deepEqual(topicSummariesDb.store.get('lone'), { room: 'lone', postCount: 0, lastHeight: 0 }) + + assert.deepEqual(topicRecencyDb.store.get(topicRecencyKey(600200, 'bitcoin')), { room: 'bitcoin', blockHeight: 600200 }) + assert.deepEqual(topicRecencyDb.store.get(topicRecencyKey(600250, 'cash')), { room: 'cash', blockHeight: 600250 }) + assert.deepEqual(topicRecencyDb.store.get(topicRecencyKey(0, 'lone')), { room: 'lone', blockHeight: 0 }) + assert.equal(topicRecencyDb.store.size, 3) + }) + + it('should be idempotent across repeated runs', async () => { + const roomsDb = makeDb([ + ['bitcoin:post-200', { room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 600200 }], + ['lone:addr-f', { room: 'lone', addr: 'addr-f', type: 'follow', unfollow: false }] + ]) + const topicSummariesDb = makeDb() + const topicRecencyDb = makeDb() + + await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb }) + const firstSummaries = new Map(topicSummariesDb.store) + const firstRecency = new Map(topicRecencyDb.store) + + await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb }) + + assert.deepEqual(topicSummariesDb.store, firstSummaries) + assert.deepEqual(topicRecencyDb.store, firstRecency) + }) + + it('should remove stale recency records from a previous run', async () => { + const roomsDb = makeDb([ + ['bitcoin:post-200', { room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 600200 }] + ]) + const topicSummariesDb = makeDb() + const topicRecencyDb = makeDb([ + [topicRecencyKey(600100, 'bitcoin'), { room: 'bitcoin', blockHeight: 600100 }], + [topicRecencyKey(0, 'stale'), { room: 'stale', blockHeight: 0 }] + ]) + + await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb }) + + assert.isFalse(topicRecencyDb.store.has(topicRecencyKey(600100, 'bitcoin'))) + assert.isFalse(topicRecencyDb.store.has(topicRecencyKey(0, 'stale'))) + assert.isTrue(topicRecencyDb.store.has(topicRecencyKey(600200, 'bitcoin'))) + }) +}) diff --git a/psf-memo-db/test/unit/use-cases/list-topics.unit.js b/psf-memo-db/test/unit/use-cases/list-topics.unit.js index d4c1b5a..862a40b 100644 --- a/psf-memo-db/test/unit/use-cases/list-topics.unit.js +++ b/psf-memo-db/test/unit/use-cases/list-topics.unit.js @@ -7,13 +7,18 @@ describe('#ListTopics', () => { let sandbox let topicQuery + const adapterResult = { + topics: [ + { room: 'bitcoin', postCount: 2 }, + { room: 'cash', postCount: 1 } + ], + pagination: { limit: 100, offset: 0, total: 2, hasMore: false } + } + beforeEach(() => { sandbox = sinon.createSandbox() topicQuery = { - listTopics: sandbox.stub().resolves([ - { room: 'bitcoin', postCount: 2 }, - { room: 'cash', postCount: 1 } - ]) + listTopics: sandbox.stub().resolves(adapterResult) } uut = new ListTopics({ adapters: { topicQuery } @@ -42,12 +47,36 @@ describe('#ListTopics', () => { } }) - it('should return topics from the adapter', async () => { + it('should return the adapter page with default limit and offset', async () => { const result = await uut.execute() - assert.deepEqual(result.topics, [ - { room: 'bitcoin', postCount: 2 }, - { room: 'cash', postCount: 1 } - ]) + assert.deepEqual(result, adapterResult) + assert.deepEqual(topicQuery.listTopics.firstCall.args[0], { limit: 100, offset: 0 }) + }) + + it('should forward limit and offset to the adapter', async () => { + await uut.execute({ limit: 2, offset: 4 }) + + assert.deepEqual(topicQuery.listTopics.firstCall.args[0], { limit: 2, offset: 4 }) + }) + + it('should reject a non-positive limit', async () => { + try { + await uut.execute({ limit: 0 }) + assert.fail('Expected error') + } catch (err) { + assert.include(err.message, 'limit must be a positive integer') + assert.equal(err.status, 400) + } + }) + + it('should reject a negative offset', async () => { + try { + await uut.execute({ offset: -1 }) + assert.fail('Expected error') + } catch (err) { + assert.include(err.message, 'offset must be a non-negative integer') + assert.equal(err.status, 400) + } }) }) diff --git a/psf-memo-db/util/room/backfill-topic-indexes.js b/psf-memo-db/util/room/backfill-topic-indexes.js new file mode 100644 index 0000000..756e710 --- /dev/null +++ b/psf-memo-db/util/room/backfill-topic-indexes.js @@ -0,0 +1,71 @@ +/* + Utility: build the topicSummaries and topicRecency indexes for an existing + psf-memo-db. + + New deployments write these indexes while indexing live topic messages, but + existing databases that were populated before the topic recency feature need + a one-time backfill. + + Run from the psf-memo-db repo root on the host that owns the LevelDB files: + + node util/room/backfill-topic-indexes.js + + The script is idempotent: re-running it produces the same indexes. Progress + and a summary are printed to stderr. + + WARNING: + - This script opens the LevelDB files directly. psf-memo-db must NOT be + running, or another process must not hold the database locks. + - Make a backup of leveldb/current before running on a production server: + cp -r leveldb/current leveldb/current-pre-topic-index-backup +*/ + +import level from 'level' +import * as fs from 'fs' +import * as path from 'path' +import * as url from 'url' +import { backfillTopicIndexes } from '../../src/lib/backfill-topic-indexes.js' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) + +const DATA_DIR = process.env.PSF_MEMO_DB_DATA_DIR + ? path.resolve(process.env.PSF_MEMO_DB_DATA_DIR) + : path.resolve(__dirname, '../../leveldb/current') + +function requiredStorePath (dir, name) { + const storePath = path.join(dir, name) + if (!fs.existsSync(storePath)) { + throw new Error(`Required LevelDB store not found: ${storePath}. Set PSF_MEMO_DB_DATA_DIR to the directory containing the rooms store.`) + } + return storePath +} + +async function main () { + console.error(`Using LevelDB data directory: ${DATA_DIR}`) + + const roomsPath = requiredStorePath(DATA_DIR, 'rooms') + const topicSummariesPath = path.join(DATA_DIR, 'topicSummaries') + const topicRecencyPath = path.join(DATA_DIR, 'topicRecency') + + console.error('Opening LevelDB stores...') + const roomsDb = level(roomsPath, { valueEncoding: 'json' }) + const topicSummariesDb = level(topicSummariesPath, { valueEncoding: 'json', createIfMissing: true }) + const topicRecencyDb = level(topicRecencyPath, { valueEncoding: 'json', createIfMissing: true }) + + try { + console.error('Backfilling topicSummaries and topicRecency from rooms...') + const summary = await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb }) + + console.error('\nTopic index backfill complete.') + console.error(` rooms summarized: ${summary.rooms}`) + } catch (err) { + console.error('\nTopic index backfill failed:', err.message) + process.exitCode = 1 + } finally { + await topicRecencyDb.close().catch(() => {}) + await topicSummariesDb.close().catch(() => {}) + await roomsDb.close().catch(() => {}) + } +} + +main() diff --git a/psf-memo-indexer/acceptance/lib/handlers.js b/psf-memo-indexer/acceptance/lib/handlers.js index 90b2425..1bc9b37 100644 --- a/psf-memo-indexer/acceptance/lib/handlers.js +++ b/psf-memo-indexer/acceptance/lib/handlers.js @@ -14,6 +14,9 @@ import { handleCreatePoll } from '../../src/use-cases/action-types/poll-create.j import { handleAddPollOption } from '../../src/use-cases/action-types/poll-option.js' import { handlePollVote } from '../../src/use-cases/action-types/poll-vote.js' import { handleMute } from '../../src/use-cases/action-types/mute.js' +import { handleTopicMessage } from '../../src/use-cases/action-types/topic-message.js' +import { handleTopicFollow } from '../../src/use-cases/action-types/topic-follow.js' +import { topicRecencyKey } from '../../src/use-cases/action-types/helpers.js' import BackupDb from '../../src/use-cases/backup-db.js' function makeInMemoryDb () { @@ -85,6 +88,9 @@ async function createWorld () { const pollOptionDb = makeInMemoryDb() const pollVoteDb = makeInMemoryDb() const muteDb = makeInMemoryDb() + const roomDb = makeInMemoryDb() + const topicSummaryDb = makeInMemoryDb() + const topicRecencyDb = makeInMemoryDb() const adapters = { postDb: postsDb, @@ -98,6 +104,9 @@ async function createWorld () { pollOptionDb, pollVoteDb, muteDb, + roomDb, + topicSummaryDb, + topicRecencyDb, processErrorDb: makeInMemoryDb(), dbCtrl: { backupDb: async (height, epoch) => { @@ -121,6 +130,9 @@ async function createWorld () { pollOptionsDb: pollOptionDb, pollVotesDb: pollVoteDb, mutesDb: muteDb, + roomsDb: roomDb, + topicSummariesDb: topicSummaryDb, + topicRecencyDb, txidMap: new Map(), lastTxid: null, lastHeight: null, @@ -697,6 +709,112 @@ const muteHandlers = [ throw new Error(`Expected no mute document for txid ${txid}, but one was stored`) } } + }, + { + name: 'db instance with rooms and topic index stores', + pattern: /^a psf-memo-db instance with rooms, topicSummaries, and topicRecency stores$/, + async run () { + // World is already created with the room and topic index stores. + } + }, + { + name: 'process a Memo topic message', + pattern: /^the indexer processes a Memo topic message (.+) in room "(.+)" from (.+) at block height (.+) with text "(.+)"$/, + async run (m, example, world) { + const txid = resolveTxid(m[1], example, world) + const room = m[2] + const addr = resolveParam(m[3], example) + const height = parseInt(resolveParam(m[4], example), 10) + const text = m[5] + + world.lastTxid = txid + world.lastHeight = height + world.lastAddr = addr + + const prefix = Buffer.from('6d0c', 'hex') + const ctx = { + adapters: world.adapters, + txid, + signerAddr: addr, + seen: Date.now(), + blockHeight: height, + decoded: { + action: 'topicMessage', + prefix, + pushDatas: [prefix, Buffer.from(room, 'utf8'), Buffer.from(text, 'utf8')] + } + } + world.lastTopicMessage = ctx + + await handleTopicMessage(ctx) + } + }, + { + name: 'reprocess the last Memo topic message', + pattern: /^the indexer processes the same Memo topic message (.+) again$/, + async run (m, example, world) { + if (!world.lastTopicMessage) { + throw new Error('No previous topic message to reprocess') + } + await handleTopicMessage(world.lastTopicMessage) + } + }, + { + name: 'process a Memo topic follow', + pattern: /^the indexer processes a Memo topic follow for room "(.+)" from (.+)$/, + async run (m, example, world) { + const room = m[1] + const addr = resolveParam(m[2], example) + const txid = deriveTxid(`topic-follow-${room}-${addr}`) + + world.lastTxid = txid + world.lastAddr = addr + + const prefix = Buffer.from('6d0d', 'hex') + await handleTopicFollow({ + adapters: world.adapters, + txid, + signerAddr: addr, + seen: Date.now(), + blockHeight: 600000, + decoded: { + action: 'topicFollow', + prefix, + pushDatas: [prefix, Buffer.from(room, 'utf8')] + } + }) + } + }, + { + name: 'topicSummaries contains room with postCount and lastHeight', + pattern: /^the topicSummaries store contains the room "(.+)" with postCount (.+) and lastHeight (.+)$/, + async run (m, example, world) { + const room = m[1] + const postCount = parseInt(resolveParam(m[2], example), 10) + const lastHeight = parseInt(resolveParam(m[3], example), 10) + const record = await world.topicSummariesDb.get(room) + if (!record) { + throw new Error(`No topicSummaries record for ${room}`) + } + if (record.room !== room || record.postCount !== postCount || record.lastHeight !== lastHeight) { + throw new Error(`Expected ${room} postCount ${postCount} lastHeight ${lastHeight}, got ${JSON.stringify(record)}`) + } + } + }, + { + name: 'topicRecency records room at height', + pattern: /^the topicRecency store records the room "(.+)" at block height (.+)$/, + async run (m, example, world) { + const room = m[1] + const height = parseInt(resolveParam(m[2], example), 10) + const record = await world.topicRecencyDb.get(topicRecencyKey(height, room)) + if (!record) { + throw new Error(`No topicRecency record for ${room} at ${height}`) + } + if (record.room !== room || record.blockHeight !== height) { + throw new Error(`Expected topicRecency ${room} at ${height}, got ${JSON.stringify(record)}`) + } + } } ] diff --git a/psf-memo-indexer/src/adapters/adapters-index.js b/psf-memo-indexer/src/adapters/adapters-index.js index acd5e1c..f7360ca 100644 --- a/psf-memo-indexer/src/adapters/adapters-index.js +++ b/psf-memo-indexer/src/adapters/adapters-index.js @@ -32,6 +32,8 @@ class Adapters { this.followDb = createEntityDb('follow', 'key', 'followData') this.muteDb = createEntityDb('mute', 'key', 'muteData') this.roomDb = createEntityDb('room', 'key', 'roomData') + this.topicSummaryDb = createEntityDb('topicsummary', 'key', 'topicSummaryData') + this.topicRecencyDb = createEntityDb('topicrecency', 'key', 'topicRecencyData') this.pollDb = createEntityDb('poll', 'txid', 'pollData') this.pollOptionDb = createEntityDb('polloption', 'txid', 'optionData') this.pollVoteDb = createEntityDb('pollvote', 'txid', 'voteData') diff --git a/psf-memo-indexer/src/use-cases/action-types/helpers.js b/psf-memo-indexer/src/use-cases/action-types/helpers.js index 078df73..f1602d3 100644 --- a/psf-memo-indexer/src/use-cases/action-types/helpers.js +++ b/psf-memo-indexer/src/use-cases/action-types/helpers.js @@ -62,6 +62,30 @@ export function roomKey (roomName, txid) { return `${roomName}:${txid}` } +// The topicRecency store is keyed with an inverted height so a plain iterator +// yields rooms ordered by most recent post height and, within a height, by +// room name ascending. +export function topicRecencyKey (blockHeight, roomName) { + const inverted = 999999999999 - (blockHeight ?? 0) + return `${String(inverted).padStart(12, '0')}:${roomName}` +} + +// True for the LevelDB not-found error, its in-memory equivalent, and the +// 404 surfaced by the psf-memo-db entity routes. +export function isNotFound (err) { + return Boolean(err && (err.notFound || err.code === 'LEVEL_NOT_FOUND' || err.response?.status === 404)) +} + +// Read a record, or null when it does not exist. Rethrows real errors. +export async function getIfPresent (db, key) { + try { + return await db.get(key) + } catch (err) { + if (isNotFound(err)) return null + throw err + } +} + export function postHeightKey (blockHeight, txid) { const padded = String(blockHeight).padStart(12, '0') return `${padded}:${txid}` diff --git a/psf-memo-indexer/src/use-cases/action-types/topic-follow.js b/psf-memo-indexer/src/use-cases/action-types/topic-follow.js index b11975f..67f7bf5 100644 --- a/psf-memo-indexer/src/use-cases/action-types/topic-follow.js +++ b/psf-memo-indexer/src/use-cases/action-types/topic-follow.js @@ -1,5 +1,6 @@ import { utf8FromPush, logProcessError, roomKey } from './helpers.js' import { PREFIX_TOPIC_UNFOLLOW } from '../../lib/memo-codes.js' +import { ensureTopicRoom } from './topic-indexing.js' export async function handleTopicFollow (ctx) { const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx @@ -22,4 +23,8 @@ export async function handleTopicFollow (ctx) { type: 'follow', blockHeight }) + + if (!unfollow) { + await ensureTopicRoom(adapters, room) + } } diff --git a/psf-memo-indexer/src/use-cases/action-types/topic-indexing.js b/psf-memo-indexer/src/use-cases/action-types/topic-indexing.js new file mode 100644 index 0000000..8542d66 --- /dev/null +++ b/psf-memo-indexer/src/use-cases/action-types/topic-indexing.js @@ -0,0 +1,50 @@ +/* + Maintain the topicSummaries and topicRecency indexes for Memo topic activity. + + topicSummaries: one record per room keyed by room, value + { room, postCount, lastHeight }. + topicRecency: one record per room keyed by `${invertedHeight}:${room}`, value + { room, blockHeight }. Follow-only rooms live at height 0. The height is + inverted so an ascending scan yields newest-first. + + Both indexes are written with the generic entity `update`, which upserts, so + re-running an action is safe. +*/ + +import { getIfPresent, topicRecencyKey } from './helpers.js' + +// Add a post to the room's summary and move its recency record to the newest +// height. A room that is reprocessed is filtered out before this runs, so the +// post count is not double-counted. +export async function recordTopicPost (adapters, room, blockHeight) { + const height = blockHeight ?? 0 + const summary = await getIfPresent(adapters.topicSummaryDb, room) + const lastHeight = Math.max(summary?.lastHeight ?? 0, height) + + await adapters.topicSummaryDb.update(room, { + room, + postCount: (summary?.postCount ?? 0) + 1, + lastHeight + }) + + // When the room moved to a newer height, delete the stale recency record so + // each room has exactly one recency entry. + if (summary && (summary.lastHeight ?? 0) !== lastHeight) { + await adapters.topicRecencyDb.delete(topicRecencyKey(summary.lastHeight ?? 0, room)) + } + + await adapters.topicRecencyDb.update(topicRecencyKey(lastHeight, room), { + room, + blockHeight: lastHeight + }) +} + +// Give a follow-only room a zero-post summary and recency record. A room that +// already has a summary (because it has posts) is left unchanged. +export async function ensureTopicRoom (adapters, room) { + const summary = await getIfPresent(adapters.topicSummaryDb, room) + if (summary) return + + await adapters.topicSummaryDb.update(room, { room, postCount: 0, lastHeight: 0 }) + await adapters.topicRecencyDb.update(topicRecencyKey(0, room), { room, blockHeight: 0 }) +} diff --git a/psf-memo-indexer/src/use-cases/action-types/topic-message.js b/psf-memo-indexer/src/use-cases/action-types/topic-message.js index 5dd928b..edb492c 100644 --- a/psf-memo-indexer/src/use-cases/action-types/topic-message.js +++ b/psf-memo-indexer/src/use-cases/action-types/topic-message.js @@ -1,6 +1,7 @@ -import { utf8FromPush, logProcessError, roomKey } from './helpers.js' +import { utf8FromPush, logProcessError, roomKey, getIfPresent } from './helpers.js' import { MAX_POST_SIZE } from '../../lib/memo-codes.js' import { handlePost } from './post.js' +import { recordTopicPost } from './topic-indexing.js' export async function handleTopicMessage (ctx) { const { adapters, txid, decoded, seen, blockHeight } = ctx @@ -18,10 +19,20 @@ export async function handleTopicMessage (ctx) { return } + // The room record is the marker that this topic message was already + // indexed. Check it before writing so reprocessing does not double-count the + // room's post summary. + const key = roomKey(room, txid) + const alreadyIndexed = (await getIfPresent(adapters.roomDb, key)) !== null + await handlePost({ ...ctx, decoded: { ...decoded, pushDatas: [pushDatas[0], pushDatas[2]] } }) - await adapters.roomDb.create(roomKey(room, txid), { room, txid, seen, type: 'post', blockHeight }) + await adapters.roomDb.create(key, { room, txid, seen, type: 'post', blockHeight }) + + if (!alreadyIndexed) { + await recordTopicPost(adapters, room, blockHeight) + } } diff --git a/psf-memo-indexer/test/unit/use-cases/action-types/topic-follow.unit.js b/psf-memo-indexer/test/unit/use-cases/action-types/topic-follow.unit.js new file mode 100644 index 0000000..98d4d21 --- /dev/null +++ b/psf-memo-indexer/test/unit/use-cases/action-types/topic-follow.unit.js @@ -0,0 +1,106 @@ +import { assert } from 'chai' +import { handleTopicFollow } from '../../../../src/use-cases/action-types/topic-follow.js' +import { PREFIX_TOPIC_FOLLOW, PREFIX_TOPIC_UNFOLLOW } from '../../../../src/lib/memo-codes.js' +import { topicRecencyKey } from '../../../../src/use-cases/action-types/helpers.js' + +function makeDb () { + const store = new Map() + return { + store, + async get (key) { + if (!store.has(key)) { + const err = new Error('not found') + err.notFound = true + throw err + } + return store.get(key) + }, + async create (key, value) { + store.set(key, value) + return { success: true } + }, + async update (key, value) { + store.set(key, value) + return { success: true } + }, + async delete (key) { + store.delete(key) + return { success: true } + } + } +} + +function makeAdapters () { + return { + roomDb: makeDb(), + topicSummaryDb: makeDb(), + topicRecencyDb: makeDb(), + processErrorDb: makeDb() + } +} + +async function processFollow (adapters, { txid, room, addr, prefix = PREFIX_TOPIC_FOLLOW }) { + await handleTopicFollow({ + adapters, + txid, + signerAddr: addr, + seen: 1, + blockHeight: 600000, + decoded: { + action: 'topic-follow', + prefix, + pushDatas: [prefix, Buffer.from(room, 'utf8')] + } + }) +} + +describe('#handleTopicFollow topic indexes', () => { + it('should record a zero-post room for a follow-only room', async () => { + const adapters = makeAdapters() + + await processFollow(adapters, { txid: 'follow-1', room: 'lone', addr: 'bitcoincash:qaddr-a' }) + + assert.deepEqual(adapters.topicSummaryDb.store.get('lone'), { + room: 'lone', + postCount: 0, + lastHeight: 0 + }) + assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(0, 'lone')), { + room: 'lone', + blockHeight: 0 + }) + }) + + it('should leave an existing room summary unchanged', async () => { + const adapters = makeAdapters() + adapters.topicSummaryDb.store.set('bitcoin', { room: 'bitcoin', postCount: 1, lastHeight: 600400 }) + adapters.topicRecencyDb.store.set(topicRecencyKey(600400, 'bitcoin'), { room: 'bitcoin', blockHeight: 600400 }) + + await processFollow(adapters, { txid: 'follow-2', room: 'bitcoin', addr: 'bitcoincash:qaddr-a' }) + + assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), { + room: 'bitcoin', + postCount: 1, + lastHeight: 600400 + }) + assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(600400, 'bitcoin')), { + room: 'bitcoin', + blockHeight: 600400 + }) + assert.isFalse(adapters.topicRecencyDb.store.has(topicRecencyKey(0, 'bitcoin'))) + }) + + it('should not create a zero-post room for an unfollow', async () => { + const adapters = makeAdapters() + + await processFollow(adapters, { + txid: 'unfollow-1', + room: 'lone', + addr: 'bitcoincash:qaddr-a', + prefix: PREFIX_TOPIC_UNFOLLOW + }) + + assert.equal(adapters.topicSummaryDb.store.size, 0) + assert.equal(adapters.topicRecencyDb.store.size, 0) + }) +}) diff --git a/psf-memo-indexer/test/unit/use-cases/action-types/topic-message.unit.js b/psf-memo-indexer/test/unit/use-cases/action-types/topic-message.unit.js new file mode 100644 index 0000000..18acbb8 --- /dev/null +++ b/psf-memo-indexer/test/unit/use-cases/action-types/topic-message.unit.js @@ -0,0 +1,135 @@ +import { assert } from 'chai' +import { handleTopicMessage } from '../../../../src/use-cases/action-types/topic-message.js' +import { topicRecencyKey } from '../../../../src/use-cases/action-types/helpers.js' + +function makeDb () { + const store = new Map() + return { + store, + async get (key) { + if (!store.has(key)) { + const err = new Error('not found') + err.notFound = true + throw err + } + return store.get(key) + }, + async create (key, value) { + store.set(key, value) + return { success: true } + }, + async update (key, value) { + store.set(key, value) + return { success: true } + }, + async delete (key) { + store.delete(key) + return { success: true } + }, + entries () { + return Array.from(store.entries()) + } + } +} + +function makeAdapters () { + return { + postDb: makeDb(), + postHeightDb: makeDb(), + addrPostHeightDb: makeDb(), + roomDb: makeDb(), + topicSummaryDb: makeDb(), + topicRecencyDb: makeDb(), + processErrorDb: makeDb() + } +} + +async function processTopicMessage (adapters, { txid, room, addr, height, text }) { + const prefix = Buffer.from('6d0c', 'hex') + await handleTopicMessage({ + adapters, + txid, + signerAddr: addr, + seen: 1, + blockHeight: height, + decoded: { + action: 'topic-message', + prefix, + pushDatas: [prefix, Buffer.from(room, 'utf8'), Buffer.from(text, 'utf8')] + } + }) +} + +describe('#handleTopicMessage topic indexes', () => { + it('should record a room summary and a recency record', async () => { + const adapters = makeAdapters() + + await processTopicMessage(adapters, { + txid: 'topic-a1', + room: 'bitcoin', + addr: 'bitcoincash:qaddr-a', + height: 600100, + text: 'hello' + }) + + assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), { + room: 'bitcoin', + postCount: 1, + lastHeight: 600100 + }) + assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(600100, 'bitcoin')), { + room: 'bitcoin', + blockHeight: 600100 + }) + }) + + it('should accumulate postCount and keep the newest height', async () => { + const adapters = makeAdapters() + + await processTopicMessage(adapters, { txid: 'topic-a1', room: 'bitcoin', addr: 'bitcoincash:qaddr-a', height: 600100, text: 'hello' }) + await processTopicMessage(adapters, { txid: 'topic-a2', room: 'bitcoin', addr: 'bitcoincash:qaddr-a', height: 600200, text: 'again' }) + + assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), { + room: 'bitcoin', + postCount: 2, + lastHeight: 600200 + }) + assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(600200, 'bitcoin')), { + room: 'bitcoin', + blockHeight: 600200 + }) + assert.equal(adapters.topicRecencyDb.store.size, 1) + }) + + it('should keep the newest height when a later message has an earlier height', async () => { + const adapters = makeAdapters() + + await processTopicMessage(adapters, { txid: 'topic-a3', room: 'bitcoin', addr: 'bitcoincash:qaddr-a', height: 600200, text: 'later' }) + await processTopicMessage(adapters, { txid: 'topic-a4', room: 'bitcoin', addr: 'bitcoincash:qaddr-a', height: 600100, text: 'earlier' }) + + assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), { + room: 'bitcoin', + postCount: 2, + lastHeight: 600200 + }) + assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(600200, 'bitcoin')), { + room: 'bitcoin', + blockHeight: 600200 + }) + assert.equal(adapters.topicRecencyDb.store.size, 1) + }) + + it('should not double-count a reprocessed topic message', async () => { + const adapters = makeAdapters() + + await processTopicMessage(adapters, { txid: 'topic-c1', room: 'bitcoin', addr: 'bitcoincash:qaddr-c', height: 600300, text: 'repeated' }) + await processTopicMessage(adapters, { txid: 'topic-c1', room: 'bitcoin', addr: 'bitcoincash:qaddr-c', height: 600300, text: 'repeated' }) + + assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), { + room: 'bitcoin', + postCount: 1, + lastHeight: 600300 + }) + assert.equal(adapters.topicRecencyDb.store.size, 1) + }) +})