Implement topic recency ordering and pagination

- Indexer maintains topicSummaries and topicRecency for topic messages and
  follows, idempotently, moving each room's recency record to its newest post.
- DB adds the two index stores and serves GET /topics from them with
  limit/offset pagination, without iterating the rooms store. Adds
  util/room/backfill-topic-indexes.js to build the indexes idempotently.
- Client topics page loads 50 per page with Previous/Next.

By coder.
This commit is contained in:
Chris Troutner
2026-09-17 07:24:45 -07:00
parent e2c0bbe523
commit e02bea5460
28 changed files with 1397 additions and 134 deletions
+55 -2
View File
@@ -361,13 +361,18 @@ function makeMemoDb () {
async getMuteState (muterAddr, muteeAddr) {
return muteState[`${muterAddr}:${muteeAddr}`] || false
},
async getTopics () {
async getTopics ({ limit = 50, offset = 0 } = {}) {
const list = []
for (const [room, postCount] of topicCounts.entries()) {
list.push({ room, postCount })
}
list.sort((a, b) => a.room.localeCompare(b.room))
return { topics: list }
const total = list.length
const page = list.slice(offset, offset + limit)
return {
topics: page,
pagination: { limit, offset, total, hasMore: offset + page.length < total }
}
},
async getTopicPosts (room, { limit = 50, offset = 0, viewer = null } = {}) {
let all = topicPosts[room] || []
@@ -2141,6 +2146,54 @@ const handlers = [
world.currentPath = TopicDiscoveryPage.TOPICS_PATH
}
},
{
name: 'open topics page at offset',
pattern: /^I open the topics page at offset (<[A-Za-z0-9_]+>)$/,
async run (m, example, world) {
const offset = parseInt(resolveParam(m[1], example), 10)
await world.topicDiscoveryPage.load({ limit: 50, offset })
world.currentPath = TopicDiscoveryPage.TOPICS_PATH
}
},
{
name: 'API serves N topics',
pattern: /^the psf-memo-db API serves (<[A-Za-z0-9_]+>) topics$/,
run (m, example, world) {
const count = parseInt(resolveParam(m[1], example), 10)
for (let i = 0; i < count; i++) {
world.memoDb.addTopic(`topic-${String(i + 1).padStart(3, '0')}`, 1)
}
}
},
{
name: 'topics page shows N topics',
pattern: /^the topics page shows (<[A-Za-z0-9_]+>) topics$/,
run (m, example, world) {
const expected = parseInt(resolveParam(m[1], example), 10)
const actual = world.topicDiscoveryPage.topics.length
if (actual !== expected) {
throw new Error(`Expected ${expected} topics on the topics page, got ${actual}.`)
}
}
},
{
name: 'topics page can load more topics',
pattern: /^the topics page can load more topics$/,
run (m, example, world) {
if (!world.topicDiscoveryPage.canLoadMore()) {
throw new Error('Expected the topics page to have more topics, but pagination says there are none.')
}
}
},
{
name: 'topics page has no more topics',
pattern: /^the topics page has no more topics$/,
run (m, example, world) {
if (world.topicDiscoveryPage.canLoadMore()) {
throw new Error('Expected the topics page to have no more topics, but pagination says there are more.')
}
}
},
{
name: 'topics page shows topic count',
pattern: /^the topics page shows the topic (<topic>) with (<count>) posts$/,
@@ -1,10 +1,10 @@
/*
Display the list of Memo topics served by psf-memo-db.
Display a page of Memo topics served by psf-memo-db.
*/
// Global npm libraries
import React, { useState, useEffect } from 'react'
import { Container, Row, Col, Spinner, ListGroup } from 'react-bootstrap'
import { Container, Row, Col, Spinner, ListGroup, Button } from 'react-bootstrap'
import { useNavigate } from 'react-router-dom'
// Local libraries
@@ -12,11 +12,15 @@ import MemoDb from '../../../services/memo-db'
import TopicDiscoveryPage from '../../../services/topic-discovery-page'
import '../../../App.css'
const PAGE_SIZE = 50
function Topics (props) {
const navigate = useNavigate()
const [loading, setLoading] = useState(true)
const [error, setError] = useState(null)
const [topics, setTopics] = useState([])
const [pagination, setPagination] = useState(null)
const [offset, setOffset] = useState(0)
useEffect(() => {
const loadTopics = async () => {
@@ -26,23 +30,36 @@ function Topics (props) {
try {
const memoDb = new MemoDb()
const page = new TopicDiscoveryPage({ memoDb, navigate })
const result = await page.load()
const result = await page.load({ limit: PAGE_SIZE, offset })
setTopics(result.topics || [])
setPagination(result.pagination || null)
} catch (err) {
setError(err.message || 'Failed to load topics')
setTopics([])
setPagination(null)
}
setLoading(false)
}
loadTopics()
}, [navigate])
}, [navigate, offset])
const handleClick = (room) => {
navigate(TopicDiscoveryPage.topicFeedPath(room))
}
const canGoBack = offset > 0
const canGoNext = pagination?.hasMore ?? false
const handlePrevious = () => {
setOffset((prev) => Math.max(0, prev - PAGE_SIZE))
}
const handleNext = () => {
setOffset((prev) => prev + PAGE_SIZE)
}
return (
<Container className='topics-page'>
<Row className='justify-content-center'>
@@ -50,6 +67,13 @@ function Topics (props) {
<header className='topics-heading'>
<h1>Topics</h1>
<p>Discover Memo conversations organized by topic.</p>
{pagination && topics.length > 0 && (
<span className='topics-count'>
Showing {pagination.offset + 1}
{pagination.offset + topics.length} of {pagination.total}
</span>
)}
</header>
{error && (
@@ -84,6 +108,26 @@ function Topics (props) {
))}
</ListGroup>
)}
{!loading && !error && (pagination || offset > 0) && (
<div className='topics-pagination mt-3'>
<Button
variant='outline-dark'
onClick={handlePrevious}
disabled={!canGoBack}
>
Previous
</Button>
<Button
variant='outline-dark'
onClick={handleNext}
disabled={!canGoNext}
>
Next
</Button>
</div>
)}
</Col>
</Row>
</Container>
+2 -2
View File
@@ -44,8 +44,8 @@ class MemoDb {
return this._getList(`/mute/muted/${encodeURIComponent(muterAddr)}`, 'getMuted', 'muted')
}
async getTopics () {
return this.getRecent('/topics', 'getTopics', {})
async getTopics ({ limit = 50, offset = 0 } = {}) {
return this.getRecent('/topics', 'getTopics', { limit, offset })
}
async getTopicPosts (room, { limit = 50, offset = 0, viewer = null } = {}) {
@@ -1,29 +1,23 @@
/*
Topic Discovery Page behavior: load and display the list of Memo topics.
Topic Discovery Page behavior: load and display a page of Memo topics.
This is the testable controller behind the React "Topics" page. It wraps
the MemoDb client and exposes the loaded topics so the view can render each
topic name and post count.
the MemoDb client and exposes the loaded topics and pagination so the view
can render each topic name and post count and move between pages.
*/
const PaginatedPage = require('./paginated-page')
const TOPICS_PATH = '/topics'
class TopicDiscoveryPage {
class TopicDiscoveryPage extends PaginatedPage {
constructor (deps = {}) {
this.memoDb = deps.memoDb || null
super(deps, {
listField: 'topics',
loadMethod: 'getTopics',
errorMessage: 'Topic discovery page requires a memo db client.'
})
this.navigate = deps.navigate || (() => {})
this.topics = []
}
async load () {
if (!this.memoDb) {
throw new Error('Topic discovery page requires a memo db client.')
}
const data = await this.memoDb.getTopics()
this.topics = data.topics || []
return { topics: this.topics }
}
getTopic (room) {
@@ -8,10 +8,13 @@ const test = require('node:test')
const assert = require('node:assert/strict')
const TopicDiscoveryPage = require('../../src/services/topic-discovery-page')
function makeMemoDb (topics) {
function makeMemoDb ({ topics = [], pagination = null } = {}) {
const calls = []
return {
async getTopics () {
return { topics }
calls,
async getTopics (opts) {
calls.push(opts)
return { topics, pagination }
}
}
}
@@ -21,13 +24,56 @@ test('load returns topics with post counts', async () => {
{ room: 'bitcoin', postCount: 2 },
{ room: 'cash', postCount: 1 }
]
const page = new TopicDiscoveryPage({ memoDb: makeMemoDb(topics) })
const page = new TopicDiscoveryPage({ memoDb: makeMemoDb({ topics }) })
const result = await page.load()
assert.deepEqual(result.topics, topics)
})
test('load defaults to a page of 50 from offset 0', async () => {
const memoDb = makeMemoDb({ topics: [] })
const page = new TopicDiscoveryPage({ memoDb })
await page.load()
assert.deepEqual(memoDb.calls[0], { limit: 50, offset: 0 })
})
test('load forwards limit and offset and stores pagination', async () => {
const memoDb = makeMemoDb({
topics: [{ room: 'bitcoin', postCount: 2 }],
pagination: { limit: 50, offset: 50, total: 60, hasMore: false }
})
const page = new TopicDiscoveryPage({ memoDb })
const result = await page.load({ limit: 50, offset: 50 })
assert.deepEqual(memoDb.calls[0], { limit: 50, offset: 50 })
assert.deepEqual(result.pagination, { limit: 50, offset: 50, total: 60, hasMore: false })
assert.equal(page.pagination.offset, 50)
})
test('canLoadMore reflects the pagination hasMore flag', async () => {
const more = new TopicDiscoveryPage({
memoDb: makeMemoDb({
topics: [{ room: 'a', postCount: 0 }],
pagination: { limit: 50, offset: 0, total: 60, hasMore: true }
})
})
await more.load()
assert.equal(more.canLoadMore(), true)
const last = new TopicDiscoveryPage({
memoDb: makeMemoDb({
topics: [{ room: 'a', postCount: 0 }],
pagination: { limit: 50, offset: 50, total: 60, hasMore: false }
})
})
await last.load()
assert.equal(last.canLoadMore(), false)
})
test('load throws when no memo db client is provided', async () => {
const page = new TopicDiscoveryPage({})
@@ -39,7 +85,7 @@ test('load throws when no memo db client is provided', async () => {
test('stores the provided navigate function', () => {
const navigate = () => {}
const page = new TopicDiscoveryPage({ memoDb: makeMemoDb([]), navigate })
const page = new TopicDiscoveryPage({ memoDb: makeMemoDb({ topics: [] }), navigate })
assert.equal(page.navigate, navigate)
})
@@ -49,7 +95,7 @@ test('getTopic returns the matching topic', async () => {
{ room: 'bitcoin', postCount: 2 },
{ room: 'cash', postCount: 1 }
]
const page = new TopicDiscoveryPage({ memoDb: makeMemoDb(topics) })
const page = new TopicDiscoveryPage({ memoDb: makeMemoDb({ topics }) })
await page.load()
@@ -57,7 +103,7 @@ test('getTopic returns the matching topic', async () => {
})
test('getTopic returns null when the topic is not loaded', async () => {
const page = new TopicDiscoveryPage({ memoDb: makeMemoDb([]) })
const page = new TopicDiscoveryPage({ memoDb: makeMemoDb({ topics: [] }) })
await page.load()
+219 -1
View File
@@ -28,6 +28,7 @@ import GetPoll from '../../src/use-cases/get-poll.js'
import GetPollOptions from '../../src/use-cases/get-poll-options.js'
import GetPollVotes from '../../src/use-cases/get-poll-votes.js'
import { repairTxidEncoding } from '../../src/lib/repair-txid-encoding.js'
import { backfillTopicIndexes, topicRecencyKey } from '../../src/lib/backfill-topic-indexes.js'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const tmpDir = path.resolve(__dirname, '..', '..', 'tmp', 'acceptance')
@@ -97,12 +98,16 @@ async function createWorld () {
const postsGetCounter = { calls: 0 }
const likesIteratorCounter = { calls: 0 }
const postLikesIteratorCounter = { calls: 0 }
const roomsIteratorCounter = { calls: 0, entries: 0 }
const topicRecencyIteratorCounter = { calls: 0, entries: 0 }
wrapIterator(adapters.level.postHeightsDb, postHeightsIteratorCounter)
wrapIterator(adapters.level.addrPostHeightsDb, addrPostHeightsIteratorCounter)
wrapIterator(adapters.level.postChildrenDb, postChildrenIteratorCounter)
wrapGet(adapters.level.postsDb, postsGetCounter)
wrapIterator(adapters.level.likesDb, likesIteratorCounter)
wrapIterator(adapters.level.postLikesDb, postLikesIteratorCounter)
wrapIterator(adapters.level.roomsDb, roomsIteratorCounter)
wrapIterator(adapters.level.topicRecencyDb, topicRecencyIteratorCounter)
const listRecentPosts = new ListRecentPosts({ adapters })
const listPostsByAddr = new ListPostsByAddr({ adapters })
@@ -145,6 +150,8 @@ async function createWorld () {
postsGetCounter,
likesIteratorCounter,
postLikesIteratorCounter,
roomsIteratorCounter,
topicRecencyIteratorCounter,
getLastResponse: () => lastResponse,
setLastResponse: (resp) => { lastResponse = resp },
close: async () => {
@@ -195,6 +202,16 @@ async function loadFixture (world, name) {
return
}
if (name === 'topic-indexes') {
await loadTopicIndexes(world)
return
}
if (name === 'rooms-with-topics-and-follows') {
await loadRoomsWithTopicsAndFollows(world)
return
}
if (name === 'topic-follows') {
await loadTopicFollows(world)
return
@@ -502,6 +519,56 @@ async function loadTopicsWithPosts (world) {
for (const [txid, post] of Object.entries(posts)) {
await world.adapters.level.postsDb.put(txid, post)
}
// Derived topic indexes (see topic-read.feature).
const summaries = [
{ room: 'bitcoin', postCount: 2, lastHeight: 300 },
{ room: 'cash', postCount: 1, lastHeight: 250 },
{ room: 'dev', postCount: 1, lastHeight: 400 },
{ room: 'lone', postCount: 0, lastHeight: 0 }
]
for (const summary of summaries) {
await world.adapters.level.topicSummariesDb.put(summary.room, summary)
await world.adapters.level.topicRecencyDb.put(
topicRecencyKey(summary.lastHeight, summary.room),
{ room: summary.room, blockHeight: summary.lastHeight }
)
}
}
// Fixture "topic-indexes" from topic-pagination.feature: indexes only.
async function loadTopicIndexes (world) {
const summaries = [
{ room: 'memo', postCount: 5, lastHeight: 600500 },
{ room: 'cash', postCount: 2, lastHeight: 600400 },
{ room: 'dance', postCount: 3, lastHeight: 600400 },
{ room: 'anime', postCount: 1, lastHeight: 600300 },
{ room: 'lone', postCount: 0, lastHeight: 0 },
{ room: 'quiet', postCount: 0, lastHeight: 0 }
]
for (const summary of summaries) {
await world.adapters.level.topicSummariesDb.put(summary.room, summary)
await world.adapters.level.topicRecencyDb.put(
topicRecencyKey(summary.lastHeight, summary.room),
{ room: summary.room, blockHeight: summary.lastHeight }
)
}
}
// Fixture "rooms-with-topics-and-follows" from backfill-topic-indexes.feature.
async function loadRoomsWithTopicsAndFollows (world) {
const entries = [
{ key: 'bitcoin:post-100', room: 'bitcoin', txid: 'post-100', type: 'post', blockHeight: 600100 },
{ key: 'bitcoin:post-200', room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 600200 },
{ key: 'bitcoin:addr-f', room: 'bitcoin', addr: 'addr-f', type: 'follow', unfollow: false },
{ key: 'cash:post-250', room: 'cash', txid: 'post-250', type: 'post', blockHeight: 600250 },
{ key: 'lone:addr-f', room: 'lone', addr: 'addr-f', type: 'follow', unfollow: false }
]
for (const entry of entries) {
await world.adapters.level.roomsDb.put(entry.key, entry)
}
}
async function loadTopicFollows (world) {
@@ -1119,7 +1186,7 @@ const handlers = [
},
{
name: 'response lists topics in order',
pattern: /^the response lists topics in order (<expected_order>)$/,
pattern: /^the response lists topics in order (<[A-Za-z0-9_]+>)$/,
run (m, example, world) {
const expected = resolveParam(m[1], example).split(',').map((s) => s.trim())
const actual = world.getLastResponse().topics.map((t) => t.room)
@@ -1163,6 +1230,157 @@ const handlers = [
await loadFixture(world, m[1])
}
},
{
name: 'db instance with rooms, posts, and topic index stores',
pattern: /^a psf-memo-db instance with rooms, posts, topicSummaries, and topicRecency stores$/,
async run () {
// World is already created with all stores.
}
},
{
name: 'load fixture into rooms, posts, and topic index stores',
pattern: /^the fixture "(.+)" is loaded into the rooms, posts, topicSummaries, and topicRecency stores$/,
async run (m, example, world) {
await loadFixture(world, m[1])
}
},
{
name: 'db instance with topic index stores',
pattern: /^a psf-memo-db instance with topicSummaries and topicRecency stores$/,
async run () {
// World is already created with both topic index stores.
}
},
{
name: 'load fixture into topic index stores',
pattern: /^the fixture "(.+)" is loaded into the topic index stores$/,
async run (m, example, world) {
await loadFixture(world, m[1])
}
},
{
name: 'db instance with rooms and topic index stores',
pattern: /^a psf-memo-db instance with rooms, topicSummaries, and topicRecency stores$/,
async run () {
// World is already created with all stores.
}
},
{
name: 'request topics with limit and offset',
pattern: /^the client requests \/topics with limit (<limit>) and offset (<offset>)$/,
async run (m, example, world) {
const limit = parseInt(resolveParam(m[1], example), 10)
const offset = parseInt(resolveParam(m[2], example), 10)
const resp = await world.listTopics.execute({ limit, offset })
world.setLastResponse(resp)
}
},
{
name: 'response contains quoted topic with post count',
pattern: /^the response contains the topic "(<topic>)" with post count (<postCount>)$/,
run (m, example, world) {
const topic = resolveParam(m[1], example)
const expectedCount = parseInt(resolveParam(m[2], example), 10)
const found = world.getLastResponse().topics.find((t) => t.room === topic)
if (!found) {
throw new Error(`Topic ${topic} not found in response`)
}
if (found.postCount !== expectedCount) {
throw new Error(`Expected post count ${expectedCount} for ${topic}, got ${found.postCount}`)
}
}
},
{
name: 'response pagination shows total and hasMore',
pattern: /^the response pagination shows total (<total>) and hasMore (<hasMore>)$/,
run (m, example, world) {
const expectedTotal = parseInt(resolveParam(m[1], example), 10)
const expectedHasMore = resolveParam(m[2], example) === 'true'
const pagination = world.getLastResponse().pagination
if (!pagination) {
throw new Error('Response has no pagination metadata')
}
if (pagination.total !== expectedTotal) {
throw new Error(`Expected total ${expectedTotal}, got ${pagination.total}`)
}
if (pagination.hasMore !== expectedHasMore) {
throw new Error(`Expected hasMore ${expectedHasMore}, got ${pagination.hasMore}`)
}
}
},
{
name: 'topicRecency store read count',
pattern: /^the topicRecency store was read exactly (<read_count>) records$/,
run (m, example, world) {
const expected = parseInt(resolveParam(m[1], example), 10)
const actual = world.topicRecencyIteratorCounter.entries || 0
if (actual !== expected) {
throw new Error(`Expected topicRecency to be read ${expected} times, got ${actual}`)
}
}
},
{
name: 'rooms store was not iterated',
pattern: /^the rooms store was not iterated$/,
run (m, example, world) {
if (world.roomsIteratorCounter.calls !== 0) {
throw new Error(`Expected rooms store not to be iterated, got ${world.roomsIteratorCounter.calls} call(s)`)
}
}
},
{
name: 'run topic backfill utility',
pattern: /^the topic backfill utility is run$/,
async run (m, example, world) {
await backfillTopicIndexes({
roomsDb: world.adapters.level.roomsDb,
topicSummariesDb: world.adapters.level.topicSummariesDb,
topicRecencyDb: world.adapters.level.topicRecencyDb
})
}
},
{
name: 'run topic backfill utility again',
pattern: /^the topic backfill utility is run again$/,
async run (m, example, world) {
await backfillTopicIndexes({
roomsDb: world.adapters.level.roomsDb,
topicSummariesDb: world.adapters.level.topicSummariesDb,
topicRecencyDb: world.adapters.level.topicRecencyDb
})
}
},
{
name: 'topicSummaries contains room with postCount and lastHeight',
pattern: /^the topicSummaries store contains the room "(<room>)" with postCount (<postCount>) and lastHeight (<[A-Za-z0-9_]+>)$/,
async run (m, example, world) {
const room = resolveParam(m[1], example)
const postCount = parseInt(resolveParam(m[2], example), 10)
const lastHeight = parseInt(resolveParam(m[3], example), 10)
const record = await world.adapters.level.topicSummariesDb.get(room)
if (!record) {
throw new Error(`No topicSummaries record for ${room}`)
}
if (record.room !== room || record.postCount !== postCount || record.lastHeight !== lastHeight) {
throw new Error(`Expected ${room} postCount ${postCount} lastHeight ${lastHeight}, got ${JSON.stringify(record)}`)
}
}
},
{
name: 'topicRecency records room at height',
pattern: /^the topicRecency store records the room "(<room>)" at block height (<height>)$/,
async run (m, example, world) {
const room = resolveParam(m[1], example)
const height = parseInt(resolveParam(m[2], example), 10)
const record = await world.adapters.level.topicRecencyDb.get(topicRecencyKey(height, room))
if (!record) {
throw new Error(`No topicRecency record for ${room} at ${height}`)
}
if (record.room !== room || record.blockHeight !== height) {
throw new Error(`Expected topicRecency ${room} at ${height}, got ${JSON.stringify(record)}`)
}
}
},
{
name: 'request topic follow state',
pattern: /^the client requests the topic follow state for room (<[A-Za-z0-9_]+>) and address (<[A-Za-z0-9_]+>)$/,
+2
View File
@@ -50,6 +50,8 @@ class Adapters {
this.topicQuery = new TopicQuery({
roomsDb: level.roomsDb,
postsDb: level.postsDb,
topicSummariesDb: level.topicSummariesDb,
topicRecencyDb: level.topicRecencyDb,
muteQuery: this.muteQuery
})
this.pollQuery = new PollQuery({
+2
View File
@@ -24,6 +24,8 @@ const DB_NAMES = [
'follows',
'mutes',
'rooms',
'topicSummaries',
'topicRecency',
'processErrors',
'ptxs',
'polls',
+61 -28
View File
@@ -5,8 +5,16 @@
- Topic posts are keyed `${room}:${txid}` with type 'post'.
- Topic follows are keyed `${room}:${addr}` with type 'follow'.
Topic listing is served from two indexes so it can order and paginate
without iterating the rooms store:
- topicSummaries: one record per room keyed by room, value
{ room, postCount, lastHeight }.
- topicRecency: one record per room keyed `${invertedHeight}:${room}`, value
{ room, blockHeight }. Follow-only rooms live at height 0. The height is
inverted so an ascending scan yields newest-first.
This adapter exposes:
- listTopics() - distinct rooms with post counts
- listTopics() - ordered, paginated distinct rooms with counts
- getTopicPostTxids() - paginated txids for a room sorted by block height
*/
@@ -14,15 +22,23 @@ import { loadMutedAddrs, isMutedPost } from './lib/muted-posts.js'
class TopicQuery {
constructor (localConfig = {}) {
const { roomsDb, postsDb, muteQuery } = localConfig
const { roomsDb, postsDb, topicSummariesDb, topicRecencyDb, muteQuery } = localConfig
if (!roomsDb) {
throw new Error('roomsDb required when instantiating TopicQuery adapter.')
}
if (!postsDb) {
throw new Error('postsDb required when instantiating TopicQuery adapter.')
}
if (!topicSummariesDb) {
throw new Error('topicSummariesDb required when instantiating TopicQuery adapter.')
}
if (!topicRecencyDb) {
throw new Error('topicRecencyDb required when instantiating TopicQuery adapter.')
}
this.roomsDb = roomsDb
this.postsDb = postsDb
this.topicSummariesDb = topicSummariesDb
this.topicRecencyDb = topicRecencyDb
this.muteQuery = muteQuery || null
this.listTopics = this.listTopics.bind(this)
@@ -44,35 +60,52 @@ class TopicQuery {
return parts[parts.length - 1]
}
async listTopics () {
const topics = new Map()
// The topicSummaries key is the room name; fall back to the key when the
// stored value omits it.
summaryRoom (key, value) {
if (value && typeof value.room === 'string') return value.room
return String(key)
}
for await (const [key, value] of this.roomsDb.iterator()) {
const room = this.roomFromKey(key, value)
if (!topics.has(room)) {
topics.set(room, { postCount: 0, lastHeight: 0 })
}
const topic = topics.get(room)
if (value?.type === 'post') {
topic.postCount++
const height = value?.blockHeight ?? 0
if (height > topic.lastHeight) {
topic.lastHeight = height
}
}
// The topicRecency key is `${invertedHeight}:${room}`; the stored value
// carries the room name, but fall back to the key for robustness.
recencyRoom (key, value) {
if (value && typeof value.room === 'string') return value.room
const parts = String(key).split(':')
return parts.slice(1).join(':')
}
// Return a page of topics ordered by their most recent post, descending,
// with rooms at the same height ordered by name ascending. Counts and the
// total come from topicSummaries; ordering and pagination come from
// topicRecency, which is read only through offset + limit records.
async listTopics ({ limit = 100, offset = 0 } = {}) {
const postCounts = new Map()
for await (const [key, value] of this.topicSummariesDb.iterator()) {
postCounts.set(this.summaryRoom(key, value), value?.postCount ?? 0)
}
const total = postCounts.size
const recencyRooms = []
for await (const [key, value] of this.topicRecencyDb.iterator({ limit: offset + limit })) {
recencyRooms.push(this.recencyRoom(key, value))
}
return Array.from(topics.entries())
.map(([room, { postCount, lastHeight }]) => ({ room, postCount, lastHeight }))
.sort((a, b) => {
if (b.lastHeight !== a.lastHeight) {
return b.lastHeight - a.lastHeight
}
return a.room.localeCompare(b.room)
})
// lastHeight is an internal ordering key; keep it out of the public
// API contract so the response exposes only room and postCount.
.map(({ room, postCount }) => ({ room, postCount }))
const pageRooms = recencyRooms.slice(offset, offset + limit)
const topics = pageRooms.map((room) => ({
room,
postCount: postCounts.get(room) ?? 0
}))
return {
topics,
pagination: {
limit,
offset,
total,
hasMore: offset + topics.length < total
}
}
}
async getTopicPostTxids (room, { limit, offset, viewerAddr = null }) {
@@ -47,6 +47,8 @@ export const ENTITY_CONFIG = [
{ route: 'profilepic', dbProp: 'profilePicsDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'profilePicData' },
{ route: 'follow', dbProp: 'followsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'followData' },
{ route: 'room', dbProp: 'roomsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'roomData' },
{ route: 'topicsummary', dbProp: 'topicSummariesDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'topicSummaryData' },
{ route: 'topicrecency', dbProp: 'topicRecencyDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'topicRecencyData' },
{ route: 'processerror', dbProp: 'processErrorsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'errorData' },
{ route: 'ptx', dbProp: 'ptxsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'ptxData' },
{ route: 'poll', dbProp: 'pollsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'pollData' },
@@ -32,7 +32,11 @@ class TopicsRESTControllerLib {
* @apiName GetTopics
* @apiGroup REST Topics
*
* @apiDescription Returns all distinct Memo topics with their post counts.
* @apiDescription Returns a page of distinct Memo topics ordered by their
* most recent post, newest first. Supports limit and offset.
*
* @apiQuery {Number} [limit=100] Page size (max 100)
* @apiQuery {Number} [offset=0] Number of topics to skip after sorting
*
* @apiExample Example usage:
* curl -X GET "localhost:5021/topics"
@@ -40,10 +44,12 @@ class TopicsRESTControllerLib {
* @apiSuccess {Object[]} topics Array of topic objects
* @apiSuccess {String} topics.room Topic name
* @apiSuccess {Number} topics.postCount Number of posts in the topic
* @apiSuccess {Object} pagination Pagination metadata
*/
async getTopics (ctx) {
try {
ctx.body = await this.useCases.listTopics.execute()
const { limit, offset } = ctx.query
ctx.body = await this.useCases.listTopics.execute({ limit, offset })
} catch (err) {
this.handleError(ctx, err)
}
@@ -0,0 +1,76 @@
/*
Library to build the topicSummaries and topicRecency indexes from an
existing rooms store.
The read side serves GET /topics from these two indexes:
- topicSummaries: one record per room keyed by room, value
{ room, postCount, lastHeight }.
- topicRecency: one record per room keyed `${invertedHeight}:${room}`,
value { room, blockHeight }. Follow-only rooms live at height 0. The
height is inverted so an ascending scan yields newest-first.
The backfill recomputes both indexes from the rooms store, so running it
more than once is idempotent. Stale index records for rooms that no longer
have any room entry, or that moved to a new height, are removed.
The LevelDB handles are injected so the logic stays testable and free of
file-system concerns; the CLI wrapper in util/room opens the real stores.
*/
const HEIGHT_PAD = 12
// Heights are inverted in the recency key so a plain ascending iterator yields
// rooms from most recent to least recent. Rooms at the same height still sort
// by room name ascending.
const MAX_HEIGHT = 999999999999
export function topicRecencyKey (blockHeight, room) {
const inverted = MAX_HEIGHT - (blockHeight ?? 0)
return `${String(inverted).padStart(HEIGHT_PAD, '0')}:${room}`
}
async function collectSummaries (roomsDb) {
const summaries = new Map()
for await (const [key, value] of roomsDb.iterator()) {
const room = (value && typeof value.room === 'string') ? value.room : String(key).split(':')[0]
if (!summaries.has(room)) {
summaries.set(room, { room, postCount: 0, lastHeight: 0 })
}
if (value?.type === 'post') {
const summary = summaries.get(room)
summary.postCount++
const height = value.blockHeight ?? 0
if (height > summary.lastHeight) summary.lastHeight = height
}
}
return summaries
}
// Remove index records that no longer have a desired key.
async function removeStale (db, desiredKeys) {
for await (const [key] of db.iterator()) {
if (!desiredKeys.has(key)) {
await db.del(key)
}
}
}
export async function backfillTopicIndexes ({ roomsDb, topicSummariesDb, topicRecencyDb }) {
const summaries = await collectSummaries(roomsDb)
const desiredRecencyKeys = new Set()
for (const summary of summaries.values()) {
await topicSummariesDb.put(summary.room, summary)
const key = topicRecencyKey(summary.lastHeight, summary.room)
desiredRecencyKeys.add(key)
await topicRecencyDb.put(key, { room: summary.room, blockHeight: summary.lastHeight })
}
await removeStale(topicRecencyDb, desiredRecencyKeys)
await removeStale(topicSummariesDb, new Set(summaries.keys()))
return { rooms: summaries.size }
}
+5 -3
View File
@@ -1,8 +1,9 @@
/*
Use case: list all distinct Memo topics with their post counts.
Use case: list a page of distinct Memo topics ordered by most recent post.
*/
import { ListUseCase } from './lib/use-case.js'
import { parseLimit, parseOffset } from './lib/pagination.js'
class ListTopics extends ListUseCase {
constructor (localConfig = {}) {
@@ -10,8 +11,9 @@ class ListTopics extends ListUseCase {
}
async execute (inObj = {}) {
const topics = await this.adapters.topicQuery.listTopics()
return { topics }
const limit = parseLimit(inObj.limit)
const offset = parseOffset(inObj.offset)
return this.adapters.topicQuery.listTopics({ limit, offset })
}
}
@@ -47,10 +47,19 @@ function makeRoomsDb (entries) {
}
}
// Empty index store; this suite exercises the follow read side, not listTopics.
function makeEmptyIndexDb () {
return {
async * iterator () {}
}
}
function makeQuery (entries) {
return new TopicQuery({
roomsDb: makeRoomsDb(entries),
postsDb: {}
postsDb: {},
topicSummariesDb: makeEmptyIndexDb(),
topicRecencyDb: makeEmptyIndexDb()
})
}
@@ -19,6 +19,7 @@ import test from 'node:test'
import { seededRandom, forAll, intGen, txidGen } from './harness.js'
import TopicQuery from '../../src/adapters/topic-query.js'
import { topicRecencyKey } from '../../src/lib/backfill-topic-indexes.js'
const rng = seededRandom(20260828)
@@ -45,10 +46,55 @@ function makeRoomsDb (entries) {
function makeQuery (entries) {
return new TopicQuery({
roomsDb: makeRoomsDb(entries),
postsDb: {}
postsDb: {},
topicSummariesDb: makeIndexDb([]),
topicRecencyDb: makeIndexDb([])
})
}
// In-memory index store honoring the LevelDB `limit` option used by listTopics.
function makeIndexDb (records) {
const store = new Map(records)
return {
async * iterator (opts = {}) {
const keys = Array.from(store.keys()).sort()
const limit = opts.limit === undefined ? keys.length : opts.limit
for (const key of keys.slice(0, limit)) {
yield [key, store.get(key)]
}
}
}
}
// Derive topicSummaries and topicRecency records from rooms-store entries the
// way the indexer and backfill do.
function indexDbsFromEntries (entries) {
const summaries = new Map()
for (const e of entries) {
const room = e.value.room
if (!summaries.has(room)) summaries.set(room, { room, postCount: 0, lastHeight: 0 })
if (e.value.type === 'post') {
const summary = summaries.get(room)
summary.postCount++
const height = e.value.blockHeight ?? 0
if (height > summary.lastHeight) summary.lastHeight = height
}
}
const recency = []
for (const summary of summaries.values()) {
recency.push([
topicRecencyKey(summary.lastHeight, summary.room),
{ room: summary.room, blockHeight: summary.lastHeight }
])
}
return {
topicSummariesDb: makeIndexDb(Array.from(summaries.entries())),
topicRecencyDb: makeIndexDb(recency)
}
}
// In-memory posts store exposing the LevelDB `get` contract used by the mute
// post lookup. Returns the post record or throws a LEVEL_NOT_FOUND-style error.
function makePostsDb (entries) {
@@ -112,14 +158,15 @@ function fixtureGen () {
test('listTopics conserves post counts and returns rooms sorted by most recent post', async () => {
await forAll(
fixtureGen(),
async ({ entries, rooms }) => {
const query = makeQuery(entries)
const topics = await query.listTopics()
async ({ entries, limit, offset }) => {
const query = new TopicQuery({
roomsDb: makeRoomsDb(entries),
postsDb: {},
...indexDbsFromEntries(entries)
})
const { topics, pagination } = await query.listTopics({ limit, offset })
const postEntries = entries.filter((e) => e.value.type === 'post')
const totalPosts = topics.reduce((sum, t) => sum + t.postCount, 0)
if (totalPosts !== postEntries.length) return false
const expectedTopics = [...new Set(entries.map((e) => e.value.room))]
.map((room) => {
const heights = entries
@@ -131,7 +178,12 @@ test('listTopics conserves post counts and returns rooms sorted by most recent p
if (b.lastHeight !== a.lastHeight) return b.lastHeight - a.lastHeight
return a.room.localeCompare(b.room)
})
if (JSON.stringify(topics.map((t) => t.room)) !== JSON.stringify(expectedTopics.map((t) => t.room))) return false
if (pagination.total !== expectedTopics.length) return false
const expectedPage = expectedTopics.slice(offset, offset + limit)
if (JSON.stringify(topics.map((t) => t.room)) !== JSON.stringify(expectedPage.map((t) => t.room))) return false
if (pagination.hasMore !== (offset + topics.length < pagination.total)) return false
for (const topic of topics) {
const roomPosts = postEntries.filter((e) => e.value.room === topic.room).length
@@ -139,7 +191,7 @@ test('listTopics conserves post counts and returns rooms sorted by most recent p
}
return true
},
{ label: 'listTopics conservation and ordering' }
{ label: 'listTopics conservation, ordering, and pagination' }
)
})
@@ -204,6 +256,8 @@ test('getTopicPostTxids excludes muted addresses and conserves total and paginat
const query = new TopicQuery({
roomsDb: makeRoomsDb(entries),
postsDb: makePostsDb(entries),
topicSummariesDb: makeIndexDb([]),
topicRecencyDb: makeIndexDb([]),
muteQuery
})
@@ -1,6 +1,7 @@
import { assert } from 'chai'
import sinon from 'sinon'
import TopicQuery from '../../../src/adapters/topic-query.js'
import { topicRecencyKey } from '../../../src/lib/backfill-topic-indexes.js'
function makeRoomsDb (records = {}) {
const store = new Map(Object.entries(records))
@@ -37,11 +38,38 @@ function makeRoomsDb (records = {}) {
}
}
// In-memory index store whose iterator honors the LevelDB `limit` option, so
// the read path's bounded recency reads can be asserted.
function makeIteratorDb (records = []) {
const store = new Map(records)
return {
async get (key) {
if (!store.has(key)) {
const err = new Error('not found')
err.notFound = true
throw err
}
return store.get(key)
},
async * iterator (opts = {}) {
let keys = Array.from(store.keys()).sort()
if (opts.gte !== undefined) keys = keys.filter((k) => k >= opts.gte)
if (opts.lte !== undefined) keys = keys.filter((k) => k <= opts.lte)
const limit = opts.limit === undefined ? keys.length : opts.limit
for (const key of keys.slice(0, limit)) {
yield [key, store.get(key)]
}
}
}
}
describe('#TopicQuery', () => {
let uut
let sandbox
let roomsDb
let postsDb
let topicSummariesDb
let topicRecencyDb
beforeEach(() => {
sandbox = sinon.createSandbox()
@@ -52,8 +80,10 @@ describe('#TopicQuery', () => {
postsDb = {
get: sandbox.stub()
}
topicSummariesDb = makeIteratorDb()
topicRecencyDb = makeIteratorDb()
uut = new TopicQuery({ roomsDb, postsDb })
uut = new TopicQuery({ roomsDb, postsDb, topicSummariesDb, topicRecencyDb })
})
afterEach(() => sandbox.restore())
@@ -71,13 +101,33 @@ describe('#TopicQuery', () => {
it('should throw when postsDb is missing', () => {
try {
// eslint-disable-next-line no-new
new TopicQuery({ roomsDb })
new TopicQuery({ roomsDb, topicSummariesDb, topicRecencyDb })
assert.fail('Expected error')
} catch (err) {
assert.include(err.message, 'postsDb required')
}
})
it('should throw when topicSummariesDb is missing', () => {
try {
// eslint-disable-next-line no-new
new TopicQuery({ roomsDb, postsDb, topicRecencyDb })
assert.fail('Expected error')
} catch (err) {
assert.include(err.message, 'topicSummariesDb required')
}
})
it('should throw when topicRecencyDb is missing', () => {
try {
// eslint-disable-next-line no-new
new TopicQuery({ roomsDb, postsDb, topicSummariesDb })
assert.fail('Expected error')
} catch (err) {
assert.include(err.message, 'topicRecencyDb required')
}
})
describe('#roomFromKey', () => {
it('should return the room from the value when present', () => {
assert.equal(uut.roomFromKey('ignored:post-1', { room: 'bitcoin' }), 'bitcoin')
@@ -95,60 +145,85 @@ describe('#TopicQuery', () => {
})
describe('#listTopics', () => {
it('should return distinct topics with post counts', async () => {
async function * mockRooms () {
yield ['bitcoin:post-300', { room: 'bitcoin', txid: 'post-300', type: 'post', blockHeight: 300 }]
yield ['bitcoin:post-200', { room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 200 }]
yield ['cash:post-250', { room: 'cash', txid: 'post-250', type: 'post', blockHeight: 250 }]
yield ['dev:post-100', { room: 'dev', txid: 'post-100', type: 'post', blockHeight: 100 }]
yield ['lone:addr-f', { room: 'lone', addr: 'addr-f', type: 'follow', unfollow: false }]
}
roomsDb.iterator.returns(mockRooms())
const summaries = [
['memo', { room: 'memo', postCount: 5, lastHeight: 600500 }],
['cash', { room: 'cash', postCount: 2, lastHeight: 600400 }],
['dance', { room: 'dance', postCount: 3, lastHeight: 600400 }],
['anime', { room: 'anime', postCount: 1, lastHeight: 600300 }],
['lone', { room: 'lone', postCount: 0, lastHeight: 0 }],
['quiet', { room: 'quiet', postCount: 0, lastHeight: 0 }]
]
const recency = [
[topicRecencyKey(600500, 'memo'), { room: 'memo', blockHeight: 600500 }],
[topicRecencyKey(600400, 'cash'), { room: 'cash', blockHeight: 600400 }],
[topicRecencyKey(600400, 'dance'), { room: 'dance', blockHeight: 600400 }],
[topicRecencyKey(600300, 'anime'), { room: 'anime', blockHeight: 600300 }],
[topicRecencyKey(0, 'lone'), { room: 'lone', blockHeight: 0 }],
[topicRecencyKey(0, 'quiet'), { room: 'quiet', blockHeight: 0 }]
]
const result = await uut.listTopics()
it('should return topics ordered by recency with counts from summaries', async () => {
uut = new TopicQuery({
roomsDb,
postsDb,
topicSummariesDb: makeIteratorDb(summaries),
topicRecencyDb: makeIteratorDb(recency)
})
assert.deepEqual(result, [
{ room: 'bitcoin', postCount: 2 },
{ room: 'cash', postCount: 1 },
{ room: 'dev', postCount: 1 },
{ room: 'lone', postCount: 0 }
const result = await uut.listTopics({ limit: 100, offset: 0 })
assert.deepEqual(result.topics, [
{ room: 'memo', postCount: 5 },
{ room: 'cash', postCount: 2 },
{ room: 'dance', postCount: 3 },
{ room: 'anime', postCount: 1 },
{ room: 'lone', postCount: 0 },
{ room: 'quiet', postCount: 0 }
])
assert.deepEqual(result.pagination, { limit: 100, offset: 0, total: 6, hasMore: false })
})
it('should sort topics by most recent post descending', async () => {
async function * mockRooms () {
yield ['zoo:post-1', { room: 'zoo', txid: 'post-1', type: 'post', blockHeight: 1 }]
yield ['alpha:post-1', { room: 'alpha', txid: 'post-1', type: 'post', blockHeight: 2 }]
}
roomsDb.iterator.returns(mockRooms())
it('should paginate using recency order and report total and hasMore', async () => {
uut = new TopicQuery({
roomsDb,
postsDb,
topicSummariesDb: makeIteratorDb(summaries),
topicRecencyDb: makeIteratorDb(recency)
})
const result = await uut.listTopics()
const result = await uut.listTopics({ limit: 2, offset: 2 })
assert.deepEqual(result.map((t) => t.room), ['alpha', 'zoo'])
assert.deepEqual(result.topics.map((t) => t.room), ['dance', 'anime'])
assert.equal(result.pagination.total, 6)
assert.equal(result.pagination.hasMore, true)
})
it('should treat a post at block height 0 as the least recent', async () => {
async function * mockRooms () {
yield ['a:post-1', { room: 'a', txid: 'post-1', type: 'post', blockHeight: 0 }]
yield ['b:post-1', { room: 'b', txid: 'post-1', type: 'post', blockHeight: 1 }]
}
roomsDb.iterator.returns(mockRooms())
it('should report hasMore false on the last page', async () => {
uut = new TopicQuery({
roomsDb,
postsDb,
topicSummariesDb: makeIteratorDb(summaries),
topicRecencyDb: makeIteratorDb(recency)
})
const result = await uut.listTopics()
const result = await uut.listTopics({ limit: 2, offset: 4 })
assert.deepEqual(result.map((t) => t.room), ['b', 'a'])
assert.deepEqual(result.topics.map((t) => t.room), ['lone', 'quiet'])
assert.equal(result.pagination.total, 6)
assert.equal(result.pagination.hasMore, false)
})
it('should treat a post with no block height as height 0', async () => {
async function * mockRooms () {
yield ['a:post-1', { room: 'a', txid: 'post-1', type: 'post' }]
yield ['b:post-1', { room: 'b', txid: 'post-1', type: 'post', blockHeight: 1 }]
}
roomsDb.iterator.returns(mockRooms())
it('should read the recency index without iterating the rooms store', async () => {
uut = new TopicQuery({
roomsDb,
postsDb,
topicSummariesDb: makeIteratorDb(summaries),
topicRecencyDb: makeIteratorDb(recency)
})
const result = await uut.listTopics()
await uut.listTopics({ limit: 2, offset: 2 })
assert.deepEqual(result.map((t) => t.room), ['b', 'a'])
assert.equal(roomsDb.iterator.callCount, 0)
})
})
@@ -258,7 +333,7 @@ describe('#TopicQuery', () => {
const muteQuery = {
listMuted: sandbox.stub().resolves(['muted-addr'])
}
uut = new TopicQuery({ roomsDb, postsDb, muteQuery })
uut = new TopicQuery({ roomsDb, postsDb, topicSummariesDb, topicRecencyDb, muteQuery })
const result = await uut.getTopicPostTxids('bitcoin', { limit: 100, offset: 0, viewerAddr: 'viewer-addr' })
@@ -320,7 +395,9 @@ describe('#TopicQuery', () => {
'bitcoin:addr-c': { room: 'bitcoin', addr: 'addr-c', type: 'follow', unfollow: true },
'cash:addr-a': { room: 'cash', addr: 'addr-a', type: 'follow', unfollow: false }
}),
postsDb
postsDb,
topicSummariesDb,
topicRecencyDb
})
const result = await query.listRoomFollowers('bitcoin')
@@ -332,7 +409,9 @@ describe('#TopicQuery', () => {
roomsDb: makeRoomsDb({
'bitcoin:addr-a': { room: 'bitcoin', type: 'follow', unfollow: false }
}),
postsDb
postsDb,
topicSummariesDb,
topicRecencyDb
})
const result = await query.listRoomFollowers('bitcoin')
@@ -342,7 +421,9 @@ describe('#TopicQuery', () => {
it('should return an empty array for a room with no followers', async () => {
const query = new TopicQuery({
roomsDb: makeRoomsDb({}),
postsDb
postsDb,
topicSummariesDb,
topicRecencyDb
})
const result = await query.listRoomFollowers('lone')
@@ -45,10 +45,11 @@ describe('#TopicsRESTController', () => {
afterEach(() => sandbox.restore())
it('should return topics from use case', async () => {
const ctx = { body: null, throw: sandbox.stub() }
const ctx = { query: { limit: '2', offset: '4' }, body: null, throw: sandbox.stub() }
await uut.getTopics(ctx)
assert.equal(uut.useCases.listTopics.execute.callCount, 1)
assert.deepEqual(uut.useCases.listTopics.execute.firstCall.args[0], { limit: '2', offset: '4' })
assert.equal(ctx.body.topics.length, 2)
assert.equal(ctx.body.topics[0].room, 'bitcoin')
})
@@ -106,7 +107,7 @@ describe('#TopicsRESTController', () => {
it('should throw a 500 when the use case fails without a status', async () => {
uut.useCases.listTopics.execute = sandbox.stub().rejects(new Error('boom'))
const ctx = { body: null, throw: sandbox.stub() }
const ctx = { query: {}, body: null, throw: sandbox.stub() }
await uut.getTopics(ctx)
@@ -0,0 +1,89 @@
import { assert } from 'chai'
import { backfillTopicIndexes, topicRecencyKey } from '../../../src/lib/backfill-topic-indexes.js'
function makeDb (records = []) {
const store = new Map(records)
return {
store,
async get (key) {
if (!store.has(key)) {
const err = new Error('not found')
err.notFound = true
throw err
}
return store.get(key)
},
async put (key, value) {
store.set(key, value)
},
async del (key) {
store.delete(key)
},
async * iterator () {
for (const key of Array.from(store.keys()).sort()) {
yield [key, store.get(key)]
}
}
}
}
describe('#backfillTopicIndexes', () => {
it('should summarize rooms with posts and follow-only rooms', async () => {
const roomsDb = makeDb([
['bitcoin:post-100', { room: 'bitcoin', txid: 'post-100', type: 'post', blockHeight: 600100 }],
['bitcoin:post-200', { room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 600200 }],
['bitcoin:addr-f', { room: 'bitcoin', addr: 'addr-f', type: 'follow', unfollow: false }],
['cash:post-250', { room: 'cash', txid: 'post-250', type: 'post', blockHeight: 600250 }],
['lone:addr-f', { room: 'lone', addr: 'addr-f', type: 'follow', unfollow: false }]
])
const topicSummariesDb = makeDb()
const topicRecencyDb = makeDb()
const result = await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb })
assert.equal(result.rooms, 3)
assert.deepEqual(topicSummariesDb.store.get('bitcoin'), { room: 'bitcoin', postCount: 2, lastHeight: 600200 })
assert.deepEqual(topicSummariesDb.store.get('cash'), { room: 'cash', postCount: 1, lastHeight: 600250 })
assert.deepEqual(topicSummariesDb.store.get('lone'), { room: 'lone', postCount: 0, lastHeight: 0 })
assert.deepEqual(topicRecencyDb.store.get(topicRecencyKey(600200, 'bitcoin')), { room: 'bitcoin', blockHeight: 600200 })
assert.deepEqual(topicRecencyDb.store.get(topicRecencyKey(600250, 'cash')), { room: 'cash', blockHeight: 600250 })
assert.deepEqual(topicRecencyDb.store.get(topicRecencyKey(0, 'lone')), { room: 'lone', blockHeight: 0 })
assert.equal(topicRecencyDb.store.size, 3)
})
it('should be idempotent across repeated runs', async () => {
const roomsDb = makeDb([
['bitcoin:post-200', { room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 600200 }],
['lone:addr-f', { room: 'lone', addr: 'addr-f', type: 'follow', unfollow: false }]
])
const topicSummariesDb = makeDb()
const topicRecencyDb = makeDb()
await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb })
const firstSummaries = new Map(topicSummariesDb.store)
const firstRecency = new Map(topicRecencyDb.store)
await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb })
assert.deepEqual(topicSummariesDb.store, firstSummaries)
assert.deepEqual(topicRecencyDb.store, firstRecency)
})
it('should remove stale recency records from a previous run', async () => {
const roomsDb = makeDb([
['bitcoin:post-200', { room: 'bitcoin', txid: 'post-200', type: 'post', blockHeight: 600200 }]
])
const topicSummariesDb = makeDb()
const topicRecencyDb = makeDb([
[topicRecencyKey(600100, 'bitcoin'), { room: 'bitcoin', blockHeight: 600100 }],
[topicRecencyKey(0, 'stale'), { room: 'stale', blockHeight: 0 }]
])
await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb })
assert.isFalse(topicRecencyDb.store.has(topicRecencyKey(600100, 'bitcoin')))
assert.isFalse(topicRecencyDb.store.has(topicRecencyKey(0, 'stale')))
assert.isTrue(topicRecencyDb.store.has(topicRecencyKey(600200, 'bitcoin')))
})
})
@@ -7,13 +7,18 @@ describe('#ListTopics', () => {
let sandbox
let topicQuery
const adapterResult = {
topics: [
{ room: 'bitcoin', postCount: 2 },
{ room: 'cash', postCount: 1 }
],
pagination: { limit: 100, offset: 0, total: 2, hasMore: false }
}
beforeEach(() => {
sandbox = sinon.createSandbox()
topicQuery = {
listTopics: sandbox.stub().resolves([
{ room: 'bitcoin', postCount: 2 },
{ room: 'cash', postCount: 1 }
])
listTopics: sandbox.stub().resolves(adapterResult)
}
uut = new ListTopics({
adapters: { topicQuery }
@@ -42,12 +47,36 @@ describe('#ListTopics', () => {
}
})
it('should return topics from the adapter', async () => {
it('should return the adapter page with default limit and offset', async () => {
const result = await uut.execute()
assert.deepEqual(result.topics, [
{ room: 'bitcoin', postCount: 2 },
{ room: 'cash', postCount: 1 }
])
assert.deepEqual(result, adapterResult)
assert.deepEqual(topicQuery.listTopics.firstCall.args[0], { limit: 100, offset: 0 })
})
it('should forward limit and offset to the adapter', async () => {
await uut.execute({ limit: 2, offset: 4 })
assert.deepEqual(topicQuery.listTopics.firstCall.args[0], { limit: 2, offset: 4 })
})
it('should reject a non-positive limit', async () => {
try {
await uut.execute({ limit: 0 })
assert.fail('Expected error')
} catch (err) {
assert.include(err.message, 'limit must be a positive integer')
assert.equal(err.status, 400)
}
})
it('should reject a negative offset', async () => {
try {
await uut.execute({ offset: -1 })
assert.fail('Expected error')
} catch (err) {
assert.include(err.message, 'offset must be a non-negative integer')
assert.equal(err.status, 400)
}
})
})
@@ -0,0 +1,71 @@
/*
Utility: build the topicSummaries and topicRecency indexes for an existing
psf-memo-db.
New deployments write these indexes while indexing live topic messages, but
existing databases that were populated before the topic recency feature need
a one-time backfill.
Run from the psf-memo-db repo root on the host that owns the LevelDB files:
node util/room/backfill-topic-indexes.js
The script is idempotent: re-running it produces the same indexes. Progress
and a summary are printed to stderr.
WARNING:
- This script opens the LevelDB files directly. psf-memo-db must NOT be
running, or another process must not hold the database locks.
- Make a backup of leveldb/current before running on a production server:
cp -r leveldb/current leveldb/current-pre-topic-index-backup
*/
import level from 'level'
import * as fs from 'fs'
import * as path from 'path'
import * as url from 'url'
import { backfillTopicIndexes } from '../../src/lib/backfill-topic-indexes.js'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const DATA_DIR = process.env.PSF_MEMO_DB_DATA_DIR
? path.resolve(process.env.PSF_MEMO_DB_DATA_DIR)
: path.resolve(__dirname, '../../leveldb/current')
function requiredStorePath (dir, name) {
const storePath = path.join(dir, name)
if (!fs.existsSync(storePath)) {
throw new Error(`Required LevelDB store not found: ${storePath}. Set PSF_MEMO_DB_DATA_DIR to the directory containing the rooms store.`)
}
return storePath
}
async function main () {
console.error(`Using LevelDB data directory: ${DATA_DIR}`)
const roomsPath = requiredStorePath(DATA_DIR, 'rooms')
const topicSummariesPath = path.join(DATA_DIR, 'topicSummaries')
const topicRecencyPath = path.join(DATA_DIR, 'topicRecency')
console.error('Opening LevelDB stores...')
const roomsDb = level(roomsPath, { valueEncoding: 'json' })
const topicSummariesDb = level(topicSummariesPath, { valueEncoding: 'json', createIfMissing: true })
const topicRecencyDb = level(topicRecencyPath, { valueEncoding: 'json', createIfMissing: true })
try {
console.error('Backfilling topicSummaries and topicRecency from rooms...')
const summary = await backfillTopicIndexes({ roomsDb, topicSummariesDb, topicRecencyDb })
console.error('\nTopic index backfill complete.')
console.error(` rooms summarized: ${summary.rooms}`)
} catch (err) {
console.error('\nTopic index backfill failed:', err.message)
process.exitCode = 1
} finally {
await topicRecencyDb.close().catch(() => {})
await topicSummariesDb.close().catch(() => {})
await roomsDb.close().catch(() => {})
}
}
main()
+118
View File
@@ -14,6 +14,9 @@ import { handleCreatePoll } from '../../src/use-cases/action-types/poll-create.j
import { handleAddPollOption } from '../../src/use-cases/action-types/poll-option.js'
import { handlePollVote } from '../../src/use-cases/action-types/poll-vote.js'
import { handleMute } from '../../src/use-cases/action-types/mute.js'
import { handleTopicMessage } from '../../src/use-cases/action-types/topic-message.js'
import { handleTopicFollow } from '../../src/use-cases/action-types/topic-follow.js'
import { topicRecencyKey } from '../../src/use-cases/action-types/helpers.js'
import BackupDb from '../../src/use-cases/backup-db.js'
function makeInMemoryDb () {
@@ -85,6 +88,9 @@ async function createWorld () {
const pollOptionDb = makeInMemoryDb()
const pollVoteDb = makeInMemoryDb()
const muteDb = makeInMemoryDb()
const roomDb = makeInMemoryDb()
const topicSummaryDb = makeInMemoryDb()
const topicRecencyDb = makeInMemoryDb()
const adapters = {
postDb: postsDb,
@@ -98,6 +104,9 @@ async function createWorld () {
pollOptionDb,
pollVoteDb,
muteDb,
roomDb,
topicSummaryDb,
topicRecencyDb,
processErrorDb: makeInMemoryDb(),
dbCtrl: {
backupDb: async (height, epoch) => {
@@ -121,6 +130,9 @@ async function createWorld () {
pollOptionsDb: pollOptionDb,
pollVotesDb: pollVoteDb,
mutesDb: muteDb,
roomsDb: roomDb,
topicSummariesDb: topicSummaryDb,
topicRecencyDb,
txidMap: new Map(),
lastTxid: null,
lastHeight: null,
@@ -697,6 +709,112 @@ const muteHandlers = [
throw new Error(`Expected no mute document for txid ${txid}, but one was stored`)
}
}
},
{
name: 'db instance with rooms and topic index stores',
pattern: /^a psf-memo-db instance with rooms, topicSummaries, and topicRecency stores$/,
async run () {
// World is already created with the room and topic index stores.
}
},
{
name: 'process a Memo topic message',
pattern: /^the indexer processes a Memo topic message (.+) in room "(.+)" from (.+) at block height (.+) with text "(.+)"$/,
async run (m, example, world) {
const txid = resolveTxid(m[1], example, world)
const room = m[2]
const addr = resolveParam(m[3], example)
const height = parseInt(resolveParam(m[4], example), 10)
const text = m[5]
world.lastTxid = txid
world.lastHeight = height
world.lastAddr = addr
const prefix = Buffer.from('6d0c', 'hex')
const ctx = {
adapters: world.adapters,
txid,
signerAddr: addr,
seen: Date.now(),
blockHeight: height,
decoded: {
action: 'topicMessage',
prefix,
pushDatas: [prefix, Buffer.from(room, 'utf8'), Buffer.from(text, 'utf8')]
}
}
world.lastTopicMessage = ctx
await handleTopicMessage(ctx)
}
},
{
name: 'reprocess the last Memo topic message',
pattern: /^the indexer processes the same Memo topic message (.+) again$/,
async run (m, example, world) {
if (!world.lastTopicMessage) {
throw new Error('No previous topic message to reprocess')
}
await handleTopicMessage(world.lastTopicMessage)
}
},
{
name: 'process a Memo topic follow',
pattern: /^the indexer processes a Memo topic follow for room "(.+)" from (.+)$/,
async run (m, example, world) {
const room = m[1]
const addr = resolveParam(m[2], example)
const txid = deriveTxid(`topic-follow-${room}-${addr}`)
world.lastTxid = txid
world.lastAddr = addr
const prefix = Buffer.from('6d0d', 'hex')
await handleTopicFollow({
adapters: world.adapters,
txid,
signerAddr: addr,
seen: Date.now(),
blockHeight: 600000,
decoded: {
action: 'topicFollow',
prefix,
pushDatas: [prefix, Buffer.from(room, 'utf8')]
}
})
}
},
{
name: 'topicSummaries contains room with postCount and lastHeight',
pattern: /^the topicSummaries store contains the room "(.+)" with postCount (.+) and lastHeight (.+)$/,
async run (m, example, world) {
const room = m[1]
const postCount = parseInt(resolveParam(m[2], example), 10)
const lastHeight = parseInt(resolveParam(m[3], example), 10)
const record = await world.topicSummariesDb.get(room)
if (!record) {
throw new Error(`No topicSummaries record for ${room}`)
}
if (record.room !== room || record.postCount !== postCount || record.lastHeight !== lastHeight) {
throw new Error(`Expected ${room} postCount ${postCount} lastHeight ${lastHeight}, got ${JSON.stringify(record)}`)
}
}
},
{
name: 'topicRecency records room at height',
pattern: /^the topicRecency store records the room "(.+)" at block height (.+)$/,
async run (m, example, world) {
const room = m[1]
const height = parseInt(resolveParam(m[2], example), 10)
const record = await world.topicRecencyDb.get(topicRecencyKey(height, room))
if (!record) {
throw new Error(`No topicRecency record for ${room} at ${height}`)
}
if (record.room !== room || record.blockHeight !== height) {
throw new Error(`Expected topicRecency ${room} at ${height}, got ${JSON.stringify(record)}`)
}
}
}
]
@@ -32,6 +32,8 @@ class Adapters {
this.followDb = createEntityDb('follow', 'key', 'followData')
this.muteDb = createEntityDb('mute', 'key', 'muteData')
this.roomDb = createEntityDb('room', 'key', 'roomData')
this.topicSummaryDb = createEntityDb('topicsummary', 'key', 'topicSummaryData')
this.topicRecencyDb = createEntityDb('topicrecency', 'key', 'topicRecencyData')
this.pollDb = createEntityDb('poll', 'txid', 'pollData')
this.pollOptionDb = createEntityDb('polloption', 'txid', 'optionData')
this.pollVoteDb = createEntityDb('pollvote', 'txid', 'voteData')
@@ -62,6 +62,30 @@ export function roomKey (roomName, txid) {
return `${roomName}:${txid}`
}
// The topicRecency store is keyed with an inverted height so a plain iterator
// yields rooms ordered by most recent post height and, within a height, by
// room name ascending.
export function topicRecencyKey (blockHeight, roomName) {
const inverted = 999999999999 - (blockHeight ?? 0)
return `${String(inverted).padStart(12, '0')}:${roomName}`
}
// True for the LevelDB not-found error, its in-memory equivalent, and the
// 404 surfaced by the psf-memo-db entity routes.
export function isNotFound (err) {
return Boolean(err && (err.notFound || err.code === 'LEVEL_NOT_FOUND' || err.response?.status === 404))
}
// Read a record, or null when it does not exist. Rethrows real errors.
export async function getIfPresent (db, key) {
try {
return await db.get(key)
} catch (err) {
if (isNotFound(err)) return null
throw err
}
}
export function postHeightKey (blockHeight, txid) {
const padded = String(blockHeight).padStart(12, '0')
return `${padded}:${txid}`
@@ -1,5 +1,6 @@
import { utf8FromPush, logProcessError, roomKey } from './helpers.js'
import { PREFIX_TOPIC_UNFOLLOW } from '../../lib/memo-codes.js'
import { ensureTopicRoom } from './topic-indexing.js'
export async function handleTopicFollow (ctx) {
const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx
@@ -22,4 +23,8 @@ export async function handleTopicFollow (ctx) {
type: 'follow',
blockHeight
})
if (!unfollow) {
await ensureTopicRoom(adapters, room)
}
}
@@ -0,0 +1,50 @@
/*
Maintain the topicSummaries and topicRecency indexes for Memo topic activity.
topicSummaries: one record per room keyed by room, value
{ room, postCount, lastHeight }.
topicRecency: one record per room keyed by `${invertedHeight}:${room}`, value
{ room, blockHeight }. Follow-only rooms live at height 0. The height is
inverted so an ascending scan yields newest-first.
Both indexes are written with the generic entity `update`, which upserts, so
re-running an action is safe.
*/
import { getIfPresent, topicRecencyKey } from './helpers.js'
// Add a post to the room's summary and move its recency record to the newest
// height. A room that is reprocessed is filtered out before this runs, so the
// post count is not double-counted.
export async function recordTopicPost (adapters, room, blockHeight) {
const height = blockHeight ?? 0
const summary = await getIfPresent(adapters.topicSummaryDb, room)
const lastHeight = Math.max(summary?.lastHeight ?? 0, height)
await adapters.topicSummaryDb.update(room, {
room,
postCount: (summary?.postCount ?? 0) + 1,
lastHeight
})
// When the room moved to a newer height, delete the stale recency record so
// each room has exactly one recency entry.
if (summary && (summary.lastHeight ?? 0) !== lastHeight) {
await adapters.topicRecencyDb.delete(topicRecencyKey(summary.lastHeight ?? 0, room))
}
await adapters.topicRecencyDb.update(topicRecencyKey(lastHeight, room), {
room,
blockHeight: lastHeight
})
}
// Give a follow-only room a zero-post summary and recency record. A room that
// already has a summary (because it has posts) is left unchanged.
export async function ensureTopicRoom (adapters, room) {
const summary = await getIfPresent(adapters.topicSummaryDb, room)
if (summary) return
await adapters.topicSummaryDb.update(room, { room, postCount: 0, lastHeight: 0 })
await adapters.topicRecencyDb.update(topicRecencyKey(0, room), { room, blockHeight: 0 })
}
@@ -1,6 +1,7 @@
import { utf8FromPush, logProcessError, roomKey } from './helpers.js'
import { utf8FromPush, logProcessError, roomKey, getIfPresent } from './helpers.js'
import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
import { handlePost } from './post.js'
import { recordTopicPost } from './topic-indexing.js'
export async function handleTopicMessage (ctx) {
const { adapters, txid, decoded, seen, blockHeight } = ctx
@@ -18,10 +19,20 @@ export async function handleTopicMessage (ctx) {
return
}
// The room record is the marker that this topic message was already
// indexed. Check it before writing so reprocessing does not double-count the
// room's post summary.
const key = roomKey(room, txid)
const alreadyIndexed = (await getIfPresent(adapters.roomDb, key)) !== null
await handlePost({
...ctx,
decoded: { ...decoded, pushDatas: [pushDatas[0], pushDatas[2]] }
})
await adapters.roomDb.create(roomKey(room, txid), { room, txid, seen, type: 'post', blockHeight })
await adapters.roomDb.create(key, { room, txid, seen, type: 'post', blockHeight })
if (!alreadyIndexed) {
await recordTopicPost(adapters, room, blockHeight)
}
}
@@ -0,0 +1,106 @@
import { assert } from 'chai'
import { handleTopicFollow } from '../../../../src/use-cases/action-types/topic-follow.js'
import { PREFIX_TOPIC_FOLLOW, PREFIX_TOPIC_UNFOLLOW } from '../../../../src/lib/memo-codes.js'
import { topicRecencyKey } from '../../../../src/use-cases/action-types/helpers.js'
function makeDb () {
const store = new Map()
return {
store,
async get (key) {
if (!store.has(key)) {
const err = new Error('not found')
err.notFound = true
throw err
}
return store.get(key)
},
async create (key, value) {
store.set(key, value)
return { success: true }
},
async update (key, value) {
store.set(key, value)
return { success: true }
},
async delete (key) {
store.delete(key)
return { success: true }
}
}
}
function makeAdapters () {
return {
roomDb: makeDb(),
topicSummaryDb: makeDb(),
topicRecencyDb: makeDb(),
processErrorDb: makeDb()
}
}
async function processFollow (adapters, { txid, room, addr, prefix = PREFIX_TOPIC_FOLLOW }) {
await handleTopicFollow({
adapters,
txid,
signerAddr: addr,
seen: 1,
blockHeight: 600000,
decoded: {
action: 'topic-follow',
prefix,
pushDatas: [prefix, Buffer.from(room, 'utf8')]
}
})
}
describe('#handleTopicFollow topic indexes', () => {
it('should record a zero-post room for a follow-only room', async () => {
const adapters = makeAdapters()
await processFollow(adapters, { txid: 'follow-1', room: 'lone', addr: 'bitcoincash:qaddr-a' })
assert.deepEqual(adapters.topicSummaryDb.store.get('lone'), {
room: 'lone',
postCount: 0,
lastHeight: 0
})
assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(0, 'lone')), {
room: 'lone',
blockHeight: 0
})
})
it('should leave an existing room summary unchanged', async () => {
const adapters = makeAdapters()
adapters.topicSummaryDb.store.set('bitcoin', { room: 'bitcoin', postCount: 1, lastHeight: 600400 })
adapters.topicRecencyDb.store.set(topicRecencyKey(600400, 'bitcoin'), { room: 'bitcoin', blockHeight: 600400 })
await processFollow(adapters, { txid: 'follow-2', room: 'bitcoin', addr: 'bitcoincash:qaddr-a' })
assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), {
room: 'bitcoin',
postCount: 1,
lastHeight: 600400
})
assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(600400, 'bitcoin')), {
room: 'bitcoin',
blockHeight: 600400
})
assert.isFalse(adapters.topicRecencyDb.store.has(topicRecencyKey(0, 'bitcoin')))
})
it('should not create a zero-post room for an unfollow', async () => {
const adapters = makeAdapters()
await processFollow(adapters, {
txid: 'unfollow-1',
room: 'lone',
addr: 'bitcoincash:qaddr-a',
prefix: PREFIX_TOPIC_UNFOLLOW
})
assert.equal(adapters.topicSummaryDb.store.size, 0)
assert.equal(adapters.topicRecencyDb.store.size, 0)
})
})
@@ -0,0 +1,135 @@
import { assert } from 'chai'
import { handleTopicMessage } from '../../../../src/use-cases/action-types/topic-message.js'
import { topicRecencyKey } from '../../../../src/use-cases/action-types/helpers.js'
function makeDb () {
const store = new Map()
return {
store,
async get (key) {
if (!store.has(key)) {
const err = new Error('not found')
err.notFound = true
throw err
}
return store.get(key)
},
async create (key, value) {
store.set(key, value)
return { success: true }
},
async update (key, value) {
store.set(key, value)
return { success: true }
},
async delete (key) {
store.delete(key)
return { success: true }
},
entries () {
return Array.from(store.entries())
}
}
}
function makeAdapters () {
return {
postDb: makeDb(),
postHeightDb: makeDb(),
addrPostHeightDb: makeDb(),
roomDb: makeDb(),
topicSummaryDb: makeDb(),
topicRecencyDb: makeDb(),
processErrorDb: makeDb()
}
}
async function processTopicMessage (adapters, { txid, room, addr, height, text }) {
const prefix = Buffer.from('6d0c', 'hex')
await handleTopicMessage({
adapters,
txid,
signerAddr: addr,
seen: 1,
blockHeight: height,
decoded: {
action: 'topic-message',
prefix,
pushDatas: [prefix, Buffer.from(room, 'utf8'), Buffer.from(text, 'utf8')]
}
})
}
describe('#handleTopicMessage topic indexes', () => {
it('should record a room summary and a recency record', async () => {
const adapters = makeAdapters()
await processTopicMessage(adapters, {
txid: 'topic-a1',
room: 'bitcoin',
addr: 'bitcoincash:qaddr-a',
height: 600100,
text: 'hello'
})
assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), {
room: 'bitcoin',
postCount: 1,
lastHeight: 600100
})
assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(600100, 'bitcoin')), {
room: 'bitcoin',
blockHeight: 600100
})
})
it('should accumulate postCount and keep the newest height', async () => {
const adapters = makeAdapters()
await processTopicMessage(adapters, { txid: 'topic-a1', room: 'bitcoin', addr: 'bitcoincash:qaddr-a', height: 600100, text: 'hello' })
await processTopicMessage(adapters, { txid: 'topic-a2', room: 'bitcoin', addr: 'bitcoincash:qaddr-a', height: 600200, text: 'again' })
assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), {
room: 'bitcoin',
postCount: 2,
lastHeight: 600200
})
assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(600200, 'bitcoin')), {
room: 'bitcoin',
blockHeight: 600200
})
assert.equal(adapters.topicRecencyDb.store.size, 1)
})
it('should keep the newest height when a later message has an earlier height', async () => {
const adapters = makeAdapters()
await processTopicMessage(adapters, { txid: 'topic-a3', room: 'bitcoin', addr: 'bitcoincash:qaddr-a', height: 600200, text: 'later' })
await processTopicMessage(adapters, { txid: 'topic-a4', room: 'bitcoin', addr: 'bitcoincash:qaddr-a', height: 600100, text: 'earlier' })
assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), {
room: 'bitcoin',
postCount: 2,
lastHeight: 600200
})
assert.deepEqual(adapters.topicRecencyDb.store.get(topicRecencyKey(600200, 'bitcoin')), {
room: 'bitcoin',
blockHeight: 600200
})
assert.equal(adapters.topicRecencyDb.store.size, 1)
})
it('should not double-count a reprocessed topic message', async () => {
const adapters = makeAdapters()
await processTopicMessage(adapters, { txid: 'topic-c1', room: 'bitcoin', addr: 'bitcoincash:qaddr-c', height: 600300, text: 'repeated' })
await processTopicMessage(adapters, { txid: 'topic-c1', room: 'bitcoin', addr: 'bitcoincash:qaddr-c', height: 600300, text: 'repeated' })
assert.deepEqual(adapters.topicSummaryDb.store.get('bitcoin'), {
room: 'bitcoin',
postCount: 1,
lastHeight: 600300
})
assert.equal(adapters.topicRecencyDb.store.size, 1)
})
})