Adding endpoint for most recent posts

This commit is contained in:
Chris Troutner
2026-06-03 12:00:28 -07:00
parent eac6fcb66c
commit 638207a8e1
19 changed files with 802 additions and 0 deletions
+5
View File
@@ -5,6 +5,7 @@
import LevelDb from './level-db.js'
import DbBackup from './db-backup.js'
import ProfileQuery from './profile-query.js'
import PostQuery from './post-query.js'
class Adapters {
constructor () {
@@ -22,6 +23,10 @@ class Adapters {
profilesDb: level.profilesDb,
ptxsDb: level.ptxsDb
})
this.postQuery = new PostQuery({
postsDb: level.postsDb,
ptxsDb: level.ptxsDb
})
return true
}
+48
View File
@@ -0,0 +1,48 @@
/*
Adapter for scanning posts and resolving block height from ptx records.
*/
class PostQuery {
constructor (localConfig = {}) {
const { postsDb, ptxsDb } = localConfig
if (!postsDb) {
throw new Error('postsDb required when instantiating PostQuery adapter.')
}
if (!ptxsDb) {
throw new Error('ptxsDb required when instantiating PostQuery adapter.')
}
this.postsDb = postsDb
this.ptxsDb = ptxsDb
this.scanPostsWithBlockHeight = this.scanPostsWithBlockHeight.bind(this)
}
async getBlockHeightForTxid (txid) {
if (!txid) return 0
try {
const ptx = await this.ptxsDb.get(txid)
const height = ptx && ptx.blockHeight
return typeof height === 'number' ? height : parseInt(height, 10) || 0
} catch (err) {
return 0
}
}
async scanPostsWithBlockHeight () {
const posts = []
for await (const [txid, post] of this.postsDb.iterator()) {
const blockHeight = await this.getBlockHeightForTxid(txid)
posts.push({
txid,
addr: post.addr,
text: post.text,
seen: post.seen,
blockHeight
})
}
return posts
}
}
export default PostQuery
+4
View File
@@ -5,6 +5,7 @@
import LevelRESTController from './level/index.js'
import HealthRouter from './health/index.js'
import ProfileRouter from './profile/index.js'
import PostsRouter from './posts/index.js'
class RESTControllers {
constructor (localConfig = {}) {
@@ -27,6 +28,9 @@ class RESTControllers {
const profileRouter = new ProfileRouter(dependencies)
profileRouter.attach(app)
const postsRouter = new PostsRouter(dependencies)
postsRouter.attach(app)
}
}
@@ -0,0 +1,46 @@
/*
REST API controller for /posts routes.
*/
import wlogger from '../../../adapters/wlogger.js'
class PostsRESTControllerLib {
constructor (localConfig = {}) {
this.adapters = localConfig.adapters
this.useCases = localConfig.useCases
if (!this.adapters) {
throw new Error('Adapters required for Posts REST Controller.')
}
if (!this.useCases) {
throw new Error('Use Cases required for Posts REST Controller.')
}
this.getRecentPosts = this.getRecentPosts.bind(this)
this.handleError = this.handleError.bind(this)
}
handleError (ctx, err) {
if (err.status) {
ctx.throw(err.status, err.message || err)
} else {
wlogger.error('Error in posts controller: ', err)
ctx.throw(500, err.message || 'Internal server error')
}
}
/**
* @api {get} /posts/recent List recent posts
* @apiQuery {Number} [limit=100] Page size (max 100)
* @apiQuery {Number} [offset=0] Number of posts to skip after sorting
*/
async getRecentPosts (ctx) {
try {
const { limit, offset } = ctx.query
ctx.body = await this.useCases.listRecentPosts.execute({ limit, offset })
} catch (err) {
this.handleError(ctx, err)
}
}
}
export default PostsRESTControllerLib
+33
View File
@@ -0,0 +1,33 @@
/*
REST API router for /posts routes.
*/
import Router from 'koa-router'
import PostsRESTControllerLib from './controller.js'
class PostsRouter {
constructor (localConfig = {}) {
this.adapters = localConfig.adapters
this.useCases = localConfig.useCases
if (!this.adapters) {
throw new Error('Adapters required when instantiating Posts REST Controller.')
}
if (!this.useCases) {
throw new Error('Use Cases required when instantiating Posts REST Controller.')
}
this.postsRESTController = new PostsRESTControllerLib({
adapters: this.adapters,
useCases: this.useCases
})
this.router = new Router({ prefix: '/posts' })
}
attach (app) {
this.router.get('/recent', this.postsRESTController.getRecentPosts)
app.use(this.router.routes())
app.use(this.router.allowedMethods())
}
}
export default PostsRouter
+3
View File
@@ -3,6 +3,7 @@
*/
import ListRecentProfiles from './list-recent-profiles.js'
import ListRecentPosts from './list-recent-posts.js'
class UseCases {
constructor (localConfig = {}) {
@@ -12,10 +13,12 @@ class UseCases {
}
this.listRecentProfiles = null
this.listRecentPosts = null
}
async start () {
this.listRecentProfiles = new ListRecentProfiles({ adapters: this.adapters })
this.listRecentPosts = new ListRecentPosts({ adapters: this.adapters })
console.log('Use cases initialized.')
return true
}
+81
View File
@@ -0,0 +1,81 @@
/*
Use case: list posts ordered by block height (most recent first), paginated.
*/
const DEFAULT_LIMIT = 100
const MAX_LIMIT = 100
class ListRecentPosts {
constructor (localConfig = {}) {
this.adapters = localConfig.adapters
if (!this.adapters) {
throw new Error('Adapters required when instantiating ListRecentPosts use case.')
}
if (!this.adapters.postQuery) {
throw new Error('postQuery adapter required for ListRecentPosts use case.')
}
this.execute = this.execute.bind(this)
}
parseLimit (limit) {
if (limit === undefined || limit === null || limit === '') {
return DEFAULT_LIMIT
}
const parsed = parseInt(limit, 10)
if (Number.isNaN(parsed) || parsed < 1) {
const err = new Error('limit must be a positive integer')
err.status = 400
throw err
}
if (parsed > MAX_LIMIT) {
const err = new Error(`limit cannot exceed ${MAX_LIMIT}`)
err.status = 400
throw err
}
return parsed
}
parseOffset (offset) {
if (offset === undefined || offset === null || offset === '') {
return 0
}
const parsed = parseInt(offset, 10)
if (Number.isNaN(parsed) || parsed < 0) {
const err = new Error('offset must be a non-negative integer')
err.status = 400
throw err
}
return parsed
}
sortPosts (posts) {
return posts.sort((a, b) => {
if (b.blockHeight !== a.blockHeight) {
return b.blockHeight - a.blockHeight
}
return (b.seen || 0) - (a.seen || 0)
})
}
async execute (inObj = {}) {
const limit = this.parseLimit(inObj.limit)
const offset = this.parseOffset(inObj.offset)
const allPosts = await this.adapters.postQuery.scanPostsWithBlockHeight()
const sorted = this.sortPosts(allPosts)
const total = sorted.length
const posts = sorted.slice(offset, offset + limit)
return {
posts,
pagination: {
limit,
offset,
total,
hasMore: offset + posts.length < total
}
}
}
}
export default ListRecentPosts
+53
View File
@@ -0,0 +1,53 @@
import { assert } from 'chai'
import sinon from 'sinon'
import PostQuery from '../../../src/adapters/post-query.js'
describe('#PostQuery', () => {
let uut
let sandbox
let postsDb
let ptxsDb
beforeEach(() => {
sandbox = sinon.createSandbox()
postsDb = {
iterator: sandbox.stub()
}
ptxsDb = {
get: sandbox.stub()
}
uut = new PostQuery({ postsDb, ptxsDb })
})
afterEach(() => sandbox.restore())
it('should scan posts and attach block height from ptx', async () => {
async function * mockIterator () {
yield ['tx1', { addr: 'addr1', text: 'hello', seen: 1000 }]
yield ['tx2', { addr: 'addr2', text: 'world', seen: 2000 }]
}
postsDb.iterator.returns(mockIterator())
ptxsDb.get.withArgs('tx1').resolves({ blockHeight: 600100 })
ptxsDb.get.withArgs('tx2').resolves({ blockHeight: 600200 })
const result = await uut.scanPostsWithBlockHeight()
assert.equal(result.length, 2)
assert.equal(result[0].txid, 'tx1')
assert.equal(result[0].blockHeight, 600100)
assert.equal(result[1].txid, 'tx2')
assert.equal(result[1].blockHeight, 600200)
})
it('should use block height 0 when ptx is missing', async () => {
async function * mockIterator () {
yield ['tx-missing', { addr: 'addr1', text: 'hi', seen: 1000 }]
}
postsDb.iterator.returns(mockIterator())
ptxsDb.get.rejects(new Error('not found'))
const result = await uut.scanPostsWithBlockHeight()
assert.equal(result[0].blockHeight, 0)
})
})
@@ -0,0 +1,38 @@
import { assert } from 'chai'
import sinon from 'sinon'
import PostsRESTControllerLib from '../../../src/controllers/rest-api/posts/controller.js'
describe('#PostsRESTController', () => {
let uut
let sandbox
beforeEach(() => {
sandbox = sinon.createSandbox()
uut = new PostsRESTControllerLib({
adapters: {},
useCases: {
listRecentPosts: {
execute: sandbox.stub().resolves({
posts: [{ txid: 'tx1', blockHeight: 600000 }],
pagination: { limit: 100, offset: 0, total: 1, hasMore: false }
})
}
}
})
})
afterEach(() => sandbox.restore())
it('should return recent posts from use case', async () => {
const ctx = { query: { limit: '50', offset: '0' }, body: null, throw: sandbox.stub() }
await uut.getRecentPosts(ctx)
assert.equal(uut.useCases.listRecentPosts.execute.callCount, 1)
assert.deepEqual(uut.useCases.listRecentPosts.execute.firstCall.args[0], {
limit: '50',
offset: '0'
})
assert.equal(ctx.body.posts.length, 1)
assert.equal(ctx.body.pagination.total, 1)
})
})
@@ -0,0 +1,65 @@
import { assert } from 'chai'
import sinon from 'sinon'
import ListRecentPosts from '../../../src/use-cases/list-recent-posts.js'
describe('#ListRecentPosts', () => {
let uut
let sandbox
const mockPosts = [
{ txid: 'tx-a', addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
{ txid: 'tx-b', addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
{ txid: 'tx-c', addr: 'addr-c', text: 'c', seen: 50, blockHeight: 600200 }
]
beforeEach(() => {
sandbox = sinon.createSandbox()
uut = new ListRecentPosts({
adapters: {
postQuery: {
scanPostsWithBlockHeight: sandbox.stub().resolves([...mockPosts])
}
}
})
})
afterEach(() => sandbox.restore())
it('should return posts sorted by block height descending', async () => {
const result = await uut.execute({ limit: 10, offset: 0 })
assert.equal(result.posts.length, 3)
assert.equal(result.posts[0].txid, 'tx-b')
assert.equal(result.posts[1].txid, 'tx-c')
assert.equal(result.posts[2].txid, 'tx-a')
assert.equal(result.pagination.total, 3)
assert.equal(result.pagination.hasMore, false)
})
it('should paginate with limit and offset', async () => {
const result = await uut.execute({ limit: 1, offset: 1 })
assert.equal(result.posts.length, 1)
assert.equal(result.posts[0].txid, 'tx-c')
assert.equal(result.pagination.limit, 1)
assert.equal(result.pagination.offset, 1)
assert.equal(result.pagination.hasMore, true)
})
it('should default limit to 100 and offset to 0', async () => {
const result = await uut.execute({})
assert.equal(result.pagination.limit, 100)
assert.equal(result.pagination.offset, 0)
})
it('should reject limit over 100', async () => {
try {
await uut.execute({ limit: 101 })
assert.fail('Expected error')
} catch (err) {
assert.equal(err.status, 400)
assert.include(err.message, 'limit cannot exceed')
}
})
})
+39
View File
@@ -0,0 +1,39 @@
/*
Utility: read all follows from the Memo indexer LevelDB and print to the terminal.
Run from the psf-memo-db repo root:
node util/follow/get_follows.js
*/
import level from 'level'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const followsDb = level(`${__dirname}/../../leveldb/current/follows`, {
valueEncoding: 'json'
})
async function getFollows () {
try {
let count = 0
for await (const [key, follow] of followsDb.iterator()) {
count++
console.log(`${key} = ${JSON.stringify(follow, null, 2)}`)
}
console.log(`\nTotal follows: ${count}`)
await followsDb.close()
} catch (err) {
console.error('Error reading follows:', err.message)
try {
await followsDb.close()
} catch (closeErr) {
// ignore close errors after read failure
}
process.exit(1)
}
}
getFollows()
+39
View File
@@ -0,0 +1,39 @@
/*
Utility: read all likes from the Memo indexer LevelDB and print to the terminal.
Run from the psf-memo-db repo root:
node util/like/get_likes.js
*/
import level from 'level'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const likesDb = level(`${__dirname}/../../leveldb/current/likes`, {
valueEncoding: 'json'
})
async function getLikes () {
try {
let count = 0
for await (const [txid, like] of likesDb.iterator()) {
count++
console.log(`${txid} = ${JSON.stringify(like, null, 2)}`)
}
console.log(`\nTotal likes: ${count}`)
await likesDb.close()
} catch (err) {
console.error('Error reading likes:', err.message)
try {
await likesDb.close()
} catch (closeErr) {
// ignore close errors after read failure
}
process.exit(1)
}
}
getLikes()
+39
View File
@@ -0,0 +1,39 @@
/*
Utility: read all names from the Memo indexer LevelDB and print to the terminal.
Run from the psf-memo-db repo root:
node util/name/get_names.js
*/
import level from 'level'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const namesDb = level(`${__dirname}/../../leveldb/current/names`, {
valueEncoding: 'json'
})
async function getNames () {
try {
let count = 0
for await (const [addr, name] of namesDb.iterator()) {
count++
console.log(`${addr} = ${JSON.stringify(name, null, 2)}`)
}
console.log(`\nTotal names: ${count}`)
await namesDb.close()
} catch (err) {
console.error('Error reading names:', err.message)
try {
await namesDb.close()
} catch (closeErr) {
// ignore close errors after read failure
}
process.exit(1)
}
}
getNames()
+39
View File
@@ -0,0 +1,39 @@
/*
Utility: read all postChildren from the Memo indexer LevelDB and print to the terminal.
Run from the psf-memo-db repo root:
node util/postchild/get_post_children.js
*/
import level from 'level'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const postChildrenDb = level(`${__dirname}/../../leveldb/current/postChildren`, {
valueEncoding: 'json'
})
async function getPostChildren () {
try {
let count = 0
for await (const [txid, child] of postChildrenDb.iterator()) {
count++
console.log(`${txid} = ${JSON.stringify(child, null, 2)}`)
}
console.log(`\nTotal post children: ${count}`)
await postChildrenDb.close()
} catch (err) {
console.error('Error reading postChildren:', err.message)
try {
await postChildrenDb.close()
} catch (closeErr) {
// ignore close errors after read failure
}
process.exit(1)
}
}
getPostChildren()
+39
View File
@@ -0,0 +1,39 @@
/*
Utility: read all postParents from the Memo indexer LevelDB and print to the terminal.
Run from the psf-memo-db repo root:
node util/postparent/get_post_parents.js
*/
import level from 'level'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const postParentsDb = level(`${__dirname}/../../leveldb/current/postParents`, {
valueEncoding: 'json'
})
async function getPostParents () {
try {
let count = 0
for await (const [txid, parent] of postParentsDb.iterator()) {
count++
console.log(`${txid} = ${JSON.stringify(parent, null, 2)}`)
}
console.log(`\nTotal post parents: ${count}`)
await postParentsDb.close()
} catch (err) {
console.error('Error reading postParents:', err.message)
try {
await postParentsDb.close()
} catch (closeErr) {
// ignore close errors after read failure
}
process.exit(1)
}
}
getPostParents()
+39
View File
@@ -0,0 +1,39 @@
/*
Utility: read all profilePics from the Memo indexer LevelDB and print to the terminal.
Run from the psf-memo-db repo root:
node util/profilepic/get_profile_pics.js
*/
import level from 'level'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const profilePicsDb = level(`${__dirname}/../../leveldb/current/profilePics`, {
valueEncoding: 'json'
})
async function getProfilePics () {
try {
let count = 0
for await (const [addr, profilePic] of profilePicsDb.iterator()) {
count++
console.log(`${addr} = ${JSON.stringify(profilePic, null, 2)}`)
}
console.log(`\nTotal profile pics: ${count}`)
await profilePicsDb.close()
} catch (err) {
console.error('Error reading profilePics:', err.message)
try {
await profilePicsDb.close()
} catch (closeErr) {
// ignore close errors after read failure
}
process.exit(1)
}
}
getProfilePics()
+114
View File
@@ -0,0 +1,114 @@
/*
Utility: backfill profile `seen` timestamps from on-chain block header times.
Reads each profile's txid, looks up blockHeight in ptxs, fetches block header
time via BCH RPC, and writes seen = block.time * 1000 (Unix ms).
Run from the psf-memo-db repo root (memo-db must be stopped to avoid lock conflicts):
node util/profiles/backfill_seen.js
Environment (same as psf-memo-indexer):
RPC_IP, RPC_PORT, RPC_USER, RPC_PASS
*/
import 'dotenv/config'
import level from 'level'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const rpcIp = process.env.RPC_IP || '172.17.0.1'
const rpcPort = process.env.RPC_PORT || '8332'
const rpcUser = process.env.RPC_USER || 'bitcoin'
const rpcPass = process.env.RPC_PASS || 'password'
const profilesDb = level(`${__dirname}/../../leveldb/current/profiles`, {
valueEncoding: 'json'
})
const ptxsDb = level(`${__dirname}/../../leveldb/current/ptxs`, {
valueEncoding: 'json'
})
const blockTimeCache = new Map()
async function rpcCall (method, params = []) {
const response = await fetch(`http://${rpcIp}:${rpcPort}/`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Basic ${Buffer.from(`${rpcUser}:${rpcPass}`).toString('base64')}`
},
body: JSON.stringify({
jsonrpc: '1.0',
id: method,
method,
params
})
})
const body = await response.json()
if (body.error) {
throw new Error(body.error.message || `RPC ${method} failed`)
}
return body.result
}
async function getBlockTimeMs (blockHeight) {
if (blockTimeCache.has(blockHeight)) {
return blockTimeCache.get(blockHeight)
}
const hash = await rpcCall('getblockhash', [blockHeight])
const header = await rpcCall('getblockheader', [hash, true])
const seen = header.time * 1000
blockTimeCache.set(blockHeight, seen)
return seen
}
async function backfillProfiles () {
let updated = 0
let skipped = 0
let errors = 0
try {
for await (const [addr, profile] of profilesDb.iterator()) {
try {
if (!profile.txid) {
skipped++
continue
}
const ptx = await ptxsDb.get(profile.txid)
if (!ptx || ptx.blockHeight === undefined) {
console.warn(`No ptx blockHeight for ${addr} txid ${profile.txid}`)
skipped++
continue
}
const seen = await getBlockTimeMs(ptx.blockHeight)
if (profile.seen === seen) {
skipped++
continue
}
await profilesDb.put(addr, { ...profile, seen })
updated++
console.log(`${addr}: seen ${profile.seen} -> ${seen} (block ${ptx.blockHeight})`)
} catch (err) {
errors++
console.error(`Error updating ${addr}:`, err.message)
}
}
console.log(`\nBackfill complete. Updated: ${updated}, skipped: ${skipped}, errors: ${errors}`)
} finally {
await profilesDb.close()
await ptxsDb.close()
}
}
backfillProfiles().catch((err) => {
console.error('Backfill failed:', err.message)
process.exit(1)
})
+39
View File
@@ -0,0 +1,39 @@
/*
Utility: read all ptxs (processed transactions) from the Memo indexer LevelDB and print to the terminal.
Run from the psf-memo-db repo root:
node util/ptx/get_ptxs.js
*/
import level from 'level'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const ptxsDb = level(`${__dirname}/../../leveldb/current/ptxs`, {
valueEncoding: 'json'
})
async function getPtxs () {
try {
let count = 0
for await (const [txid, ptx] of ptxsDb.iterator()) {
count++
console.log(`${txid} = ${JSON.stringify(ptx, null, 2)}`)
}
console.log(`\nTotal ptxs: ${count}`)
await ptxsDb.close()
} catch (err) {
console.error('Error reading ptxs:', err.message)
try {
await ptxsDb.close()
} catch (closeErr) {
// ignore close errors after read failure
}
process.exit(1)
}
}
getPtxs()
+39
View File
@@ -0,0 +1,39 @@
/*
Utility: read all rooms from the Memo indexer LevelDB and print to the terminal.
Run from the psf-memo-db repo root:
node util/room/get_rooms.js
*/
import level from 'level'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const roomsDb = level(`${__dirname}/../../leveldb/current/rooms`, {
valueEncoding: 'json'
})
async function getRooms () {
try {
let count = 0
for await (const [key, room] of roomsDb.iterator()) {
count++
console.log(`${key} = ${JSON.stringify(room, null, 2)}`)
}
console.log(`\nTotal rooms: ${count}`)
await roomsDb.close()
} catch (err) {
console.error('Error reading rooms:', err.message)
try {
await roomsDb.close()
} catch (closeErr) {
// ignore close errors after read failure
}
process.exit(1)
}
}
getRooms()