mirror of
https://github.com/Permissionless-Software-Foundation/psf-memo-db.git
synced 2026-09-21 16:52:02 -07:00
Adding endpoint for most recent posts
This commit is contained in:
@@ -5,6 +5,7 @@
|
||||
import LevelDb from './level-db.js'
|
||||
import DbBackup from './db-backup.js'
|
||||
import ProfileQuery from './profile-query.js'
|
||||
import PostQuery from './post-query.js'
|
||||
|
||||
class Adapters {
|
||||
constructor () {
|
||||
@@ -22,6 +23,10 @@ class Adapters {
|
||||
profilesDb: level.profilesDb,
|
||||
ptxsDb: level.ptxsDb
|
||||
})
|
||||
this.postQuery = new PostQuery({
|
||||
postsDb: level.postsDb,
|
||||
ptxsDb: level.ptxsDb
|
||||
})
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
Adapter for scanning posts and resolving block height from ptx records.
|
||||
*/
|
||||
|
||||
class PostQuery {
|
||||
constructor (localConfig = {}) {
|
||||
const { postsDb, ptxsDb } = localConfig
|
||||
if (!postsDb) {
|
||||
throw new Error('postsDb required when instantiating PostQuery adapter.')
|
||||
}
|
||||
if (!ptxsDb) {
|
||||
throw new Error('ptxsDb required when instantiating PostQuery adapter.')
|
||||
}
|
||||
this.postsDb = postsDb
|
||||
this.ptxsDb = ptxsDb
|
||||
this.scanPostsWithBlockHeight = this.scanPostsWithBlockHeight.bind(this)
|
||||
}
|
||||
|
||||
async getBlockHeightForTxid (txid) {
|
||||
if (!txid) return 0
|
||||
try {
|
||||
const ptx = await this.ptxsDb.get(txid)
|
||||
const height = ptx && ptx.blockHeight
|
||||
return typeof height === 'number' ? height : parseInt(height, 10) || 0
|
||||
} catch (err) {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
async scanPostsWithBlockHeight () {
|
||||
const posts = []
|
||||
|
||||
for await (const [txid, post] of this.postsDb.iterator()) {
|
||||
const blockHeight = await this.getBlockHeightForTxid(txid)
|
||||
posts.push({
|
||||
txid,
|
||||
addr: post.addr,
|
||||
text: post.text,
|
||||
seen: post.seen,
|
||||
blockHeight
|
||||
})
|
||||
}
|
||||
|
||||
return posts
|
||||
}
|
||||
}
|
||||
|
||||
export default PostQuery
|
||||
@@ -5,6 +5,7 @@
|
||||
import LevelRESTController from './level/index.js'
|
||||
import HealthRouter from './health/index.js'
|
||||
import ProfileRouter from './profile/index.js'
|
||||
import PostsRouter from './posts/index.js'
|
||||
|
||||
class RESTControllers {
|
||||
constructor (localConfig = {}) {
|
||||
@@ -27,6 +28,9 @@ class RESTControllers {
|
||||
|
||||
const profileRouter = new ProfileRouter(dependencies)
|
||||
profileRouter.attach(app)
|
||||
|
||||
const postsRouter = new PostsRouter(dependencies)
|
||||
postsRouter.attach(app)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
REST API controller for /posts routes.
|
||||
*/
|
||||
|
||||
import wlogger from '../../../adapters/wlogger.js'
|
||||
|
||||
class PostsRESTControllerLib {
|
||||
constructor (localConfig = {}) {
|
||||
this.adapters = localConfig.adapters
|
||||
this.useCases = localConfig.useCases
|
||||
if (!this.adapters) {
|
||||
throw new Error('Adapters required for Posts REST Controller.')
|
||||
}
|
||||
if (!this.useCases) {
|
||||
throw new Error('Use Cases required for Posts REST Controller.')
|
||||
}
|
||||
|
||||
this.getRecentPosts = this.getRecentPosts.bind(this)
|
||||
this.handleError = this.handleError.bind(this)
|
||||
}
|
||||
|
||||
handleError (ctx, err) {
|
||||
if (err.status) {
|
||||
ctx.throw(err.status, err.message || err)
|
||||
} else {
|
||||
wlogger.error('Error in posts controller: ', err)
|
||||
ctx.throw(500, err.message || 'Internal server error')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @api {get} /posts/recent List recent posts
|
||||
* @apiQuery {Number} [limit=100] Page size (max 100)
|
||||
* @apiQuery {Number} [offset=0] Number of posts to skip after sorting
|
||||
*/
|
||||
async getRecentPosts (ctx) {
|
||||
try {
|
||||
const { limit, offset } = ctx.query
|
||||
ctx.body = await this.useCases.listRecentPosts.execute({ limit, offset })
|
||||
} catch (err) {
|
||||
this.handleError(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default PostsRESTControllerLib
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
REST API router for /posts routes.
|
||||
*/
|
||||
|
||||
import Router from 'koa-router'
|
||||
import PostsRESTControllerLib from './controller.js'
|
||||
|
||||
class PostsRouter {
|
||||
constructor (localConfig = {}) {
|
||||
this.adapters = localConfig.adapters
|
||||
this.useCases = localConfig.useCases
|
||||
if (!this.adapters) {
|
||||
throw new Error('Adapters required when instantiating Posts REST Controller.')
|
||||
}
|
||||
if (!this.useCases) {
|
||||
throw new Error('Use Cases required when instantiating Posts REST Controller.')
|
||||
}
|
||||
|
||||
this.postsRESTController = new PostsRESTControllerLib({
|
||||
adapters: this.adapters,
|
||||
useCases: this.useCases
|
||||
})
|
||||
this.router = new Router({ prefix: '/posts' })
|
||||
}
|
||||
|
||||
attach (app) {
|
||||
this.router.get('/recent', this.postsRESTController.getRecentPosts)
|
||||
app.use(this.router.routes())
|
||||
app.use(this.router.allowedMethods())
|
||||
}
|
||||
}
|
||||
|
||||
export default PostsRouter
|
||||
@@ -3,6 +3,7 @@
|
||||
*/
|
||||
|
||||
import ListRecentProfiles from './list-recent-profiles.js'
|
||||
import ListRecentPosts from './list-recent-posts.js'
|
||||
|
||||
class UseCases {
|
||||
constructor (localConfig = {}) {
|
||||
@@ -12,10 +13,12 @@ class UseCases {
|
||||
}
|
||||
|
||||
this.listRecentProfiles = null
|
||||
this.listRecentPosts = null
|
||||
}
|
||||
|
||||
async start () {
|
||||
this.listRecentProfiles = new ListRecentProfiles({ adapters: this.adapters })
|
||||
this.listRecentPosts = new ListRecentPosts({ adapters: this.adapters })
|
||||
console.log('Use cases initialized.')
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
Use case: list posts ordered by block height (most recent first), paginated.
|
||||
*/
|
||||
|
||||
const DEFAULT_LIMIT = 100
|
||||
const MAX_LIMIT = 100
|
||||
|
||||
class ListRecentPosts {
|
||||
constructor (localConfig = {}) {
|
||||
this.adapters = localConfig.adapters
|
||||
if (!this.adapters) {
|
||||
throw new Error('Adapters required when instantiating ListRecentPosts use case.')
|
||||
}
|
||||
if (!this.adapters.postQuery) {
|
||||
throw new Error('postQuery adapter required for ListRecentPosts use case.')
|
||||
}
|
||||
this.execute = this.execute.bind(this)
|
||||
}
|
||||
|
||||
parseLimit (limit) {
|
||||
if (limit === undefined || limit === null || limit === '') {
|
||||
return DEFAULT_LIMIT
|
||||
}
|
||||
const parsed = parseInt(limit, 10)
|
||||
if (Number.isNaN(parsed) || parsed < 1) {
|
||||
const err = new Error('limit must be a positive integer')
|
||||
err.status = 400
|
||||
throw err
|
||||
}
|
||||
if (parsed > MAX_LIMIT) {
|
||||
const err = new Error(`limit cannot exceed ${MAX_LIMIT}`)
|
||||
err.status = 400
|
||||
throw err
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
parseOffset (offset) {
|
||||
if (offset === undefined || offset === null || offset === '') {
|
||||
return 0
|
||||
}
|
||||
const parsed = parseInt(offset, 10)
|
||||
if (Number.isNaN(parsed) || parsed < 0) {
|
||||
const err = new Error('offset must be a non-negative integer')
|
||||
err.status = 400
|
||||
throw err
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
sortPosts (posts) {
|
||||
return posts.sort((a, b) => {
|
||||
if (b.blockHeight !== a.blockHeight) {
|
||||
return b.blockHeight - a.blockHeight
|
||||
}
|
||||
return (b.seen || 0) - (a.seen || 0)
|
||||
})
|
||||
}
|
||||
|
||||
async execute (inObj = {}) {
|
||||
const limit = this.parseLimit(inObj.limit)
|
||||
const offset = this.parseOffset(inObj.offset)
|
||||
|
||||
const allPosts = await this.adapters.postQuery.scanPostsWithBlockHeight()
|
||||
const sorted = this.sortPosts(allPosts)
|
||||
const total = sorted.length
|
||||
const posts = sorted.slice(offset, offset + limit)
|
||||
|
||||
return {
|
||||
posts,
|
||||
pagination: {
|
||||
limit,
|
||||
offset,
|
||||
total,
|
||||
hasMore: offset + posts.length < total
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default ListRecentPosts
|
||||
@@ -0,0 +1,53 @@
|
||||
import { assert } from 'chai'
|
||||
import sinon from 'sinon'
|
||||
import PostQuery from '../../../src/adapters/post-query.js'
|
||||
|
||||
describe('#PostQuery', () => {
|
||||
let uut
|
||||
let sandbox
|
||||
let postsDb
|
||||
let ptxsDb
|
||||
|
||||
beforeEach(() => {
|
||||
sandbox = sinon.createSandbox()
|
||||
postsDb = {
|
||||
iterator: sandbox.stub()
|
||||
}
|
||||
ptxsDb = {
|
||||
get: sandbox.stub()
|
||||
}
|
||||
uut = new PostQuery({ postsDb, ptxsDb })
|
||||
})
|
||||
|
||||
afterEach(() => sandbox.restore())
|
||||
|
||||
it('should scan posts and attach block height from ptx', async () => {
|
||||
async function * mockIterator () {
|
||||
yield ['tx1', { addr: 'addr1', text: 'hello', seen: 1000 }]
|
||||
yield ['tx2', { addr: 'addr2', text: 'world', seen: 2000 }]
|
||||
}
|
||||
postsDb.iterator.returns(mockIterator())
|
||||
ptxsDb.get.withArgs('tx1').resolves({ blockHeight: 600100 })
|
||||
ptxsDb.get.withArgs('tx2').resolves({ blockHeight: 600200 })
|
||||
|
||||
const result = await uut.scanPostsWithBlockHeight()
|
||||
|
||||
assert.equal(result.length, 2)
|
||||
assert.equal(result[0].txid, 'tx1')
|
||||
assert.equal(result[0].blockHeight, 600100)
|
||||
assert.equal(result[1].txid, 'tx2')
|
||||
assert.equal(result[1].blockHeight, 600200)
|
||||
})
|
||||
|
||||
it('should use block height 0 when ptx is missing', async () => {
|
||||
async function * mockIterator () {
|
||||
yield ['tx-missing', { addr: 'addr1', text: 'hi', seen: 1000 }]
|
||||
}
|
||||
postsDb.iterator.returns(mockIterator())
|
||||
ptxsDb.get.rejects(new Error('not found'))
|
||||
|
||||
const result = await uut.scanPostsWithBlockHeight()
|
||||
|
||||
assert.equal(result[0].blockHeight, 0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,38 @@
|
||||
import { assert } from 'chai'
|
||||
import sinon from 'sinon'
|
||||
import PostsRESTControllerLib from '../../../src/controllers/rest-api/posts/controller.js'
|
||||
|
||||
describe('#PostsRESTController', () => {
|
||||
let uut
|
||||
let sandbox
|
||||
|
||||
beforeEach(() => {
|
||||
sandbox = sinon.createSandbox()
|
||||
uut = new PostsRESTControllerLib({
|
||||
adapters: {},
|
||||
useCases: {
|
||||
listRecentPosts: {
|
||||
execute: sandbox.stub().resolves({
|
||||
posts: [{ txid: 'tx1', blockHeight: 600000 }],
|
||||
pagination: { limit: 100, offset: 0, total: 1, hasMore: false }
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(() => sandbox.restore())
|
||||
|
||||
it('should return recent posts from use case', async () => {
|
||||
const ctx = { query: { limit: '50', offset: '0' }, body: null, throw: sandbox.stub() }
|
||||
await uut.getRecentPosts(ctx)
|
||||
|
||||
assert.equal(uut.useCases.listRecentPosts.execute.callCount, 1)
|
||||
assert.deepEqual(uut.useCases.listRecentPosts.execute.firstCall.args[0], {
|
||||
limit: '50',
|
||||
offset: '0'
|
||||
})
|
||||
assert.equal(ctx.body.posts.length, 1)
|
||||
assert.equal(ctx.body.pagination.total, 1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,65 @@
|
||||
import { assert } from 'chai'
|
||||
import sinon from 'sinon'
|
||||
import ListRecentPosts from '../../../src/use-cases/list-recent-posts.js'
|
||||
|
||||
describe('#ListRecentPosts', () => {
|
||||
let uut
|
||||
let sandbox
|
||||
|
||||
const mockPosts = [
|
||||
{ txid: 'tx-a', addr: 'addr-a', text: 'a', seen: 100, blockHeight: 600100 },
|
||||
{ txid: 'tx-b', addr: 'addr-b', text: 'b', seen: 200, blockHeight: 600200 },
|
||||
{ txid: 'tx-c', addr: 'addr-c', text: 'c', seen: 50, blockHeight: 600200 }
|
||||
]
|
||||
|
||||
beforeEach(() => {
|
||||
sandbox = sinon.createSandbox()
|
||||
uut = new ListRecentPosts({
|
||||
adapters: {
|
||||
postQuery: {
|
||||
scanPostsWithBlockHeight: sandbox.stub().resolves([...mockPosts])
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(() => sandbox.restore())
|
||||
|
||||
it('should return posts sorted by block height descending', async () => {
|
||||
const result = await uut.execute({ limit: 10, offset: 0 })
|
||||
|
||||
assert.equal(result.posts.length, 3)
|
||||
assert.equal(result.posts[0].txid, 'tx-b')
|
||||
assert.equal(result.posts[1].txid, 'tx-c')
|
||||
assert.equal(result.posts[2].txid, 'tx-a')
|
||||
assert.equal(result.pagination.total, 3)
|
||||
assert.equal(result.pagination.hasMore, false)
|
||||
})
|
||||
|
||||
it('should paginate with limit and offset', async () => {
|
||||
const result = await uut.execute({ limit: 1, offset: 1 })
|
||||
|
||||
assert.equal(result.posts.length, 1)
|
||||
assert.equal(result.posts[0].txid, 'tx-c')
|
||||
assert.equal(result.pagination.limit, 1)
|
||||
assert.equal(result.pagination.offset, 1)
|
||||
assert.equal(result.pagination.hasMore, true)
|
||||
})
|
||||
|
||||
it('should default limit to 100 and offset to 0', async () => {
|
||||
const result = await uut.execute({})
|
||||
|
||||
assert.equal(result.pagination.limit, 100)
|
||||
assert.equal(result.pagination.offset, 0)
|
||||
})
|
||||
|
||||
it('should reject limit over 100', async () => {
|
||||
try {
|
||||
await uut.execute({ limit: 101 })
|
||||
assert.fail('Expected error')
|
||||
} catch (err) {
|
||||
assert.equal(err.status, 400)
|
||||
assert.include(err.message, 'limit cannot exceed')
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Utility: read all follows from the Memo indexer LevelDB and print to the terminal.
|
||||
|
||||
Run from the psf-memo-db repo root:
|
||||
node util/follow/get_follows.js
|
||||
*/
|
||||
|
||||
import level from 'level'
|
||||
import * as url from 'url'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const followsDb = level(`${__dirname}/../../leveldb/current/follows`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
async function getFollows () {
|
||||
try {
|
||||
let count = 0
|
||||
|
||||
for await (const [key, follow] of followsDb.iterator()) {
|
||||
count++
|
||||
console.log(`${key} = ${JSON.stringify(follow, null, 2)}`)
|
||||
}
|
||||
|
||||
console.log(`\nTotal follows: ${count}`)
|
||||
await followsDb.close()
|
||||
} catch (err) {
|
||||
console.error('Error reading follows:', err.message)
|
||||
try {
|
||||
await followsDb.close()
|
||||
} catch (closeErr) {
|
||||
// ignore close errors after read failure
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
getFollows()
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Utility: read all likes from the Memo indexer LevelDB and print to the terminal.
|
||||
|
||||
Run from the psf-memo-db repo root:
|
||||
node util/like/get_likes.js
|
||||
*/
|
||||
|
||||
import level from 'level'
|
||||
import * as url from 'url'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const likesDb = level(`${__dirname}/../../leveldb/current/likes`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
async function getLikes () {
|
||||
try {
|
||||
let count = 0
|
||||
|
||||
for await (const [txid, like] of likesDb.iterator()) {
|
||||
count++
|
||||
console.log(`${txid} = ${JSON.stringify(like, null, 2)}`)
|
||||
}
|
||||
|
||||
console.log(`\nTotal likes: ${count}`)
|
||||
await likesDb.close()
|
||||
} catch (err) {
|
||||
console.error('Error reading likes:', err.message)
|
||||
try {
|
||||
await likesDb.close()
|
||||
} catch (closeErr) {
|
||||
// ignore close errors after read failure
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
getLikes()
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Utility: read all names from the Memo indexer LevelDB and print to the terminal.
|
||||
|
||||
Run from the psf-memo-db repo root:
|
||||
node util/name/get_names.js
|
||||
*/
|
||||
|
||||
import level from 'level'
|
||||
import * as url from 'url'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const namesDb = level(`${__dirname}/../../leveldb/current/names`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
async function getNames () {
|
||||
try {
|
||||
let count = 0
|
||||
|
||||
for await (const [addr, name] of namesDb.iterator()) {
|
||||
count++
|
||||
console.log(`${addr} = ${JSON.stringify(name, null, 2)}`)
|
||||
}
|
||||
|
||||
console.log(`\nTotal names: ${count}`)
|
||||
await namesDb.close()
|
||||
} catch (err) {
|
||||
console.error('Error reading names:', err.message)
|
||||
try {
|
||||
await namesDb.close()
|
||||
} catch (closeErr) {
|
||||
// ignore close errors after read failure
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
getNames()
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Utility: read all postChildren from the Memo indexer LevelDB and print to the terminal.
|
||||
|
||||
Run from the psf-memo-db repo root:
|
||||
node util/postchild/get_post_children.js
|
||||
*/
|
||||
|
||||
import level from 'level'
|
||||
import * as url from 'url'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const postChildrenDb = level(`${__dirname}/../../leveldb/current/postChildren`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
async function getPostChildren () {
|
||||
try {
|
||||
let count = 0
|
||||
|
||||
for await (const [txid, child] of postChildrenDb.iterator()) {
|
||||
count++
|
||||
console.log(`${txid} = ${JSON.stringify(child, null, 2)}`)
|
||||
}
|
||||
|
||||
console.log(`\nTotal post children: ${count}`)
|
||||
await postChildrenDb.close()
|
||||
} catch (err) {
|
||||
console.error('Error reading postChildren:', err.message)
|
||||
try {
|
||||
await postChildrenDb.close()
|
||||
} catch (closeErr) {
|
||||
// ignore close errors after read failure
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
getPostChildren()
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Utility: read all postParents from the Memo indexer LevelDB and print to the terminal.
|
||||
|
||||
Run from the psf-memo-db repo root:
|
||||
node util/postparent/get_post_parents.js
|
||||
*/
|
||||
|
||||
import level from 'level'
|
||||
import * as url from 'url'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const postParentsDb = level(`${__dirname}/../../leveldb/current/postParents`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
async function getPostParents () {
|
||||
try {
|
||||
let count = 0
|
||||
|
||||
for await (const [txid, parent] of postParentsDb.iterator()) {
|
||||
count++
|
||||
console.log(`${txid} = ${JSON.stringify(parent, null, 2)}`)
|
||||
}
|
||||
|
||||
console.log(`\nTotal post parents: ${count}`)
|
||||
await postParentsDb.close()
|
||||
} catch (err) {
|
||||
console.error('Error reading postParents:', err.message)
|
||||
try {
|
||||
await postParentsDb.close()
|
||||
} catch (closeErr) {
|
||||
// ignore close errors after read failure
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
getPostParents()
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Utility: read all profilePics from the Memo indexer LevelDB and print to the terminal.
|
||||
|
||||
Run from the psf-memo-db repo root:
|
||||
node util/profilepic/get_profile_pics.js
|
||||
*/
|
||||
|
||||
import level from 'level'
|
||||
import * as url from 'url'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const profilePicsDb = level(`${__dirname}/../../leveldb/current/profilePics`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
async function getProfilePics () {
|
||||
try {
|
||||
let count = 0
|
||||
|
||||
for await (const [addr, profilePic] of profilePicsDb.iterator()) {
|
||||
count++
|
||||
console.log(`${addr} = ${JSON.stringify(profilePic, null, 2)}`)
|
||||
}
|
||||
|
||||
console.log(`\nTotal profile pics: ${count}`)
|
||||
await profilePicsDb.close()
|
||||
} catch (err) {
|
||||
console.error('Error reading profilePics:', err.message)
|
||||
try {
|
||||
await profilePicsDb.close()
|
||||
} catch (closeErr) {
|
||||
// ignore close errors after read failure
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
getProfilePics()
|
||||
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
Utility: backfill profile `seen` timestamps from on-chain block header times.
|
||||
|
||||
Reads each profile's txid, looks up blockHeight in ptxs, fetches block header
|
||||
time via BCH RPC, and writes seen = block.time * 1000 (Unix ms).
|
||||
|
||||
Run from the psf-memo-db repo root (memo-db must be stopped to avoid lock conflicts):
|
||||
node util/profiles/backfill_seen.js
|
||||
|
||||
Environment (same as psf-memo-indexer):
|
||||
RPC_IP, RPC_PORT, RPC_USER, RPC_PASS
|
||||
*/
|
||||
|
||||
import 'dotenv/config'
|
||||
import level from 'level'
|
||||
import * as url from 'url'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const rpcIp = process.env.RPC_IP || '172.17.0.1'
|
||||
const rpcPort = process.env.RPC_PORT || '8332'
|
||||
const rpcUser = process.env.RPC_USER || 'bitcoin'
|
||||
const rpcPass = process.env.RPC_PASS || 'password'
|
||||
|
||||
const profilesDb = level(`${__dirname}/../../leveldb/current/profiles`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
const ptxsDb = level(`${__dirname}/../../leveldb/current/ptxs`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
const blockTimeCache = new Map()
|
||||
|
||||
async function rpcCall (method, params = []) {
|
||||
const response = await fetch(`http://${rpcIp}:${rpcPort}/`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Basic ${Buffer.from(`${rpcUser}:${rpcPass}`).toString('base64')}`
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: '1.0',
|
||||
id: method,
|
||||
method,
|
||||
params
|
||||
})
|
||||
})
|
||||
|
||||
const body = await response.json()
|
||||
if (body.error) {
|
||||
throw new Error(body.error.message || `RPC ${method} failed`)
|
||||
}
|
||||
return body.result
|
||||
}
|
||||
|
||||
async function getBlockTimeMs (blockHeight) {
|
||||
if (blockTimeCache.has(blockHeight)) {
|
||||
return blockTimeCache.get(blockHeight)
|
||||
}
|
||||
|
||||
const hash = await rpcCall('getblockhash', [blockHeight])
|
||||
const header = await rpcCall('getblockheader', [hash, true])
|
||||
const seen = header.time * 1000
|
||||
blockTimeCache.set(blockHeight, seen)
|
||||
return seen
|
||||
}
|
||||
|
||||
async function backfillProfiles () {
|
||||
let updated = 0
|
||||
let skipped = 0
|
||||
let errors = 0
|
||||
|
||||
try {
|
||||
for await (const [addr, profile] of profilesDb.iterator()) {
|
||||
try {
|
||||
if (!profile.txid) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
const ptx = await ptxsDb.get(profile.txid)
|
||||
if (!ptx || ptx.blockHeight === undefined) {
|
||||
console.warn(`No ptx blockHeight for ${addr} txid ${profile.txid}`)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
const seen = await getBlockTimeMs(ptx.blockHeight)
|
||||
if (profile.seen === seen) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
await profilesDb.put(addr, { ...profile, seen })
|
||||
updated++
|
||||
console.log(`${addr}: seen ${profile.seen} -> ${seen} (block ${ptx.blockHeight})`)
|
||||
} catch (err) {
|
||||
errors++
|
||||
console.error(`Error updating ${addr}:`, err.message)
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nBackfill complete. Updated: ${updated}, skipped: ${skipped}, errors: ${errors}`)
|
||||
} finally {
|
||||
await profilesDb.close()
|
||||
await ptxsDb.close()
|
||||
}
|
||||
}
|
||||
|
||||
backfillProfiles().catch((err) => {
|
||||
console.error('Backfill failed:', err.message)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Utility: read all ptxs (processed transactions) from the Memo indexer LevelDB and print to the terminal.
|
||||
|
||||
Run from the psf-memo-db repo root:
|
||||
node util/ptx/get_ptxs.js
|
||||
*/
|
||||
|
||||
import level from 'level'
|
||||
import * as url from 'url'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const ptxsDb = level(`${__dirname}/../../leveldb/current/ptxs`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
async function getPtxs () {
|
||||
try {
|
||||
let count = 0
|
||||
|
||||
for await (const [txid, ptx] of ptxsDb.iterator()) {
|
||||
count++
|
||||
console.log(`${txid} = ${JSON.stringify(ptx, null, 2)}`)
|
||||
}
|
||||
|
||||
console.log(`\nTotal ptxs: ${count}`)
|
||||
await ptxsDb.close()
|
||||
} catch (err) {
|
||||
console.error('Error reading ptxs:', err.message)
|
||||
try {
|
||||
await ptxsDb.close()
|
||||
} catch (closeErr) {
|
||||
// ignore close errors after read failure
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
getPtxs()
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
Utility: read all rooms from the Memo indexer LevelDB and print to the terminal.
|
||||
|
||||
Run from the psf-memo-db repo root:
|
||||
node util/room/get_rooms.js
|
||||
*/
|
||||
|
||||
import level from 'level'
|
||||
import * as url from 'url'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const roomsDb = level(`${__dirname}/../../leveldb/current/rooms`, {
|
||||
valueEncoding: 'json'
|
||||
})
|
||||
|
||||
async function getRooms () {
|
||||
try {
|
||||
let count = 0
|
||||
|
||||
for await (const [key, room] of roomsDb.iterator()) {
|
||||
count++
|
||||
console.log(`${key} = ${JSON.stringify(room, null, 2)}`)
|
||||
}
|
||||
|
||||
console.log(`\nTotal rooms: ${count}`)
|
||||
await roomsDb.close()
|
||||
} catch (err) {
|
||||
console.error('Error reading rooms:', err.message)
|
||||
try {
|
||||
await roomsDb.close()
|
||||
} catch (closeErr) {
|
||||
// ignore close errors after read failure
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
getRooms()
|
||||
Reference in New Issue
Block a user