diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 6aee8f8..0000000 --- a/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM node:20-alpine - -WORKDIR /app - -COPY package*.json ./ -RUN npm ci --omit=dev - -COPY . . - -EXPOSE 5021 - -CMD ["node", "index.js"] diff --git a/README.md b/README.md index e4c81df..3212fbb 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ LevelDB REST API for the Memo protocol indexer. Architecture mirrors [psf-slp-db](https://github.com/Permissionless-Software-Foundation/psf-slp-db). +Database-focused architecture notes live in the indexer repo: [psf-memo-indexer/dev-docs/psf-memo-db.md](../psf-memo-indexer/dev-docs/psf-memo-db.md). + ## Requirements - node ^20 @@ -22,16 +24,33 @@ Default port: **5021** All indexer data is exposed under `/level/*` with CRUD routes per entity (`post`, `like`, `name`, `profile`, `status`, etc.) plus: +- `GET /profile/recent` — paginated list of profiles, sorted by block height (newest first) - `POST /level/backup` — zip database snapshot - `POST /level/restore` — restore from snapshot (exits process) - `GET /health` — health check +### List recent profiles + +```bash +# First page (default limit 100) +curl -sS "http://localhost:5021/profile/recent" + +# Page size and offset +curl -sS "http://localhost:5021/profile/recent?limit=50&offset=50" +``` + +Query parameters: `limit` (default `100`, max `100`), `offset` (default `0`). Block height comes from the profile tx’s `ptx` record. + ## Tests ```bash npm test ``` +## Production (Docker) + +The production Docker setup is in [psf-memo-indexer/production/docker](../psf-memo-indexer/production/docker/memo-db/) (compose builds `memo-db` from this repo). + ## License MIT diff --git a/src/adapters/index.js b/src/adapters/index.js index 14b04f6..2c7902d 100644 --- a/src/adapters/index.js +++ b/src/adapters/index.js @@ -4,6 +4,7 @@ import LevelDb from './level-db.js' import DbBackup from './db-backup.js' +import ProfileQuery from './profile-query.js' class Adapters { constructor () { @@ -17,6 +18,10 @@ class Adapters { const level = this.levelDb.openDbs() this.level = level this.dbBackup = new DbBackup(level) + this.profileQuery = new ProfileQuery({ + profilesDb: level.profilesDb, + ptxsDb: level.ptxsDb + }) return true } diff --git a/src/adapters/profile-query.js b/src/adapters/profile-query.js new file mode 100644 index 0000000..5700443 --- /dev/null +++ b/src/adapters/profile-query.js @@ -0,0 +1,48 @@ +/* + Adapter for scanning profiles and resolving block height from ptx records. +*/ + +class ProfileQuery { + constructor (localConfig = {}) { + const { profilesDb, ptxsDb } = localConfig + if (!profilesDb) { + throw new Error('profilesDb required when instantiating ProfileQuery adapter.') + } + if (!ptxsDb) { + throw new Error('ptxsDb required when instantiating ProfileQuery adapter.') + } + this.profilesDb = profilesDb + this.ptxsDb = ptxsDb + this.scanProfilesWithBlockHeight = this.scanProfilesWithBlockHeight.bind(this) + } + + async getBlockHeightForTxid (txid) { + if (!txid) return 0 + try { + const ptx = await this.ptxsDb.get(txid) + const height = ptx && ptx.blockHeight + return typeof height === 'number' ? height : parseInt(height, 10) || 0 + } catch (err) { + return 0 + } + } + + async scanProfilesWithBlockHeight () { + const profiles = [] + + for await (const [addr, profile] of this.profilesDb.iterator()) { + const blockHeight = await this.getBlockHeightForTxid(profile.txid) + profiles.push({ + addr, + text: profile.text, + txid: profile.txid, + seen: profile.seen, + blockHeight + }) + } + + return profiles + } +} + +export default ProfileQuery diff --git a/src/controllers/rest-api/index.js b/src/controllers/rest-api/index.js index 9bfef65..c90ff30 100644 --- a/src/controllers/rest-api/index.js +++ b/src/controllers/rest-api/index.js @@ -4,6 +4,7 @@ import LevelRESTController from './level/index.js' import HealthRouter from './health/index.js' +import ProfileRouter from './profile/index.js' class RESTControllers { constructor (localConfig = {}) { @@ -23,6 +24,9 @@ class RESTControllers { const healthRouter = new HealthRouter() healthRouter.attach(app) + + const profileRouter = new ProfileRouter(dependencies) + profileRouter.attach(app) } } diff --git a/src/controllers/rest-api/profile/controller.js b/src/controllers/rest-api/profile/controller.js new file mode 100644 index 0000000..4f0a9b1 --- /dev/null +++ b/src/controllers/rest-api/profile/controller.js @@ -0,0 +1,46 @@ +/* + REST API controller for /profile routes. +*/ + +import wlogger from '../../../adapters/wlogger.js' + +class ProfileRESTControllerLib { + constructor (localConfig = {}) { + this.adapters = localConfig.adapters + this.useCases = localConfig.useCases + if (!this.adapters) { + throw new Error('Adapters required for Profile REST Controller.') + } + if (!this.useCases) { + throw new Error('Use Cases required for Profile REST Controller.') + } + + this.getRecentProfiles = this.getRecentProfiles.bind(this) + this.handleError = this.handleError.bind(this) + } + + handleError (ctx, err) { + if (err.status) { + ctx.throw(err.status, err.message || err) + } else { + wlogger.error('Error in profile controller: ', err) + ctx.throw(500, err.message || 'Internal server error') + } + } + + /** + * @api {get} /profile/recent List recent profiles + * @apiQuery {Number} [limit=100] Page size (max 100) + * @apiQuery {Number} [offset=0] Number of profiles to skip after sorting + */ + async getRecentProfiles (ctx) { + try { + const { limit, offset } = ctx.query + ctx.body = await this.useCases.listRecentProfiles.execute({ limit, offset }) + } catch (err) { + this.handleError(ctx, err) + } + } +} + +export default ProfileRESTControllerLib diff --git a/src/controllers/rest-api/profile/index.js b/src/controllers/rest-api/profile/index.js new file mode 100644 index 0000000..45312fe --- /dev/null +++ b/src/controllers/rest-api/profile/index.js @@ -0,0 +1,33 @@ +/* + REST API router for /profile routes. +*/ + +import Router from 'koa-router' +import ProfileRESTControllerLib from './controller.js' + +class ProfileRouter { + constructor (localConfig = {}) { + this.adapters = localConfig.adapters + this.useCases = localConfig.useCases + if (!this.adapters) { + throw new Error('Adapters required when instantiating Profile REST Controller.') + } + if (!this.useCases) { + throw new Error('Use Cases required when instantiating Profile REST Controller.') + } + + this.profileRESTController = new ProfileRESTControllerLib({ + adapters: this.adapters, + useCases: this.useCases + }) + this.router = new Router({ prefix: '/profile' }) + } + + attach (app) { + this.router.get('/recent', this.profileRESTController.getRecentProfiles) + app.use(this.router.routes()) + app.use(this.router.allowedMethods()) + } +} + +export default ProfileRouter diff --git a/src/use-cases/index.js b/src/use-cases/index.js index d3378c5..6a08d75 100644 --- a/src/use-cases/index.js +++ b/src/use-cases/index.js @@ -1,16 +1,21 @@ /* - Use cases for psf-memo-db (minimal for v1). + Use cases for psf-memo-db. */ +import ListRecentProfiles from './list-recent-profiles.js' + class UseCases { constructor (localConfig = {}) { this.adapters = localConfig.adapters if (!this.adapters) { throw new Error('Adapters required when instantiating UseCases.') } + + this.listRecentProfiles = null } async start () { + this.listRecentProfiles = new ListRecentProfiles({ adapters: this.adapters }) console.log('Use cases initialized.') return true } diff --git a/src/use-cases/list-recent-profiles.js b/src/use-cases/list-recent-profiles.js new file mode 100644 index 0000000..f6dee8e --- /dev/null +++ b/src/use-cases/list-recent-profiles.js @@ -0,0 +1,81 @@ +/* + Use case: list profiles ordered by block height (most recent first), paginated. +*/ + +const DEFAULT_LIMIT = 100 +const MAX_LIMIT = 100 + +class ListRecentProfiles { + constructor (localConfig = {}) { + this.adapters = localConfig.adapters + if (!this.adapters) { + throw new Error('Adapters required when instantiating ListRecentProfiles use case.') + } + if (!this.adapters.profileQuery) { + throw new Error('profileQuery adapter required for ListRecentProfiles use case.') + } + this.execute = this.execute.bind(this) + } + + parseLimit (limit) { + if (limit === undefined || limit === null || limit === '') { + return DEFAULT_LIMIT + } + const parsed = parseInt(limit, 10) + if (Number.isNaN(parsed) || parsed < 1) { + const err = new Error('limit must be a positive integer') + err.status = 400 + throw err + } + if (parsed > MAX_LIMIT) { + const err = new Error(`limit cannot exceed ${MAX_LIMIT}`) + err.status = 400 + throw err + } + return parsed + } + + parseOffset (offset) { + if (offset === undefined || offset === null || offset === '') { + return 0 + } + const parsed = parseInt(offset, 10) + if (Number.isNaN(parsed) || parsed < 0) { + const err = new Error('offset must be a non-negative integer') + err.status = 400 + throw err + } + return parsed + } + + sortProfiles (profiles) { + return profiles.sort((a, b) => { + if (b.blockHeight !== a.blockHeight) { + return b.blockHeight - a.blockHeight + } + return (b.seen || 0) - (a.seen || 0) + }) + } + + async execute (inObj = {}) { + const limit = this.parseLimit(inObj.limit) + const offset = this.parseOffset(inObj.offset) + + const allProfiles = await this.adapters.profileQuery.scanProfilesWithBlockHeight() + const sorted = this.sortProfiles(allProfiles) + const total = sorted.length + const profiles = sorted.slice(offset, offset + limit) + + return { + profiles, + pagination: { + limit, + offset, + total, + hasMore: offset + profiles.length < total + } + } + } +} + +export default ListRecentProfiles diff --git a/test/unit/adapters/profile-query.unit.js b/test/unit/adapters/profile-query.unit.js new file mode 100644 index 0000000..22c3c6f --- /dev/null +++ b/test/unit/adapters/profile-query.unit.js @@ -0,0 +1,51 @@ +import { assert } from 'chai' +import sinon from 'sinon' +import ProfileQuery from '../../../src/adapters/profile-query.js' + +describe('#ProfileQuery', () => { + let uut + let sandbox + let profilesDb + let ptxsDb + + beforeEach(() => { + sandbox = sinon.createSandbox() + profilesDb = { + iterator: sandbox.stub() + } + ptxsDb = { + get: sandbox.stub() + } + uut = new ProfileQuery({ profilesDb, ptxsDb }) + }) + + afterEach(() => sandbox.restore()) + + it('should scan profiles and attach block height from ptx', async () => { + async function * mockIterator () { + yield ['addr1', { text: 'hi', txid: 'tx1', seen: 1000 }] + yield ['addr2', { text: 'bye', txid: 'tx2', seen: 2000 }] + } + profilesDb.iterator.returns(mockIterator()) + ptxsDb.get.withArgs('tx1').resolves({ blockHeight: 600100 }) + ptxsDb.get.withArgs('tx2').resolves({ blockHeight: 600200 }) + + const result = await uut.scanProfilesWithBlockHeight() + + assert.equal(result.length, 2) + assert.equal(result[0].blockHeight, 600100) + assert.equal(result[1].blockHeight, 600200) + }) + + it('should use block height 0 when ptx is missing', async () => { + async function * mockIterator () { + yield ['addr1', { text: 'hi', txid: 'tx-missing', seen: 1000 }] + } + profilesDb.iterator.returns(mockIterator()) + ptxsDb.get.rejects(new Error('not found')) + + const result = await uut.scanProfilesWithBlockHeight() + + assert.equal(result[0].blockHeight, 0) + }) +}) diff --git a/test/unit/controllers/profile.controller.unit.js b/test/unit/controllers/profile.controller.unit.js new file mode 100644 index 0000000..5c5a993 --- /dev/null +++ b/test/unit/controllers/profile.controller.unit.js @@ -0,0 +1,38 @@ +import { assert } from 'chai' +import sinon from 'sinon' +import ProfileRESTControllerLib from '../../../src/controllers/rest-api/profile/controller.js' + +describe('#ProfileRESTController', () => { + let uut + let sandbox + + beforeEach(() => { + sandbox = sinon.createSandbox() + uut = new ProfileRESTControllerLib({ + adapters: {}, + useCases: { + listRecentProfiles: { + execute: sandbox.stub().resolves({ + profiles: [{ addr: 'q1', blockHeight: 600000 }], + pagination: { limit: 100, offset: 0, total: 1, hasMore: false } + }) + } + } + }) + }) + + afterEach(() => sandbox.restore()) + + it('should return recent profiles from use case', async () => { + const ctx = { query: { limit: '50', offset: '0' }, body: null, throw: sandbox.stub() } + await uut.getRecentProfiles(ctx) + + assert.equal(uut.useCases.listRecentProfiles.execute.callCount, 1) + assert.deepEqual(uut.useCases.listRecentProfiles.execute.firstCall.args[0], { + limit: '50', + offset: '0' + }) + assert.equal(ctx.body.profiles.length, 1) + assert.equal(ctx.body.pagination.total, 1) + }) +}) diff --git a/test/unit/use-cases/list-recent-profiles.unit.js b/test/unit/use-cases/list-recent-profiles.unit.js new file mode 100644 index 0000000..8f9d959 --- /dev/null +++ b/test/unit/use-cases/list-recent-profiles.unit.js @@ -0,0 +1,65 @@ +import { assert } from 'chai' +import sinon from 'sinon' +import ListRecentProfiles from '../../../src/use-cases/list-recent-profiles.js' + +describe('#ListRecentProfiles', () => { + let uut + let sandbox + + const mockProfiles = [ + { addr: 'addr-a', text: 'a', txid: 'tx-a', seen: 100, blockHeight: 600100 }, + { addr: 'addr-b', text: 'b', txid: 'tx-b', seen: 200, blockHeight: 600200 }, + { addr: 'addr-c', text: 'c', txid: 'tx-c', seen: 50, blockHeight: 600200 } + ] + + beforeEach(() => { + sandbox = sinon.createSandbox() + uut = new ListRecentProfiles({ + adapters: { + profileQuery: { + scanProfilesWithBlockHeight: sandbox.stub().resolves([...mockProfiles]) + } + } + }) + }) + + afterEach(() => sandbox.restore()) + + it('should return profiles sorted by block height descending', async () => { + const result = await uut.execute({ limit: 10, offset: 0 }) + + assert.equal(result.profiles.length, 3) + assert.equal(result.profiles[0].addr, 'addr-b') + assert.equal(result.profiles[1].addr, 'addr-c') + assert.equal(result.profiles[2].addr, 'addr-a') + assert.equal(result.pagination.total, 3) + assert.equal(result.pagination.hasMore, false) + }) + + it('should paginate with limit and offset', async () => { + const result = await uut.execute({ limit: 1, offset: 1 }) + + assert.equal(result.profiles.length, 1) + assert.equal(result.profiles[0].addr, 'addr-c') + assert.equal(result.pagination.limit, 1) + assert.equal(result.pagination.offset, 1) + assert.equal(result.pagination.hasMore, true) + }) + + it('should default limit to 100 and offset to 0', async () => { + const result = await uut.execute({}) + + assert.equal(result.pagination.limit, 100) + assert.equal(result.pagination.offset, 0) + }) + + it('should reject limit over 100', async () => { + try { + await uut.execute({ limit: 101 }) + assert.fail('Expected error') + } catch (err) { + assert.equal(err.status, 400) + assert.include(err.message, 'limit cannot exceed') + } + }) +}) diff --git a/util/post/get_posts.js b/util/post/get_posts.js new file mode 100644 index 0000000..1d351bb --- /dev/null +++ b/util/post/get_posts.js @@ -0,0 +1,39 @@ +/* + Utility: read all posts from the Memo indexer LevelDB and print to the terminal. + + Run from the psf-memo-db repo root: + node util/post/get_posts.js +*/ + +import level from 'level' +import * as url from 'url' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) + +const postsDb = level(`${__dirname}/../../leveldb/current/posts`, { + valueEncoding: 'json' +}) + +async function getPosts () { + try { + let count = 0 + + for await (const [txid, post] of postsDb.iterator()) { + count++ + console.log(`${txid} = ${JSON.stringify(post, null, 2)}`) + } + + console.log(`\nTotal posts: ${count}`) + await postsDb.close() + } catch (err) { + console.error('Error reading posts:', err.message) + try { + await postsDb.close() + } catch (closeErr) { + // ignore close errors after read failure + } + process.exit(1) + } +} + +getPosts() diff --git a/util/processErrors/get_process_errors.js b/util/processErrors/get_process_errors.js new file mode 100644 index 0000000..554c966 --- /dev/null +++ b/util/processErrors/get_process_errors.js @@ -0,0 +1,39 @@ +/* + Utility: read all processErrors from the Memo indexer LevelDB and print to the terminal. + + Run from the psf-memo-db repo root: + node util/processErrors/get_process_errors.js +*/ + +import level from 'level' +import * as url from 'url' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) + +const processErrorsDb = level(`${__dirname}/../../leveldb/current/processErrors`, { + valueEncoding: 'json' +}) + +async function getProcessErrors () { + try { + let count = 0 + + for await (const [txid, errorData] of processErrorsDb.iterator()) { + count++ + console.log(`${txid} = ${JSON.stringify(errorData, null, 2)}`) + } + + console.log(`\nTotal process errors: ${count}`) + await processErrorsDb.close() + } catch (err) { + console.error('Error reading processErrors:', err.message) + try { + await processErrorsDb.close() + } catch (closeErr) { + // ignore close errors after read failure + } + process.exit(1) + } +} + +getProcessErrors() diff --git a/util/profiles/get_profiles.js b/util/profiles/get_profiles.js new file mode 100644 index 0000000..f45817b --- /dev/null +++ b/util/profiles/get_profiles.js @@ -0,0 +1,39 @@ +/* + Utility: read all profiles from the Memo indexer LevelDB and print to the terminal. + + Run from the psf-memo-db repo root: + node util/profiles/get_profiles.js +*/ + +import level from 'level' +import * as url from 'url' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) + +const profilesDb = level(`${__dirname}/../../leveldb/current/profiles`, { + valueEncoding: 'json' +}) + +async function getProfiles () { + try { + let count = 0 + + for await (const [addr, profile] of profilesDb.iterator()) { + count++ + console.log(`${addr} = ${JSON.stringify(profile, null, 2)}`) + } + + console.log(`\nTotal profiles: ${count}`) + await profilesDb.close() + } catch (err) { + console.error('Error reading profiles:', err.message) + try { + await profilesDb.close() + } catch (closeErr) { + // ignore close errors after read failure + } + process.exit(1) + } +} + +getProfiles()