Refining REST API & Utility scripts

Dockerfile

@@ -1,12 +0,0 @@
-FROM node:20-alpine
-
-WORKDIR /app
-
-COPY package*.json ./
-RUN npm ci --omit=dev
-
-COPY . .
-
-EXPOSE 5021
-
-CMD ["node", "index.js"]

README.md

@@ -2,6 +2,8 @@
 
 LevelDB REST API for the Memo protocol indexer. Architecture mirrors [psf-slp-db](https://github.com/Permissionless-Software-Foundation/psf-slp-db).
 
+Database-focused architecture notes live in the indexer repo: [psf-memo-indexer/dev-docs/psf-memo-db.md](../psf-memo-indexer/dev-docs/psf-memo-db.md).
+
 ## Requirements
 
 - node ^20
@@ -22,16 +24,33 @@ Default port: **5021**
 
 All indexer data is exposed under `/level/*` with CRUD routes per entity (`post`, `like`, `name`, `profile`, `status`, etc.) plus:
 
+- `GET /profile/recent` — paginated list of profiles, sorted by block height (newest first)
 - `POST /level/backup` — zip database snapshot
 - `POST /level/restore` — restore from snapshot (exits process)
 - `GET /health` — health check
 
+### List recent profiles
+
+```bash
+# First page (default limit 100)
+curl -sS "http://localhost:5021/profile/recent"
+
+# Page size and offset
+curl -sS "http://localhost:5021/profile/recent?limit=50&offset=50"
+```
+
+Query parameters: `limit` (default `100`, max `100`), `offset` (default `0`). Block height comes from the profile tx’s `ptx` record.
+
 ## Tests
 
 ```bash
 npm test
 ```
 
+## Production (Docker)
+
+The production Docker setup is in [psf-memo-indexer/production/docker](../psf-memo-indexer/production/docker/memo-db/) (compose builds `memo-db` from this repo).
+
 ## License
 
 MIT

src/adapters/index.js

@@ -4,6 +4,7 @@
 
 import LevelDb from './level-db.js'
 import DbBackup from './db-backup.js'
+import ProfileQuery from './profile-query.js'
 
 class Adapters {
   constructor () {
@@ -17,6 +18,10 @@ class Adapters {
     const level = this.levelDb.openDbs()
     this.level = level
     this.dbBackup = new DbBackup(level)
+    this.profileQuery = new ProfileQuery({
+      profilesDb: level.profilesDb,
+      ptxsDb: level.ptxsDb
+    })
     return true
   }
 

src/adapters/profile-query.js

@@ -0,0 +1,48 @@
+/*
+  Adapter for scanning profiles and resolving block height from ptx records.
+*/
+
+class ProfileQuery {
+  constructor (localConfig = {}) {
+    const { profilesDb, ptxsDb } = localConfig
+    if (!profilesDb) {
+      throw new Error('profilesDb required when instantiating ProfileQuery adapter.')
+    }
+    if (!ptxsDb) {
+      throw new Error('ptxsDb required when instantiating ProfileQuery adapter.')
+    }
+    this.profilesDb = profilesDb
+    this.ptxsDb = ptxsDb
+    this.scanProfilesWithBlockHeight = this.scanProfilesWithBlockHeight.bind(this)
+  }
+
+  async getBlockHeightForTxid (txid) {
+    if (!txid) return 0
+    try {
+      const ptx = await this.ptxsDb.get(txid)
+      const height = ptx && ptx.blockHeight
+      return typeof height === 'number' ? height : parseInt(height, 10) || 0
+    } catch (err) {
+      return 0
+    }
+  }
+
+  async scanProfilesWithBlockHeight () {
+    const profiles = []
+
+    for await (const [addr, profile] of this.profilesDb.iterator()) {
+      const blockHeight = await this.getBlockHeightForTxid(profile.txid)
+      profiles.push({
+        addr,
+        text: profile.text,
+        txid: profile.txid,
+        seen: profile.seen,
+        blockHeight
+      })
+    }
+
+    return profiles
+  }
+}
+
+export default ProfileQuery

src/controllers/rest-api/index.js

@@ -4,6 +4,7 @@
 
 import LevelRESTController from './level/index.js'
 import HealthRouter from './health/index.js'
+import ProfileRouter from './profile/index.js'
 
 class RESTControllers {
   constructor (localConfig = {}) {
@@ -23,6 +24,9 @@ class RESTControllers {
 
     const healthRouter = new HealthRouter()
     healthRouter.attach(app)
+
+    const profileRouter = new ProfileRouter(dependencies)
+    profileRouter.attach(app)
   }
 }
 

src/controllers/rest-api/profile/controller.js

@@ -0,0 +1,46 @@
+/*
+  REST API controller for /profile routes.
+*/
+
+import wlogger from '../../../adapters/wlogger.js'
+
+class ProfileRESTControllerLib {
+  constructor (localConfig = {}) {
+    this.adapters = localConfig.adapters
+    this.useCases = localConfig.useCases
+    if (!this.adapters) {
+      throw new Error('Adapters required for Profile REST Controller.')
+    }
+    if (!this.useCases) {
+      throw new Error('Use Cases required for Profile REST Controller.')
+    }
+
+    this.getRecentProfiles = this.getRecentProfiles.bind(this)
+    this.handleError = this.handleError.bind(this)
+  }
+
+  handleError (ctx, err) {
+    if (err.status) {
+      ctx.throw(err.status, err.message || err)
+    } else {
+      wlogger.error('Error in profile controller: ', err)
+      ctx.throw(500, err.message || 'Internal server error')
+    }
+  }
+
+  /**
+   * @api {get} /profile/recent List recent profiles
+   * @apiQuery {Number} [limit=100] Page size (max 100)
+   * @apiQuery {Number} [offset=0] Number of profiles to skip after sorting
+   */
+  async getRecentProfiles (ctx) {
+    try {
+      const { limit, offset } = ctx.query
+      ctx.body = await this.useCases.listRecentProfiles.execute({ limit, offset })
+    } catch (err) {
+      this.handleError(ctx, err)
+    }
+  }
+}
+
+export default ProfileRESTControllerLib

src/controllers/rest-api/profile/index.js

@@ -0,0 +1,33 @@
+/*
+  REST API router for /profile routes.
+*/
+
+import Router from 'koa-router'
+import ProfileRESTControllerLib from './controller.js'
+
+class ProfileRouter {
+  constructor (localConfig = {}) {
+    this.adapters = localConfig.adapters
+    this.useCases = localConfig.useCases
+    if (!this.adapters) {
+      throw new Error('Adapters required when instantiating Profile REST Controller.')
+    }
+    if (!this.useCases) {
+      throw new Error('Use Cases required when instantiating Profile REST Controller.')
+    }
+
+    this.profileRESTController = new ProfileRESTControllerLib({
+      adapters: this.adapters,
+      useCases: this.useCases
+    })
+    this.router = new Router({ prefix: '/profile' })
+  }
+
+  attach (app) {
+    this.router.get('/recent', this.profileRESTController.getRecentProfiles)
+    app.use(this.router.routes())
+    app.use(this.router.allowedMethods())
+  }
+}
+
+export default ProfileRouter

src/use-cases/index.js

@@ -1,16 +1,21 @@
 /*
-  Use cases for psf-memo-db (minimal for v1).
+  Use cases for psf-memo-db.
 */
 
+import ListRecentProfiles from './list-recent-profiles.js'
+
 class UseCases {
   constructor (localConfig = {}) {
     this.adapters = localConfig.adapters
     if (!this.adapters) {
       throw new Error('Adapters required when instantiating UseCases.')
     }
+
+    this.listRecentProfiles = null
   }
 
   async start () {
+    this.listRecentProfiles = new ListRecentProfiles({ adapters: this.adapters })
     console.log('Use cases initialized.')
     return true
   }

src/use-cases/list-recent-profiles.js

@@ -0,0 +1,81 @@
+/*
+  Use case: list profiles ordered by block height (most recent first), paginated.
+*/
+
+const DEFAULT_LIMIT = 100
+const MAX_LIMIT = 100
+
+class ListRecentProfiles {
+  constructor (localConfig = {}) {
+    this.adapters = localConfig.adapters
+    if (!this.adapters) {
+      throw new Error('Adapters required when instantiating ListRecentProfiles use case.')
+    }
+    if (!this.adapters.profileQuery) {
+      throw new Error('profileQuery adapter required for ListRecentProfiles use case.')
+    }
+    this.execute = this.execute.bind(this)
+  }
+
+  parseLimit (limit) {
+    if (limit === undefined || limit === null || limit === '') {
+      return DEFAULT_LIMIT
+    }
+    const parsed = parseInt(limit, 10)
+    if (Number.isNaN(parsed) || parsed < 1) {
+      const err = new Error('limit must be a positive integer')
+      err.status = 400
+      throw err
+    }
+    if (parsed > MAX_LIMIT) {
+      const err = new Error(`limit cannot exceed ${MAX_LIMIT}`)
+      err.status = 400
+      throw err
+    }
+    return parsed
+  }
+
+  parseOffset (offset) {
+    if (offset === undefined || offset === null || offset === '') {
+      return 0
+    }
+    const parsed = parseInt(offset, 10)
+    if (Number.isNaN(parsed) || parsed < 0) {
+      const err = new Error('offset must be a non-negative integer')
+      err.status = 400
+      throw err
+    }
+    return parsed
+  }
+
+  sortProfiles (profiles) {
+    return profiles.sort((a, b) => {
+      if (b.blockHeight !== a.blockHeight) {
+        return b.blockHeight - a.blockHeight
+      }
+      return (b.seen || 0) - (a.seen || 0)
+    })
+  }
+
+  async execute (inObj = {}) {
+    const limit = this.parseLimit(inObj.limit)
+    const offset = this.parseOffset(inObj.offset)
+
+    const allProfiles = await this.adapters.profileQuery.scanProfilesWithBlockHeight()
+    const sorted = this.sortProfiles(allProfiles)
+    const total = sorted.length
+    const profiles = sorted.slice(offset, offset + limit)
+
+    return {
+      profiles,
+      pagination: {
+        limit,
+        offset,
+        total,
+        hasMore: offset + profiles.length < total
+      }
+    }
+  }
+}
+
+export default ListRecentProfiles

test/unit/adapters/profile-query.unit.js

@@ -0,0 +1,51 @@
+import { assert } from 'chai'
+import sinon from 'sinon'
+import ProfileQuery from '../../../src/adapters/profile-query.js'
+
+describe('#ProfileQuery', () => {
+  let uut
+  let sandbox
+  let profilesDb
+  let ptxsDb
+
+  beforeEach(() => {
+    sandbox = sinon.createSandbox()
+    profilesDb = {
+      iterator: sandbox.stub()
+    }
+    ptxsDb = {
+      get: sandbox.stub()
+    }
+    uut = new ProfileQuery({ profilesDb, ptxsDb })
+  })
+
+  afterEach(() => sandbox.restore())
+
+  it('should scan profiles and attach block height from ptx', async () => {
+    async function * mockIterator () {
+      yield ['addr1', { text: 'hi', txid: 'tx1', seen: 1000 }]
+      yield ['addr2', { text: 'bye', txid: 'tx2', seen: 2000 }]
+    }
+    profilesDb.iterator.returns(mockIterator())
+    ptxsDb.get.withArgs('tx1').resolves({ blockHeight: 600100 })
+    ptxsDb.get.withArgs('tx2').resolves({ blockHeight: 600200 })
+
+    const result = await uut.scanProfilesWithBlockHeight()
+
+    assert.equal(result.length, 2)
+    assert.equal(result[0].blockHeight, 600100)
+    assert.equal(result[1].blockHeight, 600200)
+  })
+
+  it('should use block height 0 when ptx is missing', async () => {
+    async function * mockIterator () {
+      yield ['addr1', { text: 'hi', txid: 'tx-missing', seen: 1000 }]
+    }
+    profilesDb.iterator.returns(mockIterator())
+    ptxsDb.get.rejects(new Error('not found'))
+
+    const result = await uut.scanProfilesWithBlockHeight()
+
+    assert.equal(result[0].blockHeight, 0)
+  })
+})

test/unit/controllers/profile.controller.unit.js

@@ -0,0 +1,38 @@
+import { assert } from 'chai'
+import sinon from 'sinon'
+import ProfileRESTControllerLib from '../../../src/controllers/rest-api/profile/controller.js'
+
+describe('#ProfileRESTController', () => {
+  let uut
+  let sandbox
+
+  beforeEach(() => {
+    sandbox = sinon.createSandbox()
+    uut = new ProfileRESTControllerLib({
+      adapters: {},
+      useCases: {
+        listRecentProfiles: {
+          execute: sandbox.stub().resolves({
+            profiles: [{ addr: 'q1', blockHeight: 600000 }],
+            pagination: { limit: 100, offset: 0, total: 1, hasMore: false }
+          })
+        }
+      }
+    })
+  })
+
+  afterEach(() => sandbox.restore())
+
+  it('should return recent profiles from use case', async () => {
+    const ctx = { query: { limit: '50', offset: '0' }, body: null, throw: sandbox.stub() }
+    await uut.getRecentProfiles(ctx)
+
+    assert.equal(uut.useCases.listRecentProfiles.execute.callCount, 1)
+    assert.deepEqual(uut.useCases.listRecentProfiles.execute.firstCall.args[0], {
+      limit: '50',
+      offset: '0'
+    })
+    assert.equal(ctx.body.profiles.length, 1)
+    assert.equal(ctx.body.pagination.total, 1)
+  })
+})

test/unit/use-cases/list-recent-profiles.unit.js

@@ -0,0 +1,65 @@
+import { assert } from 'chai'
+import sinon from 'sinon'
+import ListRecentProfiles from '../../../src/use-cases/list-recent-profiles.js'
+
+describe('#ListRecentProfiles', () => {
+  let uut
+  let sandbox
+
+  const mockProfiles = [
+    { addr: 'addr-a', text: 'a', txid: 'tx-a', seen: 100, blockHeight: 600100 },
+    { addr: 'addr-b', text: 'b', txid: 'tx-b', seen: 200, blockHeight: 600200 },
+    { addr: 'addr-c', text: 'c', txid: 'tx-c', seen: 50, blockHeight: 600200 }
+  ]
+
+  beforeEach(() => {
+    sandbox = sinon.createSandbox()
+    uut = new ListRecentProfiles({
+      adapters: {
+        profileQuery: {
+          scanProfilesWithBlockHeight: sandbox.stub().resolves([...mockProfiles])
+        }
+      }
+    })
+  })
+
+  afterEach(() => sandbox.restore())
+
+  it('should return profiles sorted by block height descending', async () => {
+    const result = await uut.execute({ limit: 10, offset: 0 })
+
+    assert.equal(result.profiles.length, 3)
+    assert.equal(result.profiles[0].addr, 'addr-b')
+    assert.equal(result.profiles[1].addr, 'addr-c')
+    assert.equal(result.profiles[2].addr, 'addr-a')
+    assert.equal(result.pagination.total, 3)
+    assert.equal(result.pagination.hasMore, false)
+  })
+
+  it('should paginate with limit and offset', async () => {
+    const result = await uut.execute({ limit: 1, offset: 1 })
+
+    assert.equal(result.profiles.length, 1)
+    assert.equal(result.profiles[0].addr, 'addr-c')
+    assert.equal(result.pagination.limit, 1)
+    assert.equal(result.pagination.offset, 1)
+    assert.equal(result.pagination.hasMore, true)
+  })
+
+  it('should default limit to 100 and offset to 0', async () => {
+    const result = await uut.execute({})
+
+    assert.equal(result.pagination.limit, 100)
+    assert.equal(result.pagination.offset, 0)
+  })
+
+  it('should reject limit over 100', async () => {
+    try {
+      await uut.execute({ limit: 101 })
+      assert.fail('Expected error')
+    } catch (err) {
+      assert.equal(err.status, 400)
+      assert.include(err.message, 'limit cannot exceed')
+    }
+  })
+})

util/post/get_posts.js

@@ -0,0 +1,39 @@
+/*
+  Utility: read all posts from the Memo indexer LevelDB and print to the terminal.
+
+  Run from the psf-memo-db repo root:
+    node util/post/get_posts.js
+*/
+
+import level from 'level'
+import * as url from 'url'
+
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
+
+const postsDb = level(`${__dirname}/../../leveldb/current/posts`, {
+  valueEncoding: 'json'
+})
+
+async function getPosts () {
+  try {
+    let count = 0
+
+    for await (const [txid, post] of postsDb.iterator()) {
+      count++
+      console.log(`${txid} = ${JSON.stringify(post, null, 2)}`)
+    }
+
+    console.log(`\nTotal posts: ${count}`)
+    await postsDb.close()
+  } catch (err) {
+    console.error('Error reading posts:', err.message)
+    try {
+      await postsDb.close()
+    } catch (closeErr) {
+      // ignore close errors after read failure
+    }
+    process.exit(1)
+  }
+}
+
+getPosts()

util/processErrors/get_process_errors.js

@@ -0,0 +1,39 @@
+/*
+  Utility: read all processErrors from the Memo indexer LevelDB and print to the terminal.
+
+  Run from the psf-memo-db repo root:
+    node util/processErrors/get_process_errors.js
+*/
+
+import level from 'level'
+import * as url from 'url'
+
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
+
+const processErrorsDb = level(`${__dirname}/../../leveldb/current/processErrors`, {
+  valueEncoding: 'json'
+})
+
+async function getProcessErrors () {
+  try {
+    let count = 0
+
+    for await (const [txid, errorData] of processErrorsDb.iterator()) {
+      count++
+      console.log(`${txid} = ${JSON.stringify(errorData, null, 2)}`)
+    }
+
+    console.log(`\nTotal process errors: ${count}`)
+    await processErrorsDb.close()
+  } catch (err) {
+    console.error('Error reading processErrors:', err.message)
+    try {
+      await processErrorsDb.close()
+    } catch (closeErr) {
+      // ignore close errors after read failure
+    }
+    process.exit(1)
+  }
+}
+
+getProcessErrors()

util/profiles/get_profiles.js

@@ -0,0 +1,39 @@
+/*
+  Utility: read all profiles from the Memo indexer LevelDB and print to the terminal.
+
+  Run from the psf-memo-db repo root:
+    node util/profiles/get_profiles.js
+*/
+
+import level from 'level'
+import * as url from 'url'
+
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
+
+const profilesDb = level(`${__dirname}/../../leveldb/current/profiles`, {
+  valueEncoding: 'json'
+})
+
+async function getProfiles () {
+  try {
+    let count = 0
+
+    for await (const [addr, profile] of profilesDb.iterator()) {
+      count++
+      console.log(`${addr} = ${JSON.stringify(profile, null, 2)}`)
+    }
+
+    console.log(`\nTotal profiles: ${count}`)
+    await profilesDb.close()
+  } catch (err) {
+    console.error('Error reading profiles:', err.message)
+    try {
+      await profilesDb.close()
+    } catch (closeErr) {
+      // ignore close errors after read failure
+    }
+    process.exit(1)
+  }
+}
+
+getProfiles()