first commit

This commit is contained in:
Chris Troutner
2026-06-02 16:44:51 -07:00
commit 02128422af
27 changed files with 7994 additions and 0 deletions
+4
View File
@@ -0,0 +1,4 @@
PORT=5021
SVC_ENV=development
BACKUP_QTY=3
EXIT_ON_MISSING_BACKUP=false
+5
View File
@@ -0,0 +1,5 @@
node_modules/
leveldb/
logs/
.env
coverage/
+12
View File
@@ -0,0 +1,12 @@
FROM node:20-alpine
WORKDIR /app
COPY package*.json ./
RUN npm ci --omit=dev
COPY . .
EXPOSE 5021
CMD ["node", "index.js"]
+37
View File
@@ -0,0 +1,37 @@
# psf-memo-db
LevelDB REST API for the Memo protocol indexer. Architecture mirrors [psf-slp-db](https://github.com/Permissionless-Software-Foundation/psf-slp-db).
## Requirements
- node ^20
- npm ^10
## Installation
```bash
cd psf-memo-db
npm install
cp .env-example .env # optional
npm start
```
Default port: **5021**
## API
All indexer data is exposed under `/level/*` with CRUD routes per entity (`post`, `like`, `name`, `profile`, `status`, etc.) plus:
- `POST /level/backup` — zip database snapshot
- `POST /level/restore` — restore from snapshot (exits process)
- `GET /health` — health check
## Tests
```bash
npm test
```
## License
MIT
+71
View File
@@ -0,0 +1,71 @@
/*
Koa REST API for Memo indexer LevelDB storage.
Architecture mirrors psf-slp-db bin/server.js.
*/
import Koa from 'koa'
import bodyParser from 'koa-bodyparser'
import convert from 'koa-convert'
import logger from 'koa-logger'
import cors from 'kcors'
import 'dotenv/config'
import config from '../config/index.js'
import errorMiddleware from '../src/controllers/rest-api/middleware/error.js'
import wlogger from '../src/adapters/wlogger.js'
import Controllers from '../src/controllers/index.js'
class Server {
constructor () {
this.controllers = new Controllers()
this.config = config
this.process = process
}
async startServer () {
try {
const app = new Koa()
app.keys = [this.config.session]
console.log(`Starting environment: ${this.config.env}`)
app.use(convert(logger()))
app.use(bodyParser({
jsonLimit: '100mb',
formLimit: '100mb',
textLimit: '100mb'
}))
app.use(errorMiddleware())
app.use(cors({ origin: '*' }))
await this.controllers.initAdapters()
await this.controllers.initUseCases()
await this.controllers.attachRESTControllers(app)
app.controllers = this.controllers
console.log(`Running server in environment: ${this.config.env}`)
wlogger.info(`Running server in environment: ${this.config.env}`)
this.server = await app.listen(this.config.port)
console.log(`Server started on ${this.config.port}`)
if (this.config.env !== 'test') {
await this.controllers.attachControllers(app)
}
return app
} catch (err) {
console.error('Could not start server. Error: ', err)
console.log('Exiting after 5 seconds.')
await this.sleep(5000)
this.process.exit(1)
}
}
sleep (ms) {
return new Promise((resolve) => setTimeout(resolve, ms))
}
}
export default Server
+16
View File
@@ -0,0 +1,16 @@
import 'dotenv/config'
import * as url from 'url'
import { readFileSync } from 'fs'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const pkgInfo = JSON.parse(readFileSync(`${__dirname}/../../package.json`))
export default {
port: process.env.PORT ? parseInt(process.env.PORT) : 5021,
noMongo: true,
useIpfs: false,
version: pkgInfo.version,
backupQty: process.env.BACKUP_QTY ? parseInt(process.env.BACKUP_QTY) : 3,
exitOnMissingBackup: process.env.EXIT_ON_MISSING_BACKUP === 'true'
}
+4
View File
@@ -0,0 +1,4 @@
export default {
session: 'secret-memo-db-dev',
env: 'development'
}
+4
View File
@@ -0,0 +1,4 @@
export default {
session: process.env.SESSION_SECRET || 'secret-memo-db-prod',
env: 'prod'
}
+4
View File
@@ -0,0 +1,4 @@
export default {
session: 'secret-memo-db-test',
env: 'test'
}
+16
View File
@@ -0,0 +1,16 @@
import common from './env/common.js'
import development from './env/development.js'
import production from './env/production.js'
import test from './env/test.js'
const env = process.env.SVC_ENV || 'development'
console.log(`Loading config for this environment: ${env}`)
let config = development
if (env === 'test') {
config = test
} else if (env === 'prod') {
config = production
}
export default Object.assign({}, common, config)
+9
View File
@@ -0,0 +1,9 @@
import Server from './bin/server.js'
const server = new Server()
process.on('unhandledRejection', (reason) => {
console.log('Unhandled rejection:', reason)
})
server.startServer()
+7163
View File
File diff suppressed because it is too large Load Diff
+34
View File
@@ -0,0 +1,34 @@
{
"name": "psf-memo-db",
"version": "1.0.0",
"description": "LevelDB REST API for Memo protocol indexing.",
"main": "index.js",
"type": "module",
"scripts": {
"start": "node index.js",
"test": "export SVC_ENV=test && c8 --reporter=text mocha --exit --timeout 15000 --recursive test/unit/",
"lint": "standard --env mocha --fix"
},
"author": "Chris Troutner",
"license": "MIT",
"dependencies": {
"dotenv": "17.2.3",
"kcors": "2.2.2",
"koa": "2.13.1",
"koa-bodyparser": "4.3.0",
"koa-convert": "2.0.0",
"koa-logger": "3.2.1",
"koa-router": "10.0.0",
"level": "7.0.1",
"shelljs": "0.10.0",
"winston": "3.3.3",
"winston-daily-rotate-file": "4.5.0"
},
"devDependencies": {
"c8": "10.1.3",
"chai": "4.3.0",
"mocha": "10.0.0",
"sinon": "9.2.4",
"standard": "17.0.0"
}
}
+86
View File
@@ -0,0 +1,86 @@
/*
Backup and restore for Memo indexer LevelDB.
*/
import shell from 'shelljs'
import fs from 'fs'
import config from '../../config/index.js'
import { DB_NAMES, dbDir } from './level-db.js'
class DbBackup {
constructor (levelDbs = {}) {
for (const name of DB_NAMES) {
this[`${name}Db`] = levelDbs[`${name}Db`]
}
this.shell = shell
this.config = config
this.zipDb = this.zipDb.bind(this)
this.unzipDb = this.unzipDb.bind(this)
}
async closeAll () {
for (const name of DB_NAMES) {
await this[`${name}Db`].close()
}
}
async openAll () {
for (const name of DB_NAMES) {
await this[`${name}Db`].open()
}
}
async zipDb (height, epoch) {
try {
await this.closeAll()
this.shell.cd(dbDir)
this.shell.exec(`zip -r zips/memo-indexer-${height}.zip current`)
const backupQty = this.config.backupQty
if (backupQty && epoch) {
const oldHeight = height - (epoch * backupQty)
const rmStr = `zips/memo-indexer-${oldHeight}.zip`
if (this.shell.test('-f', rmStr)) {
this.shell.rm(rmStr)
}
}
await this.openAll()
return true
} catch (err) {
console.error('Error in zipDb')
throw err
}
}
async unzipDb (height) {
try {
const zipFile = `memo-indexer-${height}.zip`
const zipFilePath = `${dbDir}/zips/${zipFile}`
if (!fs.existsSync(zipFilePath)) {
console.error(`Backup file not found: ${zipFile}`)
if (this.config.exitOnMissingBackup) {
process.exit(1)
}
return false
}
await this.closeAll()
this.shell.rm('-rf', `${dbDir}/current/*`)
this.shell.cd(`${dbDir}/zips`)
this.shell.exec(`unzip -o ${zipFile}`)
this.shell.cp('-r', `${dbDir}/zips/current/*`, `${dbDir}/current/`)
await this.openAll()
return true
} catch (err) {
console.error('Error in unzipDb: ', err)
return false
}
}
}
export default DbBackup
+30
View File
@@ -0,0 +1,30 @@
/*
Top-level adapters for psf-memo-db.
*/
import LevelDb from './level-db.js'
import DbBackup from './db-backup.js'
class Adapters {
constructor () {
this.levelDb = new LevelDb()
this.openDatabases = this.openDatabases.bind(this)
this.start = this.start.bind(this)
}
openDatabases () {
this.levelDb.ensureDirectories()
const level = this.levelDb.openDbs()
this.level = level
this.dbBackup = new DbBackup(level)
return true
}
async start () {
this.openDatabases()
console.log('Adapter libraries initialized.')
return true
}
}
export default Adapters
+73
View File
@@ -0,0 +1,73 @@
/*
Adapter library for Memo indexer LevelDB instances.
*/
import level from 'level'
import shell from 'shelljs'
import * as url from 'url'
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
const dbDir = `${__dirname}/../../leveldb`
const DB_NAMES = [
'status',
'posts',
'postParents',
'postChildren',
'likes',
'names',
'profiles',
'profilePics',
'follows',
'rooms',
'processErrors',
'ptxs'
]
class LevelDb {
constructor () {
this.level = level
this.shell = shell
this.openDbs = this.openDbs.bind(this)
this.closeDbs = this.closeDbs.bind(this)
this.ensureDirectories = this.ensureDirectories.bind(this)
this.getDbList = this.getDbList.bind(this)
}
openDbs () {
console.log('Opening LevelDB databases...')
const dbs = {}
for (const name of DB_NAMES) {
const prop = `${name}Db`
dbs[prop] = this.level(`${__dirname}/../../leveldb/current/${name}`, {
valueEncoding: 'json',
cacheSize: name === 'posts' ? 512 * 1024 * 1024 : 64 * 1024 * 1024
})
this[prop] = dbs[prop]
}
return dbs
}
getDbList () {
return DB_NAMES.map((name) => this[`${name}Db`])
}
async closeDbs () {
for (const name of DB_NAMES) {
await this[`${name}Db`].close()
}
return true
}
async ensureDirectories () {
this.shell.mkdir('-p', `${dbDir}/current`)
this.shell.mkdir('-p', `${dbDir}/zips`)
this.shell.mkdir('-p', `${dbDir}/backup`)
return true
}
}
export { DB_NAMES, dbDir }
export default LevelDb
+21
View File
@@ -0,0 +1,21 @@
'use strict'
import winston from 'winston'
import config from '../../config/index.js'
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(
winston.format.timestamp(),
winston.format.json()
),
transports: [
new winston.transports.Console({
format: winston.format.simple()
})
]
})
logger.info(`Wlogger initialized for env: ${config.env}`)
export default logger
+40
View File
@@ -0,0 +1,40 @@
/*
Top-level controllers for psf-memo-db.
*/
import Adapters from '../adapters/index.js'
import UseCases from '../use-cases/index.js'
import RESTControllers from './rest-api/index.js'
class Controllers {
constructor () {
this.adapters = new Adapters()
this.useCases = new UseCases({ adapters: this.adapters })
this.initAdapters = this.initAdapters.bind(this)
this.initUseCases = this.initUseCases.bind(this)
this.attachRESTControllers = this.attachRESTControllers.bind(this)
this.attachControllers = this.attachControllers.bind(this)
}
async initAdapters () {
await this.adapters.start()
}
async initUseCases () {
await this.useCases.start()
}
attachRESTControllers (app) {
const restControllers = new RESTControllers({
adapters: this.adapters,
useCases: this.useCases
})
restControllers.attachRESTControllers(app)
}
async attachControllers () {
return true
}
}
export default Controllers
+14
View File
@@ -0,0 +1,14 @@
import Router from 'koa-router'
class HealthRouter {
attach (app) {
const router = new Router({ prefix: '/health' })
router.get('/', (ctx) => {
ctx.body = { status: 'ok' }
})
app.use(router.routes())
app.use(router.allowedMethods())
}
}
export default HealthRouter
+29
View File
@@ -0,0 +1,29 @@
/*
REST API controllers index.
*/
import LevelRESTController from './level/index.js'
import HealthRouter from './health/index.js'
class RESTControllers {
constructor (localConfig = {}) {
this.adapters = localConfig.adapters
this.useCases = localConfig.useCases
this.attachRESTControllers = this.attachRESTControllers.bind(this)
}
attachRESTControllers (app) {
const dependencies = {
adapters: this.adapters,
useCases: this.useCases
}
const levelRESTController = new LevelRESTController(dependencies)
levelRESTController.attach(app)
const healthRouter = new HealthRouter()
healthRouter.attach(app)
}
}
export default RESTControllers
@@ -0,0 +1,116 @@
/*
REST API Controller for /level routes.
*/
import wlogger from '../../../adapters/wlogger.js'
import { makeCrudHandlers, ENTITY_CONFIG } from './crud-handlers.js'
class LevelRESTControllerLib {
constructor (localConfig = {}) {
this.adapters = localConfig.adapters
this.useCases = localConfig.useCases
if (!this.adapters) {
throw new Error('Adapters required for Level REST Controller.')
}
this.handleError = this.handleError.bind(this)
this.getStatus = this.getStatus.bind(this)
this.createStatus = this.createStatus.bind(this)
this.updateStatus = this.updateStatus.bind(this)
this.deleteStatus = this.deleteStatus.bind(this)
this.backup = this.backup.bind(this)
this.restore = this.restore.bind(this)
this.entityHandlers = {}
for (const cfg of ENTITY_CONFIG) {
const handlers = makeCrudHandlers(cfg)
this.entityHandlers[cfg.route] = {
get: async (ctx) => this._wrap(handlers.get, ctx),
create: async (ctx) => this._wrap(handlers.create, ctx),
update: async (ctx) => this._wrap(handlers.update, ctx),
delete: async (ctx) => this._wrap(handlers.delete, ctx),
keyParam: cfg.keyParam
}
}
}
handleError (ctx, err) {
if (err.status) {
ctx.throw(err.status, err.message || err)
} else {
ctx.throw(422, err.message)
}
}
async _wrap (fn, ctx) {
try {
await fn(ctx, this.adapters)
} catch (err) {
wlogger.error('Error in level controller: ', err)
this.handleError(ctx, err)
}
}
async getStatus (ctx) {
try {
const { statusKey } = ctx.params
ctx.body = await this.adapters.level.statusDb.get(statusKey)
} catch (err) {
this.handleError(ctx, err)
}
}
async createStatus (ctx) {
try {
const { statusKey, statusData } = ctx.request.body
await this.adapters.level.statusDb.put(statusKey, statusData)
ctx.body = { statusKey, success: true }
} catch (err) {
this.handleError(ctx, err)
}
}
async updateStatus (ctx) {
try {
const statusKey = 'status'
const { statusData } = ctx.request.body
await this.adapters.level.statusDb.put(statusKey, statusData)
ctx.body = { statusKey, success: true }
} catch (err) {
this.handleError(ctx, err)
}
}
async deleteStatus (ctx) {
try {
const { statusKey } = ctx.params
await this.adapters.level.statusDb.del(statusKey)
ctx.body = { statusKey, success: true }
} catch (err) {
this.handleError(ctx, err)
}
}
async backup (ctx) {
try {
const { height, epoch } = ctx.request.body
await this.adapters.dbBackup.zipDb(height, epoch)
ctx.body = { success: true }
} catch (err) {
this.handleError(ctx, err)
}
}
async restore (ctx) {
try {
const { height } = ctx.request.body
await this.adapters.dbBackup.unzipDb(height)
console.log('Restore complete. Exiting for process manager restart.')
process.exit(0)
} catch (err) {
this.handleError(ctx, err)
}
}
}
export default LevelRESTControllerLib
@@ -0,0 +1,49 @@
/*
Generic LevelDB CRUD handlers for /level routes.
*/
export function makeCrudHandlers ({ dbProp, keyParam, bodyIdField, bodyDataField, label }) {
return {
async get (ctx, adapters) {
const key = ctx.params[keyParam]
const result = await adapters.level[dbProp].get(key)
ctx.body = result
},
async create (ctx, adapters) {
const key = ctx.request.body[bodyIdField]
const data = ctx.request.body[bodyDataField]
await adapters.level[dbProp].put(key, data)
ctx.body = { [bodyIdField]: key, success: true }
},
async update (ctx, adapters) {
const key = ctx.params[keyParam]
const data = ctx.request.body[bodyDataField]
await adapters.level[dbProp].put(key, data)
ctx.body = { [keyParam]: key, success: true }
},
async delete (ctx, adapters) {
const key = ctx.params[keyParam]
await adapters.level[dbProp].del(key)
ctx.body = { [keyParam]: key, success: true }
},
label
}
}
export const ENTITY_CONFIG = [
{ route: 'post', dbProp: 'postsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'postData' },
{ route: 'postparent', dbProp: 'postParentsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'parentData' },
{ route: 'postchild', dbProp: 'postChildrenDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'childData' },
{ route: 'like', dbProp: 'likesDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'likeData' },
{ route: 'name', dbProp: 'namesDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'nameData' },
{ route: 'profile', dbProp: 'profilesDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'profileData' },
{ route: 'profilepic', dbProp: 'profilePicsDb', keyParam: 'addr', bodyIdField: 'addr', bodyDataField: 'profilePicData' },
{ route: 'follow', dbProp: 'followsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'followData' },
{ route: 'room', dbProp: 'roomsDb', keyParam: 'key', bodyIdField: 'key', bodyDataField: 'roomData' },
{ route: 'processerror', dbProp: 'processErrorsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'errorData' },
{ route: 'ptx', dbProp: 'ptxsDb', keyParam: 'txid', bodyIdField: 'txid', bodyDataField: 'ptxData' }
]
+44
View File
@@ -0,0 +1,44 @@
/*
REST API router for /level routes.
*/
import Router from 'koa-router'
import LevelRESTControllerLib from './controller.js'
import { ENTITY_CONFIG } from './crud-handlers.js'
class LevelRouter {
constructor (localConfig = {}) {
this.adapters = localConfig.adapters
this.useCases = localConfig.useCases
this.levelRESTController = new LevelRESTControllerLib({
adapters: this.adapters,
useCases: this.useCases
})
this.router = new Router({ prefix: '/level' })
}
attach (app) {
const ctrl = this.levelRESTController
for (const cfg of ENTITY_CONFIG) {
const h = ctrl.entityHandlers[cfg.route]
this.router.post(`/${cfg.route}`, h.create)
this.router.get(`/${cfg.route}/:${cfg.keyParam}`, h.get)
this.router.put(`/${cfg.route}/:${cfg.keyParam}`, h.update)
this.router.delete(`/${cfg.route}/:${cfg.keyParam}`, h.delete)
}
this.router.post('/status', ctrl.createStatus)
this.router.get('/status/:statusKey', ctrl.getStatus)
this.router.put('/status', ctrl.updateStatus)
this.router.delete('/status/:statusKey', ctrl.deleteStatus)
this.router.post('/backup', ctrl.backup)
this.router.post('/restore', ctrl.restore)
app.use(this.router.routes())
app.use(this.router.allowedMethods())
}
}
export default LevelRouter
@@ -0,0 +1,11 @@
export default function errorMiddleware () {
return async (ctx, next) => {
try {
await next()
} catch (err) {
ctx.status = err.status || 500
ctx.body = err.message
ctx.app.emit('error', err, ctx)
}
}
}
+19
View File
@@ -0,0 +1,19 @@
/*
Use cases for psf-memo-db (minimal for v1).
*/
class UseCases {
constructor (localConfig = {}) {
this.adapters = localConfig.adapters
if (!this.adapters) {
throw new Error('Adapters required when instantiating UseCases.')
}
}
async start () {
console.log('Use cases initialized.')
return true
}
}
export default UseCases
@@ -0,0 +1,47 @@
/*
Unit tests for level REST controller.
*/
import { assert } from 'chai'
import sinon from 'sinon'
import LevelRESTControllerLib from '../../../src/controllers/rest-api/level/controller.js'
describe('#LevelRESTController', () => {
let uut
let sandbox
const mockDb = {
get: sinon.stub().resolves({ text: 'hello' }),
put: sinon.stub().resolves(),
del: sinon.stub().resolves()
}
beforeEach(() => {
sandbox = sinon.createSandbox()
uut = new LevelRESTControllerLib({
adapters: {
level: { postsDb: mockDb, statusDb: mockDb },
dbBackup: { zipDb: sandbox.stub().resolves(true) }
},
useCases: {}
})
})
afterEach(() => sandbox.restore())
it('should get status', async () => {
const ctx = { params: { statusKey: 'status' }, body: null }
await uut.getStatus(ctx)
assert.deepEqual(ctx.body, { text: 'hello' })
})
it('should create a post via entity handler', async () => {
const ctx = {
params: {},
request: { body: { txid: 'abc', postData: { addr: '1', text: 'hi' } } },
body: null
}
await uut.entityHandlers.post.create(ctx)
assert.equal(ctx.body.success, true)
assert.equal(ctx.body.txid, 'abc')
})
})
+36
View File
@@ -0,0 +1,36 @@
/*
Unit tests for bin/server.js
*/
import { assert } from 'chai'
import sinon from 'sinon'
import Server from '../../../bin/server.js'
describe('#server', () => {
let uut
let sandbox
beforeEach(() => {
sandbox = sinon.createSandbox()
uut = new Server()
})
afterEach(() => sandbox.restore())
describe('#startServer', () => {
it('should start the server', async () => {
sandbox.stub(uut.controllers, 'initAdapters').resolves()
sandbox.stub(uut.controllers, 'initUseCases').resolves()
sandbox.stub(uut.controllers, 'attachRESTControllers').resolves()
sandbox.stub(uut.controllers, 'attachControllers').resolves()
uut.config.env = 'dev'
uut.config.port = 5041
const result = await uut.startServer()
assert.property(result, 'env')
uut.server.close()
uut.config.env = 'test'
})
})
})