From 32e8e1f35fa6a6cd741d808415f2c1f9cd004e28 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Wed, 3 Jun 2026 12:07:40 -0700 Subject: [PATCH] Adding block height to all models --- dev-docs/psf-memo-db.md | 40 +++++++++++++++++-- src/use-cases/action-types/follow.js | 9 +++-- src/use-cases/action-types/helpers.js | 4 +- src/use-cases/action-types/like.js | 9 +++-- src/use-cases/action-types/post.js | 10 ++--- src/use-cases/action-types/reply.js | 12 +++--- src/use-cases/action-types/set-name.js | 8 ++-- src/use-cases/action-types/set-profile-pic.js | 10 ++--- src/use-cases/action-types/set-profile.js | 8 ++-- src/use-cases/action-types/topic-follow.js | 7 ++-- src/use-cases/action-types/topic-message.js | 8 ++-- .../use-cases/action-types/helpers.unit.js | 18 ++++++++- test/unit/use-cases/action-types/post.unit.js | 2 + .../action-types/set-profile-pic.unit.js | 2 + 14 files changed, 102 insertions(+), 45 deletions(-) diff --git a/dev-docs/psf-memo-db.md b/dev-docs/psf-memo-db.md index 4718fd8..8d9e44e 100644 --- a/dev-docs/psf-memo-db.md +++ b/dev-docs/psf-memo-db.md @@ -86,12 +86,13 @@ All routes are under `/level` with a consistent CRUD pattern generated from `ENT | Method | Path | Query params | |--------|------|----------------| | `GET` | `/profile/recent` | `limit` (default 100, max 100), `offset` (default 0) | +| `GET` | `/posts/recent` | `limit` (default 100, max 100), `offset` (default 0) | -Returns profiles sorted by **block height** (newest first), using each profile’s `txid` to look up `blockHeight` in `ptxs`. Tie-breaker: `seen` timestamp descending. +Returns profiles or posts sorted by **block height** (newest first), using the `blockHeight` field stored on each entity document at indexing time. Tie-breaker: `seen` timestamp descending. The `seen` field is **Unix epoch milliseconds** from the block header time (`block.time * 1000` at indexing time). -Response shape: +Response shape (`/profile/recent`): ```json { @@ -108,7 +109,24 @@ Response shape: } ``` -Implementation: `profile-query` adapter (LevelDB scan) → `list-recent-profiles` use case → `/profile` REST controller. +Response shape (`/posts/recent`): + +```json +{ + "posts": [ + { + "txid": "...", + "addr": "bitcoincash:q...", + "text": "...", + "seen": 1500000000000, + "blockHeight": 600000 + } + ], + "pagination": { "limit": 100, "offset": 0, "total": 42, "hasMore": false } +} +``` + +Implementation: `profile-query` / `post-query` adapter (LevelDB scan) → `list-recent-profiles` / `list-recent-posts` use case → REST controller. **Tradeoff:** Full scan of `profiles` on each request; suitable for moderate corpus sizes. A height-indexed store would be needed for very large archives. @@ -157,6 +175,22 @@ createEntityDb('post', 'txid', 'postData') **Not normalized like SQL.** LevelDB stores are document keyed for fast lookup by txid or address, similar to the Go `db/item/memo` objects but without sharding. +**Denormalized block height.** Every entity written by the indexer includes a `blockHeight` field (the block in which the memo transaction was confirmed, or `tip + 1` for unconfirmed txs). This avoids ptx lookups when serving `/recent` query routes. The `ptxs` store remains for idempotency only. + +Common fields on indexed documents: + +| Entity | Key | Stored fields (includes) | +|--------|-----|--------------------------| +| post | txid | `addr`, `text`, `seen`, `blockHeight` | +| profile | addr | `text`, `txid`, `seen`, `addr`, `blockHeight` | +| name | addr | `name`, `txid`, `seen`, `addr`, `blockHeight` | +| profilePic | addr | `url`, `txid`, `seen`, `addr`, `blockHeight` | +| like | txid | `addr`, `postTxid`, `seen`, `tip`, `blockHeight` | +| follow | composite key | `followerAddr`, `followeePkHash`, `unfollow`, `txid`, `seen`, `blockHeight` | +| postParent / postChild | txid | `parentTxid`, `childTxid`, `blockHeight` | +| room | composite key | `room`, `txid`, `seen`, `type`, `blockHeight` (+ `addr` for follows) | +| processError | txid | `error`, `ts`, `blockHeight` | + **No secondary indexes in v1.** Queries like “all posts by address” may require scanning or a future `memo-query` adapter—out of scope for the indexer write path. **JSON values** keep debugging simple; binary serialization would save space but break parity with psf-slp-db tooling. diff --git a/src/use-cases/action-types/follow.js b/src/use-cases/action-types/follow.js index d2de888..eca1d02 100644 --- a/src/use-cases/action-types/follow.js +++ b/src/use-cases/action-types/follow.js @@ -2,16 +2,16 @@ import { logProcessError } from './helpers.js' import { PK_HASH_LENGTH, PREFIX_UNFOLLOW } from '../../lib/memo-codes.js' export async function handleFollow (ctx) { - const { adapters, txid, signerAddr, decoded, seen } = ctx + const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx const { pushDatas, prefix } = decoded if (pushDatas.length !== 2) { - await logProcessError(adapters, txid, `invalid follow push data count ${pushDatas.length}`) + await logProcessError(adapters, txid, `invalid follow push data count ${pushDatas.length}`, blockHeight) return } if (pushDatas[1].length !== PK_HASH_LENGTH) { - await logProcessError(adapters, txid, 'follow pk hash wrong size') + await logProcessError(adapters, txid, 'follow pk hash wrong size', blockHeight) return } @@ -24,6 +24,7 @@ export async function handleFollow (ctx) { followeePkHash, unfollow, txid, - seen + seen, + blockHeight }) } diff --git a/src/use-cases/action-types/helpers.js b/src/use-cases/action-types/helpers.js index acb8d5f..0ef0f04 100644 --- a/src/use-cases/action-types/helpers.js +++ b/src/use-cases/action-types/helpers.js @@ -4,9 +4,9 @@ import { isMemoPrefix } from '../../lib/memo-codes.js' -export async function logProcessError (adapters, txid, error) { +export async function logProcessError (adapters, txid, error, blockHeight) { try { - await adapters.processErrorDb.create(txid, { error, ts: Date.now() }) + await adapters.processErrorDb.create(txid, { error, ts: Date.now(), blockHeight }) } catch (err) { console.error('Failed to log process error:', err.message) } diff --git a/src/use-cases/action-types/like.js b/src/use-cases/action-types/like.js index 707c483..1cda90c 100644 --- a/src/use-cases/action-types/like.js +++ b/src/use-cases/action-types/like.js @@ -2,16 +2,16 @@ import { txHashFromPush, logProcessError } from './helpers.js' import { TX_HASH_LENGTH } from '../../lib/memo-codes.js' export async function handleLike (ctx) { - const { adapters, txid, signerAddr, decoded, seen, txDetails } = ctx + const { adapters, txid, signerAddr, decoded, seen, txDetails, blockHeight } = ctx const { pushDatas } = decoded if (pushDatas.length !== 2) { - await logProcessError(adapters, txid, `invalid like push data count ${pushDatas.length}`) + await logProcessError(adapters, txid, `invalid like push data count ${pushDatas.length}`, blockHeight) return } if (pushDatas[1].length !== TX_HASH_LENGTH) { - await logProcessError(adapters, txid, 'like post tx hash wrong size') + await logProcessError(adapters, txid, 'like post tx hash wrong size', blockHeight) return } @@ -36,7 +36,8 @@ export async function handleLike (ctx) { addr: signerAddr, postTxid, seen, - tip + tip, + blockHeight } await adapters.likeDb.create(txid, likeData) } diff --git a/src/use-cases/action-types/post.js b/src/use-cases/action-types/post.js index b8529c6..0410424 100644 --- a/src/use-cases/action-types/post.js +++ b/src/use-cases/action-types/post.js @@ -2,25 +2,25 @@ import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './help import { MAX_POST_SIZE } from '../../lib/memo-codes.js' export async function handlePost (ctx) { - const { adapters, txid, signerAddr, decoded, seen } = ctx + const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas) if (pushDatas.length !== 2) { - await logProcessError(adapters, txid, `invalid post push data count ${pushDatas.length}`) + await logProcessError(adapters, txid, `invalid post push data count ${pushDatas.length}`, blockHeight) return } const text = utf8FromPush(pushDatas[1]) if (!text.length) { - await logProcessError(adapters, txid, 'empty post') + await logProcessError(adapters, txid, 'empty post', blockHeight) return } if (text.length > MAX_POST_SIZE) { - await logProcessError(adapters, txid, 'post too large') + await logProcessError(adapters, txid, 'post too large', blockHeight) return } - const postData = { addr: signerAddr, text, seen } + const postData = { addr: signerAddr, text, seen, blockHeight } try { await adapters.postDb.get(txid) } catch (err) { diff --git a/src/use-cases/action-types/reply.js b/src/use-cases/action-types/reply.js index 3845d18..94cb2e6 100644 --- a/src/use-cases/action-types/reply.js +++ b/src/use-cases/action-types/reply.js @@ -3,28 +3,28 @@ import { MAX_REPLY_SIZE } from '../../lib/memo-codes.js' import { handlePost } from './post.js' export async function handleReply (ctx) { - const { adapters, txid, decoded } = ctx + const { adapters, txid, decoded, blockHeight } = ctx const { pushDatas } = decoded if (pushDatas.length !== 3) { - await logProcessError(adapters, txid, `invalid reply push data count ${pushDatas.length}`) + await logProcessError(adapters, txid, `invalid reply push data count ${pushDatas.length}`, blockHeight) return } const parentTxid = txHashFromPush(pushDatas[1]) if (!parentTxid) { - await logProcessError(adapters, txid, 'invalid parent tx hash for reply') + await logProcessError(adapters, txid, 'invalid parent tx hash for reply', blockHeight) return } const text = utf8FromPush(pushDatas[2]) if (text.length > MAX_REPLY_SIZE) { - await logProcessError(adapters, txid, 'reply too large') + await logProcessError(adapters, txid, 'reply too large', blockHeight) return } - await adapters.postParentDb.create(txid, { parentTxid, childTxid: txid }) - await adapters.postChildDb.create(parentTxid, { parentTxid, childTxid: txid }) + await adapters.postParentDb.create(txid, { parentTxid, childTxid: txid, blockHeight }) + await adapters.postChildDb.create(parentTxid, { parentTxid, childTxid: txid, blockHeight }) await handlePost({ ...ctx, decoded: { ...decoded, pushDatas: [pushDatas[0], pushDatas[2]] } }) } diff --git a/src/use-cases/action-types/set-name.js b/src/use-cases/action-types/set-name.js index d9dc089..a3d16ea 100644 --- a/src/use-cases/action-types/set-name.js +++ b/src/use-cases/action-types/set-name.js @@ -2,19 +2,19 @@ import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './help import { MAX_POST_SIZE } from '../../lib/memo-codes.js' export async function handleSetName (ctx) { - const { adapters, txid, signerAddr, decoded, seen } = ctx + const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas) if (pushDatas.length !== 2) { - await logProcessError(adapters, txid, `invalid set name push data count ${pushDatas.length}`) + await logProcessError(adapters, txid, `invalid set name push data count ${pushDatas.length}`, blockHeight) return } const name = utf8FromPush(pushDatas[1]) if (name.length > MAX_POST_SIZE) { - await logProcessError(adapters, txid, 'set name too large') + await logProcessError(adapters, txid, 'set name too large', blockHeight) return } - await adapters.nameDb.create(signerAddr, { name, txid, seen, addr: signerAddr }) + await adapters.nameDb.create(signerAddr, { name, txid, seen, addr: signerAddr, blockHeight }) } diff --git a/src/use-cases/action-types/set-profile-pic.js b/src/use-cases/action-types/set-profile-pic.js index f66ea3d..53c2a96 100644 --- a/src/use-cases/action-types/set-profile-pic.js +++ b/src/use-cases/action-types/set-profile-pic.js @@ -2,23 +2,23 @@ import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './help import { MAX_POST_SIZE } from '../../lib/memo-codes.js' export async function handleSetProfilePic (ctx) { - const { adapters, txid, signerAddr, decoded, seen } = ctx + const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas) if (pushDatas.length !== 2) { - await logProcessError(adapters, txid, `invalid profile pic push data count ${pushDatas.length}`) + await logProcessError(adapters, txid, `invalid profile pic push data count ${pushDatas.length}`, blockHeight) return } const url = utf8FromPush(pushDatas[1]) if (!url.length) { - await logProcessError(adapters, txid, 'empty profile pic url') + await logProcessError(adapters, txid, 'empty profile pic url', blockHeight) return } if (url.length > MAX_POST_SIZE) { - await logProcessError(adapters, txid, 'profile pic url too large') + await logProcessError(adapters, txid, 'profile pic url too large', blockHeight) return } - await adapters.profilePicDb.create(signerAddr, { url, txid, seen, addr: signerAddr }) + await adapters.profilePicDb.create(signerAddr, { url, txid, seen, addr: signerAddr, blockHeight }) } diff --git a/src/use-cases/action-types/set-profile.js b/src/use-cases/action-types/set-profile.js index caf11e2..3d03eb7 100644 --- a/src/use-cases/action-types/set-profile.js +++ b/src/use-cases/action-types/set-profile.js @@ -2,19 +2,19 @@ import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './help import { MAX_POST_SIZE } from '../../lib/memo-codes.js' export async function handleSetProfile (ctx) { - const { adapters, txid, signerAddr, decoded, seen } = ctx + const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas) if (pushDatas.length !== 2) { - await logProcessError(adapters, txid, `invalid profile push data count ${pushDatas.length}`) + await logProcessError(adapters, txid, `invalid profile push data count ${pushDatas.length}`, blockHeight) return } const text = utf8FromPush(pushDatas[1]) if (text.length > MAX_POST_SIZE) { - await logProcessError(adapters, txid, 'profile too large') + await logProcessError(adapters, txid, 'profile too large', blockHeight) return } - await adapters.profileDb.create(signerAddr, { text, txid, seen, addr: signerAddr }) + await adapters.profileDb.create(signerAddr, { text, txid, seen, addr: signerAddr, blockHeight }) } diff --git a/src/use-cases/action-types/topic-follow.js b/src/use-cases/action-types/topic-follow.js index fa560b6..b11975f 100644 --- a/src/use-cases/action-types/topic-follow.js +++ b/src/use-cases/action-types/topic-follow.js @@ -2,11 +2,11 @@ import { utf8FromPush, logProcessError, roomKey } from './helpers.js' import { PREFIX_TOPIC_UNFOLLOW } from '../../lib/memo-codes.js' export async function handleTopicFollow (ctx) { - const { adapters, txid, signerAddr, decoded, seen } = ctx + const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx const { pushDatas, prefix } = decoded if (pushDatas.length !== 2) { - await logProcessError(adapters, txid, `invalid topic follow push data count ${pushDatas.length}`) + await logProcessError(adapters, txid, `invalid topic follow push data count ${pushDatas.length}`, blockHeight) return } @@ -19,6 +19,7 @@ export async function handleTopicFollow (ctx) { unfollow, txid, seen, - type: 'follow' + type: 'follow', + blockHeight }) } diff --git a/src/use-cases/action-types/topic-message.js b/src/use-cases/action-types/topic-message.js index 5903ea9..5dd928b 100644 --- a/src/use-cases/action-types/topic-message.js +++ b/src/use-cases/action-types/topic-message.js @@ -3,18 +3,18 @@ import { MAX_POST_SIZE } from '../../lib/memo-codes.js' import { handlePost } from './post.js' export async function handleTopicMessage (ctx) { - const { adapters, txid, decoded, seen } = ctx + const { adapters, txid, decoded, seen, blockHeight } = ctx const { pushDatas } = decoded if (pushDatas.length !== 3) { - await logProcessError(adapters, txid, `invalid topic message push data count ${pushDatas.length}`) + await logProcessError(adapters, txid, `invalid topic message push data count ${pushDatas.length}`, blockHeight) return } const room = utf8FromPush(pushDatas[1]) const message = utf8FromPush(pushDatas[2]) if ((room.length + message.length) > MAX_POST_SIZE) { - await logProcessError(adapters, txid, 'topic message too large') + await logProcessError(adapters, txid, 'topic message too large', blockHeight) return } @@ -23,5 +23,5 @@ export async function handleTopicMessage (ctx) { decoded: { ...decoded, pushDatas: [pushDatas[0], pushDatas[2]] } }) - await adapters.roomDb.create(roomKey(room, txid), { room, txid, seen, type: 'post' }) + await adapters.roomDb.create(roomKey(room, txid), { room, txid, seen, type: 'post', blockHeight }) } diff --git a/test/unit/use-cases/action-types/helpers.unit.js b/test/unit/use-cases/action-types/helpers.unit.js index b5bfda5..76dd2d8 100644 --- a/test/unit/use-cases/action-types/helpers.unit.js +++ b/test/unit/use-cases/action-types/helpers.unit.js @@ -1,7 +1,9 @@ import { assert } from 'chai' +import sinon from 'sinon' import { normalizeTwoPushMemoDatas, - stripLeadingEmptyPushes + stripLeadingEmptyPushes, + logProcessError } from '../../../../src/use-cases/action-types/helpers.js' import { PREFIX_SET_PROFILE_PIC, PREFIX_POST } from '../../../../src/lib/memo-codes.js' @@ -52,4 +54,18 @@ describe('#action-types/helpers', () => { assert.deepEqual(result[0], payload) }) }) + + describe('#logProcessError', () => { + it('should store blockHeight on process error records', async () => { + const create = sinon.stub().resolves() + const adapters = { processErrorDb: { create } } + + await logProcessError(adapters, 'tx1', 'bad data', 600100) + + assert.equal(create.callCount, 1) + assert.equal(create.firstCall.args[0], 'tx1') + assert.equal(create.firstCall.args[1].error, 'bad data') + assert.equal(create.firstCall.args[1].blockHeight, 600100) + }) + }) }) diff --git a/test/unit/use-cases/action-types/post.unit.js b/test/unit/use-cases/action-types/post.unit.js index a90ed6c..046eb6b 100644 --- a/test/unit/use-cases/action-types/post.unit.js +++ b/test/unit/use-cases/action-types/post.unit.js @@ -19,6 +19,7 @@ describe('#handlePost', () => { txid: 'abc123', signerAddr: 'bitcoincash:qptest', seen: 1000, + blockHeight: 600100, decoded: { action: 'post', prefix: PREFIX_POST, @@ -29,5 +30,6 @@ describe('#handlePost', () => { assert.equal(create.callCount, 1) assert.equal(create.firstCall.args[0], 'abc123') assert.equal(create.firstCall.args[1].text, 'hello memo') + assert.equal(create.firstCall.args[1].blockHeight, 600100) }) }) diff --git a/test/unit/use-cases/action-types/set-profile-pic.unit.js b/test/unit/use-cases/action-types/set-profile-pic.unit.js index 9733d1a..cf4c9b9 100644 --- a/test/unit/use-cases/action-types/set-profile-pic.unit.js +++ b/test/unit/use-cases/action-types/set-profile-pic.unit.js @@ -22,6 +22,7 @@ describe('#handleSetProfilePic', () => { txid: 'abc123', signerAddr: 'bitcoincash:qptest', seen: 12345, + blockHeight: 600200, decoded: { pushDatas: [combined] } }) @@ -30,5 +31,6 @@ describe('#handleSetProfilePic', () => { const [addr, data] = adapters.profilePicDb.create.firstCall.args assert.equal(addr, 'bitcoincash:qptest') assert.equal(data.url, 'https://memo.cash/img/abc') + assert.equal(data.blockHeight, 600200) }) })