Adding block height to all models

dev-docs/psf-memo-db.md

@@ -86,12 +86,13 @@ All routes are under `/level` with a consistent CRUD pattern generated from `ENT
 | Method | Path | Query params |
 |--------|------|----------------|
 | `GET` | `/profile/recent` | `limit` (default 100, max 100), `offset` (default 0) |
+| `GET` | `/posts/recent` | `limit` (default 100, max 100), `offset` (default 0) |
 
-Returns profiles sorted by **block height** (newest first), using each profile’s `txid` to look up `blockHeight` in `ptxs`. Tie-breaker: `seen` timestamp descending.
+Returns profiles or posts sorted by **block height** (newest first), using the `blockHeight` field stored on each entity document at indexing time. Tie-breaker: `seen` timestamp descending.
 
 The `seen` field is **Unix epoch milliseconds** from the block header time (`block.time * 1000` at indexing time).
 
-Response shape:
+Response shape (`/profile/recent`):
 
 ```json
 {
@@ -108,7 +109,24 @@ Response shape:
 }
 ```
 
-Implementation: `profile-query` adapter (LevelDB scan) → `list-recent-profiles` use case → `/profile` REST controller.
+Response shape (`/posts/recent`):
+
+```json
+{
+  "posts": [
+    {
+      "txid": "...",
+      "addr": "bitcoincash:q...",
+      "text": "...",
+      "seen": 1500000000000,
+      "blockHeight": 600000
+    }
+  ],
+  "pagination": { "limit": 100, "offset": 0, "total": 42, "hasMore": false }
+}
+```
+
+Implementation: `profile-query` / `post-query` adapter (LevelDB scan) → `list-recent-profiles` / `list-recent-posts` use case → REST controller.
 
 **Tradeoff:** Full scan of `profiles` on each request; suitable for moderate corpus sizes. A height-indexed store would be needed for very large archives.
 
@@ -157,6 +175,22 @@ createEntityDb('post', 'txid', 'postData')
 
 **Not normalized like SQL.** LevelDB stores are document keyed for fast lookup by txid or address, similar to the Go `db/item/memo` objects but without sharding.
 
+**Denormalized block height.** Every entity written by the indexer includes a `blockHeight` field (the block in which the memo transaction was confirmed, or `tip + 1` for unconfirmed txs). This avoids ptx lookups when serving `/recent` query routes. The `ptxs` store remains for idempotency only.
+
+Common fields on indexed documents:
+
+| Entity | Key | Stored fields (includes) |
+|--------|-----|--------------------------|
+| post | txid | `addr`, `text`, `seen`, `blockHeight` |
+| profile | addr | `text`, `txid`, `seen`, `addr`, `blockHeight` |
+| name | addr | `name`, `txid`, `seen`, `addr`, `blockHeight` |
+| profilePic | addr | `url`, `txid`, `seen`, `addr`, `blockHeight` |
+| like | txid | `addr`, `postTxid`, `seen`, `tip`, `blockHeight` |
+| follow | composite key | `followerAddr`, `followeePkHash`, `unfollow`, `txid`, `seen`, `blockHeight` |
+| postParent / postChild | txid | `parentTxid`, `childTxid`, `blockHeight` |
+| room | composite key | `room`, `txid`, `seen`, `type`, `blockHeight` (+ `addr` for follows) |
+| processError | txid | `error`, `ts`, `blockHeight` |
+
 **No secondary indexes in v1.** Queries like “all posts by address” may require scanning or a future `memo-query` adapter—out of scope for the indexer write path.
 
 **JSON values** keep debugging simple; binary serialization would save space but break parity with psf-slp-db tooling.

src/use-cases/action-types/follow.js

@@ -2,16 +2,16 @@ import { logProcessError } from './helpers.js'
 import { PK_HASH_LENGTH, PREFIX_UNFOLLOW } from '../../lib/memo-codes.js'
 
 export async function handleFollow (ctx) {
-  const { adapters, txid, signerAddr, decoded, seen } = ctx
+  const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx
   const { pushDatas, prefix } = decoded
 
   if (pushDatas.length !== 2) {
-    await logProcessError(adapters, txid, `invalid follow push data count ${pushDatas.length}`)
+    await logProcessError(adapters, txid, `invalid follow push data count ${pushDatas.length}`, blockHeight)
     return
   }
 
   if (pushDatas[1].length !== PK_HASH_LENGTH) {
-    await logProcessError(adapters, txid, 'follow pk hash wrong size')
+    await logProcessError(adapters, txid, 'follow pk hash wrong size', blockHeight)
     return
   }
 
@@ -24,6 +24,7 @@ export async function handleFollow (ctx) {
     followeePkHash,
     unfollow,
     txid,
-    seen
+    seen,
+    blockHeight
   })
 }

src/use-cases/action-types/helpers.js

@@ -4,9 +4,9 @@
 
 import { isMemoPrefix } from '../../lib/memo-codes.js'
 
-export async function logProcessError (adapters, txid, error) {
+export async function logProcessError (adapters, txid, error, blockHeight) {
   try {
-    await adapters.processErrorDb.create(txid, { error, ts: Date.now() })
+    await adapters.processErrorDb.create(txid, { error, ts: Date.now(), blockHeight })
   } catch (err) {
     console.error('Failed to log process error:', err.message)
   }

src/use-cases/action-types/like.js

@@ -2,16 +2,16 @@ import { txHashFromPush, logProcessError } from './helpers.js'
 import { TX_HASH_LENGTH } from '../../lib/memo-codes.js'
 
 export async function handleLike (ctx) {
-  const { adapters, txid, signerAddr, decoded, seen, txDetails } = ctx
+  const { adapters, txid, signerAddr, decoded, seen, txDetails, blockHeight } = ctx
   const { pushDatas } = decoded
 
   if (pushDatas.length !== 2) {
-    await logProcessError(adapters, txid, `invalid like push data count ${pushDatas.length}`)
+    await logProcessError(adapters, txid, `invalid like push data count ${pushDatas.length}`, blockHeight)
     return
   }
 
   if (pushDatas[1].length !== TX_HASH_LENGTH) {
-    await logProcessError(adapters, txid, 'like post tx hash wrong size')
+    await logProcessError(adapters, txid, 'like post tx hash wrong size', blockHeight)
     return
   }
 
@@ -36,7 +36,8 @@ export async function handleLike (ctx) {
     addr: signerAddr,
     postTxid,
     seen,
-    tip
+    tip,
+    blockHeight
   }
   await adapters.likeDb.create(txid, likeData)
 }

src/use-cases/action-types/post.js

@@ -2,25 +2,25 @@ import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './help
 import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
 
 export async function handlePost (ctx) {
-  const { adapters, txid, signerAddr, decoded, seen } = ctx
+  const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx
   const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas)
 
   if (pushDatas.length !== 2) {
-    await logProcessError(adapters, txid, `invalid post push data count ${pushDatas.length}`)
+    await logProcessError(adapters, txid, `invalid post push data count ${pushDatas.length}`, blockHeight)
     return
   }
 
   const text = utf8FromPush(pushDatas[1])
   if (!text.length) {
-    await logProcessError(adapters, txid, 'empty post')
+    await logProcessError(adapters, txid, 'empty post', blockHeight)
     return
   }
   if (text.length > MAX_POST_SIZE) {
-    await logProcessError(adapters, txid, 'post too large')
+    await logProcessError(adapters, txid, 'post too large', blockHeight)
     return
   }
 
-  const postData = { addr: signerAddr, text, seen }
+  const postData = { addr: signerAddr, text, seen, blockHeight }
   try {
     await adapters.postDb.get(txid)
   } catch (err) {

src/use-cases/action-types/reply.js

@@ -3,28 +3,28 @@ import { MAX_REPLY_SIZE } from '../../lib/memo-codes.js'
 import { handlePost } from './post.js'
 
 export async function handleReply (ctx) {
-  const { adapters, txid, decoded } = ctx
+  const { adapters, txid, decoded, blockHeight } = ctx
   const { pushDatas } = decoded
 
   if (pushDatas.length !== 3) {
-    await logProcessError(adapters, txid, `invalid reply push data count ${pushDatas.length}`)
+    await logProcessError(adapters, txid, `invalid reply push data count ${pushDatas.length}`, blockHeight)
     return
   }
 
   const parentTxid = txHashFromPush(pushDatas[1])
   if (!parentTxid) {
-    await logProcessError(adapters, txid, 'invalid parent tx hash for reply')
+    await logProcessError(adapters, txid, 'invalid parent tx hash for reply', blockHeight)
     return
   }
 
   const text = utf8FromPush(pushDatas[2])
   if (text.length > MAX_REPLY_SIZE) {
-    await logProcessError(adapters, txid, 'reply too large')
+    await logProcessError(adapters, txid, 'reply too large', blockHeight)
     return
   }
 
-  await adapters.postParentDb.create(txid, { parentTxid, childTxid: txid })
-  await adapters.postChildDb.create(parentTxid, { parentTxid, childTxid: txid })
+  await adapters.postParentDb.create(txid, { parentTxid, childTxid: txid, blockHeight })
+  await adapters.postChildDb.create(parentTxid, { parentTxid, childTxid: txid, blockHeight })
 
   await handlePost({ ...ctx, decoded: { ...decoded, pushDatas: [pushDatas[0], pushDatas[2]] } })
 }

src/use-cases/action-types/set-name.js

@@ -2,19 +2,19 @@ import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './help
 import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
 
 export async function handleSetName (ctx) {
-  const { adapters, txid, signerAddr, decoded, seen } = ctx
+  const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx
   const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas)
 
   if (pushDatas.length !== 2) {
-    await logProcessError(adapters, txid, `invalid set name push data count ${pushDatas.length}`)
+    await logProcessError(adapters, txid, `invalid set name push data count ${pushDatas.length}`, blockHeight)
     return
   }
 
   const name = utf8FromPush(pushDatas[1])
   if (name.length > MAX_POST_SIZE) {
-    await logProcessError(adapters, txid, 'set name too large')
+    await logProcessError(adapters, txid, 'set name too large', blockHeight)
     return
   }
 
-  await adapters.nameDb.create(signerAddr, { name, txid, seen, addr: signerAddr })
+  await adapters.nameDb.create(signerAddr, { name, txid, seen, addr: signerAddr, blockHeight })
 }

src/use-cases/action-types/set-profile-pic.js

@@ -2,23 +2,23 @@ import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './help
 import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
 
 export async function handleSetProfilePic (ctx) {
-  const { adapters, txid, signerAddr, decoded, seen } = ctx
+  const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx
   const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas)
 
   if (pushDatas.length !== 2) {
-    await logProcessError(adapters, txid, `invalid profile pic push data count ${pushDatas.length}`)
+    await logProcessError(adapters, txid, `invalid profile pic push data count ${pushDatas.length}`, blockHeight)
     return
   }
 
   const url = utf8FromPush(pushDatas[1])
   if (!url.length) {
-    await logProcessError(adapters, txid, 'empty profile pic url')
+    await logProcessError(adapters, txid, 'empty profile pic url', blockHeight)
     return
   }
   if (url.length > MAX_POST_SIZE) {
-    await logProcessError(adapters, txid, 'profile pic url too large')
+    await logProcessError(adapters, txid, 'profile pic url too large', blockHeight)
     return
   }
 
-  await adapters.profilePicDb.create(signerAddr, { url, txid, seen, addr: signerAddr })
+  await adapters.profilePicDb.create(signerAddr, { url, txid, seen, addr: signerAddr, blockHeight })
 }

src/use-cases/action-types/set-profile.js

@@ -2,19 +2,19 @@ import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './help
 import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
 
 export async function handleSetProfile (ctx) {
-  const { adapters, txid, signerAddr, decoded, seen } = ctx
+  const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx
   const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas)
 
   if (pushDatas.length !== 2) {
-    await logProcessError(adapters, txid, `invalid profile push data count ${pushDatas.length}`)
+    await logProcessError(adapters, txid, `invalid profile push data count ${pushDatas.length}`, blockHeight)
     return
   }
 
   const text = utf8FromPush(pushDatas[1])
   if (text.length > MAX_POST_SIZE) {
-    await logProcessError(adapters, txid, 'profile too large')
+    await logProcessError(adapters, txid, 'profile too large', blockHeight)
     return
   }
 
-  await adapters.profileDb.create(signerAddr, { text, txid, seen, addr: signerAddr })
+  await adapters.profileDb.create(signerAddr, { text, txid, seen, addr: signerAddr, blockHeight })
 }

src/use-cases/action-types/topic-follow.js

@@ -2,11 +2,11 @@ import { utf8FromPush, logProcessError, roomKey } from './helpers.js'
 import { PREFIX_TOPIC_UNFOLLOW } from '../../lib/memo-codes.js'
 
 export async function handleTopicFollow (ctx) {
-  const { adapters, txid, signerAddr, decoded, seen } = ctx
+  const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx
   const { pushDatas, prefix } = decoded
 
   if (pushDatas.length !== 2) {
-    await logProcessError(adapters, txid, `invalid topic follow push data count ${pushDatas.length}`)
+    await logProcessError(adapters, txid, `invalid topic follow push data count ${pushDatas.length}`, blockHeight)
     return
   }
 
@@ -19,6 +19,7 @@ export async function handleTopicFollow (ctx) {
     unfollow,
     txid,
     seen,
-    type: 'follow'
+    type: 'follow',
+    blockHeight
   })
 }

src/use-cases/action-types/topic-message.js

@@ -3,18 +3,18 @@ import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
 import { handlePost } from './post.js'
 
 export async function handleTopicMessage (ctx) {
-  const { adapters, txid, decoded, seen } = ctx
+  const { adapters, txid, decoded, seen, blockHeight } = ctx
   const { pushDatas } = decoded
 
   if (pushDatas.length !== 3) {
-    await logProcessError(adapters, txid, `invalid topic message push data count ${pushDatas.length}`)
+    await logProcessError(adapters, txid, `invalid topic message push data count ${pushDatas.length}`, blockHeight)
     return
   }
 
   const room = utf8FromPush(pushDatas[1])
   const message = utf8FromPush(pushDatas[2])
   if ((room.length + message.length) > MAX_POST_SIZE) {
-    await logProcessError(adapters, txid, 'topic message too large')
+    await logProcessError(adapters, txid, 'topic message too large', blockHeight)
     return
   }
 
@@ -23,5 +23,5 @@ export async function handleTopicMessage (ctx) {
     decoded: { ...decoded, pushDatas: [pushDatas[0], pushDatas[2]] }
   })
 
-  await adapters.roomDb.create(roomKey(room, txid), { room, txid, seen, type: 'post' })
+  await adapters.roomDb.create(roomKey(room, txid), { room, txid, seen, type: 'post', blockHeight })
 }

test/unit/use-cases/action-types/helpers.unit.js

@@ -1,7 +1,9 @@
 import { assert } from 'chai'
+import sinon from 'sinon'
 import {
   normalizeTwoPushMemoDatas,
-  stripLeadingEmptyPushes
+  stripLeadingEmptyPushes,
+  logProcessError
 } from '../../../../src/use-cases/action-types/helpers.js'
 import { PREFIX_SET_PROFILE_PIC, PREFIX_POST } from '../../../../src/lib/memo-codes.js'
 
@@ -52,4 +54,18 @@ describe('#action-types/helpers', () => {
       assert.deepEqual(result[0], payload)
     })
   })
+
+  describe('#logProcessError', () => {
+    it('should store blockHeight on process error records', async () => {
+      const create = sinon.stub().resolves()
+      const adapters = { processErrorDb: { create } }
+
+      await logProcessError(adapters, 'tx1', 'bad data', 600100)
+
+      assert.equal(create.callCount, 1)
+      assert.equal(create.firstCall.args[0], 'tx1')
+      assert.equal(create.firstCall.args[1].error, 'bad data')
+      assert.equal(create.firstCall.args[1].blockHeight, 600100)
+    })
+  })
 })

test/unit/use-cases/action-types/post.unit.js

@@ -19,6 +19,7 @@ describe('#handlePost', () => {
       txid: 'abc123',
       signerAddr: 'bitcoincash:qptest',
       seen: 1000,
+      blockHeight: 600100,
       decoded: {
         action: 'post',
         prefix: PREFIX_POST,
@@ -29,5 +30,6 @@ describe('#handlePost', () => {
     assert.equal(create.callCount, 1)
     assert.equal(create.firstCall.args[0], 'abc123')
     assert.equal(create.firstCall.args[1].text, 'hello memo')
+    assert.equal(create.firstCall.args[1].blockHeight, 600100)
   })
 })

test/unit/use-cases/action-types/set-profile-pic.unit.js

@@ -22,6 +22,7 @@ describe('#handleSetProfilePic', () => {
       txid: 'abc123',
       signerAddr: 'bitcoincash:qptest',
       seen: 12345,
+      blockHeight: 600200,
       decoded: { pushDatas: [combined] }
     })
 
@@ -30,5 +31,6 @@ describe('#handleSetProfilePic', () => {
     const [addr, data] = adapters.profilePicDb.create.firstCall.args
     assert.equal(addr, 'bitcoincash:qptest')
     assert.equal(data.url, 'https://memo.cash/img/abc')
+    assert.equal(data.blockHeight, 600200)
   })
 })