Adding use-case libraries

.env-example

@@ -10,3 +10,4 @@ START_BLOCK_HEIGHT=525000
 SEEN_TX_MAX=100000
 FILTER_CONCURRENCY=20
 MEMO_TX_CONCURRENCY=20
+DEBUG_LEVEL=0

README.md

@@ -63,6 +63,7 @@ See `.env-example`. Key variables:
 | `TX_REST_API_PORT` | `5455` | TX indexer control API |
 | `FILTER_CONCURRENCY` | `20` | Parallel Memo tx detection per block |
 | `MEMO_TX_CONCURRENCY` | `20` | Parallel Memo tx processing per block |
+| `DEBUG_LEVEL` | `0` | `0` = block summary only; `1` = log each Memo tx action type and success/failure |
 
 ## Tests
 

config/index.js

@@ -27,5 +27,9 @@ export default {
     ? parseInt(process.env.START_BLOCK_HEIGHT)
     : 525000,
 
-  exitOnMissingBackup: process.env.EXIT_ON_MISSING_BACKUP === 'true'
+  exitOnMissingBackup: process.env.EXIT_ON_MISSING_BACKUP === 'true',
+
+  debugLevel: process.env.DEBUG_LEVEL !== undefined
+    ? parseInt(process.env.DEBUG_LEVEL, 10)
+    : 0
 }

dev-docs/psf-memo-db.md

@@ -89,12 +89,20 @@ All routes are under `/level` with a consistent CRUD pattern generated from `ENT
 
 Returns profiles sorted by **block height** (newest first), using each profile’s `txid` to look up `blockHeight` in `ptxs`. Tie-breaker: `seen` timestamp descending.
 
+The `seen` field is **Unix epoch milliseconds** from the block header time (`block.time * 1000` at indexing time).
+
 Response shape:
 
 ```json
 {
   "profiles": [
-    { "addr": "bitcoincash:q...", "text": "...", "txid": "...", "seen": 123, "blockHeight": 600000 }
+    {
+      "addr": "bitcoincash:q...",
+      "text": "...",
+      "txid": "...",
+      "seen": 1500000000000,
+      "blockHeight": 600000
+    }
   ],
   "pagination": { "limit": 100, "offset": 0, "total": 42, "hasMore": false }
 }

dev-docs/theory-of-operation.md

@@ -83,6 +83,9 @@ sequenceDiagram
 
 Before heavy work, the indexer checks `GET /level/ptx/{txid}`. If present, the tx is skipped. After successful handling, it `POST`s a ptx record `{ processedAt, blockHeight }`.
 
+- **`seen`** on indexed entities (profiles, posts, likes, etc.) is the on-chain block header time in **Unix epoch milliseconds** (`block.time * 1000` from the BCH full node).
+- **`processedAt`** on ptx records is indexer wall-clock time in milliseconds (`Date.now()` when the tx was processed).
+
 **Nuance:** If a handler partially fails mid-tx, v1 does not roll back prior writes in that tx. The Go indexer uses richer DB transactions; PSF v1 relies on idempotency only at tx boundaries. See tradeoffs doc.
 
 ### Signer address

production/docker/block-indexer/.env-example

@@ -16,3 +16,4 @@ START_BLOCK_HEIGHT=525000
 EXIT_ON_MISSING_BACKUP=false
 FILTER_CONCURRENCY=20
 MEMO_TX_CONCURRENCY=20
+DEBUG_LEVEL=0

psf-memo-tx-indexer.js

@@ -52,7 +52,7 @@ async function start () {
         }
 
         try {
-          await useCases.indexBlocks.processMemoTx(tx, blockHeight + 1)
+          await useCases.indexBlocks.processMemoTx(tx, blockHeight + 1, Date.now())
         } catch (err) {
           console.error(`Error indexing mempool tx ${tx}:`, err.message)
         }

src/lib/debug-log.js

@@ -0,0 +1,33 @@
+/*
+  Debug logging for Memo indexer (controlled by DEBUG_LEVEL).
+*/
+
+import config from '../../config/index.js'
+
+export function getDebugLevel () {
+  const level = config.debugLevel
+  if (typeof level !== 'number' || Number.isNaN(level) || level < 0) {
+    return 0
+  }
+  return level
+}
+
+/**
+ * Level 1: per-Memo-tx line with action type(s) and outcome.
+ */
+export function logMemoTxResult ({ txid, actions = [], success, error, skipped }) {
+  if (getDebugLevel() < 1) return
+
+  const actionStr = actions.length ? actions.join(', ') : 'unknown'
+
+  if (skipped) {
+    console.log(`Memo tx ${txid} actions=[${actionStr}] skipped (already indexed)`)
+    return
+  }
+
+  if (success) {
+    console.log(`Memo tx ${txid} actions=[${actionStr}] ok`)
+  } else {
+    console.log(`Memo tx ${txid} actions=[${actionStr}] failed: ${error || 'unknown'}`)
+  }
+}

src/use-cases/action-types/helpers.js

@@ -2,6 +2,8 @@
   Shared helpers for Memo action handlers.
 */
 
+import { isMemoPrefix } from '../../lib/memo-codes.js'
+
 export async function logProcessError (adapters, txid, error) {
   try {
     await adapters.processErrorDb.create(txid, { error, ts: Date.now() })
@@ -14,6 +16,35 @@ export function utf8FromPush (buf) {
   return buf.toString('utf8')
 }
 
+/**
+ * Drop leading empty pushes (btcd txscript.PushedData compatibility).
+ */
+export function stripLeadingEmptyPushes (pushDatas) {
+  let datas = pushDatas
+  while (datas.length > 1 && datas[0] && datas[0].length === 0) {
+    datas = datas.slice(1)
+  }
+  return datas
+}
+
+/**
+ * Normalize Memo actions that use prefix + payload as two script pushes.
+ * Some wallets encode both in a single push (0x6dXX + data); split for handlers.
+ */
+export function normalizeTwoPushMemoDatas (pushDatas) {
+  const datas = stripLeadingEmptyPushes(pushDatas)
+
+  if (datas.length === 2) {
+    return datas
+  }
+
+  if (datas.length === 1 && isMemoPrefix(datas[0]) && datas[0].length > 2) {
+    return [datas[0].subarray(0, 2), datas[0].subarray(2)]
+  }
+
+  return datas
+}
+
 export function txHashFromPush (buf) {
   if (!buf || buf.length !== 32) return null
   return buf.toString('hex')

src/use-cases/action-types/post.js

@@ -1,9 +1,9 @@
-import { utf8FromPush, logProcessError } from './helpers.js'
+import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js'
 import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
 
 export async function handlePost (ctx) {
   const { adapters, txid, signerAddr, decoded, seen } = ctx
-  const { pushDatas } = decoded
+  const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas)
 
   if (pushDatas.length !== 2) {
     await logProcessError(adapters, txid, `invalid post push data count ${pushDatas.length}`)

src/use-cases/action-types/set-name.js

@@ -1,9 +1,9 @@
-import { utf8FromPush, logProcessError } from './helpers.js'
+import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js'
 import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
 
 export async function handleSetName (ctx) {
   const { adapters, txid, signerAddr, decoded, seen } = ctx
-  const { pushDatas } = decoded
+  const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas)
 
   if (pushDatas.length !== 2) {
     await logProcessError(adapters, txid, `invalid set name push data count ${pushDatas.length}`)

src/use-cases/action-types/set-profile-pic.js

@@ -1,9 +1,9 @@
-import { utf8FromPush, logProcessError } from './helpers.js'
+import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js'
 import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
 
 export async function handleSetProfilePic (ctx) {
   const { adapters, txid, signerAddr, decoded, seen } = ctx
-  const { pushDatas } = decoded
+  const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas)
 
   if (pushDatas.length !== 2) {
     await logProcessError(adapters, txid, `invalid profile pic push data count ${pushDatas.length}`)
@@ -11,6 +11,10 @@ export async function handleSetProfilePic (ctx) {
   }
 
   const url = utf8FromPush(pushDatas[1])
+  if (!url.length) {
+    await logProcessError(adapters, txid, 'empty profile pic url')
+    return
+  }
   if (url.length > MAX_POST_SIZE) {
     await logProcessError(adapters, txid, 'profile pic url too large')
     return

src/use-cases/action-types/set-profile.js

@@ -1,9 +1,9 @@
-import { utf8FromPush, logProcessError } from './helpers.js'
+import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js'
 import { MAX_POST_SIZE } from '../../lib/memo-codes.js'
 
 export async function handleSetProfile (ctx) {
   const { adapters, txid, signerAddr, decoded, seen } = ctx
-  const { pushDatas } = decoded
+  const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas)
 
   if (pushDatas.length !== 2) {
     await logProcessError(adapters, txid, `invalid profile push data count ${pushDatas.length}`)

src/use-cases/index-blocks.js

@@ -7,6 +7,7 @@ import PQueue from 'p-queue'
 import config from '../../config/index.js'
 import FilterBlock from './filter-block.js'
 import { findMemoOutputs, getSignerAddress } from '../lib/memo-parser.js'
+import { logMemoTxResult } from '../lib/debug-log.js'
 import { dispatchMemoAction } from './action-types/index.js'
 
 class IndexBlocks {
@@ -23,27 +24,43 @@ class IndexBlocks {
     this.processMemoTxs = this.processMemoTxs.bind(this)
   }
 
-  async processMemoTx (txid, blockHeight) {
+  async getProcessErrorMessage (txid) {
+    try {
+      const record = await this.adapters.processErrorDb.get(txid)
+      return record && record.error ? record.error : null
+    } catch (err) {
+      return null
+    }
+  }
+
+  async processMemoTx (txid, blockHeight, seen = Date.now()) {
+    let actions = []
+
     try {
       try {
         await this.adapters.ptxDb.get(txid)
+        logMemoTxResult({ txid, actions, skipped: true, success: true })
         return true
       } catch (err) {
         // not processed yet
       }
 
       const txDetails = await this.adapters.transaction.get(txid)
+      const memoOutputs = findMemoOutputs(txDetails)
+      actions = memoOutputs.map((output) => output.action)
+
       const signerAddr = getSignerAddress(txDetails)
       if (!signerAddr) {
+        const error = 'could not find input address for memo tx'
         await this.adapters.processErrorDb.create(txid, {
-          error: 'could not find input address for memo tx',
+          error,
           ts: Date.now()
         })
+        logMemoTxResult({ txid, actions, success: false, error })
         return false
       }
 
-      const memoOutputs = findMemoOutputs(txDetails)
+      const processedAt = Date.now()
-      const seen = Date.now()
 
       for (const decoded of memoOutputs) {
         await dispatchMemoAction({
@@ -57,17 +74,26 @@ class IndexBlocks {
         })
       }
 
-      await this.adapters.ptxDb.create(txid, { processedAt: seen, blockHeight })
+      const handlerError = await this.getProcessErrorMessage(txid)
+      if (handlerError) {
+        logMemoTxResult({ txid, actions, success: false, error: handlerError })
+        await this.adapters.ptxDb.create(txid, { processedAt, blockHeight })
+        return true
+      }
+
+      await this.adapters.ptxDb.create(txid, { processedAt, blockHeight })
+      logMemoTxResult({ txid, actions, success: true })
       return true
     } catch (err) {
+      logMemoTxResult({ txid, actions, success: false, error: err.message })
       console.error(`Error processing memo tx ${txid}:`, err.message)
       throw err
     }
   }
 
-  async processMemoTxs (txids, blockHeight) {
+  async processMemoTxs (txids, blockHeight, seen) {
     const tasks = txids.map((txid) => async () => {
-      await this.processMemoTx(txid, blockHeight)
+      await this.processMemoTx(txid, blockHeight, seen)
     })
     await this.pQueue.addAll(tasks)
     return true
@@ -92,7 +118,8 @@ class IndexBlocks {
     const memoTxs = await this.filterBlock.filterMemoTxs(txs)
     if (memoTxs.length) {
       console.log(`Memo txs in block: ${memoTxs.length}`)
-      await this.processMemoTxs(memoTxs, blockHeight)
+      const blockSeen = block.time * 1000
+      await this.processMemoTxs(memoTxs, blockHeight, blockSeen)
     }
 
     return true

test/unit/lib/debug-log.unit.js

@@ -0,0 +1,62 @@
+import { assert } from 'chai'
+import sinon from 'sinon'
+import config from '../../../config/index.js'
+import { getDebugLevel, logMemoTxResult } from '../../../src/lib/debug-log.js'
+
+describe('#debug-log', () => {
+  let sandbox
+  let originalDebugLevel
+
+  beforeEach(() => {
+    sandbox = sinon.createSandbox()
+    originalDebugLevel = config.debugLevel
+    sandbox.stub(console, 'log')
+  })
+
+  afterEach(() => {
+    config.debugLevel = originalDebugLevel
+    sandbox.restore()
+  })
+
+  describe('#getDebugLevel', () => {
+    it('should default invalid levels to 0', () => {
+      config.debugLevel = NaN
+      assert.equal(getDebugLevel(), 0)
+    })
+  })
+
+  describe('#logMemoTxResult', () => {
+    it('should not log when debug level is 0', () => {
+      config.debugLevel = 0
+      logMemoTxResult({
+        txid: 'abc',
+        actions: ['post'],
+        success: true
+      })
+      assert.equal(console.log.callCount, 0)
+    })
+
+    it('should log ok with actions at level 1', () => {
+      config.debugLevel = 1
+      logMemoTxResult({
+        txid: 'abc',
+        actions: ['post'],
+        success: true
+      })
+      assert.include(console.log.firstCall.args[0], 'actions=[post]')
+      assert.include(console.log.firstCall.args[0], 'ok')
+    })
+
+    it('should log failure with process error at level 1', () => {
+      config.debugLevel = 1
+      logMemoTxResult({
+        txid: 'abc',
+        actions: ['like'],
+        success: false,
+        error: 'invalid like push data count 1'
+      })
+      assert.include(console.log.firstCall.args[0], 'failed:')
+      assert.include(console.log.firstCall.args[0], 'invalid like')
+    })
+  })
+})

test/unit/use-cases/action-types/helpers.unit.js

@@ -0,0 +1,55 @@
+import { assert } from 'chai'
+import {
+  normalizeTwoPushMemoDatas,
+  stripLeadingEmptyPushes
+} from '../../../../src/use-cases/action-types/helpers.js'
+import { PREFIX_SET_PROFILE_PIC, PREFIX_POST } from '../../../../src/lib/memo-codes.js'
+
+describe('#action-types/helpers', () => {
+  describe('#normalizeTwoPushMemoDatas', () => {
+    it('should pass through standard two-push encoding', () => {
+      const prefix = PREFIX_SET_PROFILE_PIC
+      const url = Buffer.from('https://example.com/pic.png', 'utf8')
+      const normalized = normalizeTwoPushMemoDatas([prefix, url])
+      assert.equal(normalized.length, 2)
+      assert.equal(normalized[1].toString('utf8'), 'https://example.com/pic.png')
+    })
+
+    it('should split single-push prefix+payload (profile pic on-chain format)', () => {
+      const url = Buffer.from('-hash-or-url-bytes', 'utf8')
+      const combined = Buffer.concat([PREFIX_SET_PROFILE_PIC, url])
+      const normalized = normalizeTwoPushMemoDatas([combined])
+      assert.equal(normalized.length, 2)
+      assert.deepEqual(normalized[0], PREFIX_SET_PROFILE_PIC)
+      assert.deepEqual(normalized[1], url)
+    })
+
+    it('should handle user sample: one push with 0x6d0a prefix and 18-byte payload', () => {
+      const combined = Buffer.from(
+        '6d0a2da40232abb64740e8abeef1f102bdc92ed5',
+        'hex'
+      )
+      const normalized = normalizeTwoPushMemoDatas([combined])
+      assert.equal(normalized.length, 2)
+      assert.equal(normalized[0][1], 0x0a)
+      assert.equal(normalized[1].length, 18)
+    })
+
+    it('should strip leading empty push before normalizing', () => {
+      const message = Buffer.from('hello', 'utf8')
+      const combined = Buffer.concat([PREFIX_POST, message])
+      const normalized = normalizeTwoPushMemoDatas([Buffer.alloc(0), combined])
+      assert.equal(normalized.length, 2)
+      assert.equal(normalized[1].toString('utf8'), 'hello')
+    })
+  })
+
+  describe('#stripLeadingEmptyPushes', () => {
+    it('should remove only leading empty pushes', () => {
+      const payload = Buffer.from([0xab])
+      const result = stripLeadingEmptyPushes([Buffer.alloc(0), payload])
+      assert.equal(result.length, 1)
+      assert.deepEqual(result[0], payload)
+    })
+  })
+})

test/unit/use-cases/action-types/set-profile-pic.unit.js

@@ -0,0 +1,34 @@
+import { assert } from 'chai'
+import sinon from 'sinon'
+import { handleSetProfilePic } from '../../../../src/use-cases/action-types/set-profile-pic.js'
+import { PREFIX_SET_PROFILE_PIC } from '../../../../src/lib/memo-codes.js'
+
+describe('#handleSetProfilePic', () => {
+  let adapters
+
+  beforeEach(() => {
+    adapters = {
+      profilePicDb: { create: sinon.stub().resolves() },
+      processErrorDb: { create: sinon.stub().resolves() }
+    }
+  })
+
+  it('should index profile pic from single-push OP_RETURN', async () => {
+    const urlBytes = Buffer.from('https://memo.cash/img/abc', 'utf8')
+    const combined = Buffer.concat([PREFIX_SET_PROFILE_PIC, urlBytes])
+
+    await handleSetProfilePic({
+      adapters,
+      txid: 'abc123',
+      signerAddr: 'bitcoincash:qptest',
+      seen: 12345,
+      decoded: { pushDatas: [combined] }
+    })
+
+    assert.equal(adapters.processErrorDb.create.callCount, 0)
+    assert.equal(adapters.profilePicDb.create.callCount, 1)
+    const [addr, data] = adapters.profilePicDb.create.firstCall.args
+    assert.equal(addr, 'bitcoincash:qptest')
+    assert.equal(data.url, 'https://memo.cash/img/abc')
+  })
+})

test/unit/use-cases/index-blocks.unit.js

@@ -26,9 +26,24 @@ describe('#IndexBlocks', () => {
             }
           }]
         })
+      },
+      rpc: {
+        getBlockHash: sandbox.stub().resolves('block-hash-1'),
+        getBlock: sandbox.stub().resolves({ tx: [], time: 1500000000 })
+      },
+      filterBlock: {
+        filterMemoTxs: sandbox.stub().resolves([])
       }
     }
     uut = new IndexBlocks({ adapters })
+    uut.retryQueue = {
+      addToQueue: sandbox.stub().callsFake(async (fn, arg) => {
+        if (fn === adapters.rpc.getBlockHash) return adapters.rpc.getBlockHash(arg)
+        if (fn === adapters.rpc.getBlock) return adapters.rpc.getBlock(arg)
+        throw new Error('Unexpected addToQueue call')
+      })
+    }
+    uut.filterBlock = adapters.filterBlock
   })
 
   afterEach(() => sandbox.restore())
@@ -36,8 +51,9 @@ describe('#IndexBlocks', () => {
   describe('#processMemoTxs', () => {
     it('should process all memo txs in parallel', async () => {
       const processMemoTx = sandbox.stub(uut, 'processMemoTx').resolves(true)
+      const blockSeen = 1500000000000
 
-      await uut.processMemoTxs(['tx-a', 'tx-b', 'tx-c'], 600000)
+      await uut.processMemoTxs(['tx-a', 'tx-b', 'tx-c'], 600000, blockSeen)
 
       assert.equal(processMemoTx.callCount, 3)
       assert.deepEqual(
@@ -46,6 +62,7 @@ describe('#IndexBlocks', () => {
       )
       processMemoTx.getCalls().forEach((call) => {
         assert.equal(call.args[1], 600000)
+        assert.equal(call.args[2], blockSeen)
       })
     })
 
@@ -56,11 +73,23 @@ describe('#IndexBlocks', () => {
         .onThirdCall().resolves(true)
 
       try {
-        await uut.processMemoTxs(['tx-a', 'tx-b', 'tx-c'], 600000)
+        await uut.processMemoTxs(['tx-a', 'tx-b', 'tx-c'], 600000, 1500000000000)
         assert.fail('Expected processMemoTxs to throw')
       } catch (err) {
         assert.include(err.message, 'db error')
       }
     })
   })
+
+  describe('#processBlock', () => {
+    it('should pass block.time in milliseconds to processMemoTxs', async () => {
+      const processMemoTxs = sandbox.stub(uut, 'processMemoTxs').resolves(true)
+      adapters.filterBlock.filterMemoTxs.resolves(['tx-a', 'tx-b'])
+
+      await uut.processBlock(600000)
+
+      assert.equal(processMemoTxs.callCount, 1)
+      assert.deepEqual(processMemoTxs.firstCall.args, [['tx-a', 'tx-b'], 600000, 1500000000000])
+    })
+  })
 })