From 12748eeac0aba728648baa046d07db8323337fd1 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Wed, 3 Jun 2026 12:00:11 -0700 Subject: [PATCH] Adding use-case libraries --- .env-example | 1 + README.md | 1 + config/index.js | 6 +- dev-docs/psf-memo-db.md | 10 ++- dev-docs/theory-of-operation.md | 3 + production/docker/block-indexer/.env-example | 1 + psf-memo-tx-indexer.js | 2 +- src/lib/debug-log.js | 33 ++++++++++ src/use-cases/action-types/helpers.js | 31 ++++++++++ src/use-cases/action-types/post.js | 4 +- src/use-cases/action-types/set-name.js | 4 +- src/use-cases/action-types/set-profile-pic.js | 8 ++- src/use-cases/action-types/set-profile.js | 4 +- src/use-cases/index-blocks.js | 43 ++++++++++--- test/unit/lib/debug-log.unit.js | 62 +++++++++++++++++++ .../use-cases/action-types/helpers.unit.js | 55 ++++++++++++++++ .../action-types/set-profile-pic.unit.js | 34 ++++++++++ test/unit/use-cases/index-blocks.unit.js | 33 +++++++++- 18 files changed, 314 insertions(+), 21 deletions(-) create mode 100644 src/lib/debug-log.js create mode 100644 test/unit/lib/debug-log.unit.js create mode 100644 test/unit/use-cases/action-types/helpers.unit.js create mode 100644 test/unit/use-cases/action-types/set-profile-pic.unit.js diff --git a/.env-example b/.env-example index 436c3d6..ea92168 100644 --- a/.env-example +++ b/.env-example @@ -10,3 +10,4 @@ START_BLOCK_HEIGHT=525000 SEEN_TX_MAX=100000 FILTER_CONCURRENCY=20 MEMO_TX_CONCURRENCY=20 +DEBUG_LEVEL=0 diff --git a/README.md b/README.md index 50d7596..6fbf36f 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ See `.env-example`. Key variables: | `TX_REST_API_PORT` | `5455` | TX indexer control API | | `FILTER_CONCURRENCY` | `20` | Parallel Memo tx detection per block | | `MEMO_TX_CONCURRENCY` | `20` | Parallel Memo tx processing per block | +| `DEBUG_LEVEL` | `0` | `0` = block summary only; `1` = log each Memo tx action type and success/failure | ## Tests diff --git a/config/index.js b/config/index.js index 8d07fa0..34c52b1 100644 --- a/config/index.js +++ b/config/index.js @@ -27,5 +27,9 @@ export default { ? parseInt(process.env.START_BLOCK_HEIGHT) : 525000, - exitOnMissingBackup: process.env.EXIT_ON_MISSING_BACKUP === 'true' + exitOnMissingBackup: process.env.EXIT_ON_MISSING_BACKUP === 'true', + + debugLevel: process.env.DEBUG_LEVEL !== undefined + ? parseInt(process.env.DEBUG_LEVEL, 10) + : 0 } diff --git a/dev-docs/psf-memo-db.md b/dev-docs/psf-memo-db.md index 278d007..4718fd8 100644 --- a/dev-docs/psf-memo-db.md +++ b/dev-docs/psf-memo-db.md @@ -89,12 +89,20 @@ All routes are under `/level` with a consistent CRUD pattern generated from `ENT Returns profiles sorted by **block height** (newest first), using each profile’s `txid` to look up `blockHeight` in `ptxs`. Tie-breaker: `seen` timestamp descending. +The `seen` field is **Unix epoch milliseconds** from the block header time (`block.time * 1000` at indexing time). + Response shape: ```json { "profiles": [ - { "addr": "bitcoincash:q...", "text": "...", "txid": "...", "seen": 123, "blockHeight": 600000 } + { + "addr": "bitcoincash:q...", + "text": "...", + "txid": "...", + "seen": 1500000000000, + "blockHeight": 600000 + } ], "pagination": { "limit": 100, "offset": 0, "total": 42, "hasMore": false } } diff --git a/dev-docs/theory-of-operation.md b/dev-docs/theory-of-operation.md index acf59e1..bba6aaa 100644 --- a/dev-docs/theory-of-operation.md +++ b/dev-docs/theory-of-operation.md @@ -83,6 +83,9 @@ sequenceDiagram Before heavy work, the indexer checks `GET /level/ptx/{txid}`. If present, the tx is skipped. After successful handling, it `POST`s a ptx record `{ processedAt, blockHeight }`. +- **`seen`** on indexed entities (profiles, posts, likes, etc.) is the on-chain block header time in **Unix epoch milliseconds** (`block.time * 1000` from the BCH full node). +- **`processedAt`** on ptx records is indexer wall-clock time in milliseconds (`Date.now()` when the tx was processed). + **Nuance:** If a handler partially fails mid-tx, v1 does not roll back prior writes in that tx. The Go indexer uses richer DB transactions; PSF v1 relies on idempotency only at tx boundaries. See tradeoffs doc. ### Signer address diff --git a/production/docker/block-indexer/.env-example b/production/docker/block-indexer/.env-example index a3a18ec..f342f2f 100644 --- a/production/docker/block-indexer/.env-example +++ b/production/docker/block-indexer/.env-example @@ -16,3 +16,4 @@ START_BLOCK_HEIGHT=525000 EXIT_ON_MISSING_BACKUP=false FILTER_CONCURRENCY=20 MEMO_TX_CONCURRENCY=20 +DEBUG_LEVEL=0 diff --git a/psf-memo-tx-indexer.js b/psf-memo-tx-indexer.js index 6f6e789..1972429 100644 --- a/psf-memo-tx-indexer.js +++ b/psf-memo-tx-indexer.js @@ -52,7 +52,7 @@ async function start () { } try { - await useCases.indexBlocks.processMemoTx(tx, blockHeight + 1) + await useCases.indexBlocks.processMemoTx(tx, blockHeight + 1, Date.now()) } catch (err) { console.error(`Error indexing mempool tx ${tx}:`, err.message) } diff --git a/src/lib/debug-log.js b/src/lib/debug-log.js new file mode 100644 index 0000000..605b21b --- /dev/null +++ b/src/lib/debug-log.js @@ -0,0 +1,33 @@ +/* + Debug logging for Memo indexer (controlled by DEBUG_LEVEL). +*/ + +import config from '../../config/index.js' + +export function getDebugLevel () { + const level = config.debugLevel + if (typeof level !== 'number' || Number.isNaN(level) || level < 0) { + return 0 + } + return level +} + +/** + * Level 1: per-Memo-tx line with action type(s) and outcome. + */ +export function logMemoTxResult ({ txid, actions = [], success, error, skipped }) { + if (getDebugLevel() < 1) return + + const actionStr = actions.length ? actions.join(', ') : 'unknown' + + if (skipped) { + console.log(`Memo tx ${txid} actions=[${actionStr}] skipped (already indexed)`) + return + } + + if (success) { + console.log(`Memo tx ${txid} actions=[${actionStr}] ok`) + } else { + console.log(`Memo tx ${txid} actions=[${actionStr}] failed: ${error || 'unknown'}`) + } +} diff --git a/src/use-cases/action-types/helpers.js b/src/use-cases/action-types/helpers.js index a26acb0..acb8d5f 100644 --- a/src/use-cases/action-types/helpers.js +++ b/src/use-cases/action-types/helpers.js @@ -2,6 +2,8 @@ Shared helpers for Memo action handlers. */ +import { isMemoPrefix } from '../../lib/memo-codes.js' + export async function logProcessError (adapters, txid, error) { try { await adapters.processErrorDb.create(txid, { error, ts: Date.now() }) @@ -14,6 +16,35 @@ export function utf8FromPush (buf) { return buf.toString('utf8') } +/** + * Drop leading empty pushes (btcd txscript.PushedData compatibility). + */ +export function stripLeadingEmptyPushes (pushDatas) { + let datas = pushDatas + while (datas.length > 1 && datas[0] && datas[0].length === 0) { + datas = datas.slice(1) + } + return datas +} + +/** + * Normalize Memo actions that use prefix + payload as two script pushes. + * Some wallets encode both in a single push (0x6dXX + data); split for handlers. + */ +export function normalizeTwoPushMemoDatas (pushDatas) { + const datas = stripLeadingEmptyPushes(pushDatas) + + if (datas.length === 2) { + return datas + } + + if (datas.length === 1 && isMemoPrefix(datas[0]) && datas[0].length > 2) { + return [datas[0].subarray(0, 2), datas[0].subarray(2)] + } + + return datas +} + export function txHashFromPush (buf) { if (!buf || buf.length !== 32) return null return buf.toString('hex') diff --git a/src/use-cases/action-types/post.js b/src/use-cases/action-types/post.js index db0b483..b8529c6 100644 --- a/src/use-cases/action-types/post.js +++ b/src/use-cases/action-types/post.js @@ -1,9 +1,9 @@ -import { utf8FromPush, logProcessError } from './helpers.js' +import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js' import { MAX_POST_SIZE } from '../../lib/memo-codes.js' export async function handlePost (ctx) { const { adapters, txid, signerAddr, decoded, seen } = ctx - const { pushDatas } = decoded + const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas) if (pushDatas.length !== 2) { await logProcessError(adapters, txid, `invalid post push data count ${pushDatas.length}`) diff --git a/src/use-cases/action-types/set-name.js b/src/use-cases/action-types/set-name.js index ec24b8c..d9dc089 100644 --- a/src/use-cases/action-types/set-name.js +++ b/src/use-cases/action-types/set-name.js @@ -1,9 +1,9 @@ -import { utf8FromPush, logProcessError } from './helpers.js' +import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js' import { MAX_POST_SIZE } from '../../lib/memo-codes.js' export async function handleSetName (ctx) { const { adapters, txid, signerAddr, decoded, seen } = ctx - const { pushDatas } = decoded + const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas) if (pushDatas.length !== 2) { await logProcessError(adapters, txid, `invalid set name push data count ${pushDatas.length}`) diff --git a/src/use-cases/action-types/set-profile-pic.js b/src/use-cases/action-types/set-profile-pic.js index e9c9c5a..f66ea3d 100644 --- a/src/use-cases/action-types/set-profile-pic.js +++ b/src/use-cases/action-types/set-profile-pic.js @@ -1,9 +1,9 @@ -import { utf8FromPush, logProcessError } from './helpers.js' +import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js' import { MAX_POST_SIZE } from '../../lib/memo-codes.js' export async function handleSetProfilePic (ctx) { const { adapters, txid, signerAddr, decoded, seen } = ctx - const { pushDatas } = decoded + const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas) if (pushDatas.length !== 2) { await logProcessError(adapters, txid, `invalid profile pic push data count ${pushDatas.length}`) @@ -11,6 +11,10 @@ export async function handleSetProfilePic (ctx) { } const url = utf8FromPush(pushDatas[1]) + if (!url.length) { + await logProcessError(adapters, txid, 'empty profile pic url') + return + } if (url.length > MAX_POST_SIZE) { await logProcessError(adapters, txid, 'profile pic url too large') return diff --git a/src/use-cases/action-types/set-profile.js b/src/use-cases/action-types/set-profile.js index 2b78e0f..caf11e2 100644 --- a/src/use-cases/action-types/set-profile.js +++ b/src/use-cases/action-types/set-profile.js @@ -1,9 +1,9 @@ -import { utf8FromPush, logProcessError } from './helpers.js' +import { utf8FromPush, logProcessError, normalizeTwoPushMemoDatas } from './helpers.js' import { MAX_POST_SIZE } from '../../lib/memo-codes.js' export async function handleSetProfile (ctx) { const { adapters, txid, signerAddr, decoded, seen } = ctx - const { pushDatas } = decoded + const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas) if (pushDatas.length !== 2) { await logProcessError(adapters, txid, `invalid profile push data count ${pushDatas.length}`) diff --git a/src/use-cases/index-blocks.js b/src/use-cases/index-blocks.js index 63de326..bf125c2 100644 --- a/src/use-cases/index-blocks.js +++ b/src/use-cases/index-blocks.js @@ -7,6 +7,7 @@ import PQueue from 'p-queue' import config from '../../config/index.js' import FilterBlock from './filter-block.js' import { findMemoOutputs, getSignerAddress } from '../lib/memo-parser.js' +import { logMemoTxResult } from '../lib/debug-log.js' import { dispatchMemoAction } from './action-types/index.js' class IndexBlocks { @@ -23,27 +24,43 @@ class IndexBlocks { this.processMemoTxs = this.processMemoTxs.bind(this) } - async processMemoTx (txid, blockHeight) { + async getProcessErrorMessage (txid) { + try { + const record = await this.adapters.processErrorDb.get(txid) + return record && record.error ? record.error : null + } catch (err) { + return null + } + } + + async processMemoTx (txid, blockHeight, seen = Date.now()) { + let actions = [] + try { try { await this.adapters.ptxDb.get(txid) + logMemoTxResult({ txid, actions, skipped: true, success: true }) return true } catch (err) { // not processed yet } const txDetails = await this.adapters.transaction.get(txid) + const memoOutputs = findMemoOutputs(txDetails) + actions = memoOutputs.map((output) => output.action) + const signerAddr = getSignerAddress(txDetails) if (!signerAddr) { + const error = 'could not find input address for memo tx' await this.adapters.processErrorDb.create(txid, { - error: 'could not find input address for memo tx', + error, ts: Date.now() }) + logMemoTxResult({ txid, actions, success: false, error }) return false } - const memoOutputs = findMemoOutputs(txDetails) - const seen = Date.now() + const processedAt = Date.now() for (const decoded of memoOutputs) { await dispatchMemoAction({ @@ -57,17 +74,26 @@ class IndexBlocks { }) } - await this.adapters.ptxDb.create(txid, { processedAt: seen, blockHeight }) + const handlerError = await this.getProcessErrorMessage(txid) + if (handlerError) { + logMemoTxResult({ txid, actions, success: false, error: handlerError }) + await this.adapters.ptxDb.create(txid, { processedAt, blockHeight }) + return true + } + + await this.adapters.ptxDb.create(txid, { processedAt, blockHeight }) + logMemoTxResult({ txid, actions, success: true }) return true } catch (err) { + logMemoTxResult({ txid, actions, success: false, error: err.message }) console.error(`Error processing memo tx ${txid}:`, err.message) throw err } } - async processMemoTxs (txids, blockHeight) { + async processMemoTxs (txids, blockHeight, seen) { const tasks = txids.map((txid) => async () => { - await this.processMemoTx(txid, blockHeight) + await this.processMemoTx(txid, blockHeight, seen) }) await this.pQueue.addAll(tasks) return true @@ -92,7 +118,8 @@ class IndexBlocks { const memoTxs = await this.filterBlock.filterMemoTxs(txs) if (memoTxs.length) { console.log(`Memo txs in block: ${memoTxs.length}`) - await this.processMemoTxs(memoTxs, blockHeight) + const blockSeen = block.time * 1000 + await this.processMemoTxs(memoTxs, blockHeight, blockSeen) } return true diff --git a/test/unit/lib/debug-log.unit.js b/test/unit/lib/debug-log.unit.js new file mode 100644 index 0000000..05bfa3b --- /dev/null +++ b/test/unit/lib/debug-log.unit.js @@ -0,0 +1,62 @@ +import { assert } from 'chai' +import sinon from 'sinon' +import config from '../../../config/index.js' +import { getDebugLevel, logMemoTxResult } from '../../../src/lib/debug-log.js' + +describe('#debug-log', () => { + let sandbox + let originalDebugLevel + + beforeEach(() => { + sandbox = sinon.createSandbox() + originalDebugLevel = config.debugLevel + sandbox.stub(console, 'log') + }) + + afterEach(() => { + config.debugLevel = originalDebugLevel + sandbox.restore() + }) + + describe('#getDebugLevel', () => { + it('should default invalid levels to 0', () => { + config.debugLevel = NaN + assert.equal(getDebugLevel(), 0) + }) + }) + + describe('#logMemoTxResult', () => { + it('should not log when debug level is 0', () => { + config.debugLevel = 0 + logMemoTxResult({ + txid: 'abc', + actions: ['post'], + success: true + }) + assert.equal(console.log.callCount, 0) + }) + + it('should log ok with actions at level 1', () => { + config.debugLevel = 1 + logMemoTxResult({ + txid: 'abc', + actions: ['post'], + success: true + }) + assert.include(console.log.firstCall.args[0], 'actions=[post]') + assert.include(console.log.firstCall.args[0], 'ok') + }) + + it('should log failure with process error at level 1', () => { + config.debugLevel = 1 + logMemoTxResult({ + txid: 'abc', + actions: ['like'], + success: false, + error: 'invalid like push data count 1' + }) + assert.include(console.log.firstCall.args[0], 'failed:') + assert.include(console.log.firstCall.args[0], 'invalid like') + }) + }) +}) diff --git a/test/unit/use-cases/action-types/helpers.unit.js b/test/unit/use-cases/action-types/helpers.unit.js new file mode 100644 index 0000000..b5bfda5 --- /dev/null +++ b/test/unit/use-cases/action-types/helpers.unit.js @@ -0,0 +1,55 @@ +import { assert } from 'chai' +import { + normalizeTwoPushMemoDatas, + stripLeadingEmptyPushes +} from '../../../../src/use-cases/action-types/helpers.js' +import { PREFIX_SET_PROFILE_PIC, PREFIX_POST } from '../../../../src/lib/memo-codes.js' + +describe('#action-types/helpers', () => { + describe('#normalizeTwoPushMemoDatas', () => { + it('should pass through standard two-push encoding', () => { + const prefix = PREFIX_SET_PROFILE_PIC + const url = Buffer.from('https://example.com/pic.png', 'utf8') + const normalized = normalizeTwoPushMemoDatas([prefix, url]) + assert.equal(normalized.length, 2) + assert.equal(normalized[1].toString('utf8'), 'https://example.com/pic.png') + }) + + it('should split single-push prefix+payload (profile pic on-chain format)', () => { + const url = Buffer.from('-hash-or-url-bytes', 'utf8') + const combined = Buffer.concat([PREFIX_SET_PROFILE_PIC, url]) + const normalized = normalizeTwoPushMemoDatas([combined]) + assert.equal(normalized.length, 2) + assert.deepEqual(normalized[0], PREFIX_SET_PROFILE_PIC) + assert.deepEqual(normalized[1], url) + }) + + it('should handle user sample: one push with 0x6d0a prefix and 18-byte payload', () => { + const combined = Buffer.from( + '6d0a2da40232abb64740e8abeef1f102bdc92ed5', + 'hex' + ) + const normalized = normalizeTwoPushMemoDatas([combined]) + assert.equal(normalized.length, 2) + assert.equal(normalized[0][1], 0x0a) + assert.equal(normalized[1].length, 18) + }) + + it('should strip leading empty push before normalizing', () => { + const message = Buffer.from('hello', 'utf8') + const combined = Buffer.concat([PREFIX_POST, message]) + const normalized = normalizeTwoPushMemoDatas([Buffer.alloc(0), combined]) + assert.equal(normalized.length, 2) + assert.equal(normalized[1].toString('utf8'), 'hello') + }) + }) + + describe('#stripLeadingEmptyPushes', () => { + it('should remove only leading empty pushes', () => { + const payload = Buffer.from([0xab]) + const result = stripLeadingEmptyPushes([Buffer.alloc(0), payload]) + assert.equal(result.length, 1) + assert.deepEqual(result[0], payload) + }) + }) +}) diff --git a/test/unit/use-cases/action-types/set-profile-pic.unit.js b/test/unit/use-cases/action-types/set-profile-pic.unit.js new file mode 100644 index 0000000..9733d1a --- /dev/null +++ b/test/unit/use-cases/action-types/set-profile-pic.unit.js @@ -0,0 +1,34 @@ +import { assert } from 'chai' +import sinon from 'sinon' +import { handleSetProfilePic } from '../../../../src/use-cases/action-types/set-profile-pic.js' +import { PREFIX_SET_PROFILE_PIC } from '../../../../src/lib/memo-codes.js' + +describe('#handleSetProfilePic', () => { + let adapters + + beforeEach(() => { + adapters = { + profilePicDb: { create: sinon.stub().resolves() }, + processErrorDb: { create: sinon.stub().resolves() } + } + }) + + it('should index profile pic from single-push OP_RETURN', async () => { + const urlBytes = Buffer.from('https://memo.cash/img/abc', 'utf8') + const combined = Buffer.concat([PREFIX_SET_PROFILE_PIC, urlBytes]) + + await handleSetProfilePic({ + adapters, + txid: 'abc123', + signerAddr: 'bitcoincash:qptest', + seen: 12345, + decoded: { pushDatas: [combined] } + }) + + assert.equal(adapters.processErrorDb.create.callCount, 0) + assert.equal(adapters.profilePicDb.create.callCount, 1) + const [addr, data] = adapters.profilePicDb.create.firstCall.args + assert.equal(addr, 'bitcoincash:qptest') + assert.equal(data.url, 'https://memo.cash/img/abc') + }) +}) diff --git a/test/unit/use-cases/index-blocks.unit.js b/test/unit/use-cases/index-blocks.unit.js index 437968c..2a7404f 100644 --- a/test/unit/use-cases/index-blocks.unit.js +++ b/test/unit/use-cases/index-blocks.unit.js @@ -26,9 +26,24 @@ describe('#IndexBlocks', () => { } }] }) + }, + rpc: { + getBlockHash: sandbox.stub().resolves('block-hash-1'), + getBlock: sandbox.stub().resolves({ tx: [], time: 1500000000 }) + }, + filterBlock: { + filterMemoTxs: sandbox.stub().resolves([]) } } uut = new IndexBlocks({ adapters }) + uut.retryQueue = { + addToQueue: sandbox.stub().callsFake(async (fn, arg) => { + if (fn === adapters.rpc.getBlockHash) return adapters.rpc.getBlockHash(arg) + if (fn === adapters.rpc.getBlock) return adapters.rpc.getBlock(arg) + throw new Error('Unexpected addToQueue call') + }) + } + uut.filterBlock = adapters.filterBlock }) afterEach(() => sandbox.restore()) @@ -36,8 +51,9 @@ describe('#IndexBlocks', () => { describe('#processMemoTxs', () => { it('should process all memo txs in parallel', async () => { const processMemoTx = sandbox.stub(uut, 'processMemoTx').resolves(true) + const blockSeen = 1500000000000 - await uut.processMemoTxs(['tx-a', 'tx-b', 'tx-c'], 600000) + await uut.processMemoTxs(['tx-a', 'tx-b', 'tx-c'], 600000, blockSeen) assert.equal(processMemoTx.callCount, 3) assert.deepEqual( @@ -46,6 +62,7 @@ describe('#IndexBlocks', () => { ) processMemoTx.getCalls().forEach((call) => { assert.equal(call.args[1], 600000) + assert.equal(call.args[2], blockSeen) }) }) @@ -56,11 +73,23 @@ describe('#IndexBlocks', () => { .onThirdCall().resolves(true) try { - await uut.processMemoTxs(['tx-a', 'tx-b', 'tx-c'], 600000) + await uut.processMemoTxs(['tx-a', 'tx-b', 'tx-c'], 600000, 1500000000000) assert.fail('Expected processMemoTxs to throw') } catch (err) { assert.include(err.message, 'db error') } }) }) + + describe('#processBlock', () => { + it('should pass block.time in milliseconds to processMemoTxs', async () => { + const processMemoTxs = sandbox.stub(uut, 'processMemoTxs').resolves(true) + adapters.filterBlock.filterMemoTxs.resolves(['tx-a', 'tx-b']) + + await uut.processBlock(600000) + + assert.equal(processMemoTxs.callCount, 1) + assert.deepEqual(processMemoTxs.firstCall.args, [['tx-a', 'tx-b'], 600000, 1500000000000]) + }) + }) })