Refactor txid wire encoding and repair library

Share the txid-and-text payload builder between hex and reply, extract the
repair store loop from repairTxidEncoding to cut CRAP from 13 to 5, correct
the poll-services property test to the little-endian wire order, and add
property tests for wire round trips and repair idempotence/conservation.

By refactorer.
This commit is contained in:
Chris Troutner
2026-09-16 14:59:00 -07:00
parent dedb7a1343
commit a76fceee73
7 changed files with 549 additions and 61 deletions
+3 -3
View File
@@ -35,9 +35,9 @@ function txidToWireBytes (txid, label = 'Value') {
// Build the raw OP_RETURN payload for a txid-referencing Memo action: the
// given 32-byte txid in little-endian wire order followed by a UTF-8 encoded
// value.
function buildTxidTextPayload (txid, text) {
const txidBytes = txidToWireBytes(txid, 'Poll txid')
// value. The label customizes the invalid-txid error message.
function buildTxidTextPayload (txid, text, label = 'Poll txid') {
const txidBytes = txidToWireBytes(txid, label)
const textBytes = new TextEncoder().encode(text)
const raw = new Uint8Array(txidBytes.length + textBytes.length)
raw.set(txidBytes, 0)
+2 -13
View File
@@ -18,7 +18,7 @@
const MemoAction = require('./memo-action')
const { byteLength } = require('./utf8')
const { txidToWireBytes } = require('./hex')
const { buildTxidTextPayload } = require('./hex')
const MEMO_REPLY_PREFIX = '6d03'
const MAX_REPLY_BYTES = 184
@@ -57,7 +57,7 @@ class MemoReply extends MemoAction {
await this.wallet.getUtxos()
// Build the raw payload: parent txid bytes followed by UTF-8 message bytes.
const raw = buildReplyPayload(parentTxid, message)
const raw = buildTxidTextPayload(parentTxid, message, 'Parent txid')
const txid = await this.wallet.sendOpReturn(raw, this.prefix)
// Reflect the result on the injected thread once broadcast succeeds.
@@ -79,17 +79,6 @@ class MemoReply extends MemoAction {
}
}
// Build the raw OP_RETURN message payload for a reply.
// The protocol wire format is: <parent txid 32 bytes><reply text UTF-8 bytes>.
function buildReplyPayload (parentTxid, message) {
const parentBytes = txidToWireBytes(parentTxid, 'Parent txid')
const textBytes = new TextEncoder().encode(message)
const raw = new Uint8Array(parentBytes.length + textBytes.length)
raw.set(parentBytes, 0)
raw.set(textBytes, parentBytes.length)
return raw
}
MemoReply.MEMO_REPLY_PREFIX = MEMO_REPLY_PREFIX
MemoReply.MAX_REPLY_BYTES = MAX_REPLY_BYTES
@@ -67,12 +67,11 @@ test('buildTxidTextPayload round-trips the canonical Memo wire format', async ()
({ txid, text }) => {
const bytes = buildTxidTextPayload(txid, text)
if (bytes.length !== 32 + byteLength(text)) return false
// The payload carries the txid's literal 32 bytes in order, followed by
// the UTF-8 bytes of the value.
const expectedTxidBytes = hexToBytes(txid, 32, 'Poll txid')
for (let i = 0; i < 32; i++) {
if (bytes[i] !== expectedTxidBytes[i]) return false
}
// The payload carries the txid in little-endian wire order, followed by
// the UTF-8 bytes of the value. Reversing the wire bytes must recover
// the 64-character display txid.
const wire = Buffer.from(bytes.slice(0, 32))
if (wire.reverse().toString('hex') !== txid) return false
const storedText = Buffer.from(bytes.slice(32)).toString('utf8')
return storedText === text
},
@@ -0,0 +1,108 @@
/*
Property tests for Memo txid wire encoding.
Unit tests pin the wire bytes of a few fixed txids. These properties pin the
encoding invariants over broad random inputs:
- Round trip: reversing the wire bytes recovers the display bytes, and
reversing twice is the identity.
- Wire order: the encoded bytes are exactly the reverse of the display
txid's bytes.
- Payload shape: a txid-and-text payload starts with the wire txid and
ends with the UTF-8 text, with no bytes added or dropped.
- Determinism: the same txid always encodes to the same bytes.
*/
'use strict'
const test = require('node:test')
const assert = require('node:assert/strict')
const { seededRandom, forAll, intGen } = require('./harness')
const { txidToWireBytes, buildTxidTextPayload } = require('../../src/services/hex')
const HEX_CHARS = '0123456789abcdef'
function randomTxid (rng) {
let out = ''
for (let i = 0; i < 64; i++) {
out += HEX_CHARS[Math.floor(rng() * HEX_CHARS.length)]
}
return out
}
function randomText (rng) {
const alphabet = 'abc XYZ!\u00e9\u4e2d\ud83d\ude00'
let out = ''
const length = intGen(rng, 0, 24)()
for (let i = 0; i < length; i++) {
out += alphabet[Math.floor(rng() * alphabet.length)]
}
return out
}
test('txid wire bytes are the byte reverse of the display txid', async () => {
const rng = seededRandom(20260916)
await forAll(
() => randomTxid(rng),
(txid) => {
const wire = Buffer.from(txidToWireBytes(txid)).toString('hex')
const expected = Buffer.from(txid, 'hex').reverse().toString('hex')
return wire === expected
},
{ label: 'txidToWireBytes byte order' }
)
})
test('reversing wire bytes twice recovers the display txid', async () => {
const rng = seededRandom(20260917)
await forAll(
() => randomTxid(rng),
(txid) => {
const bytes = txidToWireBytes(txid)
return Buffer.from(Buffer.from(bytes).reverse()).toString('hex') === txid
},
{ label: 'txidToWireBytes round trip' }
)
})
test('txid encoding is deterministic', async () => {
const rng = seededRandom(20260918)
await forAll(
() => randomTxid(rng),
(txid) => {
const first = Buffer.from(txidToWireBytes(txid)).toString('hex')
const second = Buffer.from(txidToWireBytes(txid)).toString('hex')
return first === second
},
{ label: 'txidToWireBytes determinism' }
)
})
test('payload embeds the wire txid followed by the UTF-8 text', async () => {
const rng = seededRandom(20260919)
await forAll(
() => ({ txid: randomTxid(rng), text: randomText(rng) }),
({ txid, text }) => {
const raw = buildTxidTextPayload(txid, text)
const buf = Buffer.from(raw)
const wire = Buffer.from(txidToWireBytes(txid)).toString('hex')
const expectedText = Buffer.from(text, 'utf8')
if (buf.length !== 32 + expectedText.length) return false
if (buf.subarray(0, 32).toString('hex') !== wire) return false
return buf.subarray(32).equals(expectedText)
},
{ label: 'buildTxidTextPayload shape' }
)
})
test('a reply payload rejects an invalid txid with the parent label', () => {
assert.throws(
() => buildTxidTextPayload('not-a-txid', 'hi', 'Parent txid'),
/Parent txid must be a 64-character hex string/
)
})
+115
View File
@@ -60,3 +60,118 @@ test('like reflects the post txid in display order on the feed store', async ()
assert.equal(added.length, 1)
assert.equal(added[0].postTxid, POST_TXID)
})
test('like rejects a caller without a wallet', async () => {
const memoLike = new MemoLike()
await assert.rejects(() => memoLike.like(POST_TXID), /requires a wallet/)
})
test('like rejects an invalid post txid before broadcasting', async () => {
const wallet = makeWallet()
const memoLike = new MemoLike({ wallet })
await assert.rejects(
() => memoLike.like('not-a-txid'),
(err) => err.code === 'like_validation'
)
assert.equal(wallet.broadcasts.length, 0)
})
test('like rejects a balance below the dust limit', async () => {
const wallet = makeWallet()
wallet.utxos = [{ txid: 'utxo', value: 100 }]
const memoLike = new MemoLike({ wallet })
await assert.rejects(
() => memoLike.like(POST_TXID),
(err) => err.code === 'like_empty_balance'
)
})
test('like rejects a tip that exceeds the spendable balance', async () => {
const wallet = makeWallet()
wallet.utxos = [{ txid: 'utxo', value: 3000 }]
const memoLike = new MemoLike({ wallet })
await assert.rejects(
() => memoLike.like(POST_TXID, 4000, MY_ADDRESS),
(err) => err.code === 'like_balance'
)
})
test('like requires an author address when a tip is present', async () => {
const wallet = makeWallet()
const memoLike = new MemoLike({ wallet })
await assert.rejects(
() => memoLike.like(POST_TXID, 1000),
(err) => err.code === 'like_validation'
)
})
test('_validateTipAmount rejects a non-integer or negative tip', () => {
const memoLike = new MemoLike()
assert.throws(() => memoLike._validateTipAmount(1.5), (err) => err.code === 'like_validation')
assert.throws(() => memoLike._validateTipAmount(-1), (err) => err.code === 'like_validation')
})
test('_validateTipAmount rejects a tip below the dust limit', () => {
const memoLike = new MemoLike()
assert.throws(() => memoLike._validateTipAmount(1), (err) => err.code === 'like_dust')
})
test('_validateTipAmount rejects a tip above the maximum', () => {
const memoLike = new MemoLike()
assert.throws(
() => memoLike._validateTipAmount(MemoLike.MAX_TIP_SATS + 1),
(err) => err.code === 'like_maximum'
)
})
test('_validateTipAmount accepts zero and a valid tip', () => {
const memoLike = new MemoLike()
assert.equal(memoLike._validateTipAmount(0), undefined)
assert.equal(memoLike._validateTipAmount(1000), undefined)
})
test('getSpendableSats sums the utxoStore bchUtxos shape', () => {
const wallet = {
utxos: { utxoStore: { bchUtxos: [{ satoshis: 1000 }, { amount: 500 }] } }
}
const memoLike = new MemoLike({ wallet })
assert.equal(memoLike.getSpendableSats(), 1500)
})
test('getSpendableSats is zero without a wallet', () => {
assert.equal(new MemoLike().getSpendableSats(), 0)
})
test('like includes a tip output and reflects the tip on the feed', async () => {
const wallet = makeWallet()
const added = []
const feed = {
addLike (like) {
added.push(like)
},
posts: [{ txid: POST_TXID, likeCount: 2 }]
}
const memoLike = new MemoLike({ wallet, feed })
await memoLike.like(POST_TXID, 1000, MY_ADDRESS)
assert.deepEqual(wallet.broadcasts[0].bchOutput, [{ address: MY_ADDRESS, amountSat: 1000 }])
assert.equal(added[0].tipSats, 1000)
assert.equal(feed.posts[0].likeCount, 3)
})
test('_buildTipOutput omits the output for a zero tip', () => {
const memoLike = new MemoLike()
assert.deepEqual(memoLike._buildTipOutput(0, MY_ADDRESS), [])
})
+59 -39
View File
@@ -43,50 +43,70 @@ export async function correctReference (targetDb, reference) {
return reference
}
// Repair one referencing store in place. Every record whose reference can be
// corrected is rewritten; an optional secondary index rebuilds its entry for
// each corrected record. Returns the number of records corrected.
async function repairStore ({ sourceDb, referenceField, targetDb, index }) {
let correctedCount = 0
for await (const [recordTxid, record] of sourceDb.iterator()) {
const reference = record[referenceField]
if (!reference) continue
const corrected = await correctReference(targetDb, reference)
if (corrected === reference) continue
await sourceDb.put(recordTxid, { ...record, [referenceField]: corrected })
if (index) {
await index.db.del(index.key(reference, recordTxid))
await index.db.put(index.key(corrected, recordTxid), index.value(record, corrected, recordTxid))
}
correctedCount++
}
return correctedCount
}
// A secondary index keyed as <target txid>:<record txid>; both the postLikes
// and postChildren indexes use this shape.
function txidIndex (db, value) {
return {
db,
key: (targetTxid, recordTxid) => `${targetTxid}:${recordTxid}`,
value
}
}
// Repair likes, replies, poll options, and poll votes in place. Returns a
// summary of how many records of each kind were corrected.
export async function repairTxidEncoding (level) {
const summary = { likes: 0, replies: 0, pollOptions: 0, pollVotes: 0 }
const likes = await repairStore({
sourceDb: level.likesDb,
referenceField: 'postTxid',
targetDb: level.postsDb,
index: txidIndex(level.postLikesDb, (record, postTxid, likeTxid) => ({ postTxid, txid: likeTxid }))
})
for await (const [likeTxid, like] of level.likesDb.iterator()) {
if (!like.postTxid) continue
const corrected = await correctReference(level.postsDb, like.postTxid)
if (corrected === like.postTxid) continue
const replies = await repairStore({
sourceDb: level.postParentsDb,
referenceField: 'parentTxid',
targetDb: level.postsDb,
index: txidIndex(level.postChildrenDb, (record, parentTxid) => ({ ...record, parentTxid }))
})
await level.likesDb.put(likeTxid, { ...like, postTxid: corrected })
await level.postLikesDb.del(`${like.postTxid}:${likeTxid}`)
await level.postLikesDb.put(`${corrected}:${likeTxid}`, { postTxid: corrected, txid: likeTxid })
summary.likes++
}
const pollOptions = await repairStore({
sourceDb: level.pollOptionsDb,
referenceField: 'pollTxid',
targetDb: level.pollsDb
})
for await (const [replyTxid, reply] of level.postParentsDb.iterator()) {
if (!reply.parentTxid) continue
const corrected = await correctReference(level.postsDb, reply.parentTxid)
if (corrected === reply.parentTxid) continue
const pollVotes = await repairStore({
sourceDb: level.pollVotesDb,
referenceField: 'pollTxid',
targetDb: level.pollsDb
})
await level.postParentsDb.put(replyTxid, { ...reply, parentTxid: corrected })
await level.postChildrenDb.del(`${reply.parentTxid}:${replyTxid}`)
await level.postChildrenDb.put(`${corrected}:${replyTxid}`, { ...reply, parentTxid: corrected })
summary.replies++
}
for await (const [optionTxid, option] of level.pollOptionsDb.iterator()) {
if (!option.pollTxid) continue
const corrected = await correctReference(level.pollsDb, option.pollTxid)
if (corrected === option.pollTxid) continue
await level.pollOptionsDb.put(optionTxid, { ...option, pollTxid: corrected })
summary.pollOptions++
}
for await (const [voteTxid, vote] of level.pollVotesDb.iterator()) {
if (!vote.pollTxid) continue
const corrected = await correctReference(level.pollsDb, vote.pollTxid)
if (corrected === vote.pollTxid) continue
await level.pollVotesDb.put(voteTxid, { ...vote, pollTxid: corrected })
summary.pollVotes++
}
return summary
return { likes, replies, pollOptions, pollVotes }
}
@@ -0,0 +1,257 @@
/*
Property tests for the txid encoding repair library.
Unit tests probe the repair at a fixed fixture. These properties pin the
invariants over broad random inputs:
- Selection: correctReference prefers the stored reference when its target
exists, falls back to the reversed reference when only that target
exists, and leaves an unknown reference unchanged.
- Involution: reversing a txid twice returns the original display txid.
- Idempotence: a second repair run changes nothing.
- Conservation: repair never adds or drops primary records.
- Resolution: after repair every reference points at an existing target
when either byte order had one.
*/
import test from 'node:test'
import { seededRandom, forAll, intGen, txidGen } from './harness.js'
import {
reverseTxid,
correctReference,
repairTxidEncoding
} from '../../src/lib/repair-txid-encoding.js'
const rng = seededRandom(20260916)
class FakeDb {
constructor () {
this.map = new Map()
}
async get (key) {
if (!this.map.has(key)) {
const err = new Error(`not found: ${key}`)
err.notFound = true
throw err
}
return this.map.get(key)
}
async put (key, value) {
this.map.set(key, value)
}
async del (key) {
this.map.delete(key)
}
async * iterator () {
for (const [key, value] of this.map) {
yield [key, value]
}
}
}
function makeLevel () {
return {
postsDb: new FakeDb(),
pollsDb: new FakeDb(),
likesDb: new FakeDb(),
postLikesDb: new FakeDb(),
postParentsDb: new FakeDb(),
postChildrenDb: new FakeDb(),
pollOptionsDb: new FakeDb(),
pollVotesDb: new FakeDb()
}
}
// Snapshot every store as a stable string so a second run can be compared.
function snapshot (level) {
const stores = {}
for (const [name, db] of Object.entries(level)) {
stores[name] = [...db.map.entries()].sort(([a], [b]) => a.localeCompare(b))
}
return JSON.stringify(stores)
}
// A reference that is the display target, its reverse, or an unrelated txid.
function referenceTo (rng, targets, unknown) {
if (targets.length === 0) return unknown
const roll = rng()
const target = targets[intGen(rng, 0, targets.length - 1)()]
if (roll < 1 / 3) return target
if (roll < 2 / 3) return reverseTxid(target)
return unknown
}
// Build a random level with display posts/polls and a mix of correct,
// reversed, and unknown references in each primary store. FakeDb.put mutates
// synchronously, so this builder is synchronous for the property harness.
function buildFixture () {
const level = makeLevel()
const postTxids = []
const pollTxids = []
const postCount = intGen(rng, 0, 5)()
for (let i = 0; i < postCount; i++) {
const txid = txidGen(rng)
postTxids.push(txid)
level.postsDb.put(txid, { addr: `addr-${i}`, text: 'post', blockHeight: i })
}
const pollCount = intGen(rng, 0, 4)()
for (let i = 0; i < pollCount; i++) {
const txid = txidGen(rng)
pollTxids.push(txid)
level.pollsDb.put(txid, { question: 'q', pollType: 1, optionCount: 2, blockHeight: i })
}
const likeCount = intGen(rng, 0, 6)()
for (let i = 0; i < likeCount; i++) {
const likeTxid = `like-${i}`
const postTxid = referenceTo(rng, postTxids, txidGen(rng))
level.likesDb.put(likeTxid, { postTxid, addr: 'addr', blockHeight: i })
level.postLikesDb.put(`${postTxid}:${likeTxid}`, { postTxid, txid: likeTxid })
}
const replyCount = intGen(rng, 0, 6)()
for (let i = 0; i < replyCount; i++) {
const replyTxid = `reply-${i}`
const parentTxid = referenceTo(rng, postTxids, txidGen(rng))
level.postParentsDb.put(replyTxid, { parentTxid, childTxid: replyTxid, blockHeight: i })
level.postChildrenDb.put(`${parentTxid}:${replyTxid}`, { parentTxid, childTxid: replyTxid })
}
const optionCount = intGen(rng, 0, 5)()
for (let i = 0; i < optionCount; i++) {
const pollTxid = referenceTo(rng, pollTxids, txidGen(rng))
level.pollOptionsDb.put(`option-${i}`, { pollTxid, option: `opt-${i}`, blockHeight: i })
}
const voteCount = intGen(rng, 0, 5)()
for (let i = 0; i < voteCount; i++) {
const pollTxid = referenceTo(rng, pollTxids, txidGen(rng))
level.pollVotesDb.put(`vote-${i}`, { pollTxid, comment: `vote-${i}`, blockHeight: i })
}
return { level, postTxids, pollTxids }
}
function primaryCounts (level) {
return {
likes: level.likesDb.map.size,
replies: level.postParentsDb.map.size,
pollOptions: level.pollOptionsDb.map.size,
pollVotes: level.pollVotesDb.map.size
}
}
// True when every reference in a store is either a known target or unknown in
// both byte orders. Repair must not leave a resolvable reference unresolved.
function allResolvableOrUnknown (db, targetDb, field) {
for (const record of db.map.values()) {
const reference = record[field]
if (targetDb.map.has(reference)) continue
if (targetDb.map.has(reverseTxid(reference))) return false
}
return true
}
test('reverseTxid is an involution on valid txids', async () => {
await forAll(
() => txidGen(rng),
(txid) => reverseTxid(reverseTxid(txid)) === txid,
{ label: 'reverseTxid involution' }
)
})
test('correctReference selects an existing target in either byte order', async () => {
await forAll(
() => txidGen(rng),
async (txid) => {
const display = makeLevel()
await display.postsDb.put(txid, {})
if (await correctReference(display.postsDb, txid) !== txid) return false
const reversed = makeLevel()
await reversed.postsDb.put(txid, {})
if (await correctReference(reversed.postsDb, reverseTxid(txid)) !== txid) return false
const unknown = makeLevel()
const missing = txid
if (await correctReference(unknown.postsDb, missing) !== missing) return false
return true
},
{ label: 'correctReference selection' }
)
})
test('repairTxidEncoding is idempotent and conserves records', async () => {
await forAll(
() => buildFixture(),
async ({ level }) => {
const before = primaryCounts(level)
await repairTxidEncoding(level)
const afterFirst = snapshot(level)
await repairTxidEncoding(level)
if (snapshot(level) !== afterFirst) return false
const after = primaryCounts(level)
return JSON.stringify(before) === JSON.stringify(after)
},
{ label: 'repairTxidEncoding idempotence' }
)
})
test('repairTxidEncoding resolves every resolvable reference', async () => {
await forAll(
() => buildFixture(),
async ({ level }) => {
await repairTxidEncoding(level)
return allResolvableOrUnknown(level.likesDb, level.postsDb, 'postTxid') &&
allResolvableOrUnknown(level.postParentsDb, level.postsDb, 'parentTxid') &&
allResolvableOrUnknown(level.pollOptionsDb, level.pollsDb, 'pollTxid') &&
allResolvableOrUnknown(level.pollVotesDb, level.pollsDb, 'pollTxid')
},
{ label: 'repairTxidEncoding resolution' }
)
})
test('repairTxidEncoding rebuilds the secondary indexes for corrected records', async () => {
await forAll(
() => buildFixture(),
async ({ level }) => {
await repairTxidEncoding(level)
for (const [likeTxid, like] of level.likesDb.map) {
if (!level.postLikesDb.map.has(`${like.postTxid}:${likeTxid}`)) return false
}
for (const [replyTxid, reply] of level.postParentsDb.map) {
if (!level.postChildrenDb.map.has(`${reply.parentTxid}:${replyTxid}`)) return false
}
return true
},
{ label: 'repairTxidEncoding index rebuild' }
)
})
test('an unknown reference is left unchanged by repair', async () => {
await forAll(
() => txidGen(rng),
async (unknown) => {
const level = makeLevel()
await level.likesDb.put('like-unknown', { postTxid: unknown, addr: 'addr' })
await repairTxidEncoding(level)
return (await level.likesDb.get('like-unknown')).postTxid === unknown
},
{ label: 'repairTxidEncoding unknown reference' }
)
})