diff --git a/psf-memo-client/acceptance/lib/handlers.js b/psf-memo-client/acceptance/lib/handlers.js index 4c5fa34..b46f596 100644 --- a/psf-memo-client/acceptance/lib/handlers.js +++ b/psf-memo-client/acceptance/lib/handlers.js @@ -545,16 +545,18 @@ function createWorld () { } // Decode a raw reply payload into its parent txid (hex) and reply text. +// The wire stores the txid little-endian, so reverse it into display order. function decodeReplyPayload (raw) { const buf = Buffer.from(raw) - const parentTxid = buf.slice(0, 32).toString('hex') + const parentTxid = Buffer.from(buf.slice(0, 32)).reverse().toString('hex') const text = buf.slice(32).toString('utf8') return { parentTxid, text } } -// Decode a raw like payload back into the liked post txid (hex). +// Decode a raw like payload back into the liked post txid (hex) in display +// order. The wire stores the txid little-endian, so reverse it. function decodeLikeTxid (raw) { - return Buffer.from(raw).toString('hex') + return Buffer.from(raw).reverse().toString('hex') } // Resolve a literal value or a placeholder from the example store. diff --git a/psf-memo-client/src/services/hex.js b/psf-memo-client/src/services/hex.js index 5c35922..0d91383 100644 --- a/psf-memo-client/src/services/hex.js +++ b/psf-memo-client/src/services/hex.js @@ -25,10 +25,19 @@ function hexToBytes (hex, byteLength = 32, label = 'Value') { return bytes } +// Decode a 32-byte txid and reverse it into Memo protocol wire order. Bitcoin +// txids are displayed big-endian but embedded in OP_RETURN payloads as +// little-endian bytes, so callers pass the display hex and receive the wire +// bytes. +function txidToWireBytes (txid, label = 'Value') { + return hexToBytes(txid, 32, label).reverse() +} + // Build the raw OP_RETURN payload for a txid-referencing Memo action: the -// given 32-byte txid followed by a UTF-8 encoded value. +// given 32-byte txid in little-endian wire order followed by a UTF-8 encoded +// value. function buildTxidTextPayload (txid, text) { - const txidBytes = hexToBytes(txid, 32, 'Poll txid') + const txidBytes = txidToWireBytes(txid, 'Poll txid') const textBytes = new TextEncoder().encode(text) const raw = new Uint8Array(txidBytes.length + textBytes.length) raw.set(txidBytes, 0) @@ -36,7 +45,7 @@ function buildTxidTextPayload (txid, text) { return raw } -module.exports = { hexToBytes, buildTxidTextPayload } +module.exports = { hexToBytes, txidToWireBytes, buildTxidTextPayload } // mutate4javascript-manifest-begin // {"version":1,"tested_at":"2026-08-28T22:43:42.484Z","module_hash":"7f3445b7df4d792113f2736232f69faee044f71f6b4a5985224615b36909607b","functions":[{"id":"func/hexToBytes","name":"hexToBytes","line":12,"end_line":26,"hash":"dbf7e0a434598f85365f5a60a0ca227a17a1bcd6718a78bb5aaf7afb8eb2487b"},{"id":"func/buildTxidTextPayload","name":"buildTxidTextPayload","line":30,"end_line":37,"hash":"542a47a13d30c845b7ac9a04079d7a131ece3368a604840ffef75adf97b11f7d"}]} diff --git a/psf-memo-client/src/services/memo-like.js b/psf-memo-client/src/services/memo-like.js index cff2127..55488c7 100644 --- a/psf-memo-client/src/services/memo-like.js +++ b/psf-memo-client/src/services/memo-like.js @@ -23,7 +23,7 @@ */ const MemoAction = require('./memo-action') -const { hexToBytes } = require('./hex') +const { hexToBytes, txidToWireBytes } = require('./hex') const MEMO_LIKE_PREFIX = '6d04' const DUST_LIMIT_SATS = 3000 @@ -136,7 +136,7 @@ class MemoLike extends MemoAction { this.validateTip(tipSats, spendable) this._requireTipAddress(tipSats, authorAddress) - const raw = hexToBytes(postTxid, PARENT_TXID_BYTES, 'Post txid') + const raw = txidToWireBytes(postTxid, 'Post txid') const bchOutput = this._buildTipOutput(tipSats, authorAddress) const txid = await this.wallet.sendOpReturn(raw, this.prefix, bchOutput) diff --git a/psf-memo-client/src/services/memo-reply.js b/psf-memo-client/src/services/memo-reply.js index ec9f0ce..214b982 100644 --- a/psf-memo-client/src/services/memo-reply.js +++ b/psf-memo-client/src/services/memo-reply.js @@ -18,11 +18,10 @@ const MemoAction = require('./memo-action') const { byteLength } = require('./utf8') -const { hexToBytes } = require('./hex') +const { txidToWireBytes } = require('./hex') const MEMO_REPLY_PREFIX = '6d03' const MAX_REPLY_BYTES = 184 -const PARENT_TXID_BYTES = 32 class MemoReply extends MemoAction { static config = { @@ -83,7 +82,7 @@ class MemoReply extends MemoAction { // Build the raw OP_RETURN message payload for a reply. // The protocol wire format is: . function buildReplyPayload (parentTxid, message) { - const parentBytes = hexToBytes(parentTxid, PARENT_TXID_BYTES, 'Parent txid') + const parentBytes = txidToWireBytes(parentTxid, 'Parent txid') const textBytes = new TextEncoder().encode(message) const raw = new Uint8Array(parentBytes.length + textBytes.length) raw.set(parentBytes, 0) diff --git a/psf-memo-client/test/unit/hex.test.js b/psf-memo-client/test/unit/hex.test.js index b94c511..a907922 100644 --- a/psf-memo-client/test/unit/hex.test.js +++ b/psf-memo-client/test/unit/hex.test.js @@ -11,7 +11,11 @@ const test = require('node:test') const assert = require('node:assert/strict') -const { hexToBytes, buildTxidTextPayload } = require('../../src/services/hex') +const { hexToBytes, buildTxidTextPayload, txidToWireBytes } = require('../../src/services/hex') + +// A non-palindromic txid so a missing byte reversal is observable. +const DISPLAY_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' +const WIRE_HEX = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301' test('hexToBytes decodes exactly 64 hex characters into 32 bytes', () => { const bytes = hexToBytes('ab'.repeat(32)) @@ -61,3 +65,25 @@ test('buildTxidTextPayload prefixes the raw txid bytes', () => { assert.equal(buf[0], 0xab) assert.equal(buf.slice(32).toString('utf8'), 'hi') }) + +test('txidToWireBytes reverses the display txid into little-endian wire order', () => { + const bytes = txidToWireBytes(DISPLAY_TXID) + + assert.ok(bytes instanceof Uint8Array) + assert.equal(Buffer.from(bytes).toString('hex'), WIRE_HEX) +}) + +test('txidToWireBytes rejects an invalid txid', () => { + assert.throws( + () => txidToWireBytes('zz'.repeat(32)), + /valid hex string/ + ) +}) + +test('buildTxidTextPayload embeds the txid in little-endian wire order', () => { + const raw = buildTxidTextPayload(DISPLAY_TXID, 'hi') + const buf = Buffer.from(raw) + + assert.equal(buf.slice(0, 32).toString('hex'), WIRE_HEX) + assert.equal(buf.slice(32).toString('utf8'), 'hi') +}) diff --git a/psf-memo-client/test/unit/memo-like.test.js b/psf-memo-client/test/unit/memo-like.test.js new file mode 100644 index 0000000..f2f8dc4 --- /dev/null +++ b/psf-memo-client/test/unit/memo-like.test.js @@ -0,0 +1,62 @@ +/* + Unit tests for the Memo like wire encoding. + + A like action embeds the liked post's transaction txid as 32 raw bytes in + little-endian wire order: the reverse of the 64-character display txid. The + indexer reverses those bytes back into display order, so the client must + reverse before embedding. +*/ + +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const MemoLike = require('../../src/services/memo-like') + +const MY_ADDRESS = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d' +const POST_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' +const WIRE_HEX = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301' + +function makeWallet () { + return { + walletInfo: { cashAddress: MY_ADDRESS }, + utxos: [{ txid: 'utxo', value: 100000 }], + broadcasts: [], + async getUtxos () { + return this.utxos + }, + async sendOpReturn (msg, prefix, bchOutput = []) { + this.broadcasts.push({ msg, prefix, bchOutput }) + return 'aa'.repeat(32) + } + } +} + +test('like broadcasts the post txid in little-endian wire order', async () => { + const wallet = makeWallet() + const memoLike = new MemoLike({ wallet }) + + await memoLike.like(POST_TXID) + + assert.equal(wallet.broadcasts.length, 1) + assert.equal(wallet.broadcasts[0].prefix, MemoLike.MEMO_LIKE_PREFIX) + const raw = wallet.broadcasts[0].msg + assert.equal(Buffer.from(raw).toString('hex'), WIRE_HEX) +}) + +test('like reflects the post txid in display order on the feed store', async () => { + const wallet = makeWallet() + const added = [] + const feed = { + addLike (like) { + added.push(like) + }, + posts: [] + } + const memoLike = new MemoLike({ wallet, feed }) + + await memoLike.like(POST_TXID) + + assert.equal(added.length, 1) + assert.equal(added[0].postTxid, POST_TXID) +}) diff --git a/psf-memo-client/test/unit/memo-poll-option.test.js b/psf-memo-client/test/unit/memo-poll-option.test.js index 0f48425..ded56f9 100644 --- a/psf-memo-client/test/unit/memo-poll-option.test.js +++ b/psf-memo-client/test/unit/memo-poll-option.test.js @@ -13,7 +13,8 @@ const assert = require('node:assert/strict') const MemoPollOption = require('../../src/services/memo-poll-option') const MY_ADDRESS = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d' -const POLL_TXID = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +// A non-palindromic txid so a missing byte reversal is observable. +const POLL_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' function makeWallet (address = MY_ADDRESS) { return { diff --git a/psf-memo-client/test/unit/memo-poll-vote.test.js b/psf-memo-client/test/unit/memo-poll-vote.test.js index 42bb964..7c4674f 100644 --- a/psf-memo-client/test/unit/memo-poll-vote.test.js +++ b/psf-memo-client/test/unit/memo-poll-vote.test.js @@ -13,7 +13,8 @@ const assert = require('node:assert/strict') const MemoPollVote = require('../../src/services/memo-poll-vote') const MY_ADDRESS = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d' -const POLL_TXID = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +// A non-palindromic txid so a missing byte reversal is observable. +const POLL_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' function makeWallet (address = MY_ADDRESS) { return { diff --git a/psf-memo-client/test/unit/memo-reply.test.js b/psf-memo-client/test/unit/memo-reply.test.js new file mode 100644 index 0000000..adc9768 --- /dev/null +++ b/psf-memo-client/test/unit/memo-reply.test.js @@ -0,0 +1,61 @@ +/* + Unit tests for the Memo reply wire encoding. + + A reply action embeds the parent transaction txid as 32 raw bytes in + little-endian wire order: the reverse of the 64-character display txid. The + indexer reverses those bytes back into display order, so the client must + reverse before embedding. +*/ + +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const MemoReply = require('../../src/services/memo-reply') + +const MY_ADDRESS = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d' +const PARENT_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' +const WIRE_HEX = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301' + +function makeWallet () { + return { + walletInfo: { cashAddress: MY_ADDRESS }, + broadcasts: [], + async getUtxos () { + return [] + }, + async sendOpReturn (msg, prefix) { + this.broadcasts.push({ msg, prefix }) + return 'aa'.repeat(32) + } + } +} + +test('reply broadcasts the parent txid in little-endian wire order', async () => { + const wallet = makeWallet() + const memoReply = new MemoReply({ wallet }) + + await memoReply.reply('hello memo', PARENT_TXID) + + assert.equal(wallet.broadcasts.length, 1) + assert.equal(wallet.broadcasts[0].prefix, MemoReply.MEMO_REPLY_PREFIX) + const buf = Buffer.from(wallet.broadcasts[0].msg) + assert.equal(buf.slice(0, 32).toString('hex'), WIRE_HEX) + assert.equal(buf.slice(32).toString('utf8'), 'hello memo') +}) + +test('reply reflects the parent txid in display order on the thread store', async () => { + const wallet = makeWallet() + const added = [] + const thread = { + addReply (reply) { + added.push(reply) + } + } + const memoReply = new MemoReply({ wallet, thread }) + + await memoReply.reply('hello memo', PARENT_TXID) + + assert.equal(added.length, 1) + assert.equal(added[0].parentTxid, PARENT_TXID) +}) diff --git a/psf-memo-db/acceptance/lib/handlers.js b/psf-memo-db/acceptance/lib/handlers.js index 9ae0880..c103aa2 100644 --- a/psf-memo-db/acceptance/lib/handlers.js +++ b/psf-memo-db/acceptance/lib/handlers.js @@ -27,6 +27,7 @@ import ListMuted from '../../src/use-cases/list-muted.js' import GetPoll from '../../src/use-cases/get-poll.js' import GetPollOptions from '../../src/use-cases/get-poll-options.js' import GetPollVotes from '../../src/use-cases/get-poll-votes.js' +import { repairTxidEncoding } from '../../src/lib/repair-txid-encoding.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const tmpDir = path.resolve(__dirname, '..', '..', 'tmp', 'acceptance') @@ -204,6 +205,11 @@ async function loadFixture (world, name) { return } + if (name === 'db-with-reversed-txid-references') { + await loadReversedTxidFixture(world) + return + } + if (name === 'many-posts-with-replies') { await loadManyPostsWithReplies(world) return @@ -548,6 +554,37 @@ async function loadPollWithOptionsAndVote (world) { }) } +async function loadReversedTxidFixture (world) { + const displayPost = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + const reversedPost = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301' + const displayPoll = '00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff' + const reversedPoll = 'ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100' + const unknownPost = 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef' + + await world.adapters.level.postsDb.put(displayPost, { + addr: 'bitcoincash:qaddr-a', text: 'referenced post', seen: 1, blockHeight: 600100 + }) + await world.adapters.level.pollsDb.put(displayPoll, { + addr: 'bitcoincash:qaddr-a', pollType: 1, optionCount: 2, question: 'which?', seen: 2, blockHeight: 600101 + }) + + await world.adapters.level.likesDb.put('like-1', { addr: 'bitcoincash:liker-1', postTxid: reversedPost, seen: 3, blockHeight: 600200 }) + await world.adapters.level.likesDb.put('like-2', { addr: 'bitcoincash:liker-2', postTxid: displayPost, seen: 4, blockHeight: 600201 }) + await world.adapters.level.likesDb.put('like-3', { addr: 'bitcoincash:liker-3', postTxid: unknownPost, seen: 5, blockHeight: 600202 }) + await world.adapters.level.postLikesDb.put(`${reversedPost}:like-1`, { postTxid: reversedPost, txid: 'like-1' }) + await world.adapters.level.postLikesDb.put(`${displayPost}:like-2`, { postTxid: displayPost, txid: 'like-2' }) + + await world.adapters.level.postParentsDb.put('reply-1', { parentTxid: reversedPost, childTxid: 'reply-1', blockHeight: 600050 }) + await world.adapters.level.postParentsDb.put('reply-2', { parentTxid: displayPost, childTxid: 'reply-2', blockHeight: 600051 }) + await world.adapters.level.postChildrenDb.put(`${reversedPost}:reply-1`, { parentTxid: reversedPost, childTxid: 'reply-1' }) + await world.adapters.level.postChildrenDb.put(`${displayPost}:reply-2`, { parentTxid: displayPost, childTxid: 'reply-2' }) + + await world.adapters.level.pollOptionsDb.put('option-1', { addr: 'bitcoincash:qaddr-a', pollTxid: reversedPoll, option: 'yes', seen: 6, blockHeight: 600300 }) + await world.adapters.level.pollOptionsDb.put('option-2', { addr: 'bitcoincash:qaddr-b', pollTxid: displayPoll, option: 'no', seen: 7, blockHeight: 600301 }) + await world.adapters.level.pollVotesDb.put('vote-1', { addr: 'bitcoincash:qaddr-a', pollTxid: reversedPoll, comment: 'yes', seen: 8, blockHeight: 600302 }) + await world.adapters.level.pollVotesDb.put('vote-2', { addr: 'bitcoincash:qaddr-b', pollTxid: displayPoll, comment: 'no', seen: 9, blockHeight: 600303 }) +} + async function loadMutes (world) { const muter1 = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d' const muter2 = 'bitcoincash:qpm2qsznhks23z7629mms6s4cwef74vcwvy22gdx6a' @@ -592,6 +629,13 @@ const handlers = [ // World is already created with all stores. } }, + { + name: 'db instance with txid repair stores', + pattern: /^a psf-memo-db instance with posts, likes, postLikes, postParents, postChildren, polls, pollOptions, and pollVotes stores$/, + async run () { + // World is already created with all stores. + } + }, { name: 'load fixture', pattern: /^the fixture "(.+)" is loaded into the posts (?:store|and likes stores)$/, @@ -606,6 +650,13 @@ const handlers = [ await loadFixture(world, m[1]) } }, + { + name: 'load bare fixture', + pattern: /^the fixture "(.+)" is loaded$/, + async run (m, example, world) { + await loadFixture(world, m[1]) + } + }, { name: 'run backfill utility', pattern: /^the backfill utility is run$/, @@ -620,6 +671,20 @@ const handlers = [ await backfillIndexes(world) } }, + { + name: 'run txid repair utility', + pattern: /^the txid repair utility is run$/, + async run (m, example, world) { + await repairTxidEncoding(world.adapters.level) + } + }, + { + name: 'run txid repair utility again', + pattern: /^the txid repair utility is run again$/, + async run (m, example, world) { + await repairTxidEncoding(world.adapters.level) + } + }, { name: 'request recent posts', pattern: /^the client requests \/posts\/recent with limit () and offset ()$/, @@ -859,7 +924,7 @@ const handlers = [ }, { name: 'postLikes contains entry', - pattern: /^the postLikes store contains () entry whose key starts with () and ends with ()$/, + pattern: /^the postLikes store contains (<[A-Za-z0-9_]+>|[0-9]+) entry whose key starts with (<[A-Za-z0-9_]+>) and ends with (<[A-Za-z0-9_]+>)$/, async run (m, example, world) { const expectedCount = parseInt(resolveParam(m[1], example), 10) const postTxid = resolveParam(m[2], example) @@ -873,6 +938,70 @@ const handlers = [ } } }, + { + name: 'postChildren contains entry', + pattern: /^the postChildren store contains (<[A-Za-z0-9_]+>|[0-9]+) entry whose key starts with (<[A-Za-z0-9_]+>) and ends with (<[A-Za-z0-9_]+>)$/, + async run (m, example, world) { + const expectedCount = parseInt(resolveParam(m[1], example), 10) + const postTxid = resolveParam(m[2], example) + const replyTxid = resolveParam(m[3], example) + let count = 0 + for await (const [key] of world.adapters.level.postChildrenDb.iterator()) { + if (key.startsWith(`${postTxid}:`) && key.endsWith(`:${replyTxid}`)) count++ + } + if (count !== expectedCount) { + throw new Error(`Expected ${expectedCount} postChildren entry/entries for ${postTxid}/${replyTxid}, got ${count}`) + } + } + }, + { + name: 'likes store maps like to post', + pattern: /^the likes store maps like (<[A-Za-z0-9_]+>) to post (<[A-Za-z0-9_]+>)$/, + async run (m, example, world) { + const likeTxid = resolveParam(m[1], example) + const postTxid = resolveParam(m[2], example) + const like = await world.adapters.level.likesDb.get(likeTxid) + if (like.postTxid !== postTxid) { + throw new Error(`Expected like ${likeTxid} to map to post ${postTxid}, got ${like.postTxid}.`) + } + } + }, + { + name: 'postParents store maps reply to parent', + pattern: /^the postParents store maps reply (<[A-Za-z0-9_]+>) to parent (<[A-Za-z0-9_]+>)$/, + async run (m, example, world) { + const replyTxid = resolveParam(m[1], example) + const postTxid = resolveParam(m[2], example) + const reply = await world.adapters.level.postParentsDb.get(replyTxid) + if (reply.parentTxid !== postTxid) { + throw new Error(`Expected reply ${replyTxid} to map to parent ${postTxid}, got ${reply.parentTxid}.`) + } + } + }, + { + name: 'pollOptions store maps option to poll', + pattern: /^the pollOptions store maps option (<[A-Za-z0-9_]+>) to poll (<[A-Za-z0-9_]+>)$/, + async run (m, example, world) { + const optionTxid = resolveParam(m[1], example) + const pollTxid = resolveParam(m[2], example) + const option = await world.adapters.level.pollOptionsDb.get(optionTxid) + if (option.pollTxid !== pollTxid) { + throw new Error(`Expected poll option ${optionTxid} to map to poll ${pollTxid}, got ${option.pollTxid}.`) + } + } + }, + { + name: 'pollVotes store maps vote to poll', + pattern: /^the pollVotes store maps vote (<[A-Za-z0-9_]+>) to poll (<[A-Za-z0-9_]+>)$/, + async run (m, example, world) { + const voteTxid = resolveParam(m[1], example) + const pollTxid = resolveParam(m[2], example) + const vote = await world.adapters.level.pollVotesDb.get(voteTxid) + if (vote.pollTxid !== pollTxid) { + throw new Error(`Expected poll vote ${voteTxid} to map to poll ${pollTxid}, got ${vote.pollTxid}.`) + } + } + }, { name: 'db instance with follows store', pattern: /^a psf-memo-db instance with a follows store$/, diff --git a/psf-memo-db/src/lib/repair-txid-encoding.js b/psf-memo-db/src/lib/repair-txid-encoding.js new file mode 100644 index 0000000..984a4a7 --- /dev/null +++ b/psf-memo-db/src/lib/repair-txid-encoding.js @@ -0,0 +1,92 @@ +/* + Library to repair byte-reversed txid references in a psf-memo-db. + + Older psf-memo-client broadcasts embedded a referenced txid in big-endian + display order. The indexer expected little-endian wire order, so it reversed + those bytes and stored a reference that is the reverse of the display txid. + This library rewrites a reversed reference to display order and rebuilds the + affected secondary index. + + It leaves records whose reference already points at a known target + untouched, leaves references whose target is unknown in either byte order + untouched, and is idempotent: a second run makes no changes. + + The LevelDB handles are injected so the repair logic stays testable and free + of file-system concerns; the CLI wrapper in util/txid opens the real stores. +*/ + +// Reverse the byte order of a 64-character display txid. Returns null for any +// value that is not a 64-character hex txid. +export function reverseTxid (txid) { + if (typeof txid !== 'string' || !/^[0-9a-fA-F]{64}$/.test(txid)) return null + return Buffer.from(txid, 'hex').reverse().toString('hex') +} + +// True when a store has a record at the given key. +async function hasRecord (db, key) { + try { + await db.get(key) + return true + } catch (err) { + if (err.notFound || err.code === 'LEVEL_NOT_FOUND') return false + throw err + } +} + +// Choose the reference in display order. Keep the stored reference when its +// target exists. Otherwise use the reversed reference when that target +// exists. Otherwise return the stored reference unchanged. +export async function correctReference (targetDb, reference) { + if (await hasRecord(targetDb, reference)) return reference + const reversed = reverseTxid(reference) + if (reversed && await hasRecord(targetDb, reversed)) return reversed + return reference +} + +// Repair likes, replies, poll options, and poll votes in place. Returns a +// summary of how many records of each kind were corrected. +export async function repairTxidEncoding (level) { + const summary = { likes: 0, replies: 0, pollOptions: 0, pollVotes: 0 } + + for await (const [likeTxid, like] of level.likesDb.iterator()) { + if (!like.postTxid) continue + const corrected = await correctReference(level.postsDb, like.postTxid) + if (corrected === like.postTxid) continue + + await level.likesDb.put(likeTxid, { ...like, postTxid: corrected }) + await level.postLikesDb.del(`${like.postTxid}:${likeTxid}`) + await level.postLikesDb.put(`${corrected}:${likeTxid}`, { postTxid: corrected, txid: likeTxid }) + summary.likes++ + } + + for await (const [replyTxid, reply] of level.postParentsDb.iterator()) { + if (!reply.parentTxid) continue + const corrected = await correctReference(level.postsDb, reply.parentTxid) + if (corrected === reply.parentTxid) continue + + await level.postParentsDb.put(replyTxid, { ...reply, parentTxid: corrected }) + await level.postChildrenDb.del(`${reply.parentTxid}:${replyTxid}`) + await level.postChildrenDb.put(`${corrected}:${replyTxid}`, { ...reply, parentTxid: corrected }) + summary.replies++ + } + + for await (const [optionTxid, option] of level.pollOptionsDb.iterator()) { + if (!option.pollTxid) continue + const corrected = await correctReference(level.pollsDb, option.pollTxid) + if (corrected === option.pollTxid) continue + + await level.pollOptionsDb.put(optionTxid, { ...option, pollTxid: corrected }) + summary.pollOptions++ + } + + for await (const [voteTxid, vote] of level.pollVotesDb.iterator()) { + if (!vote.pollTxid) continue + const corrected = await correctReference(level.pollsDb, vote.pollTxid) + if (corrected === vote.pollTxid) continue + + await level.pollVotesDb.put(voteTxid, { ...vote, pollTxid: corrected }) + summary.pollVotes++ + } + + return summary +} diff --git a/psf-memo-db/test/unit/lib/repair-txid-encoding.unit.js b/psf-memo-db/test/unit/lib/repair-txid-encoding.unit.js new file mode 100644 index 0000000..dc30a54 --- /dev/null +++ b/psf-memo-db/test/unit/lib/repair-txid-encoding.unit.js @@ -0,0 +1,225 @@ +/* + Unit tests for the txid encoding repair library. + + Older psf-memo-client broadcasts embedded a referenced txid in big-endian + display order. The indexer expected little-endian wire order, so it stored a + byte-reversed reference. The repair library rewrites a reversed reference to + display order and rebuilds the affected secondary index. It leaves + correctly-encoded records untouched, leaves references whose target is + unknown in either byte order untouched, and is idempotent. +*/ + +import { assert } from 'chai' +import { + reverseTxid, + correctReference, + repairTxidEncoding +} from '../../../src/lib/repair-txid-encoding.js' + +const DISPLAY_POST = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' +const REVERSED_POST = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301' +const DISPLAY_POLL = '00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff' +const REVERSED_POLL = 'ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100' +const UNKNOWN = 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef' + +class FakeDb { + constructor () { + this.map = new Map() + } + + async get (key) { + if (!this.map.has(key)) { + const err = new Error(`not found: ${key}`) + err.notFound = true + throw err + } + return this.map.get(key) + } + + async put (key, value) { + this.map.set(key, value) + } + + async del (key) { + this.map.delete(key) + } + + async * iterator () { + for (const [key, value] of this.map) { + yield [key, value] + } + } + + keys () { + return [...this.map.keys()].sort() + } +} + +function makeLevel () { + return { + postsDb: new FakeDb(), + likesDb: new FakeDb(), + postLikesDb: new FakeDb(), + postParentsDb: new FakeDb(), + postChildrenDb: new FakeDb(), + pollsDb: new FakeDb(), + pollOptionsDb: new FakeDb(), + pollVotesDb: new FakeDb() + } +} + +function loadReversedFixture (level) { + level.postsDb.put(DISPLAY_POST, { addr: 'addr-a', text: 'post', blockHeight: 1 }) + level.pollsDb.put(DISPLAY_POLL, { question: 'q', pollType: 1, optionCount: 2, blockHeight: 2 }) + + level.likesDb.put('like-1', { postTxid: REVERSED_POST, addr: 'addr-b', blockHeight: 3 }) + level.likesDb.put('like-2', { postTxid: DISPLAY_POST, addr: 'addr-c', blockHeight: 4 }) + level.likesDb.put('like-3', { postTxid: UNKNOWN, addr: 'addr-d', blockHeight: 5 }) + level.postLikesDb.put(`${REVERSED_POST}:like-1`, { postTxid: REVERSED_POST, txid: 'like-1' }) + level.postLikesDb.put(`${DISPLAY_POST}:like-2`, { postTxid: DISPLAY_POST, txid: 'like-2' }) + + level.postParentsDb.put('reply-1', { parentTxid: REVERSED_POST, childTxid: 'reply-1', blockHeight: 6 }) + level.postParentsDb.put('reply-2', { parentTxid: DISPLAY_POST, childTxid: 'reply-2', blockHeight: 7 }) + level.postChildrenDb.put(`${REVERSED_POST}:reply-1`, { parentTxid: REVERSED_POST, childTxid: 'reply-1' }) + level.postChildrenDb.put(`${DISPLAY_POST}:reply-2`, { parentTxid: DISPLAY_POST, childTxid: 'reply-2' }) + + level.pollOptionsDb.put('option-1', { pollTxid: REVERSED_POLL, option: 'yes', blockHeight: 8 }) + level.pollOptionsDb.put('option-2', { pollTxid: DISPLAY_POLL, option: 'no', blockHeight: 9 }) + level.pollVotesDb.put('vote-1', { pollTxid: REVERSED_POLL, comment: 'yes', blockHeight: 10 }) + level.pollVotesDb.put('vote-2', { pollTxid: DISPLAY_POLL, comment: 'no', blockHeight: 11 }) +} + +describe('#RepairTxidEncoding', () => { + describe('reverseTxid', () => { + it('reverses the byte order of a display txid', () => { + assert.equal(reverseTxid(DISPLAY_POST), REVERSED_POST) + }) + + it('returns null for a value that is not a 64-character hex txid', () => { + assert.isNull(reverseTxid('not-a-txid')) + assert.isNull(reverseTxid(null)) + assert.isNull(reverseTxid('zz'.repeat(32))) + }) + }) + + describe('correctReference', () => { + it('keeps a reference whose target exists in display order', async () => { + const level = makeLevel() + level.postsDb.put(DISPLAY_POST, {}) + assert.equal(await correctReference(level.postsDb, DISPLAY_POST), DISPLAY_POST) + }) + + it('reverses a reference whose reversed form is the existing target', async () => { + const level = makeLevel() + level.postsDb.put(DISPLAY_POST, {}) + assert.equal(await correctReference(level.postsDb, REVERSED_POST), DISPLAY_POST) + }) + + it('keeps an unknown reference unchanged', async () => { + const level = makeLevel() + assert.equal(await correctReference(level.postsDb, UNKNOWN), UNKNOWN) + }) + + it('propagates an unexpected store error', async () => { + const targetDb = { + async get () { + const err = new Error('io failure') + err.code = 'EIO' + throw err + } + } + + let caught = null + try { + await correctReference(targetDb, DISPLAY_POST) + } catch (err) { + caught = err + } + + assert.isNotNull(caught) + assert.equal(caught.code, 'EIO') + }) + }) + + describe('repairTxidEncoding', () => { + it('corrects a reversed like reference and rebuilds postLikes', async () => { + const level = makeLevel() + loadReversedFixture(level) + + await repairTxidEncoding(level) + + assert.equal((await level.likesDb.get('like-1')).postTxid, DISPLAY_POST) + assert.include(level.postLikesDb.keys(), `${DISPLAY_POST}:like-1`) + assert.notInclude(level.postLikesDb.keys(), `${REVERSED_POST}:like-1`) + }) + + it('corrects a reversed reply reference and rebuilds postChildren', async () => { + const level = makeLevel() + loadReversedFixture(level) + + await repairTxidEncoding(level) + + assert.equal((await level.postParentsDb.get('reply-1')).parentTxid, DISPLAY_POST) + assert.include(level.postChildrenDb.keys(), `${DISPLAY_POST}:reply-1`) + assert.notInclude(level.postChildrenDb.keys(), `${REVERSED_POST}:reply-1`) + }) + + it('corrects reversed poll option and vote references', async () => { + const level = makeLevel() + loadReversedFixture(level) + + await repairTxidEncoding(level) + + assert.equal((await level.pollOptionsDb.get('option-1')).pollTxid, DISPLAY_POLL) + assert.equal((await level.pollVotesDb.get('vote-1')).pollTxid, DISPLAY_POLL) + }) + + it('leaves correctly-encoded references unchanged', async () => { + const level = makeLevel() + loadReversedFixture(level) + + await repairTxidEncoding(level) + + assert.equal((await level.likesDb.get('like-2')).postTxid, DISPLAY_POST) + assert.equal((await level.postParentsDb.get('reply-2')).parentTxid, DISPLAY_POST) + assert.equal((await level.pollOptionsDb.get('option-2')).pollTxid, DISPLAY_POLL) + assert.equal((await level.pollVotesDb.get('vote-2')).pollTxid, DISPLAY_POLL) + }) + + it('leaves an unknown reference unchanged', async () => { + const level = makeLevel() + loadReversedFixture(level) + + await repairTxidEncoding(level) + + assert.equal((await level.likesDb.get('like-3')).postTxid, UNKNOWN) + }) + + it('skips records without a reference field', async () => { + const level = makeLevel() + level.likesDb.put('like-empty', { addr: 'addr' }) + level.postParentsDb.put('reply-empty', { childTxid: 'reply-empty' }) + level.pollOptionsDb.put('option-empty', { option: 'x' }) + level.pollVotesDb.put('vote-empty', { comment: 'x' }) + + const summary = await repairTxidEncoding(level) + + assert.deepEqual(summary, { likes: 0, replies: 0, pollOptions: 0, pollVotes: 0 }) + }) + + it('is idempotent', async () => { + const level = makeLevel() + loadReversedFixture(level) + + await repairTxidEncoding(level) + await repairTxidEncoding(level) + + const postLikes = level.postLikesDb.keys() + assert.include(postLikes, `${DISPLAY_POST}:like-1`) + assert.notInclude(postLikes, `${REVERSED_POST}:like-1`) + const postChildren = level.postChildrenDb.keys() + assert.include(postChildren, `${DISPLAY_POST}:reply-1`) + assert.notInclude(postChildren, `${REVERSED_POST}:reply-1`) + }) + }) +}) diff --git a/psf-memo-db/util/txid/repair-txid-encoding.js b/psf-memo-db/util/txid/repair-txid-encoding.js new file mode 100644 index 0000000..4da8bd2 --- /dev/null +++ b/psf-memo-db/util/txid/repair-txid-encoding.js @@ -0,0 +1,92 @@ +/* + Utility: repair byte-reversed txid references in an existing psf-memo-db and + rebuild the affected secondary indexes. + + Older psf-memo-client broadcasts embedded a referenced txid in big-endian + display order. The indexer expected little-endian wire order, so it stored a + reference that is the reverse of the display txid. This utility rewrites + those references to display order for likes, replies, poll options, and poll + votes, and rebuilds the postLikes and postChildren indexes. + + Run from the psf-memo-db repo root on the host that owns the LevelDB files: + + node util/txid/repair-txid-encoding.js + + The repair is idempotent: re-running it makes no further changes. Progress + and a summary are printed to stderr. + + WARNING: + - This script opens the LevelDB files directly. psf-memo-db must NOT be + running, or another process must not hold the database locks. + - Make a backup of leveldb/current before running on a production server: + cp -r leveldb/current leveldb/current-pre-txid-repair-backup +*/ + +import level from 'level' +import * as fs from 'fs' +import * as path from 'path' +import * as url from 'url' +import { repairTxidEncoding } from '../../src/lib/repair-txid-encoding.js' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) + +const DATA_DIR = process.env.PSF_MEMO_DB_DATA_DIR + ? path.resolve(process.env.PSF_MEMO_DB_DATA_DIR) + : path.resolve(__dirname, '../../leveldb/current') + +// Stores that hold records to scan. They must already exist. +const RECORD_STORES = ['posts', 'polls', 'likes', 'postParents', 'pollOptions', 'pollVotes'] +// Secondary indexes rebuilt by the repair. Create them when missing. +const INDEX_STORES = ['postLikes', 'postChildren'] + +function requiredStorePath (dir, name) { + const storePath = path.join(dir, name) + if (!fs.existsSync(storePath)) { + throw new Error(`Required LevelDB store not found: ${storePath}. Set PSF_MEMO_DB_DATA_DIR to the directory containing the psf-memo-db stores.`) + } + return storePath +} + +async function main () { + console.error(`Using LevelDB data directory: ${DATA_DIR}`) + + const paths = {} + for (const name of RECORD_STORES) { + paths[name] = requiredStorePath(DATA_DIR, name) + } + for (const name of INDEX_STORES) { + paths[name] = path.join(DATA_DIR, name) + } + + console.error('Opening LevelDB stores...') + const dbs = { + postsDb: level(paths.posts, { valueEncoding: 'json' }), + pollsDb: level(paths.polls, { valueEncoding: 'json' }), + likesDb: level(paths.likes, { valueEncoding: 'json' }), + postParentsDb: level(paths.postParents, { valueEncoding: 'json' }), + pollOptionsDb: level(paths.pollOptions, { valueEncoding: 'json' }), + pollVotesDb: level(paths.pollVotes, { valueEncoding: 'json' }), + postLikesDb: level(paths.postLikes, { valueEncoding: 'json', createIfMissing: true }), + postChildrenDb: level(paths.postChildren, { valueEncoding: 'json', createIfMissing: true }) + } + + try { + console.error('Repairing reversed txid references...') + const summary = await repairTxidEncoding(dbs) + + console.error('\nTxid encoding repair complete.') + console.error(` likes corrected: ${summary.likes}`) + console.error(` replies corrected: ${summary.replies}`) + console.error(` poll options corrected: ${summary.pollOptions}`) + console.error(` poll votes corrected: ${summary.pollVotes}`) + } catch (err) { + console.error('\nTxid encoding repair failed:', err.message) + process.exitCode = 1 + } finally { + for (const db of Object.values(dbs)) { + await db.close().catch(() => {}) + } + } +} + +main()