mirror of
https://github.com/Permissionless-Software-Foundation/psf-memo.git
synced 2026-09-21 16:52:01 -07:00
Fix txid wire encoding and add txid repair utility
Client likes, replies, poll options, and poll votes now embed the referenced txid in little-endian wire order, matching the indexer's byte reversal. Add a psf-memo-db repair library and CLI that rewrites byte-reversed references in likes, postParents, pollOptions, and pollVotes and rebuilds the postLikes and postChildren indexes, leaving correct and unknown references untouched and staying idempotent. By coder.
This commit is contained in:
Chris Troutner
@@ -545,16 +545,18 @@ function createWorld () {
|
||||
}
|
||||
|
||||
// Decode a raw reply payload into its parent txid (hex) and reply text.
|
||||
// The wire stores the txid little-endian, so reverse it into display order.
|
||||
function decodeReplyPayload (raw) {
|
||||
const buf = Buffer.from(raw)
|
||||
const parentTxid = buf.slice(0, 32).toString('hex')
|
||||
const parentTxid = Buffer.from(buf.slice(0, 32)).reverse().toString('hex')
|
||||
const text = buf.slice(32).toString('utf8')
|
||||
return { parentTxid, text }
|
||||
}
|
||||
|
||||
// Decode a raw like payload back into the liked post txid (hex).
|
||||
// Decode a raw like payload back into the liked post txid (hex) in display
|
||||
// order. The wire stores the txid little-endian, so reverse it.
|
||||
function decodeLikeTxid (raw) {
|
||||
return Buffer.from(raw).toString('hex')
|
||||
return Buffer.from(raw).reverse().toString('hex')
|
||||
}
|
||||
|
||||
// Resolve a literal value or a <parameter> placeholder from the example store.
|
||||
|
||||
@@ -25,10 +25,19 @@ function hexToBytes (hex, byteLength = 32, label = 'Value') {
|
||||
return bytes
|
||||
}
|
||||
|
||||
// Decode a 32-byte txid and reverse it into Memo protocol wire order. Bitcoin
|
||||
// txids are displayed big-endian but embedded in OP_RETURN payloads as
|
||||
// little-endian bytes, so callers pass the display hex and receive the wire
|
||||
// bytes.
|
||||
function txidToWireBytes (txid, label = 'Value') {
|
||||
return hexToBytes(txid, 32, label).reverse()
|
||||
}
|
||||
|
||||
// Build the raw OP_RETURN payload for a txid-referencing Memo action: the
|
||||
// given 32-byte txid followed by a UTF-8 encoded value.
|
||||
// given 32-byte txid in little-endian wire order followed by a UTF-8 encoded
|
||||
// value.
|
||||
function buildTxidTextPayload (txid, text) {
|
||||
const txidBytes = hexToBytes(txid, 32, 'Poll txid')
|
||||
const txidBytes = txidToWireBytes(txid, 'Poll txid')
|
||||
const textBytes = new TextEncoder().encode(text)
|
||||
const raw = new Uint8Array(txidBytes.length + textBytes.length)
|
||||
raw.set(txidBytes, 0)
|
||||
@@ -36,7 +45,7 @@ function buildTxidTextPayload (txid, text) {
|
||||
return raw
|
||||
}
|
||||
|
||||
module.exports = { hexToBytes, buildTxidTextPayload }
|
||||
module.exports = { hexToBytes, txidToWireBytes, buildTxidTextPayload }
|
||||
|
||||
// mutate4javascript-manifest-begin
|
||||
// {"version":1,"tested_at":"2026-08-28T22:43:42.484Z","module_hash":"7f3445b7df4d792113f2736232f69faee044f71f6b4a5985224615b36909607b","functions":[{"id":"func/hexToBytes","name":"hexToBytes","line":12,"end_line":26,"hash":"dbf7e0a434598f85365f5a60a0ca227a17a1bcd6718a78bb5aaf7afb8eb2487b"},{"id":"func/buildTxidTextPayload","name":"buildTxidTextPayload","line":30,"end_line":37,"hash":"542a47a13d30c845b7ac9a04079d7a131ece3368a604840ffef75adf97b11f7d"}]}
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
*/
|
||||
|
||||
const MemoAction = require('./memo-action')
|
||||
const { hexToBytes } = require('./hex')
|
||||
const { hexToBytes, txidToWireBytes } = require('./hex')
|
||||
|
||||
const MEMO_LIKE_PREFIX = '6d04'
|
||||
const DUST_LIMIT_SATS = 3000
|
||||
@@ -136,7 +136,7 @@ class MemoLike extends MemoAction {
|
||||
this.validateTip(tipSats, spendable)
|
||||
this._requireTipAddress(tipSats, authorAddress)
|
||||
|
||||
const raw = hexToBytes(postTxid, PARENT_TXID_BYTES, 'Post txid')
|
||||
const raw = txidToWireBytes(postTxid, 'Post txid')
|
||||
const bchOutput = this._buildTipOutput(tipSats, authorAddress)
|
||||
|
||||
const txid = await this.wallet.sendOpReturn(raw, this.prefix, bchOutput)
|
||||
|
||||
@@ -18,11 +18,10 @@
|
||||
|
||||
const MemoAction = require('./memo-action')
|
||||
const { byteLength } = require('./utf8')
|
||||
const { hexToBytes } = require('./hex')
|
||||
const { txidToWireBytes } = require('./hex')
|
||||
|
||||
const MEMO_REPLY_PREFIX = '6d03'
|
||||
const MAX_REPLY_BYTES = 184
|
||||
const PARENT_TXID_BYTES = 32
|
||||
|
||||
class MemoReply extends MemoAction {
|
||||
static config = {
|
||||
@@ -83,7 +82,7 @@ class MemoReply extends MemoAction {
|
||||
// Build the raw OP_RETURN message payload for a reply.
|
||||
// The protocol wire format is: <parent txid 32 bytes><reply text UTF-8 bytes>.
|
||||
function buildReplyPayload (parentTxid, message) {
|
||||
const parentBytes = hexToBytes(parentTxid, PARENT_TXID_BYTES, 'Parent txid')
|
||||
const parentBytes = txidToWireBytes(parentTxid, 'Parent txid')
|
||||
const textBytes = new TextEncoder().encode(message)
|
||||
const raw = new Uint8Array(parentBytes.length + textBytes.length)
|
||||
raw.set(parentBytes, 0)
|
||||
|
||||
@@ -11,7 +11,11 @@
|
||||
|
||||
const test = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
const { hexToBytes, buildTxidTextPayload } = require('../../src/services/hex')
|
||||
const { hexToBytes, buildTxidTextPayload, txidToWireBytes } = require('../../src/services/hex')
|
||||
|
||||
// A non-palindromic txid so a missing byte reversal is observable.
|
||||
const DISPLAY_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
|
||||
const WIRE_HEX = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301'
|
||||
|
||||
test('hexToBytes decodes exactly 64 hex characters into 32 bytes', () => {
|
||||
const bytes = hexToBytes('ab'.repeat(32))
|
||||
@@ -61,3 +65,25 @@ test('buildTxidTextPayload prefixes the raw txid bytes', () => {
|
||||
assert.equal(buf[0], 0xab)
|
||||
assert.equal(buf.slice(32).toString('utf8'), 'hi')
|
||||
})
|
||||
|
||||
test('txidToWireBytes reverses the display txid into little-endian wire order', () => {
|
||||
const bytes = txidToWireBytes(DISPLAY_TXID)
|
||||
|
||||
assert.ok(bytes instanceof Uint8Array)
|
||||
assert.equal(Buffer.from(bytes).toString('hex'), WIRE_HEX)
|
||||
})
|
||||
|
||||
test('txidToWireBytes rejects an invalid txid', () => {
|
||||
assert.throws(
|
||||
() => txidToWireBytes('zz'.repeat(32)),
|
||||
/valid hex string/
|
||||
)
|
||||
})
|
||||
|
||||
test('buildTxidTextPayload embeds the txid in little-endian wire order', () => {
|
||||
const raw = buildTxidTextPayload(DISPLAY_TXID, 'hi')
|
||||
const buf = Buffer.from(raw)
|
||||
|
||||
assert.equal(buf.slice(0, 32).toString('hex'), WIRE_HEX)
|
||||
assert.equal(buf.slice(32).toString('utf8'), 'hi')
|
||||
})
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
Unit tests for the Memo like wire encoding.
|
||||
|
||||
A like action embeds the liked post's transaction txid as 32 raw bytes in
|
||||
little-endian wire order: the reverse of the 64-character display txid. The
|
||||
indexer reverses those bytes back into display order, so the client must
|
||||
reverse before embedding.
|
||||
*/
|
||||
|
||||
'use strict'
|
||||
|
||||
const test = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
const MemoLike = require('../../src/services/memo-like')
|
||||
|
||||
const MY_ADDRESS = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d'
|
||||
const POST_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
|
||||
const WIRE_HEX = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301'
|
||||
|
||||
function makeWallet () {
|
||||
return {
|
||||
walletInfo: { cashAddress: MY_ADDRESS },
|
||||
utxos: [{ txid: 'utxo', value: 100000 }],
|
||||
broadcasts: [],
|
||||
async getUtxos () {
|
||||
return this.utxos
|
||||
},
|
||||
async sendOpReturn (msg, prefix, bchOutput = []) {
|
||||
this.broadcasts.push({ msg, prefix, bchOutput })
|
||||
return 'aa'.repeat(32)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test('like broadcasts the post txid in little-endian wire order', async () => {
|
||||
const wallet = makeWallet()
|
||||
const memoLike = new MemoLike({ wallet })
|
||||
|
||||
await memoLike.like(POST_TXID)
|
||||
|
||||
assert.equal(wallet.broadcasts.length, 1)
|
||||
assert.equal(wallet.broadcasts[0].prefix, MemoLike.MEMO_LIKE_PREFIX)
|
||||
const raw = wallet.broadcasts[0].msg
|
||||
assert.equal(Buffer.from(raw).toString('hex'), WIRE_HEX)
|
||||
})
|
||||
|
||||
test('like reflects the post txid in display order on the feed store', async () => {
|
||||
const wallet = makeWallet()
|
||||
const added = []
|
||||
const feed = {
|
||||
addLike (like) {
|
||||
added.push(like)
|
||||
},
|
||||
posts: []
|
||||
}
|
||||
const memoLike = new MemoLike({ wallet, feed })
|
||||
|
||||
await memoLike.like(POST_TXID)
|
||||
|
||||
assert.equal(added.length, 1)
|
||||
assert.equal(added[0].postTxid, POST_TXID)
|
||||
})
|
||||
@@ -13,7 +13,8 @@ const assert = require('node:assert/strict')
|
||||
const MemoPollOption = require('../../src/services/memo-poll-option')
|
||||
|
||||
const MY_ADDRESS = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d'
|
||||
const POLL_TXID = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
|
||||
// A non-palindromic txid so a missing byte reversal is observable.
|
||||
const POLL_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
|
||||
|
||||
function makeWallet (address = MY_ADDRESS) {
|
||||
return {
|
||||
|
||||
@@ -13,7 +13,8 @@ const assert = require('node:assert/strict')
|
||||
const MemoPollVote = require('../../src/services/memo-poll-vote')
|
||||
|
||||
const MY_ADDRESS = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d'
|
||||
const POLL_TXID = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
|
||||
// A non-palindromic txid so a missing byte reversal is observable.
|
||||
const POLL_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
|
||||
|
||||
function makeWallet (address = MY_ADDRESS) {
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
Unit tests for the Memo reply wire encoding.
|
||||
|
||||
A reply action embeds the parent transaction txid as 32 raw bytes in
|
||||
little-endian wire order: the reverse of the 64-character display txid. The
|
||||
indexer reverses those bytes back into display order, so the client must
|
||||
reverse before embedding.
|
||||
*/
|
||||
|
||||
'use strict'
|
||||
|
||||
const test = require('node:test')
|
||||
const assert = require('node:assert/strict')
|
||||
const MemoReply = require('../../src/services/memo-reply')
|
||||
|
||||
const MY_ADDRESS = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d'
|
||||
const PARENT_TXID = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
|
||||
const WIRE_HEX = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301'
|
||||
|
||||
function makeWallet () {
|
||||
return {
|
||||
walletInfo: { cashAddress: MY_ADDRESS },
|
||||
broadcasts: [],
|
||||
async getUtxos () {
|
||||
return []
|
||||
},
|
||||
async sendOpReturn (msg, prefix) {
|
||||
this.broadcasts.push({ msg, prefix })
|
||||
return 'aa'.repeat(32)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test('reply broadcasts the parent txid in little-endian wire order', async () => {
|
||||
const wallet = makeWallet()
|
||||
const memoReply = new MemoReply({ wallet })
|
||||
|
||||
await memoReply.reply('hello memo', PARENT_TXID)
|
||||
|
||||
assert.equal(wallet.broadcasts.length, 1)
|
||||
assert.equal(wallet.broadcasts[0].prefix, MemoReply.MEMO_REPLY_PREFIX)
|
||||
const buf = Buffer.from(wallet.broadcasts[0].msg)
|
||||
assert.equal(buf.slice(0, 32).toString('hex'), WIRE_HEX)
|
||||
assert.equal(buf.slice(32).toString('utf8'), 'hello memo')
|
||||
})
|
||||
|
||||
test('reply reflects the parent txid in display order on the thread store', async () => {
|
||||
const wallet = makeWallet()
|
||||
const added = []
|
||||
const thread = {
|
||||
addReply (reply) {
|
||||
added.push(reply)
|
||||
}
|
||||
}
|
||||
const memoReply = new MemoReply({ wallet, thread })
|
||||
|
||||
await memoReply.reply('hello memo', PARENT_TXID)
|
||||
|
||||
assert.equal(added.length, 1)
|
||||
assert.equal(added[0].parentTxid, PARENT_TXID)
|
||||
})
|
||||
@@ -27,6 +27,7 @@ import ListMuted from '../../src/use-cases/list-muted.js'
|
||||
import GetPoll from '../../src/use-cases/get-poll.js'
|
||||
import GetPollOptions from '../../src/use-cases/get-poll-options.js'
|
||||
import GetPollVotes from '../../src/use-cases/get-poll-votes.js'
|
||||
import { repairTxidEncoding } from '../../src/lib/repair-txid-encoding.js'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const tmpDir = path.resolve(__dirname, '..', '..', 'tmp', 'acceptance')
|
||||
@@ -204,6 +205,11 @@ async function loadFixture (world, name) {
|
||||
return
|
||||
}
|
||||
|
||||
if (name === 'db-with-reversed-txid-references') {
|
||||
await loadReversedTxidFixture(world)
|
||||
return
|
||||
}
|
||||
|
||||
if (name === 'many-posts-with-replies') {
|
||||
await loadManyPostsWithReplies(world)
|
||||
return
|
||||
@@ -548,6 +554,37 @@ async function loadPollWithOptionsAndVote (world) {
|
||||
})
|
||||
}
|
||||
|
||||
async function loadReversedTxidFixture (world) {
|
||||
const displayPost = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
|
||||
const reversedPost = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301'
|
||||
const displayPoll = '00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff'
|
||||
const reversedPoll = 'ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100'
|
||||
const unknownPost = 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef'
|
||||
|
||||
await world.adapters.level.postsDb.put(displayPost, {
|
||||
addr: 'bitcoincash:qaddr-a', text: 'referenced post', seen: 1, blockHeight: 600100
|
||||
})
|
||||
await world.adapters.level.pollsDb.put(displayPoll, {
|
||||
addr: 'bitcoincash:qaddr-a', pollType: 1, optionCount: 2, question: 'which?', seen: 2, blockHeight: 600101
|
||||
})
|
||||
|
||||
await world.adapters.level.likesDb.put('like-1', { addr: 'bitcoincash:liker-1', postTxid: reversedPost, seen: 3, blockHeight: 600200 })
|
||||
await world.adapters.level.likesDb.put('like-2', { addr: 'bitcoincash:liker-2', postTxid: displayPost, seen: 4, blockHeight: 600201 })
|
||||
await world.adapters.level.likesDb.put('like-3', { addr: 'bitcoincash:liker-3', postTxid: unknownPost, seen: 5, blockHeight: 600202 })
|
||||
await world.adapters.level.postLikesDb.put(`${reversedPost}:like-1`, { postTxid: reversedPost, txid: 'like-1' })
|
||||
await world.adapters.level.postLikesDb.put(`${displayPost}:like-2`, { postTxid: displayPost, txid: 'like-2' })
|
||||
|
||||
await world.adapters.level.postParentsDb.put('reply-1', { parentTxid: reversedPost, childTxid: 'reply-1', blockHeight: 600050 })
|
||||
await world.adapters.level.postParentsDb.put('reply-2', { parentTxid: displayPost, childTxid: 'reply-2', blockHeight: 600051 })
|
||||
await world.adapters.level.postChildrenDb.put(`${reversedPost}:reply-1`, { parentTxid: reversedPost, childTxid: 'reply-1' })
|
||||
await world.adapters.level.postChildrenDb.put(`${displayPost}:reply-2`, { parentTxid: displayPost, childTxid: 'reply-2' })
|
||||
|
||||
await world.adapters.level.pollOptionsDb.put('option-1', { addr: 'bitcoincash:qaddr-a', pollTxid: reversedPoll, option: 'yes', seen: 6, blockHeight: 600300 })
|
||||
await world.adapters.level.pollOptionsDb.put('option-2', { addr: 'bitcoincash:qaddr-b', pollTxid: displayPoll, option: 'no', seen: 7, blockHeight: 600301 })
|
||||
await world.adapters.level.pollVotesDb.put('vote-1', { addr: 'bitcoincash:qaddr-a', pollTxid: reversedPoll, comment: 'yes', seen: 8, blockHeight: 600302 })
|
||||
await world.adapters.level.pollVotesDb.put('vote-2', { addr: 'bitcoincash:qaddr-b', pollTxid: displayPoll, comment: 'no', seen: 9, blockHeight: 600303 })
|
||||
}
|
||||
|
||||
async function loadMutes (world) {
|
||||
const muter1 = 'bitcoincash:qqlrzp23w08434twmvr4fxw672whkjy0py26r63g3d'
|
||||
const muter2 = 'bitcoincash:qpm2qsznhks23z7629mms6s4cwef74vcwvy22gdx6a'
|
||||
@@ -592,6 +629,13 @@ const handlers = [
|
||||
// World is already created with all stores.
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'db instance with txid repair stores',
|
||||
pattern: /^a psf-memo-db instance with posts, likes, postLikes, postParents, postChildren, polls, pollOptions, and pollVotes stores$/,
|
||||
async run () {
|
||||
// World is already created with all stores.
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'load fixture',
|
||||
pattern: /^the fixture "(.+)" is loaded into the posts (?:store|and likes stores)$/,
|
||||
@@ -606,6 +650,13 @@ const handlers = [
|
||||
await loadFixture(world, m[1])
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'load bare fixture',
|
||||
pattern: /^the fixture "(.+)" is loaded$/,
|
||||
async run (m, example, world) {
|
||||
await loadFixture(world, m[1])
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'run backfill utility',
|
||||
pattern: /^the backfill utility is run$/,
|
||||
@@ -620,6 +671,20 @@ const handlers = [
|
||||
await backfillIndexes(world)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'run txid repair utility',
|
||||
pattern: /^the txid repair utility is run$/,
|
||||
async run (m, example, world) {
|
||||
await repairTxidEncoding(world.adapters.level)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'run txid repair utility again',
|
||||
pattern: /^the txid repair utility is run again$/,
|
||||
async run (m, example, world) {
|
||||
await repairTxidEncoding(world.adapters.level)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'request recent posts',
|
||||
pattern: /^the client requests \/posts\/recent with limit (<limit>) and offset (<offset>)$/,
|
||||
@@ -859,7 +924,7 @@ const handlers = [
|
||||
},
|
||||
{
|
||||
name: 'postLikes contains entry',
|
||||
pattern: /^the postLikes store contains (<count>) entry whose key starts with (<postTxid>) and ends with (<likeTxid>)$/,
|
||||
pattern: /^the postLikes store contains (<[A-Za-z0-9_]+>|[0-9]+) entry whose key starts with (<[A-Za-z0-9_]+>) and ends with (<[A-Za-z0-9_]+>)$/,
|
||||
async run (m, example, world) {
|
||||
const expectedCount = parseInt(resolveParam(m[1], example), 10)
|
||||
const postTxid = resolveParam(m[2], example)
|
||||
@@ -873,6 +938,70 @@ const handlers = [
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'postChildren contains entry',
|
||||
pattern: /^the postChildren store contains (<[A-Za-z0-9_]+>|[0-9]+) entry whose key starts with (<[A-Za-z0-9_]+>) and ends with (<[A-Za-z0-9_]+>)$/,
|
||||
async run (m, example, world) {
|
||||
const expectedCount = parseInt(resolveParam(m[1], example), 10)
|
||||
const postTxid = resolveParam(m[2], example)
|
||||
const replyTxid = resolveParam(m[3], example)
|
||||
let count = 0
|
||||
for await (const [key] of world.adapters.level.postChildrenDb.iterator()) {
|
||||
if (key.startsWith(`${postTxid}:`) && key.endsWith(`:${replyTxid}`)) count++
|
||||
}
|
||||
if (count !== expectedCount) {
|
||||
throw new Error(`Expected ${expectedCount} postChildren entry/entries for ${postTxid}/${replyTxid}, got ${count}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'likes store maps like to post',
|
||||
pattern: /^the likes store maps like (<[A-Za-z0-9_]+>) to post (<[A-Za-z0-9_]+>)$/,
|
||||
async run (m, example, world) {
|
||||
const likeTxid = resolveParam(m[1], example)
|
||||
const postTxid = resolveParam(m[2], example)
|
||||
const like = await world.adapters.level.likesDb.get(likeTxid)
|
||||
if (like.postTxid !== postTxid) {
|
||||
throw new Error(`Expected like ${likeTxid} to map to post ${postTxid}, got ${like.postTxid}.`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'postParents store maps reply to parent',
|
||||
pattern: /^the postParents store maps reply (<[A-Za-z0-9_]+>) to parent (<[A-Za-z0-9_]+>)$/,
|
||||
async run (m, example, world) {
|
||||
const replyTxid = resolveParam(m[1], example)
|
||||
const postTxid = resolveParam(m[2], example)
|
||||
const reply = await world.adapters.level.postParentsDb.get(replyTxid)
|
||||
if (reply.parentTxid !== postTxid) {
|
||||
throw new Error(`Expected reply ${replyTxid} to map to parent ${postTxid}, got ${reply.parentTxid}.`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'pollOptions store maps option to poll',
|
||||
pattern: /^the pollOptions store maps option (<[A-Za-z0-9_]+>) to poll (<[A-Za-z0-9_]+>)$/,
|
||||
async run (m, example, world) {
|
||||
const optionTxid = resolveParam(m[1], example)
|
||||
const pollTxid = resolveParam(m[2], example)
|
||||
const option = await world.adapters.level.pollOptionsDb.get(optionTxid)
|
||||
if (option.pollTxid !== pollTxid) {
|
||||
throw new Error(`Expected poll option ${optionTxid} to map to poll ${pollTxid}, got ${option.pollTxid}.`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'pollVotes store maps vote to poll',
|
||||
pattern: /^the pollVotes store maps vote (<[A-Za-z0-9_]+>) to poll (<[A-Za-z0-9_]+>)$/,
|
||||
async run (m, example, world) {
|
||||
const voteTxid = resolveParam(m[1], example)
|
||||
const pollTxid = resolveParam(m[2], example)
|
||||
const vote = await world.adapters.level.pollVotesDb.get(voteTxid)
|
||||
if (vote.pollTxid !== pollTxid) {
|
||||
throw new Error(`Expected poll vote ${voteTxid} to map to poll ${pollTxid}, got ${vote.pollTxid}.`)
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'db instance with follows store',
|
||||
pattern: /^a psf-memo-db instance with a follows store$/,
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
Library to repair byte-reversed txid references in a psf-memo-db.
|
||||
|
||||
Older psf-memo-client broadcasts embedded a referenced txid in big-endian
|
||||
display order. The indexer expected little-endian wire order, so it reversed
|
||||
those bytes and stored a reference that is the reverse of the display txid.
|
||||
This library rewrites a reversed reference to display order and rebuilds the
|
||||
affected secondary index.
|
||||
|
||||
It leaves records whose reference already points at a known target
|
||||
untouched, leaves references whose target is unknown in either byte order
|
||||
untouched, and is idempotent: a second run makes no changes.
|
||||
|
||||
The LevelDB handles are injected so the repair logic stays testable and free
|
||||
of file-system concerns; the CLI wrapper in util/txid opens the real stores.
|
||||
*/
|
||||
|
||||
// Reverse the byte order of a 64-character display txid. Returns null for any
|
||||
// value that is not a 64-character hex txid.
|
||||
export function reverseTxid (txid) {
|
||||
if (typeof txid !== 'string' || !/^[0-9a-fA-F]{64}$/.test(txid)) return null
|
||||
return Buffer.from(txid, 'hex').reverse().toString('hex')
|
||||
}
|
||||
|
||||
// True when a store has a record at the given key.
|
||||
async function hasRecord (db, key) {
|
||||
try {
|
||||
await db.get(key)
|
||||
return true
|
||||
} catch (err) {
|
||||
if (err.notFound || err.code === 'LEVEL_NOT_FOUND') return false
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
// Choose the reference in display order. Keep the stored reference when its
|
||||
// target exists. Otherwise use the reversed reference when that target
|
||||
// exists. Otherwise return the stored reference unchanged.
|
||||
export async function correctReference (targetDb, reference) {
|
||||
if (await hasRecord(targetDb, reference)) return reference
|
||||
const reversed = reverseTxid(reference)
|
||||
if (reversed && await hasRecord(targetDb, reversed)) return reversed
|
||||
return reference
|
||||
}
|
||||
|
||||
// Repair likes, replies, poll options, and poll votes in place. Returns a
|
||||
// summary of how many records of each kind were corrected.
|
||||
export async function repairTxidEncoding (level) {
|
||||
const summary = { likes: 0, replies: 0, pollOptions: 0, pollVotes: 0 }
|
||||
|
||||
for await (const [likeTxid, like] of level.likesDb.iterator()) {
|
||||
if (!like.postTxid) continue
|
||||
const corrected = await correctReference(level.postsDb, like.postTxid)
|
||||
if (corrected === like.postTxid) continue
|
||||
|
||||
await level.likesDb.put(likeTxid, { ...like, postTxid: corrected })
|
||||
await level.postLikesDb.del(`${like.postTxid}:${likeTxid}`)
|
||||
await level.postLikesDb.put(`${corrected}:${likeTxid}`, { postTxid: corrected, txid: likeTxid })
|
||||
summary.likes++
|
||||
}
|
||||
|
||||
for await (const [replyTxid, reply] of level.postParentsDb.iterator()) {
|
||||
if (!reply.parentTxid) continue
|
||||
const corrected = await correctReference(level.postsDb, reply.parentTxid)
|
||||
if (corrected === reply.parentTxid) continue
|
||||
|
||||
await level.postParentsDb.put(replyTxid, { ...reply, parentTxid: corrected })
|
||||
await level.postChildrenDb.del(`${reply.parentTxid}:${replyTxid}`)
|
||||
await level.postChildrenDb.put(`${corrected}:${replyTxid}`, { ...reply, parentTxid: corrected })
|
||||
summary.replies++
|
||||
}
|
||||
|
||||
for await (const [optionTxid, option] of level.pollOptionsDb.iterator()) {
|
||||
if (!option.pollTxid) continue
|
||||
const corrected = await correctReference(level.pollsDb, option.pollTxid)
|
||||
if (corrected === option.pollTxid) continue
|
||||
|
||||
await level.pollOptionsDb.put(optionTxid, { ...option, pollTxid: corrected })
|
||||
summary.pollOptions++
|
||||
}
|
||||
|
||||
for await (const [voteTxid, vote] of level.pollVotesDb.iterator()) {
|
||||
if (!vote.pollTxid) continue
|
||||
const corrected = await correctReference(level.pollsDb, vote.pollTxid)
|
||||
if (corrected === vote.pollTxid) continue
|
||||
|
||||
await level.pollVotesDb.put(voteTxid, { ...vote, pollTxid: corrected })
|
||||
summary.pollVotes++
|
||||
}
|
||||
|
||||
return summary
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
Unit tests for the txid encoding repair library.
|
||||
|
||||
Older psf-memo-client broadcasts embedded a referenced txid in big-endian
|
||||
display order. The indexer expected little-endian wire order, so it stored a
|
||||
byte-reversed reference. The repair library rewrites a reversed reference to
|
||||
display order and rebuilds the affected secondary index. It leaves
|
||||
correctly-encoded records untouched, leaves references whose target is
|
||||
unknown in either byte order untouched, and is idempotent.
|
||||
*/
|
||||
|
||||
import { assert } from 'chai'
|
||||
import {
|
||||
reverseTxid,
|
||||
correctReference,
|
||||
repairTxidEncoding
|
||||
} from '../../../src/lib/repair-txid-encoding.js'
|
||||
|
||||
const DISPLAY_POST = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
|
||||
const REVERSED_POST = 'efcdab8967452301efcdab8967452301efcdab8967452301efcdab8967452301'
|
||||
const DISPLAY_POLL = '00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff'
|
||||
const REVERSED_POLL = 'ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100'
|
||||
const UNKNOWN = 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef'
|
||||
|
||||
class FakeDb {
|
||||
constructor () {
|
||||
this.map = new Map()
|
||||
}
|
||||
|
||||
async get (key) {
|
||||
if (!this.map.has(key)) {
|
||||
const err = new Error(`not found: ${key}`)
|
||||
err.notFound = true
|
||||
throw err
|
||||
}
|
||||
return this.map.get(key)
|
||||
}
|
||||
|
||||
async put (key, value) {
|
||||
this.map.set(key, value)
|
||||
}
|
||||
|
||||
async del (key) {
|
||||
this.map.delete(key)
|
||||
}
|
||||
|
||||
async * iterator () {
|
||||
for (const [key, value] of this.map) {
|
||||
yield [key, value]
|
||||
}
|
||||
}
|
||||
|
||||
keys () {
|
||||
return [...this.map.keys()].sort()
|
||||
}
|
||||
}
|
||||
|
||||
function makeLevel () {
|
||||
return {
|
||||
postsDb: new FakeDb(),
|
||||
likesDb: new FakeDb(),
|
||||
postLikesDb: new FakeDb(),
|
||||
postParentsDb: new FakeDb(),
|
||||
postChildrenDb: new FakeDb(),
|
||||
pollsDb: new FakeDb(),
|
||||
pollOptionsDb: new FakeDb(),
|
||||
pollVotesDb: new FakeDb()
|
||||
}
|
||||
}
|
||||
|
||||
function loadReversedFixture (level) {
|
||||
level.postsDb.put(DISPLAY_POST, { addr: 'addr-a', text: 'post', blockHeight: 1 })
|
||||
level.pollsDb.put(DISPLAY_POLL, { question: 'q', pollType: 1, optionCount: 2, blockHeight: 2 })
|
||||
|
||||
level.likesDb.put('like-1', { postTxid: REVERSED_POST, addr: 'addr-b', blockHeight: 3 })
|
||||
level.likesDb.put('like-2', { postTxid: DISPLAY_POST, addr: 'addr-c', blockHeight: 4 })
|
||||
level.likesDb.put('like-3', { postTxid: UNKNOWN, addr: 'addr-d', blockHeight: 5 })
|
||||
level.postLikesDb.put(`${REVERSED_POST}:like-1`, { postTxid: REVERSED_POST, txid: 'like-1' })
|
||||
level.postLikesDb.put(`${DISPLAY_POST}:like-2`, { postTxid: DISPLAY_POST, txid: 'like-2' })
|
||||
|
||||
level.postParentsDb.put('reply-1', { parentTxid: REVERSED_POST, childTxid: 'reply-1', blockHeight: 6 })
|
||||
level.postParentsDb.put('reply-2', { parentTxid: DISPLAY_POST, childTxid: 'reply-2', blockHeight: 7 })
|
||||
level.postChildrenDb.put(`${REVERSED_POST}:reply-1`, { parentTxid: REVERSED_POST, childTxid: 'reply-1' })
|
||||
level.postChildrenDb.put(`${DISPLAY_POST}:reply-2`, { parentTxid: DISPLAY_POST, childTxid: 'reply-2' })
|
||||
|
||||
level.pollOptionsDb.put('option-1', { pollTxid: REVERSED_POLL, option: 'yes', blockHeight: 8 })
|
||||
level.pollOptionsDb.put('option-2', { pollTxid: DISPLAY_POLL, option: 'no', blockHeight: 9 })
|
||||
level.pollVotesDb.put('vote-1', { pollTxid: REVERSED_POLL, comment: 'yes', blockHeight: 10 })
|
||||
level.pollVotesDb.put('vote-2', { pollTxid: DISPLAY_POLL, comment: 'no', blockHeight: 11 })
|
||||
}
|
||||
|
||||
describe('#RepairTxidEncoding', () => {
|
||||
describe('reverseTxid', () => {
|
||||
it('reverses the byte order of a display txid', () => {
|
||||
assert.equal(reverseTxid(DISPLAY_POST), REVERSED_POST)
|
||||
})
|
||||
|
||||
it('returns null for a value that is not a 64-character hex txid', () => {
|
||||
assert.isNull(reverseTxid('not-a-txid'))
|
||||
assert.isNull(reverseTxid(null))
|
||||
assert.isNull(reverseTxid('zz'.repeat(32)))
|
||||
})
|
||||
})
|
||||
|
||||
describe('correctReference', () => {
|
||||
it('keeps a reference whose target exists in display order', async () => {
|
||||
const level = makeLevel()
|
||||
level.postsDb.put(DISPLAY_POST, {})
|
||||
assert.equal(await correctReference(level.postsDb, DISPLAY_POST), DISPLAY_POST)
|
||||
})
|
||||
|
||||
it('reverses a reference whose reversed form is the existing target', async () => {
|
||||
const level = makeLevel()
|
||||
level.postsDb.put(DISPLAY_POST, {})
|
||||
assert.equal(await correctReference(level.postsDb, REVERSED_POST), DISPLAY_POST)
|
||||
})
|
||||
|
||||
it('keeps an unknown reference unchanged', async () => {
|
||||
const level = makeLevel()
|
||||
assert.equal(await correctReference(level.postsDb, UNKNOWN), UNKNOWN)
|
||||
})
|
||||
|
||||
it('propagates an unexpected store error', async () => {
|
||||
const targetDb = {
|
||||
async get () {
|
||||
const err = new Error('io failure')
|
||||
err.code = 'EIO'
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
let caught = null
|
||||
try {
|
||||
await correctReference(targetDb, DISPLAY_POST)
|
||||
} catch (err) {
|
||||
caught = err
|
||||
}
|
||||
|
||||
assert.isNotNull(caught)
|
||||
assert.equal(caught.code, 'EIO')
|
||||
})
|
||||
})
|
||||
|
||||
describe('repairTxidEncoding', () => {
|
||||
it('corrects a reversed like reference and rebuilds postLikes', async () => {
|
||||
const level = makeLevel()
|
||||
loadReversedFixture(level)
|
||||
|
||||
await repairTxidEncoding(level)
|
||||
|
||||
assert.equal((await level.likesDb.get('like-1')).postTxid, DISPLAY_POST)
|
||||
assert.include(level.postLikesDb.keys(), `${DISPLAY_POST}:like-1`)
|
||||
assert.notInclude(level.postLikesDb.keys(), `${REVERSED_POST}:like-1`)
|
||||
})
|
||||
|
||||
it('corrects a reversed reply reference and rebuilds postChildren', async () => {
|
||||
const level = makeLevel()
|
||||
loadReversedFixture(level)
|
||||
|
||||
await repairTxidEncoding(level)
|
||||
|
||||
assert.equal((await level.postParentsDb.get('reply-1')).parentTxid, DISPLAY_POST)
|
||||
assert.include(level.postChildrenDb.keys(), `${DISPLAY_POST}:reply-1`)
|
||||
assert.notInclude(level.postChildrenDb.keys(), `${REVERSED_POST}:reply-1`)
|
||||
})
|
||||
|
||||
it('corrects reversed poll option and vote references', async () => {
|
||||
const level = makeLevel()
|
||||
loadReversedFixture(level)
|
||||
|
||||
await repairTxidEncoding(level)
|
||||
|
||||
assert.equal((await level.pollOptionsDb.get('option-1')).pollTxid, DISPLAY_POLL)
|
||||
assert.equal((await level.pollVotesDb.get('vote-1')).pollTxid, DISPLAY_POLL)
|
||||
})
|
||||
|
||||
it('leaves correctly-encoded references unchanged', async () => {
|
||||
const level = makeLevel()
|
||||
loadReversedFixture(level)
|
||||
|
||||
await repairTxidEncoding(level)
|
||||
|
||||
assert.equal((await level.likesDb.get('like-2')).postTxid, DISPLAY_POST)
|
||||
assert.equal((await level.postParentsDb.get('reply-2')).parentTxid, DISPLAY_POST)
|
||||
assert.equal((await level.pollOptionsDb.get('option-2')).pollTxid, DISPLAY_POLL)
|
||||
assert.equal((await level.pollVotesDb.get('vote-2')).pollTxid, DISPLAY_POLL)
|
||||
})
|
||||
|
||||
it('leaves an unknown reference unchanged', async () => {
|
||||
const level = makeLevel()
|
||||
loadReversedFixture(level)
|
||||
|
||||
await repairTxidEncoding(level)
|
||||
|
||||
assert.equal((await level.likesDb.get('like-3')).postTxid, UNKNOWN)
|
||||
})
|
||||
|
||||
it('skips records without a reference field', async () => {
|
||||
const level = makeLevel()
|
||||
level.likesDb.put('like-empty', { addr: 'addr' })
|
||||
level.postParentsDb.put('reply-empty', { childTxid: 'reply-empty' })
|
||||
level.pollOptionsDb.put('option-empty', { option: 'x' })
|
||||
level.pollVotesDb.put('vote-empty', { comment: 'x' })
|
||||
|
||||
const summary = await repairTxidEncoding(level)
|
||||
|
||||
assert.deepEqual(summary, { likes: 0, replies: 0, pollOptions: 0, pollVotes: 0 })
|
||||
})
|
||||
|
||||
it('is idempotent', async () => {
|
||||
const level = makeLevel()
|
||||
loadReversedFixture(level)
|
||||
|
||||
await repairTxidEncoding(level)
|
||||
await repairTxidEncoding(level)
|
||||
|
||||
const postLikes = level.postLikesDb.keys()
|
||||
assert.include(postLikes, `${DISPLAY_POST}:like-1`)
|
||||
assert.notInclude(postLikes, `${REVERSED_POST}:like-1`)
|
||||
const postChildren = level.postChildrenDb.keys()
|
||||
assert.include(postChildren, `${DISPLAY_POST}:reply-1`)
|
||||
assert.notInclude(postChildren, `${REVERSED_POST}:reply-1`)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
Utility: repair byte-reversed txid references in an existing psf-memo-db and
|
||||
rebuild the affected secondary indexes.
|
||||
|
||||
Older psf-memo-client broadcasts embedded a referenced txid in big-endian
|
||||
display order. The indexer expected little-endian wire order, so it stored a
|
||||
reference that is the reverse of the display txid. This utility rewrites
|
||||
those references to display order for likes, replies, poll options, and poll
|
||||
votes, and rebuilds the postLikes and postChildren indexes.
|
||||
|
||||
Run from the psf-memo-db repo root on the host that owns the LevelDB files:
|
||||
|
||||
node util/txid/repair-txid-encoding.js
|
||||
|
||||
The repair is idempotent: re-running it makes no further changes. Progress
|
||||
and a summary are printed to stderr.
|
||||
|
||||
WARNING:
|
||||
- This script opens the LevelDB files directly. psf-memo-db must NOT be
|
||||
running, or another process must not hold the database locks.
|
||||
- Make a backup of leveldb/current before running on a production server:
|
||||
cp -r leveldb/current leveldb/current-pre-txid-repair-backup
|
||||
*/
|
||||
|
||||
import level from 'level'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import * as url from 'url'
|
||||
import { repairTxidEncoding } from '../../src/lib/repair-txid-encoding.js'
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
|
||||
|
||||
const DATA_DIR = process.env.PSF_MEMO_DB_DATA_DIR
|
||||
? path.resolve(process.env.PSF_MEMO_DB_DATA_DIR)
|
||||
: path.resolve(__dirname, '../../leveldb/current')
|
||||
|
||||
// Stores that hold records to scan. They must already exist.
|
||||
const RECORD_STORES = ['posts', 'polls', 'likes', 'postParents', 'pollOptions', 'pollVotes']
|
||||
// Secondary indexes rebuilt by the repair. Create them when missing.
|
||||
const INDEX_STORES = ['postLikes', 'postChildren']
|
||||
|
||||
function requiredStorePath (dir, name) {
|
||||
const storePath = path.join(dir, name)
|
||||
if (!fs.existsSync(storePath)) {
|
||||
throw new Error(`Required LevelDB store not found: ${storePath}. Set PSF_MEMO_DB_DATA_DIR to the directory containing the psf-memo-db stores.`)
|
||||
}
|
||||
return storePath
|
||||
}
|
||||
|
||||
async function main () {
|
||||
console.error(`Using LevelDB data directory: ${DATA_DIR}`)
|
||||
|
||||
const paths = {}
|
||||
for (const name of RECORD_STORES) {
|
||||
paths[name] = requiredStorePath(DATA_DIR, name)
|
||||
}
|
||||
for (const name of INDEX_STORES) {
|
||||
paths[name] = path.join(DATA_DIR, name)
|
||||
}
|
||||
|
||||
console.error('Opening LevelDB stores...')
|
||||
const dbs = {
|
||||
postsDb: level(paths.posts, { valueEncoding: 'json' }),
|
||||
pollsDb: level(paths.polls, { valueEncoding: 'json' }),
|
||||
likesDb: level(paths.likes, { valueEncoding: 'json' }),
|
||||
postParentsDb: level(paths.postParents, { valueEncoding: 'json' }),
|
||||
pollOptionsDb: level(paths.pollOptions, { valueEncoding: 'json' }),
|
||||
pollVotesDb: level(paths.pollVotes, { valueEncoding: 'json' }),
|
||||
postLikesDb: level(paths.postLikes, { valueEncoding: 'json', createIfMissing: true }),
|
||||
postChildrenDb: level(paths.postChildren, { valueEncoding: 'json', createIfMissing: true })
|
||||
}
|
||||
|
||||
try {
|
||||
console.error('Repairing reversed txid references...')
|
||||
const summary = await repairTxidEncoding(dbs)
|
||||
|
||||
console.error('\nTxid encoding repair complete.')
|
||||
console.error(` likes corrected: ${summary.likes}`)
|
||||
console.error(` replies corrected: ${summary.replies}`)
|
||||
console.error(` poll options corrected: ${summary.pollOptions}`)
|
||||
console.error(` poll votes corrected: ${summary.pollVotes}`)
|
||||
} catch (err) {
|
||||
console.error('\nTxid encoding repair failed:', err.message)
|
||||
process.exitCode = 1
|
||||
} finally {
|
||||
for (const db of Object.values(dbs)) {
|
||||
await db.close().catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
Reference in New Issue
Block a user