From 6b8da07981d06fcdaf6cc310d07b43cab1b0d77b Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Sun, 20 Sep 2026 14:45:37 -0700 Subject: [PATCH] Refactor profile-last-post: split recency logic, add property tests Split backfillProfileRecency and establishProfileRecency into small helpers so every analyzed function scores CRAP 6 or below, without changing behavior. Extract the post recency-qualification check in handlePost. Add seeded property tests covering backfill conservation and idempotence, the addrPostHeight key round trip, recent-profile read ordering and pagination, and the indexer's recency convergence. By refactorer. --- .../src/lib/backfill-profile-recency.js | 108 ++++++---- .../profile-recency-order.property.test.js | 195 ++++++++++++++++++ .../src/use-cases/action-types/post.js | 12 +- .../use-cases/action-types/profile-recency.js | 52 +++-- .../property/profile-recency.property.test.js | 150 ++++++++++++++ 5 files changed, 458 insertions(+), 59 deletions(-) create mode 100644 psf-memo-db/test/property/profile-recency-order.property.test.js create mode 100644 psf-memo-indexer/test/property/profile-recency.property.test.js diff --git a/psf-memo-db/src/lib/backfill-profile-recency.js b/psf-memo-db/src/lib/backfill-profile-recency.js index 59b88cd..ba0a327 100644 --- a/psf-memo-db/src/lib/backfill-profile-recency.js +++ b/psf-memo-db/src/lib/backfill-profile-recency.js @@ -56,50 +56,86 @@ export function partsFromAddrPostHeightKey (key) { } } -// Keep the newest confirmed qualifying post per address. -export async function backfillProfileRecency ({ - profilesDb, - postsDb, - addrPostHeightsDb, - postParentsDb, - pollsDb, - statusDb, - profileRecencyDb -}) { - const chainBlockHeight = await readChainBlockHeight(statusDb) - const best = new Map() - - for await (const [key, value] of addrPostHeightsDb.iterator()) { - const fallback = partsFromAddrPostHeightKey(key) - const addr = value?.addr ?? fallback.addr - const txid = value?.txid ?? fallback.txid - const blockHeight = value?.blockHeight ?? fallback.blockHeight - if (!addr || !txid) continue - if (chainBlockHeight !== null && blockHeight > chainBlockHeight) continue - if (await getRecord(postParentsDb, txid)) continue - if (await getRecord(pollsDb, txid)) continue - if (!(await getRecord(profilesDb, addr))) continue - - const post = await getRecord(postsDb, txid) - const seen = post?.seen ?? 0 - const current = best.get(addr) - if (!current || blockHeight > current.blockHeight || (blockHeight === current.blockHeight && seen > current.seen)) { - best.set(addr, { addr, blockHeight, seen }) - } +// Prefer the stored value fields, falling back to the key segments when a +// record predates the field being written. +function entryFields (key, value) { + const fallback = partsFromAddrPostHeightKey(key) + return { + addr: value?.addr ?? fallback.addr, + txid: value?.txid ?? fallback.txid, + blockHeight: value?.blockHeight ?? fallback.blockHeight } +} +// An entry at or below the chain tip is confirmed. With no tip every entry is +// treated as confirmed. +function isConfirmedEntry (blockHeight, chainBlockHeight) { + if (chainBlockHeight === null) return true + return blockHeight <= chainBlockHeight +} + +// A qualifying post is authored by a profile address and is neither a reply +// (tracked in postParents) nor a poll creation (tracked in polls). +async function isQualifyingPost (stores, addr, txid) { + if (!(await getRecord(stores.profilesDb, addr))) return false + if (await getRecord(stores.postParentsDb, txid)) return false + if (await getRecord(stores.pollsDb, txid)) return false + return true +} + +// The qualifying post for one addrPostHeights entry, or null when the entry +// does not qualify. +async function qualifyingCandidate (stores, key, value, chainBlockHeight) { + const { addr, txid, blockHeight } = entryFields(key, value) + if (!addr || !txid) return null + if (!isConfirmedEntry(blockHeight, chainBlockHeight)) return null + if (!(await isQualifyingPost(stores, addr, txid))) return null + const post = await getRecord(stores.postsDb, txid) + return { addr, blockHeight, seen: post?.seen ?? 0 } +} + +// True when `candidate` is newer than `current`: greater height, or equal +// height with a greater seen value. +function isNewer (candidate, current) { + if (!current) return true + if (candidate.blockHeight !== current.blockHeight) return candidate.blockHeight > current.blockHeight + return candidate.seen > current.seen +} + +// Keep the newest candidate per address. Equal height and seen keeps the first +// record seen, which makes reprocessing idempotent. +function keepNewest (best, candidate) { + if (isNewer(candidate, best.get(candidate.addr))) best.set(candidate.addr, candidate) +} + +async function collectBestRecency (stores, chainBlockHeight) { + const best = new Map() + for await (const [key, value] of stores.addrPostHeightsDb.iterator()) { + const candidate = await qualifyingCandidate(stores, key, value, chainBlockHeight) + if (candidate) keepNewest(best, candidate) + } + return best +} + +async function writeRecency (profileRecencyDb, best) { for (const record of best.values()) { await profileRecencyDb.put(record.addr, record) } +} - const desired = new Set(best.keys()) - const stale = [] +async function removeStaleRecency (profileRecencyDb, desired) { for await (const [key] of profileRecencyDb.iterator()) { - if (!desired.has(key)) stale.push(key) - } - for (const key of stale) { - await profileRecencyDb.del(key) + if (!desired.has(key)) await profileRecencyDb.del(key) } +} +// Rebuild the whole index: collect the newest confirmed qualifying post per +// profile address, write those records, and drop records that no longer +// qualify. +export async function backfillProfileRecency (stores) { + const chainBlockHeight = await readChainBlockHeight(stores.statusDb) + const best = await collectBestRecency(stores, chainBlockHeight) + await writeRecency(stores.profileRecencyDb, best) + await removeStaleRecency(stores.profileRecencyDb, new Set(best.keys())) return { profiles: best.size } } diff --git a/psf-memo-db/test/property/profile-recency-order.property.test.js b/psf-memo-db/test/property/profile-recency-order.property.test.js new file mode 100644 index 0000000..21aee00 --- /dev/null +++ b/psf-memo-db/test/property/profile-recency-order.property.test.js @@ -0,0 +1,195 @@ +/* + Property tests for the profileRecency backfill and the recent-profiles read + ordering. + + The unit tests probe fixed fixtures. These properties pin the invariants over + broad random stores: + + - Conservation: backfillProfileRecency writes exactly the newest confirmed + qualifying post per profile address, excluding replies, polls, and + unconfirmed entries, and writes nothing for addresses without a profile. + - Idempotence: a second backfill leaves the index byte-for-byte identical. + - Stale removal: pre-existing records that no longer qualify are dropped. + - Key round trip: partsFromAddrPostHeightKey recovers the address, height, + and txid from a formatted addrPostHeights key, including cash addresses + that contain colons. + - Read ordering: ProfileQuery.listRecentProfiles returns the requested page + in height-desc, seen-desc, address-asc order with conserved pagination. +*/ + +import test from 'node:test' + +import { seededRandom, forAll, intGen, txidGen } from './harness.js' +import { backfillProfileRecency, partsFromAddrPostHeightKey } from '../../src/lib/backfill-profile-recency.js' +import ProfileQuery from '../../src/adapters/profile-query.js' +import { FakeDb } from '../support/level-double.js' + +const rng = seededRandom(20260921) + +const ADDRS = [ + 'bitcoincash:qaddr-a', + 'bitcoincash:qaddr-b', + 'bitcoincash:qaddr-c', + 'bitcoincash:qaddr-d' +] + +function pad (height) { + return String(height).padStart(12, '0') +} + +function addrPostHeightKey (addr, height, txid) { + return `${addr}:${pad(height)}:${txid}` +} + +function isNewer (candidate, current) { + if (!current) return true + if (candidate.blockHeight !== current.blockHeight) return candidate.blockHeight > current.blockHeight + return candidate.seen > current.seen +} + +function recencyWorldGen () { + return () => { + const chainBlockHeight = intGen(rng, 1000, 1100)() + const stores = { + profilesDb: new FakeDb(), + postsDb: new FakeDb(), + addrPostHeightsDb: new FakeDb(), + postParentsDb: new FakeDb(), + pollsDb: new FakeDb(), + statusDb: new FakeDb([['status', { chainBlockHeight }]]), + profileRecencyDb: new FakeDb() + } + const expected = new Map() + let seq = 0 + + for (const addr of ADDRS) { + const hasProfile = rng() < 0.75 + if (hasProfile) { + stores.profilesDb.store.set(addr, { text: `bio-${addr}`, txid: `profile-${addr}` }) + } + + const postCount = intGen(rng, 0, 5)() + for (let i = 0; i < postCount; i++) { + const txid = `tx-${seq++}` + const blockHeight = chainBlockHeight + intGen(rng, -20, 20)() + const seen = intGen(rng, 0, 1000)() + stores.addrPostHeightsDb.store.set(addrPostHeightKey(addr, blockHeight, txid), { txid, addr, blockHeight }) + stores.postsDb.store.set(txid, { addr, seen, blockHeight }) + + const roll = rng() + if (roll < 0.25) { + stores.postParentsDb.store.set(txid, { txid, parentTxid: 'parent' }) + } else if (roll < 0.5) { + stores.pollsDb.store.set(txid, { txid }) + } else if (hasProfile && blockHeight <= chainBlockHeight) { + const candidate = { addr, blockHeight, seen } + if (isNewer(candidate, expected.get(addr))) expected.set(addr, candidate) + } + } + } + + // Seed a stale record for an address that never qualifies so the backfill + // must remove it as part of rebuilding the index. + const staleAddr = 'bitcoincash:qaddr-stale' + stores.profileRecencyDb.store.set(staleAddr, { addr: staleAddr, blockHeight: 999999, seen: 1 }) + + return { stores, expected } + } +} + +function snapshot (db) { + return JSON.stringify([...db.store.entries()].sort()) +} + +test('backfillProfileRecency conserves the newest confirmed qualifying post per profile', async () => { + await forAll(recencyWorldGen(), async ({ stores, expected }) => { + const result = await backfillProfileRecency(stores) + + if (result.profiles !== expected.size) return false + if (stores.profileRecencyDb.store.size !== expected.size) return false + + for (const [addr, record] of expected) { + const stored = stores.profileRecencyDb.store.get(addr) + if (!stored) return false + if (stored.addr !== record.addr) return false + if (stored.blockHeight !== record.blockHeight) return false + if (stored.seen !== record.seen) return false + } + + const before = snapshot(stores.profileRecencyDb) + await backfillProfileRecency(stores) + return snapshot(stores.profileRecencyDb) === before + }, { label: 'profile recency backfill conservation and idempotence' }) +}) + +test('partsFromAddrPostHeightKey round-trips a formatted addrPostHeights key', async () => { + await forAll( + () => ({ + addr: ADDRS[Math.floor(rng() * ADDRS.length)], + height: intGen(rng, 0, 9999999)(), + txid: txidGen(rng) + }), + ({ addr, height, txid }) => { + const parts = partsFromAddrPostHeightKey(addrPostHeightKey(addr, height, txid)) + return parts.addr === addr && parts.blockHeight === height && parts.txid === txid + }, + { label: 'addrPostHeight key round trip' } + ) +}) + +function compareRecency (a, b) { + if (b.blockHeight !== a.blockHeight) return b.blockHeight - a.blockHeight + if (b.seen !== a.seen) return b.seen - a.seen + if (a.addr === b.addr) return 0 + return a.addr < b.addr ? -1 : 1 +} + +function orderingWorldGen () { + return () => { + const count = intGen(rng, 0, 10)() + const entries = [] + const profilesDb = new FakeDb() + const profileRecencyDb = new FakeDb() + + for (let i = 0; i < count; i++) { + const addr = `bitcoincash:qaddr-${i}` + const blockHeight = intGen(rng, 0, 5)() + const seen = intGen(rng, 0, 5)() + entries.push({ addr, blockHeight, seen }) + profileRecencyDb.store.set(addr, { addr, blockHeight, seen }) + profilesDb.store.set(addr, { text: `bio ${i}`, txid: `profile-${i}` }) + } + + return { + entries, + profilesDb, + profileRecencyDb, + limit: intGen(rng, 0, 12)(), + offset: intGen(rng, 0, count + 2)() + } + } +} + +test('listRecentProfiles returns the ordering and pagination contract', async () => { + await forAll(orderingWorldGen(), async ({ entries, profilesDb, profileRecencyDb, limit, offset }) => { + const query = new ProfileQuery({ profilesDb, profileRecencyDb }) + + const result = await query.listRecentProfiles({ limit, offset }) + + const ordered = [...entries].sort(compareRecency) + const expectedPage = ordered.slice(offset, offset + limit) + + if (result.total !== entries.length) return false + if (result.profiles.length !== expectedPage.length) return false + + return result.profiles.every((profile, i) => { + const source = expectedPage[i] + const stored = profilesDb.store.get(source.addr) + return profile.addr === source.addr && + profile.blockHeight === source.blockHeight && + profile.seen === source.seen && + profile.text === stored.text && + profile.txid === stored.txid + }) + }, { label: 'recent profiles ordering and pagination' }) +}) diff --git a/psf-memo-indexer/src/use-cases/action-types/post.js b/psf-memo-indexer/src/use-cases/action-types/post.js index a01fbd5..edcb29d 100644 --- a/psf-memo-indexer/src/use-cases/action-types/post.js +++ b/psf-memo-indexer/src/use-cases/action-types/post.js @@ -11,6 +11,13 @@ async function createIfMissing (db, key, value) { } } +// Only top-level posts and topic messages qualify for profile recency. Replies +// reuse handlePost to store their post record but must not move the author's +// recency. +function qualifiesForRecency (decoded) { + return decoded.action === 'post' || decoded.action === 'topicMessage' +} + export async function handlePost (ctx) { const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas) @@ -37,10 +44,7 @@ export async function handlePost (ctx) { await createIfMissing(adapters.postHeightDb, heightKey, { txid, blockHeight }) await createIfMissing(adapters.addrPostHeightDb, addrHeightKey, { txid, addr: signerAddr, blockHeight }) - // Only top-level posts and topic messages qualify for profile recency. - // Replies reuse handlePost to store their post record but must not move the - // author's recency. - if (decoded.action === 'post' || decoded.action === 'topicMessage') { + if (qualifiesForRecency(decoded)) { await recordProfileRecency(adapters, signerAddr, blockHeight, seen) } } diff --git a/psf-memo-indexer/src/use-cases/action-types/profile-recency.js b/psf-memo-indexer/src/use-cases/action-types/profile-recency.js index 7b85485..bda2761 100644 --- a/psf-memo-indexer/src/use-cases/action-types/profile-recency.js +++ b/psf-memo-indexer/src/use-cases/action-types/profile-recency.js @@ -63,6 +63,38 @@ export async function recordProfileRecency (adapters, addr, blockHeight, seen) { return upsertProfileRecency(adapters, addr, blockHeight ?? 0, seen ?? 0) } +// The qualifying post for one addrPostHeights entry, or null when the entry is +// for another address, has no txid, is unconfirmed, is a reply, or is a poll. +async function qualifyingCandidate (adapters, key, value, addr, chainBlockHeight) { + if (!String(key).startsWith(`${addr}:`)) return null + const txid = value?.txid + if (!txid) return null + const blockHeight = value?.blockHeight ?? 0 + if (!isConfirmed(blockHeight, chainBlockHeight)) return null + if (await getIfPresent(adapters.postParentDb, txid)) return null + if (await getIfPresent(adapters.pollDb, txid)) return null + const post = await getIfPresent(adapters.postDb, txid) + return { blockHeight, seen: post?.seen ?? 0 } +} + +// Return whichever of two candidates is newer: greater height, or equal height +// with a greater seen value. The current candidate wins a tie. +function newerCandidate (current, candidate) { + if (!current) return candidate + if (candidate.blockHeight !== current.blockHeight) return candidate.blockHeight > current.blockHeight ? candidate : current + return candidate.seen > current.seen ? candidate : current +} + +async function findNewestQualifyingPost (adapters, addr, chainBlockHeight) { + const range = { gte: `${addr}:`, lte: `${addr}:\uffff` } + let best = null + for await (const [key, value] of adapters.addrPostHeightDb.iterator(range)) { + const candidate = await qualifyingCandidate(adapters, key, value, addr, chainBlockHeight) + if (candidate) best = newerCandidate(best, candidate) + } + return best +} + // Establish a profile's recency from the posts already indexed for its // address. Runs when a set-profile action creates the profile, so a profile // set after posting still reports the existing newest confirmed qualifying @@ -72,25 +104,7 @@ export async function establishProfileRecency (adapters, addr) { if (!adapters.profileRecencyDb || !adapters.addrPostHeightDb) return null const chainBlockHeight = await getChainBlockHeight(adapters) - const range = { gte: `${addr}:`, lte: `${addr}:\uffff` } - let best = null - - for await (const [key, value] of adapters.addrPostHeightDb.iterator(range)) { - if (!String(key).startsWith(`${addr}:`)) continue - const txid = value?.txid - const blockHeight = value?.blockHeight ?? 0 - if (!txid) continue - if (!isConfirmed(blockHeight, chainBlockHeight)) continue - if (await getIfPresent(adapters.postParentDb, txid)) continue - if (await getIfPresent(adapters.pollDb, txid)) continue - - const post = await getIfPresent(adapters.postDb, txid) - const seen = post?.seen ?? 0 - if (!best || blockHeight > best.blockHeight || (blockHeight === best.blockHeight && seen > best.seen)) { - best = { blockHeight, seen } - } - } - + const best = await findNewestQualifyingPost(adapters, addr, chainBlockHeight) if (!best) return null return upsertProfileRecency(adapters, addr, best.blockHeight, best.seen) } diff --git a/psf-memo-indexer/test/property/profile-recency.property.test.js b/psf-memo-indexer/test/property/profile-recency.property.test.js new file mode 100644 index 0000000..4a089dd --- /dev/null +++ b/psf-memo-indexer/test/property/profile-recency.property.test.js @@ -0,0 +1,150 @@ +/* + Property tests for the indexer's profileRecency maintenance. + + The unit tests probe fixed fixtures. These properties pin the invariants that + must hold across broad random, out-of-order post streams: + + - Convergence: recordProfileRecency ends on the greatest confirmed height, + with seen as the tie-breaker, regardless of processing order. + - Confirmation: posts above status.chainBlockHeight never move recency. + - Idempotence: replaying the same stream leaves the index unchanged. + - Establishment: establishProfileRecency recovers the same newest confirmed + qualifying post from addrPostHeights, excluding replies and polls. +*/ + +import test from 'node:test' + +import { seededRandom, forAll, intGen } from './harness.js' +import { makeMemoryDb } from '../support/memory-db.js' +import { + recordProfileRecency, + establishProfileRecency +} from '../../src/use-cases/action-types/profile-recency.js' + +const rng = seededRandom(20260921) +const ADDR = 'bitcoincash:qaddr-a' + +function makeAdapters (chainBlockHeight) { + return { + profileDb: makeMemoryDb(), + profileRecencyDb: makeMemoryDb(), + addrPostHeightDb: makeMemoryDb(), + postParentDb: makeMemoryDb(), + pollDb: makeMemoryDb(), + postDb: makeMemoryDb(), + statusDb: { getStatus: async () => ({ chainBlockHeight }) } + } +} + +function expectedBest (posts, chainBlockHeight) { + let best = null + for (const post of posts) { + if (post.blockHeight > chainBlockHeight) continue + if (!best || post.blockHeight > best.blockHeight || (post.blockHeight === best.blockHeight && post.seen > best.seen)) { + best = post + } + } + return best +} + +function postStreamGen () { + return () => { + const chainBlockHeight = intGen(rng, 10, 1000)() + const count = intGen(rng, 0, 12)() + const posts = [] + for (let i = 0; i < count; i++) { + posts.push({ + blockHeight: chainBlockHeight + intGen(rng, -4, 4)(), + seen: intGen(rng, 0, 50)() + }) + } + const shuffled = [...posts] + for (let i = shuffled.length - 1; i > 0; i--) { + const j = Math.floor(rng() * (i + 1)) + const swap = shuffled[i] + shuffled[i] = shuffled[j] + shuffled[j] = swap + } + return { chainBlockHeight, posts: shuffled } + } +} + +test('recordProfileRecency converges on the newest confirmed post and is idempotent', async () => { + await forAll(postStreamGen(), async ({ chainBlockHeight, posts }) => { + const adapters = makeAdapters(chainBlockHeight) + await adapters.profileDb.update(ADDR, { addr: ADDR, text: 'bio' }) + + for (const post of posts) { + await recordProfileRecency(adapters, ADDR, post.blockHeight, post.seen) + } + + const expected = expectedBest(posts, chainBlockHeight) + const stored = adapters.profileRecencyDb.store.get(ADDR) + + if (!expected) { + if (stored !== undefined) return false + } else { + if (!stored) return false + if (stored.blockHeight !== expected.blockHeight || stored.seen !== expected.seen) return false + } + + const before = JSON.stringify(stored ?? null) + for (const post of posts) { + await recordProfileRecency(adapters, ADDR, post.blockHeight, post.seen) + } + return JSON.stringify(adapters.profileRecencyDb.store.get(ADDR) ?? null) === before + }, { label: 'recordProfileRecency convergence and idempotence' }) +}) + +function establishWorldGen () { + return () => { + const chainBlockHeight = intGen(rng, 10, 1000)() + const adapters = makeAdapters(chainBlockHeight) + const entries = [] + const count = intGen(rng, 0, 10)() + + for (let i = 0; i < count; i++) { + const txid = `tx-${i}` + const blockHeight = chainBlockHeight + intGen(rng, -4, 4)() + const seen = intGen(rng, 0, 50)() + const key = `${ADDR}:${String(blockHeight).padStart(12, '0')}:${txid}` + adapters.addrPostHeightDb.store.set(key, { txid, addr: ADDR, blockHeight }) + adapters.postDb.store.set(txid, { addr: ADDR, seen, blockHeight }) + + const roll = rng() + let kind = 'post' + if (roll < 0.3) { + kind = 'reply' + adapters.postParentDb.store.set(txid, { txid, parentTxid: 'parent' }) + } else if (roll < 0.5) { + kind = 'poll' + adapters.pollDb.store.set(txid, { txid }) + } + entries.push({ txid, blockHeight, seen, kind }) + } + + return { adapters, entries, chainBlockHeight } + } +} + +test('establishProfileRecency recovers the newest confirmed qualifying post', async () => { + await forAll(establishWorldGen(), async ({ adapters, entries, chainBlockHeight }) => { + await establishProfileRecency(adapters, ADDR) + + let expected = null + for (const entry of entries) { + if (entry.kind !== 'post') continue + if (entry.blockHeight > chainBlockHeight) continue + if (!expected || entry.blockHeight > expected.blockHeight || + (entry.blockHeight === expected.blockHeight && entry.seen > expected.seen)) { + expected = entry + } + } + + const stored = adapters.profileRecencyDb.store.get(ADDR) + if (!expected) return stored === undefined + return Boolean(stored) && + stored.blockHeight === expected.blockHeight && + stored.seen === expected.seen + }, { label: 'establishProfileRecency newest qualifying post' }) +})