Refactor profile-last-post: split recency logic, add property tests

Split backfillProfileRecency and establishProfileRecency into small
helpers so every analyzed function scores CRAP 6 or below, without
changing behavior. Extract the post recency-qualification check in
handlePost.

Add seeded property tests covering backfill conservation and
idempotence, the addrPostHeight key round trip, recent-profile read
ordering and pagination, and the indexer's recency convergence.

By refactorer.

psf-memo-db/src/lib/backfill-profile-recency.js

@@ -56,50 +56,86 @@ export function partsFromAddrPostHeightKey (key) {
   }
 }
 
-// Keep the newest confirmed qualifying post per address.
-export async function backfillProfileRecency ({
-  profilesDb,
-  postsDb,
-  addrPostHeightsDb,
-  postParentsDb,
-  pollsDb,
-  statusDb,
-  profileRecencyDb
-}) {
-  const chainBlockHeight = await readChainBlockHeight(statusDb)
-  const best = new Map()
-
-  for await (const [key, value] of addrPostHeightsDb.iterator()) {
-    const fallback = partsFromAddrPostHeightKey(key)
-    const addr = value?.addr ?? fallback.addr
-    const txid = value?.txid ?? fallback.txid
-    const blockHeight = value?.blockHeight ?? fallback.blockHeight
-    if (!addr || !txid) continue
-    if (chainBlockHeight !== null && blockHeight > chainBlockHeight) continue
-    if (await getRecord(postParentsDb, txid)) continue
-    if (await getRecord(pollsDb, txid)) continue
-    if (!(await getRecord(profilesDb, addr))) continue
-
-    const post = await getRecord(postsDb, txid)
-    const seen = post?.seen ?? 0
-    const current = best.get(addr)
-    if (!current || blockHeight > current.blockHeight || (blockHeight === current.blockHeight && seen > current.seen)) {
-      best.set(addr, { addr, blockHeight, seen })
-    }
+// Prefer the stored value fields, falling back to the key segments when a
+// record predates the field being written.
+function entryFields (key, value) {
+  const fallback = partsFromAddrPostHeightKey(key)
+  return {
+    addr: value?.addr ?? fallback.addr,
+    txid: value?.txid ?? fallback.txid,
+    blockHeight: value?.blockHeight ?? fallback.blockHeight
   }
+}
 
+// An entry at or below the chain tip is confirmed. With no tip every entry is
+// treated as confirmed.
+function isConfirmedEntry (blockHeight, chainBlockHeight) {
+  if (chainBlockHeight === null) return true
+  return blockHeight <= chainBlockHeight
+}
+
+// A qualifying post is authored by a profile address and is neither a reply
+// (tracked in postParents) nor a poll creation (tracked in polls).
+async function isQualifyingPost (stores, addr, txid) {
+  if (!(await getRecord(stores.profilesDb, addr))) return false
+  if (await getRecord(stores.postParentsDb, txid)) return false
+  if (await getRecord(stores.pollsDb, txid)) return false
+  return true
+}
+
+// The qualifying post for one addrPostHeights entry, or null when the entry
+// does not qualify.
+async function qualifyingCandidate (stores, key, value, chainBlockHeight) {
+  const { addr, txid, blockHeight } = entryFields(key, value)
+  if (!addr || !txid) return null
+  if (!isConfirmedEntry(blockHeight, chainBlockHeight)) return null
+  if (!(await isQualifyingPost(stores, addr, txid))) return null
+  const post = await getRecord(stores.postsDb, txid)
+  return { addr, blockHeight, seen: post?.seen ?? 0 }
+}
+
+// True when `candidate` is newer than `current`: greater height, or equal
+// height with a greater seen value.
+function isNewer (candidate, current) {
+  if (!current) return true
+  if (candidate.blockHeight !== current.blockHeight) return candidate.blockHeight > current.blockHeight
+  return candidate.seen > current.seen
+}
+
+// Keep the newest candidate per address. Equal height and seen keeps the first
+// record seen, which makes reprocessing idempotent.
+function keepNewest (best, candidate) {
+  if (isNewer(candidate, best.get(candidate.addr))) best.set(candidate.addr, candidate)
+}
+
+async function collectBestRecency (stores, chainBlockHeight) {
+  const best = new Map()
+  for await (const [key, value] of stores.addrPostHeightsDb.iterator()) {
+    const candidate = await qualifyingCandidate(stores, key, value, chainBlockHeight)
+    if (candidate) keepNewest(best, candidate)
+  }
+  return best
+}
+
+async function writeRecency (profileRecencyDb, best) {
   for (const record of best.values()) {
     await profileRecencyDb.put(record.addr, record)
   }
+}
 
-  const desired = new Set(best.keys())
-  const stale = []
+async function removeStaleRecency (profileRecencyDb, desired) {
   for await (const [key] of profileRecencyDb.iterator()) {
-    if (!desired.has(key)) stale.push(key)
-  }
-  for (const key of stale) {
-    await profileRecencyDb.del(key)
+    if (!desired.has(key)) await profileRecencyDb.del(key)
   }
+}
 
+// Rebuild the whole index: collect the newest confirmed qualifying post per
+// profile address, write those records, and drop records that no longer
+// qualify.
+export async function backfillProfileRecency (stores) {
+  const chainBlockHeight = await readChainBlockHeight(stores.statusDb)
+  const best = await collectBestRecency(stores, chainBlockHeight)
+  await writeRecency(stores.profileRecencyDb, best)
+  await removeStaleRecency(stores.profileRecencyDb, new Set(best.keys()))
   return { profiles: best.size }
 }

psf-memo-db/test/property/profile-recency-order.property.test.js

@@ -0,0 +1,195 @@
+/*
+  Property tests for the profileRecency backfill and the recent-profiles read
+  ordering.
+
+  The unit tests probe fixed fixtures. These properties pin the invariants over
+  broad random stores:
+
+    - Conservation: backfillProfileRecency writes exactly the newest confirmed
+      qualifying post per profile address, excluding replies, polls, and
+      unconfirmed entries, and writes nothing for addresses without a profile.
+    - Idempotence: a second backfill leaves the index byte-for-byte identical.
+    - Stale removal: pre-existing records that no longer qualify are dropped.
+    - Key round trip: partsFromAddrPostHeightKey recovers the address, height,
+      and txid from a formatted addrPostHeights key, including cash addresses
+      that contain colons.
+    - Read ordering: ProfileQuery.listRecentProfiles returns the requested page
+      in height-desc, seen-desc, address-asc order with conserved pagination.
+*/
+
+import test from 'node:test'
+
+import { seededRandom, forAll, intGen, txidGen } from './harness.js'
+import { backfillProfileRecency, partsFromAddrPostHeightKey } from '../../src/lib/backfill-profile-recency.js'
+import ProfileQuery from '../../src/adapters/profile-query.js'
+import { FakeDb } from '../support/level-double.js'
+
+const rng = seededRandom(20260921)
+
+const ADDRS = [
+  'bitcoincash:qaddr-a',
+  'bitcoincash:qaddr-b',
+  'bitcoincash:qaddr-c',
+  'bitcoincash:qaddr-d'
+]
+
+function pad (height) {
+  return String(height).padStart(12, '0')
+}
+
+function addrPostHeightKey (addr, height, txid) {
+  return `${addr}:${pad(height)}:${txid}`
+}
+
+function isNewer (candidate, current) {
+  if (!current) return true
+  if (candidate.blockHeight !== current.blockHeight) return candidate.blockHeight > current.blockHeight
+  return candidate.seen > current.seen
+}
+
+function recencyWorldGen () {
+  return () => {
+    const chainBlockHeight = intGen(rng, 1000, 1100)()
+    const stores = {
+      profilesDb: new FakeDb(),
+      postsDb: new FakeDb(),
+      addrPostHeightsDb: new FakeDb(),
+      postParentsDb: new FakeDb(),
+      pollsDb: new FakeDb(),
+      statusDb: new FakeDb([['status', { chainBlockHeight }]]),
+      profileRecencyDb: new FakeDb()
+    }
+    const expected = new Map()
+    let seq = 0
+
+    for (const addr of ADDRS) {
+      const hasProfile = rng() < 0.75
+      if (hasProfile) {
+        stores.profilesDb.store.set(addr, { text: `bio-${addr}`, txid: `profile-${addr}` })
+      }
+
+      const postCount = intGen(rng, 0, 5)()
+      for (let i = 0; i < postCount; i++) {
+        const txid = `tx-${seq++}`
+        const blockHeight = chainBlockHeight + intGen(rng, -20, 20)()
+        const seen = intGen(rng, 0, 1000)()
+        stores.addrPostHeightsDb.store.set(addrPostHeightKey(addr, blockHeight, txid), { txid, addr, blockHeight })
+        stores.postsDb.store.set(txid, { addr, seen, blockHeight })
+
+        const roll = rng()
+        if (roll < 0.25) {
+          stores.postParentsDb.store.set(txid, { txid, parentTxid: 'parent' })
+        } else if (roll < 0.5) {
+          stores.pollsDb.store.set(txid, { txid })
+        } else if (hasProfile && blockHeight <= chainBlockHeight) {
+          const candidate = { addr, blockHeight, seen }
+          if (isNewer(candidate, expected.get(addr))) expected.set(addr, candidate)
+        }
+      }
+    }
+
+    // Seed a stale record for an address that never qualifies so the backfill
+    // must remove it as part of rebuilding the index.
+    const staleAddr = 'bitcoincash:qaddr-stale'
+    stores.profileRecencyDb.store.set(staleAddr, { addr: staleAddr, blockHeight: 999999, seen: 1 })
+
+    return { stores, expected }
+  }
+}
+
+function snapshot (db) {
+  return JSON.stringify([...db.store.entries()].sort())
+}
+
+test('backfillProfileRecency conserves the newest confirmed qualifying post per profile', async () => {
+  await forAll(recencyWorldGen(), async ({ stores, expected }) => {
+    const result = await backfillProfileRecency(stores)
+
+    if (result.profiles !== expected.size) return false
+    if (stores.profileRecencyDb.store.size !== expected.size) return false
+
+    for (const [addr, record] of expected) {
+      const stored = stores.profileRecencyDb.store.get(addr)
+      if (!stored) return false
+      if (stored.addr !== record.addr) return false
+      if (stored.blockHeight !== record.blockHeight) return false
+      if (stored.seen !== record.seen) return false
+    }
+
+    const before = snapshot(stores.profileRecencyDb)
+    await backfillProfileRecency(stores)
+    return snapshot(stores.profileRecencyDb) === before
+  }, { label: 'profile recency backfill conservation and idempotence' })
+})
+
+test('partsFromAddrPostHeightKey round-trips a formatted addrPostHeights key', async () => {
+  await forAll(
+    () => ({
+      addr: ADDRS[Math.floor(rng() * ADDRS.length)],
+      height: intGen(rng, 0, 9999999)(),
+      txid: txidGen(rng)
+    }),
+    ({ addr, height, txid }) => {
+      const parts = partsFromAddrPostHeightKey(addrPostHeightKey(addr, height, txid))
+      return parts.addr === addr && parts.blockHeight === height && parts.txid === txid
+    },
+    { label: 'addrPostHeight key round trip' }
+  )
+})
+
+function compareRecency (a, b) {
+  if (b.blockHeight !== a.blockHeight) return b.blockHeight - a.blockHeight
+  if (b.seen !== a.seen) return b.seen - a.seen
+  if (a.addr === b.addr) return 0
+  return a.addr < b.addr ? -1 : 1
+}
+
+function orderingWorldGen () {
+  return () => {
+    const count = intGen(rng, 0, 10)()
+    const entries = []
+    const profilesDb = new FakeDb()
+    const profileRecencyDb = new FakeDb()
+
+    for (let i = 0; i < count; i++) {
+      const addr = `bitcoincash:qaddr-${i}`
+      const blockHeight = intGen(rng, 0, 5)()
+      const seen = intGen(rng, 0, 5)()
+      entries.push({ addr, blockHeight, seen })
+      profileRecencyDb.store.set(addr, { addr, blockHeight, seen })
+      profilesDb.store.set(addr, { text: `bio ${i}`, txid: `profile-${i}` })
+    }
+
+    return {
+      entries,
+      profilesDb,
+      profileRecencyDb,
+      limit: intGen(rng, 0, 12)(),
+      offset: intGen(rng, 0, count + 2)()
+    }
+  }
+}
+
+test('listRecentProfiles returns the ordering and pagination contract', async () => {
+  await forAll(orderingWorldGen(), async ({ entries, profilesDb, profileRecencyDb, limit, offset }) => {
+    const query = new ProfileQuery({ profilesDb, profileRecencyDb })
+
+    const result = await query.listRecentProfiles({ limit, offset })
+
+    const ordered = [...entries].sort(compareRecency)
+    const expectedPage = ordered.slice(offset, offset + limit)
+
+    if (result.total !== entries.length) return false
+    if (result.profiles.length !== expectedPage.length) return false
+
+    return result.profiles.every((profile, i) => {
+      const source = expectedPage[i]
+      const stored = profilesDb.store.get(source.addr)
+      return profile.addr === source.addr &&
+        profile.blockHeight === source.blockHeight &&
+        profile.seen === source.seen &&
+        profile.text === stored.text &&
+        profile.txid === stored.txid
+    })
+  }, { label: 'recent profiles ordering and pagination' })
+})

psf-memo-indexer/src/use-cases/action-types/post.js

@@ -11,6 +11,13 @@ async function createIfMissing (db, key, value) {
   }
 }
 
+// Only top-level posts and topic messages qualify for profile recency. Replies
+// reuse handlePost to store their post record but must not move the author's
+// recency.
+function qualifiesForRecency (decoded) {
+  return decoded.action === 'post' || decoded.action === 'topicMessage'
+}
+
 export async function handlePost (ctx) {
   const { adapters, txid, signerAddr, decoded, seen, blockHeight } = ctx
   const pushDatas = normalizeTwoPushMemoDatas(decoded.pushDatas)
@@ -37,10 +44,7 @@ export async function handlePost (ctx) {
   await createIfMissing(adapters.postHeightDb, heightKey, { txid, blockHeight })
   await createIfMissing(adapters.addrPostHeightDb, addrHeightKey, { txid, addr: signerAddr, blockHeight })
 
-  // Only top-level posts and topic messages qualify for profile recency.
-  // Replies reuse handlePost to store their post record but must not move the
-  // author's recency.
-  if (decoded.action === 'post' || decoded.action === 'topicMessage') {
+  if (qualifiesForRecency(decoded)) {
     await recordProfileRecency(adapters, signerAddr, blockHeight, seen)
   }
 }

psf-memo-indexer/src/use-cases/action-types/profile-recency.js

@@ -63,6 +63,38 @@ export async function recordProfileRecency (adapters, addr, blockHeight, seen) {
   return upsertProfileRecency(adapters, addr, blockHeight ?? 0, seen ?? 0)
 }
 
+// The qualifying post for one addrPostHeights entry, or null when the entry is
+// for another address, has no txid, is unconfirmed, is a reply, or is a poll.
+async function qualifyingCandidate (adapters, key, value, addr, chainBlockHeight) {
+  if (!String(key).startsWith(`${addr}:`)) return null
+  const txid = value?.txid
+  if (!txid) return null
+  const blockHeight = value?.blockHeight ?? 0
+  if (!isConfirmed(blockHeight, chainBlockHeight)) return null
+  if (await getIfPresent(adapters.postParentDb, txid)) return null
+  if (await getIfPresent(adapters.pollDb, txid)) return null
+  const post = await getIfPresent(adapters.postDb, txid)
+  return { blockHeight, seen: post?.seen ?? 0 }
+}
+
+// Return whichever of two candidates is newer: greater height, or equal height
+// with a greater seen value. The current candidate wins a tie.
+function newerCandidate (current, candidate) {
+  if (!current) return candidate
+  if (candidate.blockHeight !== current.blockHeight) return candidate.blockHeight > current.blockHeight ? candidate : current
+  return candidate.seen > current.seen ? candidate : current
+}
+
+async function findNewestQualifyingPost (adapters, addr, chainBlockHeight) {
+  const range = { gte: `${addr}:`, lte: `${addr}:\uffff` }
+  let best = null
+  for await (const [key, value] of adapters.addrPostHeightDb.iterator(range)) {
+    const candidate = await qualifyingCandidate(adapters, key, value, addr, chainBlockHeight)
+    if (candidate) best = newerCandidate(best, candidate)
+  }
+  return best
+}
+
 // Establish a profile's recency from the posts already indexed for its
 // address. Runs when a set-profile action creates the profile, so a profile
 // set after posting still reports the existing newest confirmed qualifying
@@ -72,25 +104,7 @@ export async function establishProfileRecency (adapters, addr) {
   if (!adapters.profileRecencyDb || !adapters.addrPostHeightDb) return null
 
   const chainBlockHeight = await getChainBlockHeight(adapters)
-  const range = { gte: `${addr}:`, lte: `${addr}:\uffff` }
-  let best = null
-
-  for await (const [key, value] of adapters.addrPostHeightDb.iterator(range)) {
-    if (!String(key).startsWith(`${addr}:`)) continue
-    const txid = value?.txid
-    const blockHeight = value?.blockHeight ?? 0
-    if (!txid) continue
-    if (!isConfirmed(blockHeight, chainBlockHeight)) continue
-    if (await getIfPresent(adapters.postParentDb, txid)) continue
-    if (await getIfPresent(adapters.pollDb, txid)) continue
-
-    const post = await getIfPresent(adapters.postDb, txid)
-    const seen = post?.seen ?? 0
-    if (!best || blockHeight > best.blockHeight || (blockHeight === best.blockHeight && seen > best.seen)) {
-      best = { blockHeight, seen }
-    }
-  }
-
+  const best = await findNewestQualifyingPost(adapters, addr, chainBlockHeight)
   if (!best) return null
   return upsertProfileRecency(adapters, addr, best.blockHeight, best.seen)
 }

psf-memo-indexer/test/property/profile-recency.property.test.js

@@ -0,0 +1,150 @@
+/*
+  Property tests for the indexer's profileRecency maintenance.
+
+  The unit tests probe fixed fixtures. These properties pin the invariants that
+  must hold across broad random, out-of-order post streams:
+
+    - Convergence: recordProfileRecency ends on the greatest confirmed height,
+      with seen as the tie-breaker, regardless of processing order.
+    - Confirmation: posts above status.chainBlockHeight never move recency.
+    - Idempotence: replaying the same stream leaves the index unchanged.
+    - Establishment: establishProfileRecency recovers the same newest confirmed
+      qualifying post from addrPostHeights, excluding replies and polls.
+*/
+
+import test from 'node:test'
+
+import { seededRandom, forAll, intGen } from './harness.js'
+import { makeMemoryDb } from '../support/memory-db.js'
+import {
+  recordProfileRecency,
+  establishProfileRecency
+} from '../../src/use-cases/action-types/profile-recency.js'
+
+const rng = seededRandom(20260921)
+const ADDR = 'bitcoincash:qaddr-a'
+
+function makeAdapters (chainBlockHeight) {
+  return {
+    profileDb: makeMemoryDb(),
+    profileRecencyDb: makeMemoryDb(),
+    addrPostHeightDb: makeMemoryDb(),
+    postParentDb: makeMemoryDb(),
+    pollDb: makeMemoryDb(),
+    postDb: makeMemoryDb(),
+    statusDb: { getStatus: async () => ({ chainBlockHeight }) }
+  }
+}
+
+function expectedBest (posts, chainBlockHeight) {
+  let best = null
+  for (const post of posts) {
+    if (post.blockHeight > chainBlockHeight) continue
+    if (!best || post.blockHeight > best.blockHeight || (post.blockHeight === best.blockHeight && post.seen > best.seen)) {
+      best = post
+    }
+  }
+  return best
+}
+
+function postStreamGen () {
+  return () => {
+    const chainBlockHeight = intGen(rng, 10, 1000)()
+    const count = intGen(rng, 0, 12)()
+    const posts = []
+    for (let i = 0; i < count; i++) {
+      posts.push({
+        blockHeight: chainBlockHeight + intGen(rng, -4, 4)(),
+        seen: intGen(rng, 0, 50)()
+      })
+    }
+    const shuffled = [...posts]
+    for (let i = shuffled.length - 1; i > 0; i--) {
+      const j = Math.floor(rng() * (i + 1))
+      const swap = shuffled[i]
+      shuffled[i] = shuffled[j]
+      shuffled[j] = swap
+    }
+    return { chainBlockHeight, posts: shuffled }
+  }
+}
+
+test('recordProfileRecency converges on the newest confirmed post and is idempotent', async () => {
+  await forAll(postStreamGen(), async ({ chainBlockHeight, posts }) => {
+    const adapters = makeAdapters(chainBlockHeight)
+    await adapters.profileDb.update(ADDR, { addr: ADDR, text: 'bio' })
+
+    for (const post of posts) {
+      await recordProfileRecency(adapters, ADDR, post.blockHeight, post.seen)
+    }
+
+    const expected = expectedBest(posts, chainBlockHeight)
+    const stored = adapters.profileRecencyDb.store.get(ADDR)
+
+    if (!expected) {
+      if (stored !== undefined) return false
+    } else {
+      if (!stored) return false
+      if (stored.blockHeight !== expected.blockHeight || stored.seen !== expected.seen) return false
+    }
+
+    const before = JSON.stringify(stored ?? null)
+    for (const post of posts) {
+      await recordProfileRecency(adapters, ADDR, post.blockHeight, post.seen)
+    }
+    return JSON.stringify(adapters.profileRecencyDb.store.get(ADDR) ?? null) === before
+  }, { label: 'recordProfileRecency convergence and idempotence' })
+})
+
+function establishWorldGen () {
+  return () => {
+    const chainBlockHeight = intGen(rng, 10, 1000)()
+    const adapters = makeAdapters(chainBlockHeight)
+    const entries = []
+    const count = intGen(rng, 0, 10)()
+
+    for (let i = 0; i < count; i++) {
+      const txid = `tx-${i}`
+      const blockHeight = chainBlockHeight + intGen(rng, -4, 4)()
+      const seen = intGen(rng, 0, 50)()
+      const key = `${ADDR}:${String(blockHeight).padStart(12, '0')}:${txid}`
+      adapters.addrPostHeightDb.store.set(key, { txid, addr: ADDR, blockHeight })
+      adapters.postDb.store.set(txid, { addr: ADDR, seen, blockHeight })
+
+      const roll = rng()
+      let kind = 'post'
+      if (roll < 0.3) {
+        kind = 'reply'
+        adapters.postParentDb.store.set(txid, { txid, parentTxid: 'parent' })
+      } else if (roll < 0.5) {
+        kind = 'poll'
+        adapters.pollDb.store.set(txid, { txid })
+      }
+      entries.push({ txid, blockHeight, seen, kind })
+    }
+
+    return { adapters, entries, chainBlockHeight }
+  }
+}
+
+test('establishProfileRecency recovers the newest confirmed qualifying post', async () => {
+  await forAll(establishWorldGen(), async ({ adapters, entries, chainBlockHeight }) => {
+    await establishProfileRecency(adapters, ADDR)
+
+    let expected = null
+    for (const entry of entries) {
+      if (entry.kind !== 'post') continue
+      if (entry.blockHeight > chainBlockHeight) continue
+      if (!expected || entry.blockHeight > expected.blockHeight ||
+        (entry.blockHeight === expected.blockHeight && entry.seen > expected.seen)) {
+        expected = entry
+      }
+    }
+
+    const stored = adapters.profileRecencyDb.store.get(ADDR)
+    if (!expected) return stored === undefined
+    return Boolean(stored) &&
+      stored.blockHeight === expected.blockHeight &&
+      stored.seen === expected.seen
+  }, { label: 'establishProfileRecency newest qualifying post' })
+})