Implement profile ordering by last post

Indexer: maintain a profileRecency index keyed by address with the newest
confirmed qualifying post (top-level 0x6d02 or topic message 0x6d0c).
Replies and poll creations do not qualify, and mempool (unconfirmed) posts do
not write the index. Setting a profile after posting establishes recency from
the address's existing addrPostHeights, excluding replies and polls.

DB: GET /profile/recent now orders profiles by their most recent qualifying
post (height desc, seen desc, address asc) from profileRecency plus per-page
profile lookups, so it no longer scans addrPostHeights or sorts the whole
profiles store. Add a profileRecency backfill library and CLI that rebuilds
the index from addrPostHeights, posts, postParents, polls, and status,
ignoring unconfirmed entries and removing stale records.

Add focused unit tests, acceptance fixtures, and regex step handlers.

By coder.

psf-memo-db/test/property/recent-profile-identity.property.test.js

@@ -119,11 +119,11 @@ function recentProfilesSetGen () {
   return { profiles, identities }
 }
 
-function makeUseCase (profiles, identities, joined) {
+function makeUseCase (page, total, identities, joined) {
   return new ListRecentProfiles({
     adapters: {
       profileQuery: {
-        scanProfilesWithBlockHeight: async () => profiles.map((profile) => ({ ...profile })),
+        listRecentProfiles: async () => ({ profiles: page.map((profile) => ({ ...profile })), total }),
         getProfileIdentity: async (addr) => {
           joined.push(addr)
           return identities[addr]
@@ -133,7 +133,7 @@ function makeUseCase (profiles, identities, joined) {
   })
 }
 
-test('ListRecentProfiles enriches exactly the requested page without changing order or pagination', async () => {
+test('ListRecentProfiles enriches exactly the returned page without changing order or pagination', async () => {
   await forAll(
     (i) => {
       const { profiles, identities } = recentProfilesSetGen()
@@ -146,10 +146,9 @@ test('ListRecentProfiles enriches exactly the requested page without changing or
     },
     async ({ profiles, identities, limit, offset }) => {
       const joined = []
-      const result = await makeUseCase(profiles, identities, joined).execute({ limit, offset })
-
       const expectedOrder = [...profiles].sort(sortByHeightDesc)
       const expectedPage = expectedOrder.slice(offset, offset + limit)
+      const result = await makeUseCase(expectedPage, profiles.length, identities, joined).execute({ limit, offset })
 
       if (result.profiles.length !== expectedPage.length) return false
       if (result.pagination.limit !== limit) return false

psf-memo-db/test/unit/adapters/profile-query.unit.js

@@ -6,48 +6,121 @@ describe('#ProfileQuery', () => {
   let uut
   let sandbox
   let profilesDb
+  let profileRecencyDb
   let namesDb
   let profilePicsDb
 
   beforeEach(() => {
     sandbox = sinon.createSandbox()
-    profilesDb = {
-      iterator: sandbox.stub()
-    }
-    namesDb = {
-      get: sandbox.stub()
-    }
-    profilePicsDb = {
-      get: sandbox.stub()
-    }
-    uut = new ProfileQuery({ profilesDb, namesDb, profilePicsDb })
+    profilesDb = { get: sandbox.stub(), iterator: sandbox.stub() }
+    profileRecencyDb = { iterator: sandbox.stub() }
+    namesDb = { get: sandbox.stub() }
+    profilePicsDb = { get: sandbox.stub() }
+    uut = new ProfileQuery({ profilesDb, namesDb, profilePicsDb, profileRecencyDb })
   })
 
   afterEach(() => sandbox.restore())
 
-  it('should scan profiles and read block height from stored document', async () => {
-    async function * mockIterator () {
-      yield ['addr1', { text: 'hi', txid: 'tx1', seen: 1000, blockHeight: 600100 }]
-      yield ['addr2', { text: 'bye', txid: 'tx2', seen: 2000, blockHeight: 600200 }]
-    }
-    profilesDb.iterator.returns(mockIterator())
+  function stubRecency (records) {
+    profileRecencyDb.iterator.returns((async function * () {
+      for (const [addr, record] of Object.entries(records)) {
+        yield [addr, record]
+      }
+    })())
+  }
 
-    const result = await uut.scanProfilesWithBlockHeight()
+  function stubProfiles (profiles) {
+    profilesDb.get.callsFake(async (addr) => {
+      if (Object.prototype.hasOwnProperty.call(profiles, addr)) return profiles[addr]
+      const err = new Error('not found')
+      err.notFound = true
+      throw err
+    })
+  }
 
-    assert.equal(result.length, 2)
-    assert.equal(result[0].blockHeight, 600100)
-    assert.equal(result[1].blockHeight, 600200)
+  it('should order profiles by post height descending, then seen descending, then address ascending', async () => {
+    stubRecency({
+      'bitcoincash:qaddr-alice': { addr: 'bitcoincash:qaddr-alice', blockHeight: 600300, seen: 300 },
+      'bitcoincash:qaddr-bob': { addr: 'bitcoincash:qaddr-bob', blockHeight: 600300, seen: 200 },
+      'bitcoincash:qaddr-erin': { addr: 'bitcoincash:qaddr-erin', blockHeight: 600300, seen: 200 },
+      'bitcoincash:qaddr-carol': { addr: 'bitcoincash:qaddr-carol', blockHeight: 600200, seen: 400 }
+    })
+    stubProfiles({
+      'bitcoincash:qaddr-alice': { text: 'alice bio', txid: 'profile-alice' },
+      'bitcoincash:qaddr-bob': { text: 'bob bio', txid: 'profile-bob' },
+      'bitcoincash:qaddr-erin': { text: 'erin bio', txid: 'profile-erin' },
+      'bitcoincash:qaddr-carol': { text: 'carol bio', txid: 'profile-carol' }
+    })
+
+    const { profiles, total } = await uut.listRecentProfiles({ limit: 5, offset: 0 })
+
+    assert.equal(total, 4)
+    assert.deepEqual(profiles.map((p) => p.addr), [
+      'bitcoincash:qaddr-alice',
+      'bitcoincash:qaddr-bob',
+      'bitcoincash:qaddr-erin',
+      'bitcoincash:qaddr-carol'
+    ])
   })
 
-  it('should use block height 0 when field is missing', async () => {
-    async function * mockIterator () {
-      yield ['addr1', { text: 'hi', txid: 'tx1', seen: 1000 }]
-    }
-    profilesDb.iterator.returns(mockIterator())
+  it('should report the recency block height and seen, not the profile record values', async () => {
+    stubRecency({
+      'bitcoincash:qaddr-alice': { addr: 'bitcoincash:qaddr-alice', blockHeight: 600300, seen: 300 }
+    })
+    stubProfiles({
+      'bitcoincash:qaddr-alice': { text: 'alice bio', txid: 'profile-alice', blockHeight: 600010, seen: 10 }
+    })
 
-    const result = await uut.scanProfilesWithBlockHeight()
+    const { profiles } = await uut.listRecentProfiles({ limit: 5, offset: 0 })
 
-    assert.equal(result[0].blockHeight, 0)
+    assert.deepEqual(profiles[0], {
+      addr: 'bitcoincash:qaddr-alice',
+      text: 'alice bio',
+      txid: 'profile-alice',
+      blockHeight: 600300,
+      seen: 300
+    })
+  })
+
+  it('should paginate the ordered recency entries', async () => {
+    stubRecency({
+      'bitcoincash:qaddr-a': { addr: 'bitcoincash:qaddr-a', blockHeight: 600300, seen: 3 },
+      'bitcoincash:qaddr-b': { addr: 'bitcoincash:qaddr-b', blockHeight: 600200, seen: 2 },
+      'bitcoincash:qaddr-c': { addr: 'bitcoincash:qaddr-c', blockHeight: 600100, seen: 1 }
+    })
+    stubProfiles({
+      'bitcoincash:qaddr-a': { text: 'a bio', txid: 'profile-a' },
+      'bitcoincash:qaddr-b': { text: 'b bio', txid: 'profile-b' },
+      'bitcoincash:qaddr-c': { text: 'c bio', txid: 'profile-c' }
+    })
+
+    const { profiles, total } = await uut.listRecentProfiles({ limit: 1, offset: 1 })
+
+    assert.equal(total, 3)
+    assert.deepEqual(profiles.map((p) => p.addr), ['bitcoincash:qaddr-b'])
+  })
+
+  it('should omit a recency record whose profile is missing', async () => {
+    stubRecency({
+      'bitcoincash:qaddr-a': { addr: 'bitcoincash:qaddr-a', blockHeight: 600300, seen: 3 },
+      'bitcoincash:qaddr-gone': { addr: 'bitcoincash:qaddr-gone', blockHeight: 600200, seen: 2 }
+    })
+    stubProfiles({
+      'bitcoincash:qaddr-a': { text: 'a bio', txid: 'profile-a' }
+    })
+
+    const { profiles } = await uut.listRecentProfiles({ limit: 5, offset: 0 })
+
+    assert.deepEqual(profiles.map((p) => p.addr), ['bitcoincash:qaddr-a'])
+  })
+
+  it('should return an empty page when no recency store is configured', async () => {
+    const uutWithoutRecency = new ProfileQuery({ profilesDb })
+
+    const { profiles, total } = await uutWithoutRecency.listRecentProfiles({ limit: 5, offset: 0 })
+
+    assert.deepEqual(profiles, [])
+    assert.equal(total, 0)
   })
 
   it('should join the display name and avatar for an address', async () => {

psf-memo-db/test/unit/lib/backfill-profile-recency.unit.js

@@ -0,0 +1,104 @@
+import { assert } from 'chai'
+import { backfillProfileRecency, partsFromAddrPostHeightKey } from '../../../src/lib/backfill-profile-recency.js'
+import { FakeDb } from '../../support/level-double.js'
+
+const ALICE = 'bitcoincash:qaddr-alice'
+const BOB = 'bitcoincash:qaddr-bob'
+const NOPOST = 'bitcoincash:qaddr-nopost'
+
+function pad (height) {
+  return String(height).padStart(12, '0')
+}
+
+function addrPostHeightKey (addr, height, txid) {
+  return `${addr}:${pad(height)}:${txid}`
+}
+
+function fixtureStores () {
+  const profilesDb = new FakeDb([
+    [ALICE, { text: 'alice bio', txid: 'profile-alice' }],
+    [BOB, { text: 'bob bio', txid: 'profile-bob' }],
+    [NOPOST, { text: 'nopost bio', txid: 'profile-nopost' }]
+  ])
+
+  const postsDb = new FakeDb([
+    ['post-a1', { addr: ALICE, seen: 100, blockHeight: 600100 }],
+    ['reply-a1', { addr: ALICE, seen: 150, blockHeight: 600300 }],
+    ['post-b1', { addr: BOB, seen: 200, blockHeight: 600400 }],
+    ['post-b2', { addr: BOB, seen: 250, blockHeight: 600500 }]
+  ])
+
+  const addrPostHeightsDb = new FakeDb([
+    [addrPostHeightKey(ALICE, 600100, 'post-a1'), { txid: 'post-a1', addr: ALICE, blockHeight: 600100 }],
+    [addrPostHeightKey(ALICE, 600300, 'reply-a1'), { txid: 'reply-a1', addr: ALICE, blockHeight: 600300 }],
+    [addrPostHeightKey(ALICE, 600200, 'poll-a1'), { txid: 'poll-a1', addr: ALICE, blockHeight: 600200 }],
+    [addrPostHeightKey(BOB, 600400, 'post-b1'), { txid: 'post-b1', addr: BOB, blockHeight: 600400 }],
+    [addrPostHeightKey(BOB, 600500, 'post-b2'), { txid: 'post-b2', addr: BOB, blockHeight: 600500 }]
+  ])
+
+  const postParentsDb = new FakeDb([
+    ['reply-a1', { txid: 'reply-a1', parentTxid: 'post-a1' }]
+  ])
+  const pollsDb = new FakeDb([
+    ['poll-a1', { txid: 'poll-a1' }]
+  ])
+  const statusDb = new FakeDb([
+    ['status', { chainBlockHeight: 600450 }]
+  ])
+  const profileRecencyDb = new FakeDb()
+
+  return { profilesDb, postsDb, addrPostHeightsDb, postParentsDb, pollsDb, statusDb, profileRecencyDb }
+}
+
+describe('#backfillProfileRecency', () => {
+  it('should record each profile newest confirmed qualifying post', async () => {
+    const stores = fixtureStores()
+
+    const result = await backfillProfileRecency(stores)
+
+    assert.equal(result.profiles, 2)
+    assert.deepEqual(stores.profileRecencyDb.store.get(ALICE), {
+      addr: ALICE, blockHeight: 600100, seen: 100
+    })
+    assert.deepEqual(stores.profileRecencyDb.store.get(BOB), {
+      addr: BOB, blockHeight: 600400, seen: 200
+    })
+  })
+
+  it('should not record a profile with no qualifying post', async () => {
+    const stores = fixtureStores()
+
+    await backfillProfileRecency(stores)
+
+    assert.isFalse(stores.profileRecencyDb.store.has(NOPOST))
+  })
+
+  it('should be idempotent', async () => {
+    const stores = fixtureStores()
+
+    await backfillProfileRecency(stores)
+    const afterFirst = new Map(stores.profileRecencyDb.store)
+    await backfillProfileRecency(stores)
+
+    assert.deepEqual(stores.profileRecencyDb.store, afterFirst)
+  })
+
+  it('should remove stale recency records for addresses that no longer qualify', async () => {
+    const stores = fixtureStores()
+    stores.profileRecencyDb.put(NOPOST, { addr: NOPOST, blockHeight: 600900, seen: 9 })
+
+    await backfillProfileRecency(stores)
+
+    assert.isFalse(stores.profileRecencyDb.store.has(NOPOST))
+  })
+})
+
+describe('#partsFromAddrPostHeightKey', () => {
+  it('should recover the address, height, and txid from a key', () => {
+    assert.deepEqual(partsFromAddrPostHeightKey(addrPostHeightKey(ALICE, 600100, 'post-a1')), {
+      addr: ALICE,
+      blockHeight: 600100,
+      txid: 'post-a1'
+    })
+  })
+})

psf-memo-db/test/unit/use-cases/list-recent-profiles.unit.js

@@ -6,8 +6,7 @@ describe('#ListRecentProfiles', () => {
   let uut
   let sandbox
 
-  const mockProfiles = [
-    { addr: 'addr-a', text: 'a', txid: 'tx-a', seen: 100, blockHeight: 600100 },
+  const page = [
     { addr: 'addr-b', text: 'b', txid: 'tx-b', seen: 200, blockHeight: 600200 },
     { addr: 'addr-c', text: 'c', txid: 'tx-c', seen: 50, blockHeight: 600200 }
   ]
@@ -17,7 +16,7 @@ describe('#ListRecentProfiles', () => {
     uut = new ListRecentProfiles({
       adapters: {
         profileQuery: {
-          scanProfilesWithBlockHeight: sandbox.stub().resolves([...mockProfiles]),
+          listRecentProfiles: sandbox.stub().resolves({ profiles: [...page], total: 3 }),
           getProfileIdentity: sandbox.stub().resolves({ name: null, profilePicUrl: null })
         }
       }
@@ -26,7 +25,7 @@ describe('#ListRecentProfiles', () => {
 
   afterEach(() => sandbox.restore())
 
-  it('should join each profile display name and avatar', async () => {
+  it('should join each returned profile display name and avatar', async () => {
     uut.adapters.profileQuery.getProfileIdentity.callsFake(async (addr) => {
       if (addr === 'addr-b') {
         return { name: 'Bob', profilePicUrl: 'https://example.com/bob.jpg' }
@@ -39,44 +38,34 @@ describe('#ListRecentProfiles', () => {
     const bob = result.profiles.find((p) => p.addr === 'addr-b')
     assert.equal(bob.name, 'Bob')
     assert.equal(bob.profilePicUrl, 'https://example.com/bob.jpg')
-    const alice = result.profiles.find((p) => p.addr === 'addr-a')
-    assert.equal(alice.name, null)
-    assert.equal(alice.profilePicUrl, null)
+    const carol = result.profiles.find((p) => p.addr === 'addr-c')
+    assert.equal(carol.name, null)
+    assert.equal(carol.profilePicUrl, null)
   })
 
-  it('should only join identities for the requested page', async () => {
-    await uut.execute({ limit: 1, offset: 1 })
+  it('should only join identities for the profiles the adapter returns', async () => {
+    await uut.execute({ limit: 2, offset: 1 })
 
-    assert.equal(uut.adapters.profileQuery.getProfileIdentity.callCount, 1)
-    assert.equal(uut.adapters.profileQuery.getProfileIdentity.firstCall.args[0], 'addr-c')
+    assert.equal(uut.adapters.profileQuery.getProfileIdentity.callCount, 2)
+    assert.equal(uut.adapters.profileQuery.getProfileIdentity.firstCall.args[0], 'addr-b')
+    assert.equal(uut.adapters.profileQuery.getProfileIdentity.secondCall.args[0], 'addr-c')
   })
 
-  it('should not change pagination metadata when joining identities', async () => {
-    const result = await uut.execute({ limit: 2, offset: 0 })
+  it('should preserve adapter order and pagination metadata when joining identities', async () => {
+    const result = await uut.execute({ limit: 2, offset: 1 })
 
-    assert.equal(result.pagination.total, 3)
-    assert.equal(result.pagination.hasMore, true)
-  })
-
-  it('should return profiles sorted by block height descending', async () => {
-    const result = await uut.execute({ limit: 10, offset: 0 })
-
-    assert.equal(result.profiles.length, 3)
-    assert.equal(result.profiles[0].addr, 'addr-b')
-    assert.equal(result.profiles[1].addr, 'addr-c')
-    assert.equal(result.profiles[2].addr, 'addr-a')
+    assert.deepEqual(result.profiles.map((p) => p.addr), ['addr-b', 'addr-c'])
     assert.equal(result.pagination.total, 3)
     assert.equal(result.pagination.hasMore, false)
   })
 
-  it('should paginate with limit and offset', async () => {
-    const result = await uut.execute({ limit: 1, offset: 1 })
+  it('should pass limit and offset through to the adapter', async () => {
+    await uut.execute({ limit: 7, offset: 14 })
 
-    assert.equal(result.profiles.length, 1)
-    assert.equal(result.profiles[0].addr, 'addr-c')
-    assert.equal(result.pagination.limit, 1)
-    assert.equal(result.pagination.offset, 1)
-    assert.equal(result.pagination.hasMore, true)
+    assert.deepEqual(uut.adapters.profileQuery.listRecentProfiles.firstCall.args[0], {
+      limit: 7,
+      offset: 14
+    })
   })
 
   it('should default limit to 100 and offset to 0', async () => {
@@ -86,22 +75,6 @@ describe('#ListRecentProfiles', () => {
     assert.equal(result.pagination.offset, 0)
   })
 
-  it('should sort equal-height profiles by seen descending, falsy seen last', async () => {
-    // Same blockHeight so only the `seen` tie-break matters. The dataset mixes
-    // truthy and falsy (0) seen values; a broken comparator is observable here
-    // because the falsy profiles are not already in descending input order.
-    const ties = [
-      { addr: 'addr-a', txid: 't-a', seen: 0, blockHeight: 700000 },
-      { addr: 'addr-b', txid: 't-b', seen: 0, blockHeight: 700000 },
-      { addr: 'addr-c', txid: 't-c', seen: 1, blockHeight: 700000 }
-    ]
-    uut.adapters.profileQuery.scanProfilesWithBlockHeight.resolves(ties)
-
-    const result = await uut.execute({ limit: 10, offset: 0 })
-
-    assert.deepEqual(result.profiles.map((p) => p.addr), ['addr-c', 'addr-a', 'addr-b'])
-  })
-
   it('should reject limit over 100', async () => {
     try {
       await uut.execute({ limit: 101 })