diff --git a/psf-memo-db/util/post/backfill-post-heights.js b/psf-memo-db/util/post/backfill-post-heights.js new file mode 100644 index 0000000..35a2c99 --- /dev/null +++ b/psf-memo-db/util/post/backfill-post-heights.js @@ -0,0 +1,132 @@ +/* + Utility: backfill the postHeights secondary index for an existing psf-memo-db. + + The postHeights index is required for efficient pagination of /posts/recent and + /posts/by/:addr. New deployments write this index while indexing live blocks, + but existing databases that were populated before the postHeights feature need + a one-time backfill. + + Run from the psf-memo-db repo root on the host that owns the LevelDB files: + + node util/post/backfill-post-heights.js + + The script is idempotent: re-running it will not create duplicate index + entries. Progress and a summary are printed to stderr. + + WARNING: + - This script opens the LevelDB files directly. psf-memo-db must NOT be + running, or another process must not hold the database locks. + - Make a backup of leveldb/current before running on a production server: + cp -r leveldb/current leveldb/current-pre-postheights-backup +*/ + +import level from 'level' +import * as fs from 'fs' +import * as path from 'path' +import * as url from 'url' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) + +const DATA_DIR = process.env.PSF_MEMO_DB_DATA_DIR + ? path.resolve(process.env.PSF_MEMO_DB_DATA_DIR) + : path.resolve(__dirname, '../../leveldb/current') + +const HEIGHT_PAD = 12 +const PROGRESS_INTERVAL = 10000 + +function requiredStorePath (dir, name) { + const storePath = path.join(dir, name) + if (!fs.existsSync(storePath)) { + throw new Error(`Required LevelDB store not found: ${storePath}. Set PSF_MEMO_DB_DATA_DIR to the directory containing the posts, postHeights, and postParents stores.`) + } + return storePath +} + +function postHeightKey (blockHeight, txid) { + const padded = String(blockHeight).padStart(HEIGHT_PAD, '0') + return `${padded}:${txid}` +} + +async function createIfMissing (db, key, value) { + try { + await db.get(key) + return false + } catch (err) { + if (err.notFound || err.code === 'LEVEL_NOT_FOUND') { + await db.put(key, value) + return true + } + throw err + } +} + +async function backfill () { + console.error(`Using LevelDB data directory: ${DATA_DIR}`) + + const postsPath = requiredStorePath(DATA_DIR, 'posts') + const postParentsPath = requiredStorePath(DATA_DIR, 'postParents') + const postHeightsPath = path.join(DATA_DIR, 'postHeights') + + console.error('Opening LevelDB stores...') + const postsDb = level(postsPath, { valueEncoding: 'json' }) + const postHeightsDb = level(postHeightsPath, { valueEncoding: 'json', createIfMissing: true }) + const postParentsDb = level(postParentsPath, { valueEncoding: 'json' }) + + try { + console.error('Loading reply txids from postParents...') + const replyTxids = new Set() + let replyCount = 0 + for await (const [childTxid] of postParentsDb.iterator()) { + replyTxids.add(childTxid) + replyCount++ + } + console.error(`Loaded ${replyCount} reply txids.`) + + console.error('Scanning posts and backfilling postHeights entries...') + let postsScanned = 0 + let topLevelPosts = 0 + let indexEntriesCreated = 0 + let indexEntriesExisting = 0 + + for await (const [txid, post] of postsDb.iterator()) { + postsScanned++ + + if (replyTxids.has(txid)) { + if (postsScanned % PROGRESS_INTERVAL === 0) { + console.error(` scanned ${postsScanned} posts, ${topLevelPosts} top-level, ${indexEntriesCreated} written, ${indexEntriesExisting} skipped...`) + } + continue + } + + topLevelPosts++ + const blockHeight = post.blockHeight ?? 0 + const key = postHeightKey(blockHeight, txid) + const value = { txid, blockHeight } + const created = await createIfMissing(postHeightsDb, key, value) + if (created) { + indexEntriesCreated++ + } else { + indexEntriesExisting++ + } + + if (postsScanned % PROGRESS_INTERVAL === 0) { + console.error(` scanned ${postsScanned} posts, ${topLevelPosts} top-level, ${indexEntriesCreated} written, ${indexEntriesExisting} skipped...`) + } + } + + console.error('\nBackfill complete.') + console.error(` posts scanned: ${postsScanned}`) + console.error(` top-level posts: ${topLevelPosts}`) + console.error(` index entries created: ${indexEntriesCreated}`) + console.error(` index entries already present: ${indexEntriesExisting}`) + } catch (err) { + console.error('\nBackfill failed:', err.message) + process.exitCode = 1 + } finally { + await postHeightsDb.close().catch(() => {}) + await postParentsDb.close().catch(() => {}) + await postsDb.close().catch(() => {}) + } +} + +backfill()