custo-viewer/prune-blobs.js

#!/usr/bin/env node
//
// prune-blobs.js — delete cached blobs that none of our own feeds reference.
//
// CONTEXT
// -------
// ssb-blobs `sympathy` (default 3) made this node fetch and cache blobs on
// behalf of any peer up to 3 network hops away. With port 8008 open to the
// world, that turned it into a free CDN: 31,283 blobs / 4.6 GB, of which only
// 267 are referenced by any message we hold.
//
// Once sympathy is 0 the inflow stops, but the existing hoard stays on disk --
// and ssb-viewer will serve any of it by hash to anyone who asks. This removes
// it.
//
// SAFETY
// ------
//   * Dry run by default. Pass --delete to actually remove anything.
//   * Every deleted id is written to a manifest first. Blobs are content-
//     addressed, so anything deleted in error can be re-requested with
//     `ssb-server blobs.want <id>` if a peer still has it.
//   * The keep set is built from the raw log, so it does not depend on sbot
//     being up or on any index being in sync.
//
// Blobs are stored by multiblob at blobs/<alg>/<hex[0:2]>/<hex[2:]>, where hex
// is the base64 blob hash decoded to hex (see multiblob/index.js toPath).

var fs = require('fs')
var path = require('path')

var DELETE = process.argv.indexOf('--delete') !== -1
var HOME = process.env.HOME || '/home/cyberian'
var LOG_OFFSET = path.join(HOME, '.ssb/flume/log.offset')
var BLOB_DIR = path.join(HOME, '.ssb/blobs/sha256')
var PUSH_DIR = path.join(HOME, '.ssb/blobs_push')
var MANIFEST = path.join(HOME, 'logs/pruned-blobs.txt')

var BLOB_RE = /&[A-Za-z0-9+/]{43}=\.sha256/g
// sha256 of the empty string; ssb-blobs special-cases it and never stores it.
var EMPTY_HASH = '&47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=.sha256'

function idToHex (id) {
  return Buffer.from(id.slice(1).replace(/\.sha256$/, ''), 'base64').toString('hex')
}
function hexToId (hex) {
  return '&' + Buffer.from(hex, 'hex').toString('base64') + '.sha256'
}
function human (bytes) {
  var u = ['B', 'KB', 'MB', 'GB']
  var i = 0
  while (bytes >= 1024 && i < u.length - 1) { bytes /= 1024; i++ }
  return bytes.toFixed(1) + ' ' + u[i]
}

// ---- keep set -------------------------------------------------------------
var keep = Object.create(null)
keep[EMPTY_HASH] = true

var log = fs.readFileSync(LOG_OFFSET).toString('binary')
var refs = log.match(BLOB_RE) || []
refs.forEach(function (r) { keep[r] = true })
var fromLog = Object.keys(keep).length - 1

// Anything queued for push to the network is ours by definition. The leveldb
// is locked by the running sbot, so scrape the ids out of its files instead of
// opening it.
var fromPush = 0
try {
  fs.readdirSync(PUSH_DIR).forEach(function (f) {
    if (!/\.(log|ldb)$/.test(f)) return
    var buf = fs.readFileSync(path.join(PUSH_DIR, f)).toString('binary')
    ;(buf.match(BLOB_RE) || []).forEach(function (r) {
      if (!keep[r]) { keep[r] = true; fromPush++ }
    })
  })
} catch (e) {
  console.error('warning: could not scan blobs_push:', e.message)
}

// ---- walk the blob store --------------------------------------------------
var kept = { n: 0, bytes: 0 }
var doomed = []
var orphans = []

fs.readdirSync(BLOB_DIR).forEach(function (sub) {
  var subdir = path.join(BLOB_DIR, sub)
  var st
  try { st = fs.statSync(subdir) } catch (e) { return }
  if (!st.isDirectory()) return

  fs.readdirSync(subdir).forEach(function (name) {
    var full = path.join(subdir, name)
    var size
    try { size = fs.statSync(full).size } catch (e) { return }

    var id
    try {
      id = hexToId(sub + name)
    } catch (e) {
      orphans.push(full)
      return
    }

    if (keep[id]) { kept.n++; kept.bytes += size }
    else doomed.push({ id: id, path: full, size: size })
  })
})

var doomedBytes = doomed.reduce(function (a, b) { return a + b.size }, 0)

console.log('keep set:      ' + (fromLog + fromPush) +
            ' (' + fromLog + ' referenced in the log, ' + fromPush + ' from blobs_push)')
console.log('blobs on disk: ' + (kept.n + doomed.length))
console.log('  keeping:     ' + kept.n + '  (' + human(kept.bytes) + ')')
console.log('  removing:    ' + doomed.length + '  (' + human(doomedBytes) + ')')
if (orphans.length) console.log('  unparseable: ' + orphans.length + ' (left alone)')

// A keep-set entry we do not actually have on disk is fine -- blob-wanter has a
// standing want for it. Just report so the numbers reconcile.
var missing = Object.keys(keep).filter(function (id) {
  if (id === EMPTY_HASH) return false
  var hex = idToHex(id)
  return !fs.existsSync(path.join(BLOB_DIR, hex.substring(0, 2), hex.substring(2)))
})
if (missing.length) {
  console.log('  note: ' + missing.length + ' referenced blob(s) are not on disk; ' +
              'blob-wanter holds standing wants for them')
}

if (!DELETE) {
  console.log('\nDRY RUN — nothing deleted. Re-run with --delete to apply.')
  process.exit(0)
}

// ---- delete ---------------------------------------------------------------
// Manifest first, so a crash mid-delete still leaves a record of what went.
fs.mkdirSync(path.dirname(MANIFEST), { recursive: true })
var out = fs.createWriteStream(MANIFEST, { flags: 'a' })
out.write('# pruned ' + new Date().toISOString() + ' — ' + doomed.length +
          ' blobs, ' + human(doomedBytes) + '\n')
out.write('# re-request any of these with: ssb-server blobs.want <id>\n')
doomed.forEach(function (d) { out.write(d.id + '\t' + d.size + '\n') })
out.end()

var removed = 0, failed = 0, freed = 0
doomed.forEach(function (d) {
  try { fs.unlinkSync(d.path); removed++; freed += d.size }
  catch (e) { failed++ }
})

// Clean up the now-empty two-char shard directories.
fs.readdirSync(BLOB_DIR).forEach(function (sub) {
  var subdir = path.join(BLOB_DIR, sub)
  try {
    if (fs.statSync(subdir).isDirectory() && fs.readdirSync(subdir).length === 0) {
      fs.rmdirSync(subdir)
    }
  } catch (e) { /* not empty, or vanished */ }
})

console.log('\nremoved ' + removed + ' blobs, freed ' + human(freed) +
            (failed ? ', ' + failed + ' failed' : ''))
console.log('manifest: ' + MANIFEST)