Hub stage 8: settings drift, webhook notices, profiles

The drift check now also covers the settings baseline and the Blocked
and Allowed lists baseline; the drift dialog offers Open compare,
Accept as baseline and Revert to baseline for them. Old drift.json
files still load.

HUB_WEBHOOK_URL (and an optional HUB_WEBHOOK_SECRET for an HMAC
signature) receives drift.found, drift.cleared and write.failed
notices, only on change, without values, never blocking a request and
never logging the address beyond its host. Admins see the webhook
status and can send a test from About and from the drift dialog.

Profiles bundle the hub's app, settings and lists baselines. Applying
one to a server runs install, app configs, settings and lists in order
after a preview; a failed step stops that server's later steps and
Retry failed resumes from it. Profiles export and import as a file.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

dev/mock-api.mjs

@@ -15,10 +15,12 @@ behaviour: it exists so a design can be seen with data in it. The real harness i
 */
 import http from 'node:http'
 
-// `--port <n>` and `--scale <x>`: two instances with different numbers stand in for two servers.
+// `--port <n>`, `--scale <x>` and `--variant a|b`: two instances with different numbers (and apps) stand in for two servers.
 const arg = (name) => { const i = process.argv.indexOf(`--${name}`); return i >= 0 ? process.argv[i + 1] : undefined }
 const PORT = Number(arg('port') ?? process.env.PORT ?? 5380)
 const SCALE = Number(arg('scale') ?? 1)
+// `--variant b`: the second server of the Apps matrix (a missing app, another version, other config fields).
+const VARIANT = arg('variant') === 'b' ? 'b' : 'a'
 const now = Date.now()
 const iso = (ms) => new Date(ms).toISOString()
 const ago = (min) => iso(now - min * 60_000)
@@ -91,6 +93,18 @@ let zones = [
   { name: 'vpn.example.net', type: 'Primary', dnssecStatus: 'Unsigned', soaSerial: 12, disabled: false, notifyFailed: true, notifyFailedFor: ['ns2.example.net'], lastModified: ago(60 * 24 * 9) },
   { name: 'staging.example.net', type: 'Primary', dnssecStatus: 'Unsigned', soaSerial: 4, disabled: true, lastModified: ago(60 * 24 * 20) },
 ].map((z) => ({ notifyFailed: false, notifyFailedFor: [], ...z }))
+if (VARIANT === 'b') {
+  // The second server: a zone the first lacks, a serial that moved on, and a Secondary of the first server's Primary.
+  zones = zones.filter((z) => z.name !== 'dev.example.net' && z.name !== 'vpn.example.net')
+  zones = zones.map((z) => (z.name === 'lan' ? { ...z, soaSerial: 44 } : z.name === 'home.example.net' ? { ...z, type: 'Secondary', soaSerial: 16, catalog: null } : z))
+  zones.push({ name: 'ndns-only.example.net', type: 'Primary', dnssecStatus: 'Unsigned', soaSerial: 3, disabled: false, lastModified: ago(60 * 3), notifyFailed: false, notifyFailedFor: [] })
+}
+// Zone files (zones/export, zones/import) and options (zones/options/get, set) are kept per zone.
+const zoneFiles = {}
+const zoneOptions = {}
+const zoneFile = (name) => zoneFiles[name] ?? `$ORIGIN ${name}.\n@ IN SOA ns1.${name}. hostmaster.${name}. ${(zones.find((z) => z.name === name) ?? {}).soaSerial ?? 1} 900 300 604800 900\n@ IN NS ns1.${name}.\nwww IN A 203.0.113.${VARIANT === 'b' ? 20 : 10}\n`
+const optionsOf = (name) => ({ zoneTransfer: 'Deny', zoneTransferNetworkACL: [], zoneTransferTsigKeyNames: [], notify: 'ZoneNameServers', notifyNameServers: [], ...(zoneOptions[name] ?? {}) })
+
 
 function recordsFor(zone) {
   const z = zone.name
@@ -125,6 +139,10 @@ const cacheRecords = (domain) =>
       ]
 const allowed = new Set(['updates.example.net', 'api.github.com', 'time.cloudflare.com'])
 const blocked = new Set(['ads.tracker-example.com', 'telemetry.vendor-example.io', 'pixel.metrics-example.net', 'cdn.adserver-example.org', 'beacon.analytics-example.com'])
+if (VARIANT === 'b') {
+  blocked.delete('beacon.analytics-example.com'); blocked.add('only-ndns.tracker-example.com')
+  allowed.add('extra-allowed.example.net')
+}
 const listNode = (set, domain) => {
   const all = [...set].sort()
   return { domain, zones: domain === '' ? all : [], records: domain !== '' && set.has(domain) ? [{ name: domain, type: 'SOA', ttl: 60, rData: {}, dnssecStatus: 'Disabled' }] : [] }
@@ -164,6 +182,15 @@ const settings = {
   loggingType: 'File, Console', ignoreResolverLogs: false, logQueries: false, useLocalTime: false, logFolder: 'logs', maxLogFileDays: 365, enableInMemoryStats: false, maxStatFileDays: 365,
 }
 
+if (VARIANT === 'b') {
+  // The second server: other shared values (these show up in the compare), and its own identity and log folder (local fields).
+  Object.assign(settings, {
+    dnsServerDomain: 'ndns.example.net', dnsServerLocalEndPoints: ['10.0.0.9:53'], logFolder: 'logs-ndns', webServiceHttpPort: 5381,
+    defaultRecordTtl: 7200, forwarders: ['8.8.8.8'], blockingType: 'AnyAddress', serveStale: false,
+    tsigKeys: [{ keyName: 'transfer.example.net', sharedSecret: 'bmRucy1vdGhlci1zZWNyZXQ=', algorithmName: 'hmac-sha256' }],
+  })
+}
+
 // ── DHCP ───────────────────────────────────────────────────────────────────
 let scopes = [
   { name: 'Main', enabled: true, startingAddress: '192.168.10.10', endingAddress: '192.168.10.200', subnetMask: '255.255.255.0', networkAddress: '192.168.10.0', broadcastAddress: '192.168.10.255', interfaceAddress: '192.168.10.2' },
@@ -189,11 +216,29 @@ let apps = [
   { name: 'Advanced Blocking', description: 'Group based blocking with regex', version: '10.1', updateAvailable: true, updateVersion: '10.2', updateUrl: 'https://download.technitium.com/dns/apps/AdvancedBlockingApp-v10.2.zip', dnsApps: [{ ...app(false), classPath: 'AdvancedBlocking.App' }] },
   { name: 'Split Horizon', description: 'Different answers for different networks', version: '9.0', dnsApps: [{ ...app(false), classPath: 'SplitHorizon.SimpleAddress' }, { ...app(false), classPath: 'SplitHorizon.SimpleCNAME' }] },
 ]
-const store = [
+if (VARIANT === 'b') {
+  apps = apps.filter((a) => a.name !== 'Split Horizon')
+  apps = apps.map((a) => (a.name === 'Query Logs (Sqlite)' ? { ...a, version: '7.1', updateAvailable: true, updateVersion: '7.2', updateUrl: 'https://download.technitium.com/dns/apps/QueryLogsSqliteApp-v7.2.zip' } : a))
+}
+const appConfigs = {
+  'Advanced Blocking': VARIANT === 'b'
+    ? { enableBlocking: true, blockingAnswerTtl: 60, localGroup: 'office', networkGroupMap: { '0.0.0.0/0': 'default' }, groups: [{ name: 'default', enableBlocking: false, blockAsNxDomain: false }] }
+    : { enableBlocking: true, blockingAnswerTtl: 30, localGroup: 'home', networkGroupMap: { '0.0.0.0/0': 'default' }, groups: [{ name: 'default', enableBlocking: true, blockAsNxDomain: false }] },
+  'Query Logs (Sqlite)': { enableLogging: true, maxLogDays: 365, sqliteDbPath: VARIANT === 'b' ? '/var/lib/dns/ndns/querylogs.db' : '/var/lib/dns/mdns/querylogs.db' },
+  'Split Horizon': { domainGroupMap: { 'lan.example.net': 'internal' }, networkGroupMap: { '10.0.0.0/8': 'internal' } },
+}
+const catalogRows = [
   ['Failover', 'Fails over to a backup address when the primary is down', '8.0', '58 KB'], ['Geo Country', 'Answers by the country of the client', '7.2', '1.2 MB'],
   ['Wild IP', 'Answers with the IP address in the name', '5.1', '22 KB'], ['Auto PTR', 'Answers PTR queries from the forward zones', '6.0', '31 KB'],
   ['Drop Requests', 'Drops requests from the listed networks', '7.0', '19 KB'], ['NX Domain', 'Answers NXDOMAIN for the listed domains', '6.1', '17 KB'],
-].map(([name, description, version, size]) => ({ name, description, version, size, url: `https://download.technitium.com/dns/apps/${name.replace(/ /g, '')}App-v${version}.zip`, installed: false }))
+  ['Advanced Blocking', 'Group based blocking with regex', '10.2', '240 KB'], ['Query Logs (Sqlite)', 'Logs DNS queries into a SQLite database', '7.2', '310 KB'],
+  ['Split Horizon', 'Different answers for different networks', '9.0', '120 KB'],
+].map(([name, description, version, size]) => ({ name, description, version, size, url: `https://download.technitium.com/dns/apps/${name.replace(/[ ()]/g, '')}App-v${version}.zip` }))
+const catalog = () => catalogRows
+const storeListing = () => catalogRows.map((c) => {
+  const have = apps.find((a) => a.name === c.name)
+  return have ? { ...c, installed: true, installedVersion: have.version, updateAvailable: have.version !== c.version } : { ...c, installed: false }
+})
 
 // ── Query logs ─────────────────────────────────────────────────────────────
 const RTYPES = ['Recursive', 'Cached', 'Authoritative', 'Blocked', 'Cached', 'Recursive']
@@ -224,6 +269,14 @@ let sessions = [
 const userDetails = (u) => ({ ...u, sessionTimeoutSeconds: 1800, remotelyManagedGroups: false, memberOfGroups: u.username === 'admin' ? ['Administrators'] : ['DNS Administrators', 'Everyone'], sessions: sessions.filter((s) => s.username === u.username) })
 
 // ── Router ─────────────────────────────────────────────────────────────────
+const addedRecords = {}
+// Answers that are files, not JSON (the downloads and the Rules table's reads).
+const fileFor = (path, q = new URLSearchParams()) => ({
+  'allowed/export': [...allowed].join('\n') + '\n',
+  'blocked/export': [...blocked].join('\n') + '\n',
+  'zones/export': zones.some((z) => z.name === q.get('zone')) ? zoneFile(q.get('zone')) : undefined,
+  'logs/download': '[2026-10-26 10:00:00 UTC] [127.0.0.1:5380] [] DNS Server was started.\n',
+})[path]
 const ok = (response = {}) => ({ status: 'ok', response })
 function route(path, q) {
   const domain = q.get('domain') ?? q.get('zone') ?? ''
@@ -242,11 +295,43 @@ function route(path, q) {
       const per = Number(q.get('zonesPerPage') ?? 10), page = Math.max(1, Number(q.get('pageNumber') ?? 1))
       return ok({ zones: zones.slice((page - 1) * per, page * per), pageNumber: page, totalPages: Math.max(1, Math.ceil(zones.length / per)), totalZones: zones.length })
     }
-    case 'zones/records/get': { const z = zones.find((x) => x.name === domain) ?? zones[0]; return ok({ zone: { name: z.name, type: z.type, disabled: z.disabled, dnssecStatus: z.dnssecStatus, soaSerial: z.soaSerial, catalog: z.catalog ?? null }, records: recordsFor(z) }) }
+    case 'zones/records/get': { const z = zones.find((x) => x.name === domain) ?? zones[0]; return ok({ zone: { name: z.name, type: z.type, disabled: z.disabled, dnssecStatus: z.dnssecStatus, soaSerial: z.soaSerial, catalog: z.catalog ?? null }, records: [...recordsFor(z), ...(addedRecords[z.name] ?? [])] }) }
+    case 'zones/records/add': {
+      // One record added for the session: it comes back in the zone's list. `type` decides what the data is called.
+      const type = q.get('type') ?? 'A'
+      const rData = type === 'TXT' ? { text: q.get('text') ?? '', splitText: false } : type === 'CNAME' ? { cname: q.get('cname') ?? '' } : { ipAddress: q.get('ipAddress') ?? '' }
+      const added = { name: q.get('domain') ?? domain, type, ttl: Number(q.get('ttl') ?? 3600), ttlString: `${q.get('ttl') ?? 3600}`, disabled: false, rData, dnssecStatus: 'Disabled', lastUsedOn: '0001-01-01T00:00:00Z', lastModified: iso(Date.now()), expiryTtl: 0, expiryTtlString: '0' }
+      const z = zones.find((x) => x.name === (q.get('zone') ?? domain)) ?? zones[0]
+      addedRecords[z.name] = [...(addedRecords[z.name] ?? []), added]
+      return ok({ addedRecord: added, zone: { name: z.name, type: z.type, disabled: z.disabled } })
+    }
     case 'zones/enable': case 'zones/disable': zones = zones.map((z) => (q.get('zone') === z.name ? { ...z, disabled: path === 'zones/disable' } : z)); return ok()
     case 'zones/delete': { const gone = (q.get('zones') ?? q.get('zone') ?? '').split(',').filter(Boolean); zones = zones.filter((z) => !gone.includes(z.name)); return ok({ deleted: gone, failed: {} }) }
-    case 'zones/create': zones = [{ name: domain, type: q.get('type') ?? 'Primary', dnssecStatus: 'Unsigned', soaSerial: 1, disabled: false, lastModified: iso(Date.now()), notifyFailed: false, notifyFailedFor: [] }, ...zones]; return ok({ domain })
-    case 'zones/options/get': return ok({ name: domain, type: 'Primary', dnssecStatus: 'Unsigned', disabled: false, catalog: null, queryAccess: 'Allow', queryAccessNetworkACL: [], zoneTransfer: 'AllowOnlyZoneNameServers', zoneTransferNetworkACL: [], zoneTransferTsigKeyNames: [], notify: 'ZoneNameServers', notifyNameServers: [], update: 'Deny', updateNetworkACL: [], updateSecurityPolicies: [], availableCatalogZoneNames: [], availableTsigKeyNames: ['transfer.example.net'] })
+    case 'zones/create': {
+      if (zones.some((z) => z.name === domain)) return { status: 'error', errorMessage: `Zone already exists: ${domain}` }
+      const type = q.get('type') ?? 'Primary'
+      zones = [{ name: domain, type, dnssecStatus: type === 'Primary' ? 'Unsigned' : undefined, soaSerial: type === 'Forwarder' ? undefined : 1, disabled: false, lastModified: iso(Date.now()), notifyFailed: false, notifyFailedFor: [] }, ...zones]
+      if (type === 'Secondary') zoneOptions[domain] = { primaryNameServerAddresses: (q.get('primaryNameServerAddresses') ?? '').split(','), primaryZoneTransferProtocol: q.get('zoneTransferProtocol') ?? 'Tcp', primaryZoneTransferTsigKeyName: q.get('tsigKeyName') ?? '' }
+      return ok({ domain })
+    }
+    case 'zones/import': {
+      const z = zones.find((x) => x.name === domain)
+      if (!z) return { status: 'error', errorMessage: `No such zone was found: ${domain}` }
+      zoneFiles[domain] = q.get('__text') ?? zoneFile(domain)
+      const serial = /SOA\s+\S+\s+\S+\s+(\d+)/.exec(zoneFiles[domain])
+      if (serial) zones = zones.map((x) => (x.name === domain ? { ...x, soaSerial: Number(serial[1]), lastModified: iso(Date.now()) } : x))
+      return ok()
+    }
+    case 'zones/options/get': { const z = zones.find((x) => x.name === domain); return ok({ name: domain, type: z?.type ?? 'Primary', dnssecStatus: z?.dnssecStatus ?? 'Unsigned', disabled: false, catalog: z?.catalog ?? null, queryAccess: 'Allow', queryAccessNetworkACL: [], update: 'Deny', updateNetworkACL: [], updateSecurityPolicies: [], availableCatalogZoneNames: [], availableTsigKeyNames: ['transfer.example.net'], ...optionsOf(domain) }) }
+    case 'zones/options/set': {
+      const next = { ...(zoneOptions[domain] ?? {}) }
+      for (const [k, v] of q) {
+        if (['zone', 'node', 'token'].includes(k)) continue
+        next[k] = /ACL$|Names$|Servers$|Addresses$/.test(k) ? (v === 'false' || v === '' ? [] : v.split(',')) : v
+      }
+      zoneOptions[domain] = next
+      return ok()
+    }
     case 'zones/permissions/get': return ok({ zone: domain, userPermissions: [{ username: 'admin', canView: true, canModify: true, canDelete: true }], groupPermissions: [{ name: 'DNS Administrators', canView: true, canModify: true, canDelete: false }], users: users.map((u) => u.username), groups: groups.map((g) => g.name) })
     case 'zones/catalogs/list': return ok({ catalogZoneNames: ['catalog.example.net'] })
     case 'cache/list': return ok({ domain, zones: cacheTree[domain] ?? [], records: cacheRecords(domain) })
@@ -258,16 +343,66 @@ function route(path, q) {
     case 'allowed/flush': allowed.clear(); return ok()
     case 'blocked/flush': blocked.clear(); return ok()
     case 'settings/get': return ok(settings)
-    case 'settings/set': case 'settings/forceUpdateBlockLists': case 'settings/temporaryDisableBlocking': return ok(path === 'settings/set' ? settings : { temporaryDisableBlockingTill: iso(now + 10 * 60_000) })
-    case 'settings/getTsigKeyNames': return ok({ tsigKeyNames: ['transfer.example.net'] })
+    case 'settings/set': {
+      // Changes only what is sent and already there, by its kind (a flag, a number, a list or text); the rest is kept.
+      for (const [k, v] of q) {
+        if (!(k in settings) || k === 'node' || k === 'token') continue
+        if (k === 'tsigKeys') {
+          // Rows of keyName|sharedSecret|algorithmName joined with a pipe, or "false" for none.
+          const cells = v === 'false' ? [] : v.split('|')
+          settings.tsigKeys = Array.from({ length: Math.floor(cells.length / 3) }, (_, i) => ({ keyName: cells[i * 3], sharedSecret: cells[i * 3 + 1], algorithmName: cells[i * 3 + 2] }))
+          continue
+        }
+        const old = settings[k]
+        if (k === 'enableBlocking' && v === 'true') settings.temporaryDisableBlockingTill = null
+        settings[k] = typeof old === 'boolean' ? v === 'true' : typeof old === 'number' ? Number(v) : Array.isArray(old) ? v.split(/[,\n]/).map((x) => x.trim()).filter((x) => x && x !== 'false') : v
+      }
+      return ok(settings)
+    }
+    case 'settings/forceUpdateBlockLists': return ok()
+    case 'settings/temporaryDisableBlocking': {
+      // Blocking is off until the time given, as on a server; `settings/set enableBlocking=true` turns it back on.
+      const minutes = Number(q.get('minutes') ?? 10)
+      settings.enableBlocking = false
+      settings.temporaryDisableBlockingTill = iso(Date.now() + minutes * 60_000)
+      return ok({ temporaryDisableBlockingTill: settings.temporaryDisableBlockingTill })
+    }
+    case 'user/createSingleUseToken': return ok({ token: 'mock-once' })
+    case 'settings/getTsigKeyNames': return ok({ tsigKeyNames: settings.tsigKeys.map((k) => k.keyName) })
     case 'dhcp/leases/list': return ok({ leases })
     case 'dhcp/scopes/list': return ok({ scopes })
     case 'dhcp/scopes/get': return ok(scopeDetail(q.get('name') ?? 'Main'))
     case 'dhcp/scopes/enable': case 'dhcp/scopes/disable': scopes = scopes.map((s) => (s.name === q.get('name') ? { ...s, enabled: path === 'dhcp/scopes/enable' } : s)); return ok()
     case 'dhcp/scopes/delete': scopes = scopes.filter((s) => s.name !== q.get('name')); return ok()
     case 'apps/list': return ok({ apps })
-    case 'apps/listStoreApps': return ok({ storeApps: store })
-    case 'apps/config/get': return ok({ config: '{\n  "enabled": true\n}' })
+    case 'apps/listStoreApps': return ok({ storeApps: storeListing() })
+    case 'apps/downloadAndInstall': case 'apps/downloadAndUpdate': {
+      const entry = catalog().find((c) => c.name === q.get('name') && c.url === q.get('url'))
+      if (!entry) return { status: 'error', errorMessage: 'The app was not found in the store.' }
+      const update = path === 'apps/downloadAndUpdate'
+      const have = apps.find((a) => a.name === entry.name)
+      if (update !== Boolean(have)) return { status: 'error', errorMessage: update ? `DNS application '${entry.name}' was not found.` : `DNS application '${entry.name}' already exists.` }
+      const installed = { ...(have ?? { name: entry.name, description: entry.description, dnsApps: [app(false)] }), version: entry.version }
+      delete installed.updateAvailable; delete installed.updateVersion; delete installed.updateUrl
+      apps = update ? apps.map((a) => (a.name === entry.name ? installed : a)) : [...apps, installed]
+      return ok(update ? { updatedApp: installed } : { installedApp: installed })
+    }
+    case 'apps/install': case 'apps/update': {
+      const name = q.get('name') ?? ''
+      const have = apps.find((a) => a.name === name)
+      const update = path === 'apps/update'
+      if (update !== Boolean(have)) return { status: 'error', errorMessage: update ? `DNS application '${name}' was not found.` : `DNS application '${name}' already exists.` }
+      const installed = { ...(have ?? { name, description: 'Installed from a zip file', dnsApps: [app(false)] }), version: update ? `${have.version}.1` : '1.0' }
+      apps = update ? apps.map((a) => (a.name === name ? installed : a)) : [...apps, installed]
+      return ok(update ? { updatedApp: installed } : { installedApp: installed })
+    }
+    case 'apps/config/set': {
+      const name = q.get('name') ?? ''
+      if (!apps.some((a) => a.name === name)) return { status: 'error', errorMessage: `DNS application '${name}' was not found.` }
+      try { appConfigs[name] = JSON.parse(q.get('config') ?? '') } catch (e) { return { status: 'error', errorMessage: `The config is not valid JSON: ${e.message}` } }
+      return ok()
+    }
+    case 'apps/config/get': return ok({ config: appConfigs[q.get('name')] ? JSON.stringify(appConfigs[q.get('name')], null, 2) : JSON.stringify({ enabled: true }, null, 2) })
     case 'apps/uninstall': apps = apps.filter((a) => a.name !== q.get('name')); return ok()
     case 'logs/list': return ok({ logFiles: [{ fileName: '2026-10-26', size: '412.8 KB' }, { fileName: '2026-10-25', size: '1.2 MB' }, { fileName: '2026-10-24', size: '980.4 KB' }] })
     case 'logs/query': {
@@ -302,6 +437,14 @@ http.createServer((req, res) => {
   req.on('end', () => {
     const q = new URLSearchParams(url.search)
     if ((req.headers['content-type'] ?? '').includes('application/x-www-form-urlencoded')) for (const [k, v] of new URLSearchParams(body)) q.set(k, v)
+    if ((req.headers['content-type'] ?? '').includes('text/plain') && body !== '') q.set('__text', body)
+    const file = fileFor(path, q)
+    if (file !== undefined) {
+      res.setHeader('content-type', 'text/plain')
+      res.setHeader('content-disposition', `attachment; filename="${path.replace('/', '-')}.txt"`)
+      res.end(file)
+      return
+    }
     res.setHeader('content-type', 'application/json')
     res.end(JSON.stringify(route(path, q)))
   })

docker/Dockerfile.hub

@@ -3,7 +3,7 @@
 # See hub/README.md. Self-contained: it does not use the console image.
 #
 #   docker run --rm -p 8080:8080 -e HUB_SERVERS=dns=https://dns.example.net \
-#     -e DNS_TOKEN=... -e HUB_USER=admin -e HUB_PASSWORD=... \
+#     -e DNS_TOKEN=... -e HUB_USER=admin -e HUB_PASSWORD=... -v hub-data:/data \
 #     git.shx.one/denozord/technitium-hub:latest
 FROM node:22-alpine AS build
 WORKDIR /src
@@ -22,7 +22,10 @@ FROM node:22-alpine
 WORKDIR /app
 COPY --from=build /out/ ./
 COPY --from=build /src/dist ./dist
-ENV HUB_STATIC=/app/dist HUB_PORT=8080 NODE_ENV=production
+ENV HUB_STATIC=/app/dist HUB_PORT=8080 HUB_DATA=/data NODE_ENV=production
+# The users file and the audit log live here: mount a volume on it.
+RUN mkdir -p /data && chown node:node /data
+VOLUME /data
 USER node
 EXPOSE 8080
 # No curl in alpine: node asks the hub's public endpoint itself.

docs/compose.hub.example.yaml

@@ -11,8 +11,10 @@ version: "3.8"
 # Токены кладутся в .env рядом с этим файлом (не в git):
 #   MDNS_TOKEN=...
 #   NDNS_TOKEN=...
+#   MDNS_ADMIN_TOKEN=...   # токены hub-admin (Modify и Delete только в разделах, которыми управляете из хаба, см. docs/hub-deploy.md): нужны, чтобы хаб менял приложения и серверы
+#   NDNS_ADMIN_TOKEN=...
 #   HUB_USER=...        # логин в сам хаб
-#   HUB_PASSWORD=...    # длинный пароль: у хаба нет ограничения числа попыток входа
+#   HUB_PASSWORD=...    # первый вход; затем пользователи в /data/users.json (docs/hub-deploy.md, шаг 4a)
 #
 # Обновление: docker compose pull technitium-hub && docker compose up -d technitium-hub
 
@@ -23,13 +25,23 @@ services:
     restart: unless-stopped
     networks:
       - dns-net
+    volumes:
+      - technitium-hub-data:/data
     environment:
       # id=адрес, по порядку; токен каждого берётся из <ID>_TOKEN
       - HUB_SERVERS=mdns=https://mdns.shnt.top,ndns=https://ndns.shnt.top
       - MDNS_TOKEN=${MDNS_TOKEN}
       - NDNS_TOKEN=${NDNS_TOKEN}
+      - MDNS_ADMIN_TOKEN=${MDNS_ADMIN_TOKEN}
+      - NDNS_ADMIN_TOKEN=${NDNS_ADMIN_TOKEN}
+      # Только пока нет /data/users.json; потом пользователи создаются через users-cli (docs/hub-deploy.md)
       - HUB_USER=${HUB_USER}
       - HUB_PASSWORD=${HUB_PASSWORD}
+      - HUB_DATA=/data
+      # За Traefik: брать адрес клиента из X-Forwarded-For (для ограничения попыток входа и журнала)
+      - HUB_TRUST_PROXY=1
+      # Как часто проверять, что конфиги серверов не разошлись с эталоном (минуты, 0 выключает)
+      # - HUB_DRIFT_MINUTES=15
       - HUB_DOMAIN=dnss.shnt.top
       - TZ=Asia/Novosibirsk
     labels:
@@ -42,6 +54,9 @@ services:
       - "traefik.http.routers.hub.service=hub-svc"
       - "traefik.http.services.hub-svc.loadbalancer.server.port=8080"
 
+volumes:
+  technitium-hub-data:
+
 networks:
   dns-net:
     external: true

docs/compose.shnt.example.yaml

@@ -181,14 +181,25 @@ services:
     restart: unless-stopped
     networks:
       - dns-net
+    volumes:
+      - technitium-hub-data:/data
     environment:
       # id=адрес по порядку; токен каждого из <ID>_TOKEN. Сервер на ЭТОМ же хосте можно
       # указать изнутри сети, мимо Traefik: ndns=http://technitium-smart:5380
       - HUB_SERVERS=mdns=https://mdns.shnt.top,ndns=https://ndns.shnt.top
       - MDNS_TOKEN=${MDNS_TOKEN}
       - NDNS_TOKEN=${NDNS_TOKEN}
+      # Токены hub-admin: без них хаб только читает. Нужны для установки и обновления приложений
+      - MDNS_ADMIN_TOKEN=${MDNS_ADMIN_TOKEN}
+      - NDNS_ADMIN_TOKEN=${NDNS_ADMIN_TOKEN}
+      # Только пока нет /data/users.json; потом пользователи создаются через users-cli (docs/hub-deploy.md)
       - HUB_USER=${HUB_USER}
       - HUB_PASSWORD=${HUB_PASSWORD}
+      - HUB_DATA=/data
+      # За Traefik: брать адрес клиента из X-Forwarded-For (для ограничения попыток входа и журнала)
+      - HUB_TRUST_PROXY=1
+      # Как часто проверять, что конфиги серверов не разошлись с эталоном (минуты, 0 выключает)
+      # - HUB_DRIFT_MINUTES=15
       - HUB_DOMAIN=dnss.shnt.top
       - TZ=Asia/Novosibirsk
     labels:
@@ -209,3 +220,4 @@ networks:
 volumes:
   technitium-data:
   technitium-console:
+  technitium-hub-data:

docs/hub-deploy.md

@@ -4,7 +4,7 @@
 
 - запрос «All servers» хаб отправляет на оба сервера и складывает ответы;
 - запрос к одному серверу хаб пересылает только ему;
-- запись хаб не выполняет никогда.
+- менять хаб умеет только приложения (установка, обновление, удаление), и только для пользователей с ролью `apps-admin` или `admin`. Остальные запросы на изменение получают отказ и попадают в журнал.
 
 Серверы Technitium не меняются, кластер не нужен. В инструкции хаб ставится на хост, где уже работают Traefik и `technitium-smart` (`docs/compose.shnt.example.yaml`). Адрес хаба: `https://dnss.shnt.top`.
 
@@ -14,7 +14,9 @@
                                              └─► https://ndns.shnt.top/api/...  (NDNS_TOKEN)
 ```
 
-Что хаб показывает: Dashboard (сумма или один сервер), Logs (View Logs и Query Logs, объединённые с колонкой Server) и About. Остальные разделы скрыты, ими управляют в консоли каждого сервера.
+Что хаб показывает: Dashboard (сумма или один сервер), Logs (View Logs и Query Logs, объединённые с колонкой Server), Apps, About и разделы управления одним сервером: Zones, Cache, Blocking (Blocked и Allowed), DNS Client, Settings и DHCP. В этих разделах хаб начинает с первого сервера, а менять что-либо можно только на выбранном сервере и только ролью, у которой есть нужное право (таблица в шаге 1а).
+
+Остаётся в консоли каждого сервера: Administration (пользователи, группы, сессии, токены, SSO, LDAP), удаление логов и статистики, резервная копия и восстановление настроек (в копии лежат пользователи и токены), права на зоны и настройки веб-службы сервера (адреса, порты, HTTPS), через которую хаб сам ходит на сервер.
 
 ---
 
@@ -46,15 +48,22 @@ git tag hub-v1.0.0 && git push origin hub-v1.0.0
 Сделайте это **на обоих серверах**: `https://mdns.shnt.top` и `https://ndns.shnt.top`. Войдите в консоль сервера как администратор.
 
 1. **Группа.** Administration › Groups › **Add Group**: имя `hub-readers`, описание любое.
-2. **Права группы.** Administration › Permissions: у группы `hub-readers` отметьте только **View** в четырёх разделах:
+2. **Права группы.** Administration › Permissions: у группы `hub-readers` отметьте только **View** в каждом разделе, который хаб показывает:
 
    | Раздел | Зачем хабу |
    |---|---|
    | Dashboard | статистика, топы, график по часам, живые метрики |
    | Logs | список файлов логов и Query Logs |
    | Apps | список приложений логирования, из которых Query Logs читает записи |
-   | Settings | состояние блокировки на Dashboard |
+   | Settings | настройки и состояние блокировки |
+   | Zones | список зон, записи, экспорт зоны |
+   | Cache | дерево кеша |
+   | Allowed | список разрешённых доменов и его экспорт |
+   | Blocked | список заблокированных доменов и его экспорт |
+   | DnsClient | DNS-запрос из раздела DNS Client |
+   | DhcpServer | области и аренды DHCP |
 
+   Без View на новом разделе этот раздел в хабе отвечает «в доступе отказано». Секретные значения настроек (ключи TSIG, пароли) хаб скрывает от всех ролей, кроме `admin`.
    Modify и Delete не отмечайте нигде. Остальные разделы оставьте без прав.
 3. **Пользователь.** Administration › Users › **Add User**: имя `hub-reader`, длинный случайный пароль. Затем в карточке пользователя добавьте его в группу `hub-readers`. Из группы `Everyone` можно убрать, если у неё есть лишние права.
 4. **API-токен.** Нужен токен именно для `hub-reader`, а не для администратора:
@@ -72,6 +81,40 @@ git tag hub-v1.0.0 && git push origin hub-v1.0.0
 
 Результат шага: два токена, `MDNS_TOKEN` (с `mdns.shnt.top`) и `NDNS_TOKEN` (с `ndns.shnt.top`).
 
+### Шаг 1а. Пользователь для записи (нужен, чтобы менять приложения)
+
+Без этого шага хаб только читает. Токены записи отдельные, у отдельного пользователя. Сделайте это **на обоих серверах**:
+
+1. **Группа** `hub-admins`. Права группы (Administration › Permissions) дают только то, что хаб может сделать этим токеном:
+
+   | Раздел | Права | Что хаб пишет |
+   |---|---|---|
+   | Apps | View, Modify, Delete | установка, обновление, удаление, конфиги приложений |
+   | Zones | View, Modify, Delete | зоны, записи, DNSSEC, импорт |
+   | Cache | View, Delete | сброс кеша и удаление домена из кеша |
+   | Allowed, Blocked | View, Modify, Delete | добавить, удалить, очистить, импортировать |
+   | Settings | View, Modify | сохранение настроек и URL списков блокировки, временное отключение блокировки, включение блокировки, обновление списков. Delete не нужен: резервная копия и восстановление через хаб закрыты |
+   | DnsClient | View, Modify | Import из DNS Client пишет ответ в зону |
+   | DhcpServer | View, Modify, Delete | области и аренды |
+   | Dashboard, Logs | View | чтобы этот токен мог читать то же, что `hub-reader`. Delete не нужен: удаление логов и статистики закрыто |
+   | Administration | без прав | хаб это не использует |
+
+   Выдавайте Modify и Delete только в тех разделах, которыми собираетесь управлять из хаба; без права на сервере запись просто будет отклонена.
+2. **Пользователь** `hub-admin` в этой группе, длинный случайный пароль.
+3. **API-токен** для `hub-admin` (так же, как в шаге 1.4). Это `MDNS_ADMIN_TOKEN` и `NDNS_ADMIN_TOKEN`.
+
+Хаб использует эти токены только для записи (приложения, зоны, кеш, блокировка, DNS Client с импортом, настройки, DHCP), для чтения конфига приложения перед изменением и для сверки веб-службы перед сохранением настроек. Если для сервера токена нет, хаб показывает его «только для чтения» и не вызывает для записи: кнопки неактивны, а под выбором сервера написано «No admin token configured on the hub».
+
+Запись идёт на один выбранный сервер. В режиме «All servers» (Dashboard, Logs, Blocking › Overview) кнопки записи неактивны с подсказкой «Choose a server».
+
+Действия на несколько серверов сразу (этап 6): в «All servers» кнопки Block, Allow, Disable в Blocking › Overview, «Update lists on all servers», «Edit list URLs on all servers», «Remove from Blocked» и «Remove from Allowed», а также «Flush on all servers» и «Delete a domain on all servers» в Cache и Block/Allow в строках Query Logs открывают выбор серверов. По умолчанию отмечены все серверы с токеном записи; сервер без токена виден, но отключён, с причиной. Итог показывается по каждому серверу, «Retry failed» повторяет только неудавшиеся. Для этого нужны те же права на серверах: Blocked и Allowed (Modify, Delete), Cache (Delete), Settings (Modify: URL списков и переключатель блокировки). Роли `operator` и `admin`; в аудит попадает по строке на сервер, список URL в аудит не пишется. Заменённый список URL полностью перезаписывает прежний на каждом выбранном сервере.
+
+Зоны, настройки и блокировка на нескольких серверах (этап 7), только роль `admin`. Zones при «All servers» показывает общую таблицу: где зона есть, её тип и SOA serial, статус (одинаково, serial отличается, тип отличается, только на одном сервере; Secondary с отставшим serial отмечен, но расхождением не считается). В меню строки: «Open on …», «Copy to …» (экспорт зоны с одного сервера и импорт на другой; копируются только Primary и Forwarder, подписанные DNSSEC и Secondary не копируются; если зона уже есть, нужна галочка Overwrite) и «Make secondary on …» (хаб добавляет адрес второго сервера в передачу зоны и уведомления первого и создаёт Secondary на втором; адрес первого, как его видит второй, подставляется из адреса сервера в хабе, адрес второго для списка передачи вводится вручную; по желанию ключ TSIG, он должен быть на обоих серверах). Для этого на **обоих** серверах у `hub-admin` нужны Zones (View, Modify, Delete) и Settings (View, Modify). Кнопка «Compare servers» в Settings и в Blocking открывает сравнение настроек (вкладки Compare, Baseline, Local fields, History): эталон берётся с выбранного сервера кнопкой «Accept … as baseline», применение (Apply) отправляет только изменившиеся общие поля и перед записью сохраняет прежние настройки сервера в историю. По умолчанию **локальны** и не отправляются: `dnsServerDomain`, адреса привязки (`dnsServerLocalEndPoints`, `dnsServerIPv4SourceAddresses`, `dnsServerIPv6SourceAddresses`), все `webService*`, сертификат DNS (`dnsTlsCertificatePath`, `dnsTlsCertificatePassword`), сокеты и адреса обратного прокси этого хоста, `logFolder`; версия, время работы, данные кластера и временное отключение блокировки не сравниваются. Секреты (ключи TSIG, пароль прокси) сравниваются внутри хаба, нигде не показываются (только «secret changed») и в аудит не пишутся; файлы эталона и истории хранят их в `/data` с правами 0600. В Blocking сравниваются ещё свои списки Blocked и Allowed как множества («не хватает N, лишних M»); лишние удаляются только при галочке «Also remove extras».
+
+Какие роли хаба что могут: `operator` меняет кеш, списки Blocked и Allowed, переключатель блокировки (включить, временно отключить) и обновляет списки, `admin` меняет и зоны, настройки и DHCP; `viewer` и `apps-admin` в этих разделах только читают.
+
+Хаб отклоняет всё, чего нет в таблице путей: `admin/*` и `user/*` (кроме собственного входа хаба), `logs/delete`, `logs/deleteAll`, `dashboard/stats/deleteAll`, `settings/backup`, `settings/restore`, права на зоны (`zones/permissions/*`). Сохранение настроек с изменённым полем веб-службы сервера (`webService*`) или с полем пользователей, SSO и LDAP хаб тоже отклоняет с сообщением «Change the web service settings in that server's own console.»
+
 ---
 
 ## Шаг 2. DNS-запись для dnss.shnt.top
@@ -91,12 +134,16 @@ git tag hub-v1.0.0 && git push origin hub-v1.0.0
 MDNS_TOKEN=вставьте_токен_mdns
 NDNS_TOKEN=вставьте_токен_ndns
 
-# Вход в сам хаб (это НЕ логин Technitium)
+# Токены hub-admin с шага 1а (без них хаб не меняет приложения)
+MDNS_ADMIN_TOKEN=вставьте_токен_записи_mdns
+NDNS_ADMIN_TOKEN=вставьте_токен_записи_ndns
+
+# Первый вход в хаб (это НЕ логин Technitium). Нужен, пока не создан файл пользователей (шаг 4a)
 HUB_USER=admin
 HUB_PASSWORD=длинный_случайный_пароль
 ```
 
-Пароль хаба нужен длинный: у хаба нет ограничения числа попыток входа. Сгенерировать можно так:
+Пароль лучше длинный. После 5 неудачных попыток за 15 минут вход блокируется на 15 минут, но это не замена хорошему паролю. Сгенерировать можно так:
 ```bash
 openssl rand -base64 24
 ```
@@ -124,10 +171,16 @@ chmod 600 .env
       - HUB_SERVERS=mdns=https://mdns.shnt.top,ndns=https://ndns.shnt.top
       - MDNS_TOKEN=${MDNS_TOKEN}
       - NDNS_TOKEN=${NDNS_TOKEN}
+      - MDNS_ADMIN_TOKEN=${MDNS_ADMIN_TOKEN}
+      - NDNS_ADMIN_TOKEN=${NDNS_ADMIN_TOKEN}
       - HUB_USER=${HUB_USER}
       - HUB_PASSWORD=${HUB_PASSWORD}
+      - HUB_DATA=/data
+      - HUB_TRUST_PROXY=1
       - HUB_DOMAIN=dnss.shnt.top
       - TZ=Asia/Novosibirsk
+    volumes:
+      - technitium-hub-data:/data
     labels:
       - "traefik.enable=true"
       - "traefik.docker.network=dns-net"
@@ -138,6 +191,13 @@ chmod 600 .env
       - "traefik.http.services.hub-svc.loadbalancer.server.port=8080"
 ```
 
+Том `technitium-hub-data` нужно объявить в разделе `volumes:` того же файла:
+```yaml
+volumes:
+  technitium-hub-data:
+```
+В нём лежат файл пользователей `users.json` и журнал `audit.log`. `HUB_TRUST_PROXY=1` нужен за Traefik: хаб берёт адрес клиента из заголовка `X-Forwarded-For` для ограничения попыток входа и журнала. Если до хаба можно достучаться в обход Traefik, включать эту переменную нельзя: заголовок можно подделать.
+
 Если хаб стоит в том же файле, что Traefik, отдельный блок `networks:` с `external: true` из примера не нужен: сеть `dns-net` в этом файле уже объявлена.
 
 Про имена в `HUB_SERVERS`:
@@ -149,6 +209,32 @@ chmod 600 .env
 
 ---
 
+## Шаг 4a. Пользователи хаба
+
+У хаба свои пользователи с ролями. Они лежат в `/data/users.json`, пароли в виде хешей.
+
+| Роль | Что может |
+|---|---|
+| `viewer` | смотреть |
+| `operator` | смотреть, блокировка и кеш (после появления записи) |
+| `apps-admin` | смотреть, устанавливать, обновлять и удалять приложения на выбранных серверах |
+| `admin` | всё, включая журнал действий |
+
+Пока файла нет, в хаб входит один `admin` из `HUB_USER` / `HUB_PASSWORD`. Когда файл появился, эти две переменные игнорируются (в логе будет строка об этом). Если нет ни файла, ни переменных, хаб не запустится.
+
+После запуска (шаг 5) создайте пользователей. Пароль команда спросит дважды, при вводе он не показывается:
+```bash
+docker exec -it technitium-hub node --experimental-strip-types /app/hub/users-cli.ts add denis admin
+docker exec -it technitium-hub node --experimental-strip-types /app/hub/users-cli.ts add alice viewer
+```
+Остальные команды: `passwd <имя>`, `role <имя> <роль>`, `disable <имя>`, `enable <имя>`, `list`. Хаб замечает изменение файла на следующем запросе и завершает сессии затронутых пользователей. Перезапуск не нужен.
+
+После создания своего `admin` уберите `HUB_USER` и `HUB_PASSWORD` из `.env` и compose, затем `docker compose up -d technitium-hub`.
+
+Журнал: войдите админом и откройте `/api/hub/audit/list?limit=200` (JSON, новые записи первыми) или смотрите файл `docker exec technitium-hub tail -n 50 /data/audit.log`. Пароли и токены в журнал не пишутся.
+
+---
+
 ## Шаг 5. Запуск
 
 Если реестр закрытый, один раз войдите в него. Нужен токен Gitea с правом `read:package`:
@@ -218,11 +304,16 @@ docker compose rm -f technitium-hub     # удалить контейнер
 |---|---|---|
 | `HUB_SERVERS` | обязательна | Серверы по порядку: `id=https://адрес,...` |
 | `<ID>_TOKEN` | обязательна для каждого | API-токен сервера (`MDNS_TOKEN`, `NDNS_TOKEN`, ...) |
-| `HUB_USER`, `HUB_PASSWORD` | обязательны | Вход в хаб |
+| `HUB_USER`, `HUB_PASSWORD` | нужны, пока нет `users.json` | Первый `admin`; при наличии файла игнорируются |
+| `HUB_DATA` | `/data` | Папка `users.json` и `audit.log` (на ней том) |
+| `HUB_TRUST_PROXY` | выключено | `1`: адрес клиента из `X-Forwarded-For` (только за Traefik) |
+| `<ID>_ADMIN_TOKEN` | нет | Токен записи сервера (`MDNS_ADMIN_TOKEN`). Им хаб меняет приложения; без него сервер только для чтения. В логи не пишется |
 | `HUB_DOMAIN` | `hub` | Имя, которое консоль показывает как имя сервера |
 | `HUB_CACHE_SECONDS` | `15` | Сколько секунд хранить ответы серверов (0 выключает кеш) |
 | `HUB_TIMEOUT_MS` | `8000` | Сколько ждать ответа сервера, мс |
 | `HUB_PORT` | `8080` | Порт внутри контейнера |
+| `HUB_WEBHOOK_URL` | нет | Адрес http или https, куда хаб шлёт уведомления (см. ниже). Адрес может содержать токен, храните его как секрет |
+| `HUB_WEBHOOK_SECRET` | нет | Подписывает уведомления (`X-Hub-Signature`). В логи не пишется |
 
 Третий сервер добавляется строкой в `HUB_SERVERS` и его токеном:
 ```yaml
@@ -239,16 +330,94 @@ docker compose rm -f technitium-hub     # удалить контейнер
 | Контейнер сразу завершается | В `docker compose logs technitium-hub` будет названа недостающая переменная. Проверьте `.env` и что compose его видит (`docker compose config \| grep TOKEN`). |
 | `https://dnss.shnt.top` не открывается, ошибка сертификата | Нет DNS-записи `dnss` (шаг 2) или Traefik ещё получает сертификат: смотрите `docker logs traefik`. |
 | 404 от Traefik | Контейнер не в сети `dns-net` или опечатка в `Host(\`dnss.shnt.top\`)`. |
-| Вход в хаб: «Invalid username or password» | Неверные `HUB_USER`/`HUB_PASSWORD` в `.env`. После правки `.env` нужен `docker compose up -d technitium-hub`. |
+| Вход в хаб: «Invalid username or password» | Неверный логин или пароль. Если есть `users.json`, работают только его пользователи (`users-cli.ts list`), а `HUB_USER` игнорируется. После правки `.env` нужен `docker compose up -d technitium-hub`. |
+| «Too many failed logins. Try again in N minutes.» | 5 неудач за 15 минут с одного адреса или на одно имя. Подождите; блокировка хранится в памяти, её снимает перезапуск контейнера. |
+| Все пользователи блокируются разом | Не включён `HUB_TRUST_PROXY=1`: хаб видит только адрес Traefik. |
+| Контейнер пишет `audit log is not writable` | Нет тома на `/data` или папка не принадлежит `node`. Журнал живёт в памяти (последние 1000 записей) до исправления. |
 | В шапке «1 server not answering», в подсказке `ndns: its API token was refused` | Токен `NDNS_TOKEN` отозван или скопирован с ошибкой. Создайте новый (шаг 1.4) и перезапустите хаб. |
 | В подсказке `... did not answer: ...` или `ECONNREFUSED` / `timeout` | Хаб не достучался до сервера. Проверьте с хоста хаба: `docker exec technitium-hub node -e "fetch('https://ndns.shnt.top/api/status').then(r=>console.log(r.status))"`. |
 | В подсказке `ndns: ...` с сообщением об отказе в доступе | У группы `hub-readers` на этом сервере не хватает View в каком-то разделе (шаг 1.2). |
 | Query Logs пустой или «нет приложения» | На сервере не установлено приложение логирования (Query Logs (Sqlite) и подобные) или нет права View на Apps. |
 | Цифры «All servers» больше суммы на 1-2 клиента | Clients считается по каждому серверу отдельно: клиент, который ходит на оба сервера, учтён дважды. Это ожидаемо. |
-| Нажатие на что-то пишет «The hub is read-only.» | Хаб только читает. Изменения делаются в консоли нужного сервера. |
+| Нажатие на что-то пишет «The hub is read-only.» | Хаб меняет только приложения. Остальные изменения делаются в консоли нужного сервера. |
+
+## Конфиги приложений: эталон и синхронизация
+
+Чтобы конфиги приложений на серверах были одинаковыми, кроме полей, которые у каждого сервера свои (путь к базе, сети):
+
+1. Для чтения конфигов `hub-reader` нужно право **View на Apps** (оно уже есть в таблице шага 1). Если сервер откажет, хаб попробует токен `hub-admin`.
+2. Откройте Apps › All servers, в меню строки приложения выберите **Config…** (или нажмите значок «Different · N»). Читать сравнение могут все роли.
+3. **Эталон.** Вкладка Baseline: **Import from mdns** (или другого сервера, на котором конфиг правильный). Эталон можно поправить в редакторе и сохранить. Если текст начинается с `{` или `[`, он должен быть корректным JSON, иначе хаб покажет ошибку.
+4. **Локальные поля.** Вкладка Local fields: добавьте путь поля, например `sqliteDbPath` или `networks.0.cidr`, нажмите **Fill from the current server configs** (подставит текущие значения каждого сервера) и сохраните. Такие поля не сравниваются и не рассылаются: в матрице серверы, различающиеся только ими, показываются как «Same».
+5. **Применение.** Кнопка **Apply…**: выберите серверы, посмотрите предпросмотр (что изменится, предупреждения, например «версии отличаются, сначала выровняйте версии»), нажмите **Apply**. По каждому серверу будет итог; **Retry failed** повторит только неудавшиеся.
+
+Что хаб делает для безопасности:
+- перед записью сохраняет прежний конфиг сервера в `/data/apps/<имя>/history/` (50 последних) и не пишет, если сохранить не удалось;
+- если конфиг на сервере изменили после предпросмотра, этот сервер пропускается с пояснением;
+- после записи читает конфиг обратно и проверяет, что он совпал; если сервер отклонил конфиг, остаётся старый;
+- в журнал попадают кто, на каком сервере и размер с коротким хешем конфига, но не сам текст.
+
+Файлы эталона и локальных полей лежат в `/data/apps/<имя>/` (`baseline.json`, `overrides.json`). Они содержат конфиги, поэтому том `/data` нужно беречь так же, как `users.json`.
+
+## История, откат, контроль расхождений, перенос эталонов
+
+- **История.** Вкладка History в окне Config: все прежние версии, которые хаб сохранил (конфиг сервера перед применением или сохранением, прежний эталон). Выберите версию, чтобы прочитать её и увидеть отличие от текущего конфига сервера. Роли `apps-admin` и `admin`: **Roll back…** (выбор серверов, предпросмотр, итог по каждому) и **Restore as baseline** для прежнего эталона. Хранится по 50 версий на приложение.
+- **Контроль расхождений.** Раз в 15 минут хаб читает конфиги и сравнивает их с тем, что должно быть (эталон плюс локальные поля сервера). Если конфиг поменяли прямо в консоли сервера, в шапке появляется «N config drift». В списке у каждого расхождения есть **Open config**, а для `apps-admin` и `admin` ещё **Accept as baseline** (конфиг этого сервера становится эталоном; локальные поля остаются локальными) и **Revert to baseline** (через предпросмотр). Период меняется переменной `HUB_DRIFT_MINUTES`, `0` отключает проверку. Сервер, который не ответил, показывается как «Cannot read» и в счётчик не входит. Последний результат лежит в `/data/drift.json`.
+- **Что проверяется.** Конфиги приложений, общие поля настроек (против эталона настроек) и списки Blocked/Allowed (против эталона списков). Проверка настроек и списков идёт только если соответствующий эталон принят. Расхождение настроек или списков показывает только названия полей, без значений. В списке у него **Open compare** (окно сравнения Settings или Blocking), **Accept as baseline** и **Revert to baseline** (через тот же предпросмотр); эти действия только для роли `admin`.
+- **Перенос эталонов.** Apps › All servers: **Export baselines** скачивает один файл со всеми эталонами и локальными полями, **Import baselines** загружает его в другой хаб: сначала проверка и сводка (New, Changed, Same), запись только после **Import**. На серверы импорт ничего не отправляет. Файл содержит конфиги, храните его как секрет.
+
+## Уведомления (webhook)
+
+Если задан `HUB_WEBHOOK_URL`, хаб отправляет POST с JSON `{event, at, hub, items}`:
+
+- `drift.found`: что-то только что разошлось с эталоном (сервер, вид, имя). Шлётся один раз при изменении, а не каждые 15 минут;
+- `drift.cleared`: расхождение исчезло (сервер, который перестал отвечать, не считается исчезнувшим);
+- `write.failed`: изменение, записанное в журнал с исходом `failed` (без значений). Сбои одного момента идут одним уведомлением.
+
+Пример:
+```json
+{"event":"drift.found","at":"2026-11-01T09:15:00.000Z","hub":"dnss.shnt.top",
+ "items":[{"kind":"settings","server":"ndns","since":"2026-11-01T09:15:00.000Z","fields":["blockingType"]}]}
+```
+
+Если задан `HUB_WEBHOOK_SECRET`, заголовок `X-Hub-Signature: sha256=<hex>` содержит HMAC-SHA256 точных байт тела. Проверка на стороне получателя (Node):
+```js
+const expected = 'sha256=' + createHmac('sha256', SECRET).update(rawBody).digest('hex')
+// сравните с заголовком через timingSafeEqual
+```
+Отправка не блокирует работу хаба: таймаут 5 секунд, не больше одного повтора. Ошибка пишется в лог хаба с именем хоста (без параметров адреса). Администратор видит состояние (настроен ли, хост, результат последней отправки) и кнопку **Send test** в окне «Config drift» (оно открывается из шапки, когда есть расхождения).
+
+## Профили
+
+Профиль нужен, чтобы быстро привести новый или сброшенный сервер к общему виду. Это именованный набор в `/data/profiles/`: приложения магазина (имя и версия) с эталонным конфигом каждого, общие поля настроек (поля, относящиеся к самому хосту, не входят) и эталон списков Blocked/Allowed. Файл содержит секреты настроек (ключи TSIG), берегите том `/data`.
+
+Все действия только для роли `admin` (`apps-admin`, `operator` и `viewer` получают отказ). Кнопка **Profiles** на экране Apps › All servers:
+
+- **Create from current baselines**: профиль собирается из эталонов, которые хаб держит сейчас; можно не брать приложения, настройки или списки. Приложение не из магазина (поставленное из zip) в профиль не входит;
+- **View**: состав профиля; секреты показаны как `<secret>`, тексты конфигов не показываются;
+- **Apply to servers…**: выбор серверов, предпросмотр по шагам для каждого сервера, затем итог по каждому шагу. Шаги идут по порядку, ошибка шага останавливает следующие шаги этого сервера: 1) установка недостающих приложений из магазина самого сервера, 2) конфиги приложений (эталон плюс локальные поля сервера, прежний конфиг сохраняется в историю), 3) общие поля настроек (через те же проверки, что и обычное сохранение), 4) списки: недостающие домены добавляются, лишние удаляются только по галочке **Also remove extras**. **Retry failed** начинает заново с шага, который не удался;
+- **Export** и **Import profiles**: перенос между хабами (файл содержит секреты, храните его как секрет; сначала проверка, запись только после **Import**).
+
+## Приложения: установка, обновление, удаление
+
+Роль `apps-admin` (или `admin`) видит в разделе Apps при «All servers» меню действий в каждой строке и кнопки **App Store** и **Install from file**:
+
+- **Update on all**: обновить с сервера, где есть обновление;
+- **Install on missing servers**: поставить из магазина там, где приложения нет;
+- **Align versions**: подтянуть отстающие серверы до самой новой установленной версии (если магазин её предлагает);
+- **Uninstall from…**: удалить с выбранных серверов.
+
+Каждое действие сначала открывает выбор серверов (сервер без токена записи недоступен, с причиной), потом показывает итог по каждому серверу. Если на каком-то сервере не вышло, кнопка **Retry failed** повторит только его.
+
+Что хаб делает для безопасности:
+- ссылку на скачивание берёт из магазина самого сервера, произвольный адрес не принимает;
+- перед удалением сохраняет конфиг приложения в `/data/apps/<имя>/history/` (50 последних на приложение) и не удаляет, если сохранить не удалось. Откат: вставить сохранённый текст в Config приложения на сервере;
+- каждое действие по каждому серверу пишется в журнал (`apps.update-store`, `apps.uninstall` и т. д.).
+
+Права на запись у `viewer` и `operator` нет: они получают «Your role does not allow this.».
 
 ## Безопасность
 
-- Хаб только читает: любая запись получает отказ ещё в хабе, а токены выданы пользователю с одними правами View.
+- Читает хаб токенами с одними правами View. Менять он умеет только приложения, и только токенами `*_ADMIN_TOKEN` (права Apps) по запросу пользователя с ролью `apps-admin` или `admin`. Всё остальное получает отказ в хабе и запись в журнале.
 - Токены лежат только в `.env` на хосте и в памяти контейнера. В браузер они не попадают, в ответы и логи хаба не пишутся.
-- У хаба один общий логин и нет ограничения попыток входа. Нужен длинный пароль, а если хаб не должен быть виден из интернета, добавьте к роутеру `hub` в Traefik `ipallowlist` или `basicAuth`.
+- У каждого человека свой логин и роль. Вход защищён блокировкой после 5 неудач. Нужны длинные пароли, а если хаб не должен быть виден из интернета, добавьте к роутеру `hub` в Traefik `ipallowlist` или `basicAuth`.

docs/hub-management-plan.md

@@ -0,0 +1,181 @@
+# Хаб: централизованное управление серверами Technitium без кластера
+
+Общий план. Внутри два пласта:
+- **общее управление**: каждый сервер из одной консоли, действия «на все серверы», сравнение и выравнивание;
+- **упор на Apps**: приложения на всех серверах и синхронизация их конфигов.
+
+Статус: этапы 0 и 1 (безопасность, Apps по серверам), 2 (установка и обновление), 3 (сравнение и синхронизация конфигов), 4 (история, откат, расхождения), 5 (управление каждым сервером), 6 (действия на все серверы вне Apps), 7 (зоны, настройки и блокировка: сравнение и выравнивание) и 8 (профили, webhook, расхождения настроек и списков) сделаны. Этапы после 8 пока планы (`docs/hub-deploy.md`).
+
+---
+
+## 1. Контекст и цель
+
+- Есть два независимых сервера Technitium DNS: `mdns.shnt.top` и `ndns.shnt.top`. Кластера нет и не будет: серверы должны оставаться самостоятельными.
+- Хаб на `dnss.shnt.top` уже показывает статистику обоих: Dashboard, Logs, About, переключатель «All servers / mdns / ndns», предупреждение об отказе сервера.
+- **Цель:** управлять обоими серверами из хаба. Прежде всего приложениями (Apps): ставить, обновлять и держать их конфиги одинаковыми там, где нужно, не трогая того, что у каждого сервера своё.
+
+---
+
+## 2. Принципы
+
+1. **Два режима записи.**
+   - **На один сервер:** действие пересылается выбранному серверу (`node=mdns` или `node=ndns`). Так уже работает чтение.
+   - **На все серверы:** хаб выполняет действие на каждом выбранном сервере и показывает итог по каждому отдельно. Тихого «ок» при частичном успехе не бывает.
+2. **Чтение и запись разделены.**
+   - Токены `hub-reader` (только View) остаются для чтения.
+   - Для записи на каждом сервере заводится отдельный `hub-admin` только с теми правами, которые нужны этапу.
+3. **Хаб защищается до того, как получает запись.** Роли, ограничение попыток входа, журнал действий (этап 0).
+4. **Конфиги синхронизируются, данные нет.**
+   - Синхронизируются тексты конфигов приложений и настройки.
+   - Базы логов, кеш и статистика у каждого сервера свои.
+5. **Общая часть плюс локальные поля.**
+   - В конфигах приложений есть поля, свои для каждого сервера: сети Split Horizon, путь к базе Query Logs, локальные группы клиентов Advanced Blocking.
+   - Синхронизируется только общая часть. Локальные поля задаются по каждому серверу.
+6. **Источник правды для конфигов: эталон в хабе.**
+   - Хаб хранит желаемый конфиг в своём томе (`/data`), серверы приводятся к нему.
+   - Первый эталон берётся импортом с любого сервера.
+   - Альтернатива: ведущий сервер (mdns) как эталон. Она проще, но хуже: зависит от живучести одного сервера и не даёт истории.
+7. **Зоны DNS лучше синхронизировать средствами Technitium.** Primary-зона на одном сервере и secondary с передачей зоны (AXFR) на другом надёжнее рассылки каждой правки из хаба. Хаб помогает это настроить и показывает расхождения, но не дублирует записи сам.
+8. **Перед изменением предпросмотр, после него итог и откат.** Каждое применение конфига сохраняет прежнюю версию.
+
+---
+
+## 3. Этапы
+
+Сначала безопасность, потом Apps (главный упор), потом общее управление.
+
+### Этап 0. Безопасность записи (обязателен первым)
+
+- **Пользователи хаба с ролями**, в файле `/data/users.json`, пароли в виде хешей:
+  - `viewer`: только смотрит;
+  - `apps-admin`: управляет приложениями;
+  - `operator`: блокировка и кеш;
+  - `admin`: всё.
+- **Защита входа:**
+  - ограничение попыток входа и временная блокировка после серии неудач;
+  - по желанию: вход через ваш auth-портал (Traefik `forwardAuth`).
+- **Журнал действий** (`/data/audit.log`, раздел Audit в консоли хаба): кто, когда, на каком сервере, что сделал, итог.
+- **Токены записи** `MDNS_ADMIN_TOKEN` и `NDNS_ADMIN_TOKEN`. На старте права только Modify и Delete на Apps, другие разделы добавляются по этапам.
+- **Права в хабе:** вместо списка «только чтение»: таблица «эндпоинт → чтение / запись на один сервер / запись на все / нужная роль».
+- **Доступ к хабу по сети:** закрыть по IP (`ipallowlist` в Traefik) или через auth-портал.
+
+### Этап 1. Apps: обзор по серверам (только чтение)
+
+Матрица «приложение × сервер» в разделе Apps при «All servers»:
+- установлено или нет, версия, есть ли обновление;
+- конфиг: «совпадает», «отличается (N полей)» или «отличается только в локальных полях»;
+- фильтры: «отличаются версии», «отличаются конфиги», «есть обновление».
+
+Отвечает на вопрос «что где стоит и где расхождения» без единой записи.
+
+### Этап 2. Apps: установка, обновление, удаление на нескольких серверах
+
+- Выбор цели у действий: mdns, ndns или все серверы.
+- **Из магазина:** `apps/downloadAndInstall` и `apps/downloadAndUpdate` на каждом выбранном сервере.
+- **Свой zip:** загружается в хаб один раз, хаб отправляет его через `apps/install` или `apps/update` на каждый сервер.
+- Окно итога по серверам и кнопка «Повторить на неудавшихся».
+- **«Выровнять версии»:** обновить приложение везде до самой новой версии.
+- **Удаление на всех:** отдельное подтверждение. Хаб перед удалением сохраняет конфиг в историю.
+
+### Этап 3. Apps: сравнение и синхронизация конфигов
+
+- **Сравнение рядом:** конфиг на mdns и на ndns. Для JSON: по полям (добавлено, удалено, изменено), для не-JSON: построчно.
+- **Эталон в хабе:**
+  - импорт эталона с выбранного сервера;
+  - правка эталона в редакторе хаба;
+  - хранение в `/data/apps/<приложение>/baseline.json`.
+- **Локальные поля:**
+  - список путей полей, которые не синхронизируются (например, `networks`, `dbPath`), плюс их значения по каждому серверу;
+  - хранение в `/data/apps/<приложение>/overrides.json`;
+  - итоговый конфиг сервера = эталон + его локальные значения.
+- **Предпросмотр:** что изменится на каждом сервере, затем применение через `apps/config/set` на выбранных серверах с итогом по каждому.
+- **Проверка до записи:** JSON разбирается в хабе. Если сервер отверг конфиг, показывается его ошибка, а сервер остаётся со старым конфигом.
+
+### Этап 4. Apps: история, откат, контроль расхождений
+
+- **История:** каждое применение сохраняет прежний конфиг каждого сервера (`/data/apps/<приложение>/history/`). Можно посмотреть любую версию и откатить один сервер или все.
+- **Проверка расхождений по расписанию** (по умолчанию раз в 15 минут). Если конфиг поменяли прямо в консоли сервера, в шапке появляется предупреждение с действиями «Принять как эталон» и «Вернуть к эталону».
+- **Экспорт и импорт всех эталонов одним файлом:** резервная копия и подготовка нового сервера.
+
+### Этап 5. Управление каждым сервером из одной консоли
+
+- Запись на выбранный сервер открывается для разделов Zones, Cache, Blocking, DNS Client, Settings, DHCP, Logs (кроме удаления логов). Права токена `hub-admin` расширяются под эти разделы.
+- В «All servers» запись по-прежнему закрыта: кнопки неактивны с подсказкой «выберите сервер».
+- Администрирование пользователей, SSO и LDAP серверов остаётся в их собственных консолях.
+
+### Этап 6. Действия «на все серверы» вне Apps
+
+- **Блокировка:** добавить или удалить домен в Blocked и Allowed, временно отключить блокировку, обновить списки, изменить URL списков.
+- **Кеш:** сбросить весь кеш или удалить домен.
+- **Итог по каждому серверу** и повтор на неудавшихся.
+
+### Этап 7. Сравнение и выравнивание остального
+
+- **Zones в «All servers»:**
+  - объединённый список с отметкой, где зона есть, и признаком расхождения (разный SOA serial);
+  - действия «Скопировать зону на другой сервер» (экспорт и импорт) и «Сделать secondary на другом сервере» (передача зоны).
+- **Blocking и Settings:** отчёт о различиях и «Принять как эталон» с предпросмотром, по той же схеме «общая часть плюс локальные поля», что и для Apps.
+
+### Этап 8. Удобство (по желанию)
+
+- Профили: набор приложений с конфигами и правил, который применяется к новому серверу.
+- Уведомления о расхождениях наружу (webhook).
+
+---
+
+## 4. Что меняется в коде
+
+**Хаб (`hub/`):**
+- `server.ts`:
+  - таблица прав по эндпоинтам;
+  - пересылка POST и multipart (zip приложений, импорт зон);
+  - рассылка «на все» с итогами по серверам.
+- Новые модули:
+  - `users.ts`: роли, хеши паролей, ограничение попыток входа;
+  - `audit.ts`: журнал действий;
+  - `store.ts`: файлы в `/data` (эталоны, локальные поля, история), запись атомарно через временный файл;
+  - `apps-sync.ts`: сравнение конфигов, слияние эталона с локальными полями, предпросмотр, применение;
+  - `drift.ts`: проверка расхождений по расписанию.
+- `config.ts`: `<ID>_ADMIN_TOKEN`, `HUB_DATA` (по умолчанию `/data`), `HUB_DRIFT_MINUTES`.
+
+**Консоль (`src/`), только в режиме хаба:**
+- права разделов приходят от хаба по роли пользователя;
+- Apps: матрица по серверам, выбор цели у действий, окно итогов;
+- экран «Конфиг приложения»: сравнение рядом, эталон, локальные поля, предпросмотр, история и откат;
+- предупреждение о расхождениях в шапке;
+- раздел Audit.
+
+**Развёртывание:**
+- в compose: том `technitium-hub-data:/data` и токены `MDNS_ADMIN_TOKEN`, `NDNS_ADMIN_TOKEN` в `.env`;
+- на каждом сервере пользователь `hub-admin` в отдельной группе с правами записи только на нужные разделы.
+
+---
+
+## 5. Риски и как их закрыть
+
+| Риск | Что делаем |
+|---|---|
+| Хаб получает ключи на запись к обоим серверам | Этап 0 первым; токены только на нужные разделы; хаб закрыт по IP или через auth-портал |
+| Частичный успех при рассылке | Итог по каждому серверу, повтор на неудавшихся, откат из истории |
+| Разные версии приложения ждут разный формат конфига | Хаб показывает версии и предлагает сначала выровнять их |
+| Неверно заданные локальные поля перенесут настройки одного сервера на другой | Предпросмотр перед каждым применением, история и откат |
+| Конфиг поменяли мимо хаба | Проверка расхождений по расписанию и предупреждение |
+| Разные версии Technitium понимают API по-разному | Версия каждого сервера в хабе, предупреждение при расхождении |
+
+---
+
+## 6. Как проверяется каждый этап
+
+- **Модульные тесты хаба** (vitest): права по эндпоинтам, роли и защита входа, слияние эталона с локальными полями, сравнение JSON, история и откат.
+- **Тест с двумя фейковыми серверами:** рассылка, частичный отказ, итоги.
+- **Мок** (`dev/mock-api.mjs`) получает приложения и конфиги, разные на двух портах. В браузере проверяются матрица, сравнение, применение и откат.
+- **Порядок работы:** код по этапам пишет Sonnet, Opus проверяет дифф, тесты и браузер, коммит после каждого принятого этапа.
+- **На серверах:** сначала на одном приложении с безобидным конфигом, потом шире.
+
+---
+
+## 7. Решения перед стартом
+
+1. **Источник правды для конфигов:** эталон в хабе (рекомендация) или ведущий сервер mdns.
+2. **Вход в хаб:** свои пользователи хаба с ролями или ваш auth-портал через Traefik `forwardAuth`.
+3. **Какие приложения стоят на серверах** и какие поля в их конфигах заведомо локальные. Это станет начальными списками локальных полей.

hub/README.md

@@ -8,11 +8,11 @@ read-only, Technitium-compatible `/api/*`:
 - No `node` (or `node=cluster`) asks every server in parallel and merges the answers
   (statistics are summed, top lists and query logs are merged). If a server does not
   answer, the others are shown and the response carries `hubWarnings`.
-- Anything that would change data is refused with `403`. The hub has its own sign-in.
+- Reads go with each server's read token. Writes use a separate admin token: the app changes below (role with `apps.write`) and the per-server changes of "Managing one server" (roles with `operate` or `admin`). Anything else that would change data is refused with `403` (and written to the audit log). The hub has its own accounts with roles, a login rate limit and an audit log.
 
-The servers themselves are not changed: each only needs a read-only API token. Create it
-for a user whose group has View, and nothing else, on Dashboard, Logs, Apps (the logging
-apps Query Logs reads from) and Settings (the blocking state).
+The servers themselves are not changed: each needs a read-only API token (View, and nothing else, on Dashboard, Logs,
+Apps, Settings, Zones, Cache, Allowed, Blocked, DnsClient and DhcpServer) and, for writes, an admin token whose group has
+Modify and Delete only where the table in `docs/hub-deploy.md` grants them.
 
 ## Configuration
 
@@ -20,22 +20,325 @@ apps Query Logs reads from) and Settings (the blocking state).
 |---|---|---|
 | `HUB_SERVERS` | required | Ordered `id=url` list, e.g. `dns=https://dns.example.net,ndns=https://ndns.example.net`. The first server is shown as the Primary node. |
 | `<ID>_TOKEN` | required | API token of each server; the id is uppercased with non-alphanumerics as `_` (`dns` gives `DNS_TOKEN`, `my-dns` gives `MY_DNS_TOKEN`). |
-| `HUB_USER`, `HUB_PASSWORD` | required | The hub's own sign-in. It refuses to start without them. |
+| `HUB_USER`, `HUB_PASSWORD` | see Users | Bootstrap: one `admin`, used only while `${HUB_DATA}/users.json` does not exist. Set both or neither. |
+| `HUB_DATA` | `/data` | Folder of `users.json` and `audit.log`. Mount a volume on it. |
+| `HUB_DRIFT_MINUTES` | `15` | How often the hub checks that every server still has the config it should (see Drift). `0` turns the check off. |
+| `HUB_WEBHOOK_URL` | none | An http or https address that gets a signed JSON notice on a change of drift and on a failed write (see Webhook notices). Treat it like a secret: it may carry a token. |
+| `HUB_WEBHOOK_SECRET` | none | Signs each notice: `X-Hub-Signature: sha256=<HMAC-SHA256 of the body>`. Never logged or returned. |
+| `HUB_TRUST_PROXY` | off | `1` believes the first `X-Forwarded-For` entry as the client address (set it behind Traefik). Leave it off when clients reach the hub directly, or anyone can fake their address. |
+| `<ID>_ADMIN_TOKEN` | optional | A write-capable token per server (`MDNS_ADMIN_TOKEN`). Only app writes use it, and only it: a server without one is read-only for the hub (the matrix and the session list the writable servers). Never logged or returned. |
 | `HUB_PORT` | `8080` | Listening port. |
 | `HUB_CACHE_SECONDS` | `15` | How long successful answers of the servers are cached. |
 | `HUB_TIMEOUT_MS` | `8000` | Per-request timeout towards a server. |
 | `HUB_DOMAIN` | `hub` | Shown as the server's domain in the console. |
 | `HUB_STATIC` | `./dist` | The console's build to serve. |
 
+## Users and roles
+
+The hub has its own accounts in `${HUB_DATA}/users.json` (`{ "users": [{ "name", "role", "hash", "disabled"? }] }`,
+hash `scrypt$N$r$p$salt$key`). Names are compared case-insensitively.
+
+- If the file exists, it is the only source; `HUB_USER` / `HUB_PASSWORD` are ignored (a line says so at start).
+- If it does not exist and `HUB_USER` / `HUB_PASSWORD` are set, that one user signs in as `admin`. The file is not written.
+- If neither exists, the hub refuses to start.
+
+Manage the file with the CLI (`npm run hub:user -- <command>` locally):
+
+```sh
+docker exec -it technitium-hub node --experimental-strip-types /app/hub/users-cli.ts add alice apps-admin
+```
+
+`add <name> <role>`, `passwd <name>`, `role <name> <role>`, `disable <name>`, `enable <name>`, `list`. The password is asked
+twice without echo; for scripts use `HUB_NEW_PASSWORD=... users-cli.ts add ...` (minimum 8 characters). The file is written
+atomically with mode 0600. The running hub notices a change on the next request and ends the sessions of the users it touched
+(disabled, removed, new role or password).
+
+| Role | Capabilities |
+|---|---|
+| `viewer` | read |
+| `operator` | read, operate (blocking, cache) |
+| `apps-admin` | read, apps.write |
+| `admin` | everything, including audit.read |
+
+`apps.write` is what the app writes below need; `operate` and `admin` are what the per-server and several-server changes need. Every response also carries
+`hubRole` (and `info.hubRole`).
+
+## Login protection
+
+After 5 failed logins within 15 minutes from one client address, or for one user name, further attempts are refused for 15
+minutes with HTTP 429 and `Too many failed logins. Try again in N minutes.` (the console shows that message). A good login
+resets the user's counter. The state is in memory.
+
+## Audit log
+
+`${HUB_DATA}/audit.log` is JSON Lines: `{ts, user, role, ip, action, server?, target?, outcome, detail?}`. It records login
+(ok, failed, locked), logout, and every refused request outside the permission table as `refused-write` with its api path.
+Passwords, tokens and request bodies are never written. At 10 MB it rotates to `audit.log.1` (one kept). If the folder is not
+writable the hub says so once on stderr and keeps the last 1000 entries in memory. Admins read it with
+`GET /api/hub/audit/list?limit=200&before=<ISO time>`, newest first.
+
 ## Endpoints
 
 Public: `status`, `user/login`. After signing in: `user/session/get`, `user/logout`,
 `user/checkForUpdate` (always "no update") and these reads:
 
 `dashboard/stats/get`, `dashboard/stats/getTop`, `dashboard/metrics/json`, `apps/list`,
-`logs/query`, `logs/list`, `settings/get`, `admin/cluster/state` (synthesized).
+`logs/query`, `logs/list`, `settings/get`, `admin/cluster/state` (synthesized), and the hub's own `hub/audit/list` (role with
+`audit.read`). They are listed in `routes.ts` with a kind (`read`, `write-one`, `write-all`) and the capability they need.
 
-Everything else answers `403 The hub is read-only.`
+Everything else answers `403 The hub is read-only.` and is audited.
+
+## Changing several servers at once
+
+`POST hub/servers/run` (`write-all`, role with `operate`) runs one fixed action on the chosen servers, in parallel, each with
+that server's admin token. Parameters: `action`, `targets` (comma separated server ids) and the action's own `domain`,
+`minutes` or `urls` (lines in the grammar of the block lists, up to 300: `<url>`, `!<url>` for an allow list, `#...` for a switched-off list or a heading; http, https or file URLs; a line with a comma is refused). The answer is `{results: [{server, ok, message}]}`,
+one per server; a server with no admin token or one that does not answer fails on its own and the others still run. Empty
+targets or an unknown server are refused with `400`, and so is a bad parameter, before any server is called.
+
+| action | Technitium call | parameters sent |
+|---|---|---|
+| `blocked.add`, `blocked.delete` | `blocked/add`, `blocked/delete` | `domain` |
+| `allowed.add`, `allowed.delete` | `allowed/add`, `allowed/delete` | `domain` |
+| `blocking.disable` | `settings/temporaryDisableBlocking` | `minutes` (1 to 1440) |
+| `blocking.enable` | `settings/set` | `enableBlocking=true` only |
+| `blocking.updateLists` | `settings/forceUpdateBlockLists` | none |
+| `blocking.setListUrls` | `settings/set` | `blockListUrls` only (comma joined) |
+| `cache.flush` | `cache/flush` | none |
+| `cache.delete` | `cache/delete` | `domain` |
+
+The block list URLs are part of blocking, so `blocking.setListUrls` needs `operate` like the rest (the per-server
+`settings/set` of any other field still needs `admin`). `settings/set` with `enableBlocking` as its only field is `operate`
+too, so an operator can use the blocking switch on one server. The audit log gets one line per server with the action, the
+server, the domain or `minutes=N` or `urls=N` and the outcome, never the URL list. On the servers, the hub-admin needs
+Blocked and Allowed Modify and Delete, Cache Delete, and Settings Modify (list URLs, blocking switch).
+
+## Managing one server
+
+The sections Zones, Cache, Blocking (Blocked and Allowed), DNS Client, Settings and DHCP are managed on ONE server at a
+time. `routes.ts` lists exactly the paths the console calls; an `operate` role (`operator`, `admin`) may change the cache,
+Blocked and Allowed, the temporary blocking switch and the list update; zones (records, DNSSEC, import), settings and DHCP
+need `admin`. `viewer` and `apps-admin` only read.
+
+- A write needs `node=<id>` of a server with an admin token. No node, `cluster` or an unknown id answers `400`
+  ("Choose a server: this change is made on one server at a time."); a server without an admin token answers `403`.
+- The parameters travel to the server in a form body (only `token` stays in the query). A zone import goes on as
+  `multipart/form-data` or `text/plain` (up to 20 MB); `settings/set`, record and scope writes take up to 2 MB and the
+  Blocked/Allowed imports up to 5 MB, for a signed-in caller only.
+- `dnsClient/resolve` is a read, unless `import` is set: then it writes into a zone and needs `admin`.
+- `settings/set` is refused when it changes a `webService*` field (compared with the server's current `settings/get`; the
+  console sends the block unchanged with every save) or carries a user, SSO or LDAP field. `settings/get` blanks the TSIG
+  shared secrets and the passwords for every role but `admin`.
+- Reads that have no merger (`zones/list`, `cache/list`, `allowed|blocked/list`, `dhcp/*`, `logs/list`, `dnsClient/resolve`,
+  the exports) are refused without a node instead of being answered by the first server. `settings/get` is the exception:
+  the header's blocking mark reads it without a node.
+- Downloads (`zones/export`, `logs/download`, `logs/export`, `allowed|blocked/export`) are passed through (type, file name,
+  body) with the read token. A new tab has no Authorization header, so `user/createSingleUseToken` gives a token that works
+  once, for 60 seconds, for these paths only.
+- Refused, even for `admin`: `admin/*`, `user/*` (except the hub's own sign-in), `logs/delete`, `logs/deleteAll`,
+  `dashboard/stats/deleteAll`, `settings/backup`, `settings/restore` and `zones/permissions/*`.
+- The audit line holds the user, role, address, path (dots for slashes), server, one identifier (zone, domain, name or scope)
+  and the outcome. Never a value.
+
+## App writes
+
+Needs the `apps.write` role (`apps-admin`, `admin`); others get `403 Your role does not allow this.` and a `denied-write` audit line.
+Every write goes to a server with its `<ID>_ADMIN_TOKEN`; a server without one answers "no admin token configured" and is not called.
+
+- `POST hub/apps/run` (form or JSON): `action` = `install-store` | `update-store` | `uninstall`, `name`, `targets` (list or
+  comma-separated server ids). Runs on every target in parallel and answers `{ results: [{ server, ok, message? }] }`.
+  The download URL is taken from each target's own `apps/listStoreApps`; a `url` field is only accepted when it matches.
+- `POST hub/apps/upload` (multipart, at most 20 MB, kept in memory only): `action` = `install` | `update`, `name`, `targets`
+  (comma-separated), file field `fileAppZip`. The same zip is forwarded to every target.
+- `apps/downloadAndInstall`, `apps/downloadAndUpdate`, `apps/uninstall`, `apps/install`, `apps/update` with `node=<id>`:
+  the console's own single-server calls, forwarded to that server (store URLs checked as above). Refused without a node, and
+  with `node=cluster`.
+- `apps/listStoreApps`: the first server's store, each app tagged with `installedOn`.
+- `apps/config/get` (read, `node` required) and `apps/config/set` (`apps.write`, `node` required; the old config is saved to the
+  history first and the call is refused if that fails): the console's own per-server Config dialog.
+
+## Config sync
+
+Keeps one app's config the same on every server, except for the fields that must differ. Per app the hub stores, under
+`${HUB_DATA}/apps/<app>/`:
+
+- `baseline.json`: `{ app, text, importedFrom?, updatedAt, updatedBy }`. The config as TEXT, so non-JSON configs work too.
+- `overrides.json`: `{ paths: [...], values: { <server>: { <path>: <value> } } }`. Paths are dotted (`db.path`,
+  `networks.0.cidr`); at most 50; only for a JSON baseline.
+- `history/<time>__<server>__before-apply.txt` (and `before-uninstall`): the config a server had before the hub changed it
+  (50 files per app kept).
+
+A server's effective config is the baseline with that server's local values put in (JSON, pretty-printed with 2 spaces). A local
+path with no value for a server keeps the baseline value and is reported as a warning. Comparisons are by meaning (key order and
+spacing do not count) and leave the local fields out, so two servers that differ only there are "Same" in the matrix, and the
+matrix hash ignores them too.
+
+| Endpoint | Method | Needs | Does |
+|---|---|---|---|
+| `hub/apps/config/compare?name=` | GET | read | Each server's live config, the baseline, the local fields, `inSync`, and the differences against the baseline and against the other servers. Config texts are returned here, to roles that can read. |
+| `hub/apps/overrides/current` | POST `{name, paths}` | `apps.write` | The servers' current values of those paths. |
+| `hub/apps/baseline/import` | POST `{name, server}` | `apps.write` | Baseline := that server's live config. |
+| `hub/apps/baseline/save` | POST `{name, text}` | `apps.write` | Saves the baseline (1 MB at most). Text that starts with `{` or `[` must be valid JSON; the parse error is returned. |
+| `hub/apps/overrides/save` | POST `{name, paths, values}` | `apps.write` | Saves the local fields. |
+| `hub/apps/config/preview` | POST `{name, targets}` | `apps.write` | Per target: current and effective text, the diff, warnings (a missing local value, a version older than the newest installed, no admin token) and `currentHash`. |
+| `hub/apps/config/apply` | POST `{name, targets, expect?}` | `apps.write` | Per target: re-reads the config; refuses if its hash differs from `expect[server]`; saves the old config to the history; `apps/config/set` with the admin token; reads it back and checks it. A server that rejects the config keeps its old one. |
+
+## Zones, settings and blocking across the servers (stage 7)
+
+| Endpoint | Kind | Needs | Does |
+|---|---|---|---|
+| `hub/zones/matrix` | read | `read` | Every zone of every server, one row per name: per server `present`, `absent` or `unknown` with type, SOA serial, disabled, DNSSEC and last modified. Row `status`: `same`, `serial-differs` (Primary zones, different serials), `type-differs`, `only-on` (`onlyOn`), `missing-on`, `unknown`. A Primary with a Secondary of it is the intended setup: it stays `same` with `note: primary-secondary`, or `secondary-behind` when the Secondary lags. A server that did not answer gives `unknown` cells and a `hubWarnings` line; a zone is never called `only-on` while a server that might have it is silent. Pages of 500, internal zones left out. |
+| `hub/zones/copy/preview` | read | `admin` | `{source, zone, targets}`: the source zone (type, serial, about N record lines) and per target whether it exists (type, serial, `needsOverwrite`). |
+| `hub/zones/copy` | write-all | `admin` | `{source, zone, targets, overwrite}`. Reads `zones/export` from the source with the read token (cap 5 MB, 30 s), then per target `zones/create` (the source's type; a Forwarder with `initializeForwarder=false`) when the zone is missing and `zones/import` (`overwrite`, `overwriteZone`, `overwriteSoaSerial` all true, the text as `text/plain`: what the console's Import zone sends). A target that has the zone is touched only if its id is in `overwrite` (checked in the hub). Only Primary and Forwarder zones; Secondary, Stub, Catalog and DNSSEC-signed zones are refused with a reason; a target with the zone as another type is refused; a copy to the source is refused. If the import fails after the create, the result says an empty zone was left. |
+| `hub/zones/secondary/preview` | read | `admin` | `{source, target, zone, primaryAddress, targetAddress, tsigKeyName?}`: the two steps in words. |
+| `hub/zones/secondary` | write-all | `admin` | Step 1 on the source: `zones/options/set` with only `zoneTransfer`, `zoneTransferNetworkACL`, `zoneTransferTsigKeyNames`, `notify`, `notifyNameServers`; what is listed stays, the target address is appended once (so it can run again), a mode that cannot hold a list is widened (Deny to UseSpecifiedNetworkACL, AllowOnlyZoneNameServers to AllowZoneNameServersAndUseSpecifiedNetworkACL, notify None to SpecifiedNameServers, ZoneNameServers to BothZoneAndSpecifiedNameServers). Step 2 on the target: `zones/create` type Secondary with `primaryNameServerAddresses`, `zoneTransferProtocol=Tcp`, `tsigKeyName`. The source goes first; if the target step fails the source change is kept and the result says so. The TSIG key must exist on both servers (`settings/getTsigKeyNames`). Results are two rows, one per server. |
+| `hub/settings/compare` | read | `admin` | Per server `same`, `differs` (N fields) or `local-only`, and the differing fields with each server's value (secrets as `<secret>`, a row only says `secretChanged`). Without a baseline the servers are compared with the first. |
+| `hub/settings/baseline/import` | write-all | `admin` | `{server, group?}`: the server's settings (read with its admin token) become the baseline; `group=blocking` replaces only the blocking fields. The old baseline goes to the history. |
+| `hub/settings/baseline/save` | write-all | `admin` | `{text}`: the baseline from the editor (JSON, up to 2 MB). A secret shown as `<secret>` keeps the stored one; a placeholder with nothing to keep is refused. |
+| `hub/settings/overrides/save` | write-all | `admin` | `{paths}`: extra local field names (up to 50). |
+| `hub/settings/preview` | read | `admin` | `{targets, group?}`: per target the fields an apply would change, as `{kind:'json', changes}`, `secretFields`, `currentHash`. |
+| `hub/settings/apply` | write-all | `admin` | `{targets, expect?, group?}`. Per server: read now, refuse if it changed since the preview, send ONLY the shared fields that differ in one `settings/set` (through the checks of `guards.ts`: never `webService*`, never user, SSO or LDAP fields), after saving the server's settings to the history; read back and compare the fields sent. `enableBlocking` is left alone while blocking is paused. |
+| `hub/settings/history/list`, `hub/settings/history/get` | read | `admin` | The settings kept before an apply, the lists kept before extras were removed, and old baselines. View only, secrets masked: there is no rollback for settings (put values back by accepting a server as baseline, or by hand). |
+| `hub/blocking/lists/compare` | read | `admin` | The custom Blocked and Allowed lists (`blocked/export`, `allowed/export`, read as text with a size cap) as sets: per server `missing N, extra M` with samples. |
+| `hub/blocking/lists/baseline/import` | write-all | `admin` | `{server}`: both lists of that server become the lists baseline (up to 5000 domains each). |
+| `hub/blocking/lists/preview`, `hub/blocking/lists/apply` | read, write-all | `admin` | `{targets, removeExtras}`. Adds the missing domains, one `add` call each (at most 1000); deletes the extras only when `removeExtras=true` (at most 500, after saving the server's lists to the history); reads back. |
+
+**What is local.** Never sent by an apply, listed for information: `dnsServerDomain`, `dnsServerLocalEndPoints`,
+`dnsServerIPv4SourceAddresses`, `dnsServerIPv6SourceAddresses` (the host's identity and bind addresses), every `webService*`
+(how the hub reaches the server), `dnsTlsCertificatePath`, `dnsTlsCertificatePassword` (certificate), `dnsOverHttpUnixSocket`,
+`dnsOverHttpsUnixSocket`, `enableDnsOverHttpUnixSocket`, `enableDnsOverHttpsUnixSocket`, `dnsReverseProxyNetworkACL` (this host's sockets
+and proxy) and `logFolder` (a path). User, SSO and LDAP fields are always local. More can be added in the Local fields tab
+(`${HUB_DATA}/settings/overrides.json`); the built-in ones cannot be removed. **Never compared:** `version`, `uptimestamp`,
+`clusterInitialized`, `clusterNodes`, `temporaryDisableBlockingTill`, `blockListNextUpdatedOn`. Fields only one side reports are skipped.
+
+**Secrets.** TSIG shared secrets and the proxy password are compared inside the hub by their real value and are stored in the
+baseline and the history (mode 0600, protect the data folder like the users file). No answer carries a secret or a hash of one, to
+any role; the endpoints are `admin` only anyway. A changed secret is shown as "secret changed". The audit log has field names, counts
+and outcomes, never values. Applying a baseline with TSIG keys does send the secrets to the servers (that is the point), over the
+connection you configured for them. Files: `${HUB_DATA}/settings/{baseline,overrides}.json`, `settings/history/` (50 files),
+`${HUB_DATA}/blocking/baseline.json`. Settings and the lists are part of the drift check (see Drift).
+
+## History, rollback, drift, export
+
+**History.** Everything the hub replaces is kept in `${HUB_DATA}/apps/<app>/history/` (50 files per app):
+`<time>__<server>__before-apply.txt`, `before-set` (a single-server config save), `before-uninstall`, and
+`<time>__baseline__before-save.txt` for the baseline's earlier text on every baseline save, import or restore.
+
+| Endpoint | Method | Needs | Does |
+|---|---|---|---|
+| `hub/apps/history/list?name=` | GET | read | Newest first: `{ id, at, server, kind, size, hash }`, never the text. |
+| `hub/apps/history/get?name=&id=` | GET | read | The text. `id` has to be a name the listing returns, so nothing outside the folder can be read. |
+| `hub/apps/history/diff?name=&id=&server=` | GET | read | A version against the server's config now (`changes`, `currentHash`, `unchanged`). |
+| `hub/apps/history/rollback` | POST `{name, id, targets, expect?}` | `apps.write` | Puts that version on the targets through the same machinery as apply (old config to the history, `apps/config/set` with the admin token, read back and verify, a result per target, audit). Not for baseline entries. |
+| `hub/apps/baseline/restore` | POST `{name, id}` | `apps.write` | An old baseline entry becomes the baseline again (the current text goes to the history). |
+
+**Drift.** Every `HUB_DRIFT_MINUTES` (and once shortly after start) the hub reads each server's config of every app that has a
+baseline, with the read token, and compares it by meaning with the server's effective config (baseline plus its local values). The
+result is kept in memory and in `${HUB_DATA}/drift.json`: `{ checkedAt, apps: [{ kind, name?, server, since, status, fields? }] }`, where
+`kind` is `app`, `settings` or `lists` (stage 8), `status` is `differs`, `unreadable` (the server or the read failed; listed, not an
+alarm) or `not-installed`, and `since` is the first check that saw it. Two checks never run at once. See "Webhook notices and
+profiles" for the settings and lists kinds.
+
+- `GET hub/drift/get` (read): the state. `POST hub/drift/check` (`apps.write`): check now.
+- `user/session/get` carries `info.hubDrift`, the number of `differs` entries of every kind; the console shows it in the header.
+- `POST hub/apps/drift/accept {name, server}` (`apps.write`): "Accept as baseline". The server's config becomes the baseline; if the
+  app has local fields, they keep the baseline's old values and the server's own values go to the overrides instead.
+- "Revert to baseline" is the normal apply on that server.
+
+**Export and import.** `GET hub/apps/export` (`apps.write`, because configs can hold secrets) downloads
+`{ hub: 'technitium-hub', version: 1, exportedAt, apps: [{ app, baseline, overrides }] }`. `POST hub/apps/import {data, apply}`
+(`apps.write`, up to 5 MB, signed-in callers only) validates the file and answers per app `new`, `changed` or `same`; with
+`apply=true` it writes the baselines and overrides (the replaced baselines go to the history). Local values for servers this hub
+does not have are ignored and reported. It never writes to a server.
+
+Reading configs needs View on Apps for the `hub-reader` user (and `*_ADMIN_TOKEN` is used as a fallback if a server refuses the
+read token). The audit log records who did what, with the size and a short hash of a config, never the text.
+
+Before an uninstall the app's config is saved to `${HUB_DATA}/apps/<app>/history/<time>__<server>__before-uninstall.txt`
+(50 files per app are kept). If that cannot be done the uninstall does not happen. Every target's result is written to the
+audit log as `apps.<action>`; the history files hold configs, so protect the data folder like the users file.
+After any write the hub forgets its cached answers of that server.
+
+## Webhook notices and profiles (stage 8)
+
+**Webhook.** With `HUB_WEBHOOK_URL` (http or https) the hub POSTs a small JSON body when something changes. `HUB_WEBHOOK_SECRET`
+is optional and signs the body. Events:
+
+| Event | When |
+|---|---|
+| `drift.found` | A server/kind/name newly differs from its baseline. Sent once, not at every check. |
+| `drift.cleared` | An entry that differed is back in line. An entry that became unreadable is not cleared: it is not known. |
+| `write.failed` | An audited write ended `failed` (not `refused`, not a failed login). Failures of the same moment are one notice. |
+| `webhook.test` | The admin pressed "Send test" (`POST hub/webhook/test`). |
+
+```json
+{
+  "event": "drift.found",
+  "at": "2026-11-01T09:15:00.000Z",
+  "hub": "hub.example.net",
+  "items": [
+    { "kind": "settings", "server": "ndns", "since": "2026-11-01T09:15:00.000Z", "fields": ["blockingType", "defaultRecordTtl"] },
+    { "kind": "app", "name": "Advanced Blocking", "server": "ndns", "since": "2026-11-01T09:15:00.000Z" }
+  ]
+}
+```
+
+Items carry names and field names, never a value. A `write.failed` item has `action`, `server`, `target`, `user`, `at` and the
+audited `detail`. The first state is the one in `drift.json`, so a restart does not repeat notices.
+
+With a secret, the header `X-Hub-Signature: sha256=<hex>` is the HMAC-SHA256 of the exact request body. Verify it before parsing:
+
+```js
+import { createHmac, timingSafeEqual } from 'node:crypto'
+const expected = 'sha256=' + createHmac('sha256', SECRET).update(rawBody).digest('hex')
+const ok = expected.length === received.length && timingSafeEqual(Buffer.from(expected), Buffer.from(received))
+```
+
+Delivery is fire-and-forget: a 5 second timeout, at most one retry (after a second), redirects are not followed, and it never
+blocks or fails the request that caused it. A failure goes to the server log with the HOST only (never the query string, which may
+carry a token) and the reason (`HTTP 500`, `ECONNREFUSED`, `TimeoutError`).
+
+| Endpoint | Method | Needs | Does |
+|---|---|---|---|
+| `hub/webhook/status` | GET | `admin` | `{ configured, host, signed, last: { event, at, ok, error? } }`. The host and port only, never the address. |
+| `hub/webhook/test` | POST | `admin` | Sends a `webhook.test` notice now and waits for it; answers the same status. Audited as `webhook.test`. |
+
+**Drift kinds.** `GET hub/drift/get` answers `{ checkedAt, apps: [entry] }` (the key is kept) with
+`entry = { kind: 'app' | 'settings' | 'lists', name?, app?, server, since, status, fields? }`. `status` is `differs`, `unreadable`
+or `not-installed`; `fields` are setting names, or `blocked` / `allowed` for the lists; `name` (and `app`) are the app. A
+`drift.json` from before stage 8, where `kind` held the status and `app` the name, still loads. Settings are compared like the
+Settings compare view (shared fields against the settings baseline), lists as sets against the lists baseline, and only when that
+baseline exists. The header count is the number of `differs` entries of every kind.
+
+**Profiles.** A profile is a named bundle in `${HUB_DATA}/profiles/<name>.json` (mode 0600; it holds settings secrets as the
+settings baseline does, so protect the folder): store apps with the version they had and the hub's baseline config of each at that
+moment, the shared fields of the settings baseline (host fields never travel), and the Blocked/Allowed lists baseline. Every
+profile endpoint needs `admin`; viewer, operator and apps-admin are refused.
+
+| Endpoint | Method | Does |
+|---|---|---|
+| `hub/profiles/list` | GET | Names with counts. |
+| `hub/profiles/get?name=` | GET | One profile with secrets as `<secret>` and no app config text (only whether there is one). |
+| `hub/profiles/snapshot {name, apps?, settings, lists}` | POST | Makes or replaces a profile from the current baselines. `apps` is a comma list (absent: every app with a baseline; empty: none). A zip app, or one installed nowhere, is refused. |
+| `hub/profiles/delete {name}` | POST | Deletes it. |
+| `hub/profiles/export?name=` | GET | `{ hub: 'technitium-hub', type: 'profiles', version: 1, exportedAt, profiles }`, with the real secrets: keep the file as a secret. |
+| `hub/profiles/import {data, apply}` | POST | Same size limit as the baselines import (5 MB, signed-in callers only). Dry run first (`new`, `changed`, `same`); `apply=true` writes. A `<secret>` placeholder or a host field in the file is refused or dropped. |
+| `hub/profiles/preview {name, targets, removeExtras}` | GET/POST | Per target: apps to install, already there, or another version; app configs that would change; settings fields that would change; lists missing and extra. Writes nothing. |
+| `hub/profiles/apply {name, targets, removeExtras, from}` | POST | Runs the steps on each target. |
+
+**Apply.** The targets go in parallel; the steps of one server go in order, and a failed step stops the later ones for that server:
+
+1. `apps`: the missing apps are installed from THAT server's own `apps/listStoreApps` (the URL never comes from the request; an
+   installed app is not updated, a version difference is only shown).
+2. `configs`: the profile's config of each app plus that server's own local values, with the old config saved to the history first
+   and a read-back check (the same as the config apply).
+3. `settings`: the changed shared fields only, through the checks of `guards.ts` (no `webService*`, user, SSO or LDAP field), the
+   old settings in the settings history, and a read-back check.
+4. `lists`: the missing domains are added; extras are removed only with `removeExtras`.
+
+The answer is `{ results: [{ server, ok, message, steps: [{ step, status, message }] }] }` with `status` one of `ok`, `failed`,
+`skipped` (done by an earlier run) or `not-run`. `from` is `{ "<server>": "<step>" }`: the step to start at, so "Retry failed" does
+not repeat what worked. The audit has one `profiles.apply` line per server and step (names, counts and the outcome, no values).
 
 ## Run locally
 
@@ -44,7 +347,7 @@ npm run build
 node dev/mock-api.mjs --port 5380 &
 node dev/mock-api.mjs --port 5381 --scale 2 &
 HUB_SERVERS=a=http://127.0.0.1:5380,b=http://127.0.0.1:5381 A_TOKEN=x B_TOKEN=y \
-  HUB_USER=admin HUB_PASSWORD=admin HUB_PORT=5390 npm run hub
+  HUB_USER=admin HUB_PASSWORD=admin HUB_DATA=./hub-data HUB_PORT=5390 npm run hub
 ```
 
 Then open http://localhost:5390. Tests: `npx vitest run hub`.
@@ -54,7 +357,7 @@ Then open http://localhost:5390. Tests: `npx vitest run hub`.
 ```sh
 docker run --rm -p 8080:8080 \
   -e HUB_SERVERS=dns=https://dns.example.net,ndns=https://ndns.example.net \
-  -e DNS_TOKEN=... -e NDNS_TOKEN=... -e HUB_USER=admin -e HUB_PASSWORD=... \
+  -e DNS_TOKEN=... -e NDNS_TOKEN=... -e HUB_USER=admin -e HUB_PASSWORD=... -v hub-data:/data \
   git.shx.one/denozord/technitium-hub:latest
 ```
 

hub/apps-config.ts

@@ -0,0 +1,513 @@
+/*
+Config sync for one app across the servers: compare, baseline, local fields, preview, apply.
+
+The baseline (kept in the hub) plus a server's local values is that server's effective config.
+Apply brings the selected servers to it: it re-reads the server, refuses if the config changed
+since the preview, saves the old config to the history, writes with the server's ADMIN token and
+reads the config back to check it. Config texts are returned to the caller (it is what they edit)
+but never written to the audit log.
+*/
+import type { HubConfig, HubServer } from './config.ts'
+import {
+  MAX_CONFIG,
+  acceptServerConfig,
+  cleanValues,
+  diffConfigs,
+  effectiveConfig,
+  importLocalValues,
+  looksLikeJson,
+  parseJson,
+  sameConfig,
+  textHash,
+  validatePaths,
+  type Diff,
+} from './apps-sync.ts'
+import { Upstream, UpstreamError } from './upstream.ts'
+import {
+  listBaselineApps,
+  listHistory,
+  readBaseline,
+  readHistory,
+  readOverrides,
+  writeBaseline,
+  writeHistory,
+  writeOverrides,
+  type Baseline,
+  type HistoryEntry,
+  type Overrides,
+} from './store.ts'
+
+export class UserError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'UserError'
+  }
+}
+
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+
+export interface Live {
+  installed: boolean
+  version?: string
+  ok: boolean
+  error?: string
+  /** The server's config text; null when the app has none. */
+  text: string | null
+}
+
+export interface ServerView {
+  id: string
+  ok: boolean
+  error?: string
+  installed: boolean
+  writable: boolean
+  version?: string
+  text: string | null
+  hash: string
+  effective?: string
+  warnings: string[]
+  /** Null without a baseline. */
+  inSync: boolean | null
+  /** What applying would change: this server's config to its effective config. */
+  vsBaseline?: Diff
+  /** This server's config relative to each other server's, local fields left out. */
+  vsOthers: Record<string, Diff>
+}
+
+export interface TargetPreview {
+  server: string
+  ok: boolean
+  error?: string
+  writable: boolean
+  current: string | null
+  currentHash: string
+  effective: string
+  changes: Diff
+  unchanged: boolean
+  warnings: string[]
+}
+
+export interface TargetResult {
+  server: string
+  ok: boolean
+  message?: string
+}
+
+/* 9.10 is newer than 9.9: compared part by part, numerically. */
+export function compareVersions(a: string, b: string): number {
+  const [x, y] = [a, b].map((v) => v.split('.').map((p) => Number.parseInt(p, 10) || 0))
+  for (let i = 0; i < Math.max(x.length, y.length); i++) {
+    const d = (x[i] ?? 0) - (y[i] ?? 0)
+    if (d !== 0) return d
+  }
+  return 0
+}
+
+/* The read token first, the admin token when the server refuses it. */
+export async function readConfigText(upstream: Upstream, server: HubServer, name: string): Promise<string | null> {
+  let response: unknown
+  try {
+    response = await upstream.call(server, 'apps/config/get', { name })
+  } catch (e) {
+    if (!(e instanceof UpstreamError) || e.kind !== 'rejected' || server.adminToken === undefined) throw e
+    response = await upstream.call(server, 'apps/config/get', { name }, server.adminToken)
+  }
+  const config = (response as { config?: unknown } | null)?.config
+  return typeof config === 'string' ? config : null
+}
+
+export class AppsConfig {
+  #config: HubConfig
+  #upstream: Upstream
+  #now: () => number
+
+  constructor(config: HubConfig, upstream: Upstream, now: () => number = Date.now) {
+    this.#config = config
+    this.#upstream = upstream
+    this.#now = now
+  }
+
+  #server(id: string): HubServer {
+    const found = this.#config.servers.find((s) => s.id === id)
+    if (found === undefined) throw new UserError(`Unknown server: ${id}.`)
+    return found
+  }
+
+  #targets(ids: string[]): HubServer[] {
+    const unique = [...new Set(ids)]
+    if (unique.length === 0) throw new UserError('Choose at least one server.')
+    return unique.map((id) => this.#server(id))
+  }
+
+  async #readConfig(server: HubServer, name: string, fresh: boolean): Promise<string | null> {
+    if (fresh) this.#upstream.invalidate(server.id)
+    return readConfigText(this.#upstream, server, name)
+  }
+
+  async #live(server: HubServer, name: string, fresh: boolean): Promise<Live> {
+    let version: string | undefined
+    try {
+      if (fresh) this.#upstream.invalidate(server.id)
+      const list = (await this.#upstream.call(server, 'apps/list')) as { apps?: { name: string; version?: string }[] } | null
+      const app = (list?.apps ?? []).find((a) => a.name === name)
+      if (app === undefined) return { installed: false, ok: true, text: null }
+      version = app.version
+    } catch (e) {
+      return { installed: false, ok: false, error: reason(e), text: null }
+    }
+    try {
+      return { installed: true, ok: true, version, text: await this.#readConfig(server, name, false) }
+    } catch (e) {
+      return { installed: true, ok: false, version, error: reason(e), text: null }
+    }
+  }
+
+  async #liveAll(name: string, fresh: boolean): Promise<Map<string, Live>> {
+    const entries = await Promise.all(this.#config.servers.map(async (s): Promise<[string, Live]> => [s.id, await this.#live(s, name, fresh)]))
+    return new Map(entries)
+  }
+
+  getBaseline(name: string): Baseline | null {
+    return readBaseline(this.#config.dataDir, name)
+  }
+
+  getOverrides(name: string): Overrides {
+    return readOverrides(this.#config.dataDir, name)
+  }
+
+  async compare(name: string): Promise<unknown> {
+    const baseline = this.getBaseline(name)
+    const overrides = this.getOverrides(name)
+    const live = await this.#liveAll(name, false)
+    const views: ServerView[] = this.#config.servers.map((s) => {
+      const l = live.get(s.id) as Live
+      const text = l.text
+      const view: ServerView = {
+        id: s.id,
+        ok: l.ok,
+        installed: l.installed,
+        writable: s.adminToken !== undefined,
+        text,
+        hash: textHash(text ?? ''),
+        warnings: [],
+        inSync: null,
+        vsOthers: {},
+      }
+      if (l.error !== undefined) view.error = l.error
+      if (l.version !== undefined) view.version = l.version
+      if (baseline !== null && l.installed && l.ok) {
+        const effective = effectiveConfig(baseline.text, overrides, s.id)
+        view.effective = effective.text
+        view.warnings = effective.warnings
+        view.inSync = sameConfig(text ?? '', effective.text)
+        view.vsBaseline = diffConfigs(text ?? '', effective.text, [])
+      }
+      return view
+    })
+    for (const v of views) {
+      if (!v.installed || !v.ok) continue
+      for (const o of views) {
+        if (o.id !== v.id && o.installed && o.ok) v.vsOthers[o.id] = diffConfigs(o.text ?? '', v.text ?? '', overrides.paths)
+      }
+    }
+    return { name, baseline, overrides, servers: views }
+  }
+
+  async importBaseline(name: string, serverId: string, user: string): Promise<{ detail: string }> {
+    const server = this.#server(serverId)
+    let text: string | null
+    try {
+      text = await this.#readConfig(server, name, true)
+    } catch (e) {
+      throw new UserError(`${serverId}: ${reason(e)}`)
+    }
+    if (text === null || text.trim() === '') throw new UserError(`${serverId} has no config for "${name}" to import.`)
+    if (text.length > MAX_CONFIG) throw new UserError('The config is larger than 1 MB.')
+    this.#setBaseline(name, text, serverId, user)
+    return { detail: `from ${serverId}, ${text.length} bytes, ${textHash(text)}` }
+  }
+
+  /* Writes the baseline; the text it replaces goes to the history first (when it differs). */
+  #setBaseline(name: string, text: string, importedFrom: string | undefined, user: string): void {
+    const old = this.getBaseline(name)
+    if (old !== null && old.text !== text) {
+      try {
+        writeHistory(this.#config.dataDir, name, 'baseline', old.text, 'before-save', this.#now())
+      } catch (e) {
+        throw new UserError(`The old baseline could not be saved first (${reason(e)}); nothing was changed.`)
+      }
+    }
+    writeBaseline(this.#config.dataDir, {
+      app: name,
+      text,
+      ...(importedFrom === undefined ? {} : { importedFrom }),
+      updatedAt: new Date(this.#now()).toISOString(),
+      updatedBy: user,
+    })
+  }
+
+  saveBaseline(name: string, text: string, user: string): { detail: string } {
+    if (text.length > MAX_CONFIG) throw new UserError('The baseline is larger than 1 MB.')
+    if (looksLikeJson(text)) {
+      const parsed = parseJson(text)
+      if (!parsed.ok) throw new UserError(`The baseline is not valid JSON: ${parsed.error}`)
+    }
+    this.#setBaseline(name, text, this.getBaseline(name)?.importedFrom, user)
+    return { detail: `${text.length} bytes, ${textHash(text)}` }
+  }
+
+  saveOverrides(name: string, pathsRaw: unknown, valuesRaw: unknown): { detail: string } {
+    let paths: string[]
+    let values: Overrides['values']
+    try {
+      paths = validatePaths(pathsRaw)
+      values = cleanValues(valuesRaw ?? {}, paths, this.#config.servers.map((s) => s.id))
+    } catch (e) {
+      throw new UserError(reason(e))
+    }
+    const baseline = this.getBaseline(name)
+    if (paths.length > 0 && baseline !== null && (!looksLikeJson(baseline.text) || !parseJson(baseline.text).ok)) {
+      throw new UserError('The baseline is plain text, so it cannot have local fields.')
+    }
+    if (JSON.stringify(values).length > MAX_CONFIG) throw new UserError('The local values are larger than 1 MB.')
+    writeOverrides(this.#config.dataDir, name, { paths, values })
+    return { detail: `${paths.length} local field(s)` }
+  }
+
+  /* Reads each local path from the servers' live configs, for the "fill from current" button. */
+  async currentLocalValues(name: string, pathsRaw: unknown): Promise<Overrides['values']> {
+    let paths: string[]
+    try {
+      paths = validatePaths(pathsRaw)
+    } catch (e) {
+      throw new UserError(reason(e))
+    }
+    const live = await this.#liveAll(name, true)
+    return importLocalValues(Object.fromEntries([...live].map(([id, l]) => [id, l.installed && l.ok ? l.text : null])), paths)
+  }
+
+  async preview(name: string, ids: string[], overrideText?: string): Promise<{ targets: TargetPreview[] }> {
+    const targets = this.#targets(ids)
+    const baseline = overrideText === undefined ? this.getBaseline(name) : ({ app: name, text: overrideText, updatedAt: '', updatedBy: '' } satisfies Baseline)
+    if (baseline === null) throw new UserError('This app has no baseline yet. Import one from a server first.')
+    const overrides = this.getOverrides(name)
+    const live = await this.#liveAll(name, true)
+    const versions = [...live].filter(([, l]) => l.installed && l.version !== undefined).map(([id, l]) => ({ id, version: l.version as string }))
+    const newest = versions.reduce<{ id: string; version: string } | null>((best, v) => (best === null || compareVersions(v.version, best.version) > 0 ? v : best), null)
+
+    return {
+      targets: targets.map((s): TargetPreview => {
+        const l = live.get(s.id) as Live
+        const effective = effectiveConfig(baseline.text, overrides, s.id)
+        const warnings = [...effective.warnings]
+        if (s.adminToken === undefined) warnings.push('The hub has no admin token for this server, so it cannot write to it.')
+        if (l.installed && l.version !== undefined && newest !== null && compareVersions(l.version, newest.version) < 0) {
+          warnings.push(`Version ${l.version} here, ${newest.version} on ${newest.id}: versions differ, align versions first.`)
+        }
+        const current = l.text
+        const base = {
+          server: s.id,
+          writable: s.adminToken !== undefined,
+          current,
+          currentHash: textHash(current ?? ''),
+          effective: effective.text,
+          warnings,
+        }
+        if (!l.ok) return { ...base, ok: false, error: l.error ?? 'did not answer', changes: { kind: 'text', lines: [], truncated: false, changed: 0 }, unchanged: false }
+        if (!l.installed) return { ...base, ok: false, error: 'The app is not installed on this server.', changes: { kind: 'text', lines: [], truncated: false, changed: 0 }, unchanged: false }
+        const changes = diffConfigs(current ?? '', effective.text, [])
+        return { ...base, ok: true, changes, unchanged: sameConfig(current ?? '', effective.text) }
+      }),
+    }
+  }
+
+  /*
+  Brings one server to `desired`: re-read, refuse if it changed since `expectHash`, save the old
+  config to the history, write with the admin token, read back and check.
+  */
+  async #applyText(s: HubServer, name: string, desired: (current: string) => string, expectHash: string | undefined): Promise<{ result: TargetResult; detail?: string }> {
+    if (s.adminToken === undefined) return { result: { server: s.id, ok: false, message: 'no admin token configured' } }
+    try {
+      const l = await this.#live(s, name, true)
+      if (!l.ok) return { result: { server: s.id, ok: false, message: l.error ?? 'did not answer' } }
+      if (!l.installed) return { result: { server: s.id, ok: false, message: 'the app is not installed on this server' } }
+      const current = l.text ?? ''
+      if (expectHash !== undefined && expectHash !== textHash(current)) {
+        return { result: { server: s.id, ok: false, message: 'the config changed on the server since the preview; run the preview again' } }
+      }
+      const text = desired(current)
+      if (sameConfig(current, text)) return { result: { server: s.id, ok: true, message: 'Already in sync; nothing was written.' } }
+      if (text.length > MAX_CONFIG) return { result: { server: s.id, ok: false, message: 'the config is larger than 1 MB' } }
+      try {
+        writeHistory(this.#config.dataDir, name, s.id, current, 'before-apply', this.#now())
+      } catch (e) {
+        return { result: { server: s.id, ok: false, message: `the old config could not be saved first (${reason(e)}); nothing was changed` } }
+      }
+      await this.#upstream.write(s, 'apps/config/set', {}, new URLSearchParams({ name, config: text }))
+      const after = await this.#readConfig(s, name, true)
+      const detail = `${textHash(current)} to ${textHash(text)}, ${text.length} bytes`
+      if (after === null || !sameConfig(after, text)) {
+        return { result: { server: s.id, ok: false, message: 'the server accepted the config but reads back a different one' }, detail }
+      }
+      return { result: { server: s.id, ok: true, message: 'Applied and verified.' }, detail }
+    } catch (e) {
+      return { result: { server: s.id, ok: false, message: reason(e) } }
+    }
+  }
+
+  async #applyMany(
+    name: string,
+    ids: string[],
+    desiredFor: (s: HubServer) => (current: string) => string,
+    expect: Record<string, string>,
+  ): Promise<{ results: TargetResult[]; details: Record<string, string> }> {
+    const targets = this.#targets(ids)
+    const done = await Promise.all(targets.map((s) => this.#applyText(s, name, desiredFor(s), expect[s.id])))
+    const details: Record<string, string> = {}
+    for (const d of done) if (d.detail !== undefined) details[d.result.server] = d.detail
+    return { results: done.map((d) => d.result), details }
+  }
+
+  /* `overrideText` is a config that is not the hub's baseline (a profile's). The server's local values still apply. */
+  async apply(name: string, ids: string[], expect: Record<string, string>, overrideText?: string): Promise<{ results: TargetResult[]; details: Record<string, string> }> {
+    const baseline = overrideText === undefined ? this.getBaseline(name) : ({ app: name, text: overrideText, updatedAt: '', updatedBy: '' } satisfies Baseline)
+    if (baseline === null) throw new UserError('This app has no baseline yet. Import one from a server first.')
+    const overrides = this.getOverrides(name)
+    return this.#applyMany(name, ids, (s) => () => effectiveConfig(baseline.text, overrides, s.id).text, expect)
+  }
+
+  /* ---- History ---- */
+
+  historyList(name: string): HistoryEntry[] {
+    return listHistory(this.#config.dataDir, name)
+  }
+
+  #entry(name: string, id: string): { entry: HistoryEntry; text: string } {
+    const entry = this.historyList(name).find((e) => e.id === id)
+    const text = entry === undefined ? null : readHistory(this.#config.dataDir, name, id)
+    if (entry === undefined || text === null) throw new UserError('There is no such history entry.')
+    return { entry, text }
+  }
+
+  historyGet(name: string, id: string): { entry: HistoryEntry; text: string } {
+    return this.#entry(name, id)
+  }
+
+  /* A saved version against a server's config now: what rolling back would change. */
+  async historyDiff(name: string, id: string, serverId: string): Promise<unknown> {
+    const { entry, text } = this.#entry(name, id)
+    const server = this.#server(serverId)
+    const l = await this.#live(server, name, true)
+    if (!l.ok || !l.installed) throw new UserError(l.ok ? `The app is not installed on ${serverId}.` : `${serverId}: ${l.error ?? 'did not answer'}`)
+    const current = l.text ?? ''
+    return {
+      entry,
+      server: serverId,
+      writable: server.adminToken !== undefined,
+      current,
+      currentHash: textHash(current),
+      version: text,
+      changes: diffConfigs(current, text, []),
+      unchanged: sameConfig(current, text),
+    }
+  }
+
+  async rollback(name: string, id: string, ids: string[], expect: Record<string, string>): Promise<{ results: TargetResult[]; details: Record<string, string> }> {
+    const { entry, text } = this.#entry(name, id)
+    if (entry.kind === 'before-save') throw new UserError('That entry is an old baseline. Restore it as the baseline instead.')
+    return this.#applyMany(name, ids, () => () => text, expect)
+  }
+
+  restoreBaseline(name: string, id: string, user: string): { detail: string } {
+    const { entry, text } = this.#entry(name, id)
+    if (entry.kind !== 'before-save') throw new UserError('Only an old baseline can be restored as the baseline.')
+    this.#setBaseline(name, text, this.getBaseline(name)?.importedFrom, user)
+    return { detail: `entry ${entry.hash}, ${text.length} bytes` }
+  }
+
+  /*
+  "Accept as baseline" for a drifted server: its config becomes the baseline, except that local
+  fields keep the baseline's values and the server's own values go into the overrides.
+  */
+  async acceptDrift(name: string, serverId: string, user: string): Promise<{ detail: string }> {
+    const server = this.#server(serverId)
+    let text: string | null
+    try {
+      text = await this.#readConfig(server, name, true)
+    } catch (e) {
+      throw new UserError(`${serverId}: ${reason(e)}`)
+    }
+    if (text === null || text.trim() === '') throw new UserError(`${serverId} has no config for "${name}".`)
+    const overrides = this.getOverrides(name)
+    let next
+    try {
+      next = acceptServerConfig(text, this.getBaseline(name)?.text ?? null, overrides, serverId)
+    } catch (e) {
+      throw new UserError(reason(e))
+    }
+    this.#setBaseline(name, next.baselineText, serverId, user)
+    if (overrides.paths.length > 0) writeOverrides(this.#config.dataDir, name, next.overrides)
+    return { detail: `from ${serverId}, ${next.baselineText.length} bytes, ${textHash(next.baselineText)}` }
+  }
+
+  /* ---- Export and import of all baselines ---- */
+
+  exportAll(): { hub: string; version: number; exportedAt: string; apps: { app: string; baseline: Baseline; overrides: Overrides }[] } {
+    const apps = listBaselineApps(this.#config.dataDir).flatMap((app) => {
+      const baseline = this.getBaseline(app)
+      return baseline === null ? [] : [{ app, baseline, overrides: this.getOverrides(app) }]
+    })
+    return { hub: 'technitium-hub', version: 1, exportedAt: new Date(this.#now()).toISOString(), apps }
+  }
+
+  /* Validates a file and says what it would change; with `apply` also writes baselines and overrides (never to a server). */
+  importAll(raw: string, apply: boolean, user: string): { apps: { app: string; status: 'new' | 'changed' | 'same'; ignoredServers: string[] }[]; applied: boolean } {
+    const parsed = parseJson(raw)
+    if (!parsed.ok) throw new UserError('The file is not valid JSON.')
+    const file = parsed.value as { hub?: unknown; version?: unknown; apps?: unknown }
+    if (file === null || typeof file !== 'object' || file.hub !== 'technitium-hub') throw new UserError('This is not a baselines file of the hub.')
+    if (file.version !== 1) throw new UserError('This file has a version the hub does not know.')
+    if (!Array.isArray(file.apps) || file.apps.length > 500) throw new UserError('The file has no list of apps, or has more than 500.')
+    const known = this.#config.servers.map((s) => s.id)
+    const plan: { app: string; text: string; overrides: Overrides; status: 'new' | 'changed' | 'same'; ignoredServers: string[] }[] = []
+    const seen = new Set<string>()
+    for (const item of file.apps as Record<string, unknown>[]) {
+      const app = item?.app
+      const baseline = item?.baseline as { text?: unknown } | null | undefined
+      if (typeof app !== 'string' || app === '' || app.length > 200 || seen.has(app)) throw new UserError('An app in the file has no valid, unique name.')
+      seen.add(app)
+      if (typeof baseline?.text !== 'string') throw new UserError(`"${app}" has no baseline text.`)
+      if (baseline.text.length > MAX_CONFIG) throw new UserError(`The baseline of "${app}" is larger than 1 MB.`)
+      if (looksLikeJson(baseline.text) && !parseJson(baseline.text).ok) throw new UserError(`The baseline of "${app}" is not valid JSON.`)
+      const rawOverrides = (item.overrides ?? { paths: [], values: {} }) as { paths?: unknown; values?: unknown }
+      let paths: string[]
+      try {
+        paths = validatePaths(rawOverrides.paths ?? [])
+      } catch (e) {
+        throw new UserError(`"${app}": ${reason(e)}`)
+      }
+      if (paths.length > 0 && !looksLikeJson(baseline.text)) throw new UserError(`"${app}": a plain text baseline cannot have local fields.`)
+      const values = rawOverrides.values !== null && typeof rawOverrides.values === 'object' && !Array.isArray(rawOverrides.values) ? (rawOverrides.values as Record<string, unknown>) : {}
+      const ignoredServers = Object.keys(values).filter((id) => !known.includes(id))
+      let cleaned: Overrides['values']
+      try {
+        cleaned = cleanValues(Object.fromEntries(Object.entries(values).filter(([id]) => known.includes(id))), paths, known)
+      } catch (e) {
+        throw new UserError(`"${app}": ${reason(e)}`)
+      }
+      const overrides: Overrides = { paths, values: cleaned }
+      const old = this.getBaseline(app)
+      const oldOverrides = this.getOverrides(app)
+      const status = old === null ? 'new' : old.text === baseline.text && JSON.stringify(oldOverrides) === JSON.stringify(overrides) ? 'same' : 'changed'
+      plan.push({ app, text: baseline.text, overrides, status, ignoredServers })
+    }
+    if (apply) {
+      for (const p of plan) {
+        if (p.status === 'same') continue
+        this.#setBaseline(p.app, p.text, this.getBaseline(p.app)?.importedFrom, user)
+        writeOverrides(this.#config.dataDir, p.app, p.overrides)
+      }
+    }
+    return { apps: plan.map(({ app, status, ignoredServers }) => ({ app, status, ignoredServers })), applied: apply }
+  }
+}

hub/apps-history.test.ts

@@ -0,0 +1,439 @@
+// @vitest-environment node
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { acceptServerConfig } from './apps-sync.ts'
+import { addUser, writeUsersFile } from './users.ts'
+
+const json = (o: unknown): string => JSON.stringify(o, null, 2)
+
+interface Fake {
+  server: http.Server
+  url: string
+  apps: Record<string, { version: string; config: string | null }>
+  calls: { path: string; token: string; method: string }[]
+  rejectSet: string | null
+}
+
+function fake(): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = { server: http.createServer(), url: '', apps: {}, calls: [], rejectSet: null }
+    f.server.on('request', (req, res) => {
+      const chunks: Buffer[] = []
+      req.on('data', (c: Buffer) => chunks.push(c))
+      req.on('end', () => {
+        const url = new URL(req.url ?? '/', 'http://fake')
+        const apiPath = url.pathname.replace(/^\/api\//, '')
+        const token = url.searchParams.get('token') ?? ''
+        const form = new URLSearchParams(Buffer.concat(chunks).toString('utf8'))
+        f.calls.push({ path: apiPath, token, method: req.method ?? '' })
+        const name = url.searchParams.get('name') ?? form.get('name') ?? ''
+        res.setHeader('content-type', 'application/json')
+        const ok = (response: unknown = {}) => res.end(JSON.stringify({ status: 'ok', response }))
+        const error = (m: string) => res.end(JSON.stringify({ status: 'error', errorMessage: m }))
+        if (apiPath === 'apps/list') return ok({ apps: Object.entries(f.apps).map(([n, a]) => ({ name: n, version: a.version })) })
+        if (apiPath === 'apps/config/get') return f.apps[name] ? ok({ config: f.apps[name].config }) : error('not found')
+        if (apiPath === 'apps/config/set') {
+          if (!token.startsWith('admin-')) return error('Access was denied.')
+          if (f.rejectSet !== null) return error(f.rejectSet)
+          f.apps[name].config = form.get('config')
+          return ok()
+        }
+        ok()
+      })
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+let a: Fake
+let b: Fake
+let dataDir: string
+let staticDir: string
+let hub: http.Server | undefined
+let base: string
+const tokens: Record<string, string> = {}
+
+const V1 = json({ ttl: 30, db: '/base.db' })
+const V2 = json({ ttl: 45, db: '/base.db' })
+
+function reset(): void {
+  for (const f of [a, b]) {
+    f.calls = []
+    f.rejectSet = null
+  }
+  a.apps = { Blocker: { version: '1.0', config: V1 } }
+  b.apps = { Blocker: { version: '1.0', config: V1 } }
+}
+
+async function start(dir: string = dataDir): Promise<void> {
+  hub?.close()
+  hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      A_ADMIN_TOKEN: 'admin-a',
+      B_ADMIN_TOKEN: 'admin-b',
+      HUB_DATA: dir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      HUB_DRIFT_MINUTES: '0',
+    }),
+  )
+  await new Promise<void>((resolve) => hub!.listen(0, '127.0.0.1', resolve))
+  base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  for (const role of ['viewer', 'apps-admin', 'admin']) {
+    const res = await fetch(`${base}/api/user/login`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: `user=${role}&pass=password-${role}`,
+    })
+    tokens[role] = ((await res.json()) as { token: string }).token
+  }
+}
+
+async function raw(role: string, apiPath: string, init: { params?: Record<string, string>; json?: unknown; form?: Record<string, string> } = {}) {
+  const headers: Record<string, string> = { authorization: `Bearer ${tokens[role]}` }
+  let body: string | undefined
+  if (init.json !== undefined) {
+    headers['content-type'] = 'application/json'
+    body = JSON.stringify(init.json)
+  } else if (init.form !== undefined) {
+    headers['content-type'] = 'application/x-www-form-urlencoded'
+    body = new URLSearchParams(init.form).toString()
+  }
+  return fetch(`${base}/api/${apiPath}?${new URLSearchParams(init.params ?? {})}`, { method: body === undefined ? 'GET' : 'POST', headers, body })
+}
+
+async function api(role: string, apiPath: string, init: Parameters<typeof raw>[2] = {}) {
+  const res = await raw(role, apiPath, init)
+  return { code: res.status, body: (await res.json()) as Record<string, any> }
+}
+
+const dirOf = (d: string = dataDir) => path.join(d, 'apps', 'Blocker')
+const historyDir = (d: string = dataDir) => path.join(dirOf(d), 'history')
+const writes = (f: Fake) => f.calls.filter((c) => c.path === 'apps/config/set')
+const list = async (role = 'viewer') => (await api(role, 'hub/apps/history/list', { params: { name: 'Blocker' } })).body.response.entries as Record<string, any>[]
+
+beforeAll(async () => {
+  a = await fake()
+  b = await fake()
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-hist-data-'))
+  staticDir = mkdtempSync(path.join(tmpdir(), 'hub-hist-static-'))
+  let users = addUser([], 'viewer', 'viewer', 'password-viewer')
+  users = addUser(users, 'apps-admin', 'apps-admin', 'password-apps-admin')
+  users = addUser(users, 'admin', 'admin', 'password-admin')
+  writeUsersFile(dataDir, users)
+})
+
+afterAll(() => {
+  hub?.close()
+  a.server.close()
+  b.server.close()
+  rmSync(dataDir, { recursive: true, force: true })
+  rmSync(staticDir, { recursive: true, force: true })
+})
+
+beforeEach(async () => {
+  reset()
+  rmSync(path.join(dataDir, 'apps'), { recursive: true, force: true })
+  rmSync(path.join(dataDir, 'audit.log'), { force: true })
+  rmSync(path.join(dataDir, 'drift.json'), { force: true })
+  await start()
+})
+
+afterEach(() => {
+  hub?.close()
+})
+
+const post = (role: string, p: string, body: Record<string, unknown>) => api(role, p, { json: body })
+const importFrom = (server: string) => post('apps-admin', 'hub/apps/baseline/import', { name: 'Blocker', server })
+const saveBaseline = (text: string) => post('apps-admin', 'hub/apps/baseline/save', { name: 'Blocker', text })
+
+describe('history', () => {
+  it('keeps the old baseline text on every save and import, and lists it without the text', async () => {
+    await importFrom('a')
+    expect(await list()).toEqual([])
+    await saveBaseline(V2)
+    await importFrom('b')
+    const entries = await list()
+    expect(entries).toHaveLength(2)
+    expect(entries[0]).toMatchObject({ server: 'baseline', kind: 'before-save', size: V2.length })
+    expect(entries[0].at).toMatch(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/)
+    expect(entries[0].hash).toMatch(/^[0-9a-f]{16}$/)
+    expect(entries[0].text).toBeUndefined()
+    expect(entries[1].size).toBe(V1.length)
+    expect(entries[0].id > entries[1].id).toBe(true)
+  })
+
+  it('does not add an entry when the baseline text did not change', async () => {
+    await importFrom('a')
+    await saveBaseline(V1)
+    expect(await list()).toEqual([])
+  })
+
+  it('reads one entry by the id the listing gives and refuses anything else', async () => {
+    await importFrom('a')
+    await saveBaseline(V2)
+    const [entry] = await list()
+    const got = await api('viewer', 'hub/apps/history/get', { params: { name: 'Blocker', id: entry.id } })
+    expect(got.body.response.text).toBe(V1)
+    for (const id of ['../../../users.json', '..\\..\\users.json', entry.id.replace('2', '3'), entry.id + '/', '/etc/passwd', '', `../history/${entry.id}`, 'baseline.json']) {
+      const r = await api('viewer', 'hub/apps/history/get', { params: { name: 'Blocker', id } })
+      expect([id, r.code]).toEqual([id, 400])
+    }
+    // The app name cannot walk out either.
+    const r = await api('viewer', 'hub/apps/history/get', { params: { name: '../..', id: 'users.json' } })
+    expect(r.code).toBe(400)
+  })
+
+  it('ignores files in the folder that are not history entries', async () => {
+    await importFrom('a')
+    mkdirSync(historyDir(), { recursive: true })
+    writeFileSync(path.join(historyDir(), 'notes.txt'), 'x')
+    expect(await list()).toEqual([])
+  })
+
+  it('shows what a saved version would change on a server', async () => {
+    await importFrom('a')
+    await saveBaseline(V2)
+    const [entry] = await list()
+    const r = await api('viewer', 'hub/apps/history/diff', { params: { name: 'Blocker', id: entry.id, server: 'b' } })
+    expect(r.body.response).toMatchObject({ server: 'b', writable: true, unchanged: true, current: V1 })
+    a.apps.Blocker.config = V2
+    const d = await api('viewer', 'hub/apps/history/diff', { params: { name: 'Blocker', id: entry.id, server: 'a' } })
+    expect(d.body.response.changes.changes.map((c: any) => [c.path, c.a, c.b])).toEqual([['ttl', 45, 30]])
+    expect(d.body.response.unchanged).toBe(false)
+  })
+
+  it('keeps at most 50 entries per app', async () => {
+    await importFrom('a')
+    for (let i = 0; i < 55; i++) await saveBaseline(json({ n: i }))
+    expect((await list()).length).toBe(50)
+  })
+
+  it('also records the config a single-server set replaced', async () => {
+    await api('apps-admin', 'apps/config/set', { params: { node: 'a' }, json: { name: 'Blocker', config: '{}' } })
+    const entries = await list()
+    expect(entries[0]).toMatchObject({ server: 'a', kind: 'before-set' })
+  })
+})
+
+describe('rollback and restore', () => {
+  async function twoVersions() {
+    // Both servers were changed by the hub once: their old config (V1) is in the history.
+    await importFrom('a')
+    await saveBaseline(V2)
+    await post('apps-admin', 'hub/apps/config/apply', { name: 'Blocker', targets: ['a', 'b'] })
+    return (await list()).filter((e) => e.kind === 'before-apply')
+  }
+
+  it('rolls a version back to the chosen servers, one result each, history first, verified', async () => {
+    const applied = await twoVersions()
+    expect(applied.map((e) => e.server).sort()).toEqual(['a', 'b'])
+    expect(a.apps.Blocker.config).toBe(V2)
+    const entry = applied.find((e) => e.server === 'a') as Record<string, any>
+    b.rejectSet = 'Disk full.'
+    const r = await post('apps-admin', 'hub/apps/history/rollback', { name: 'Blocker', id: entry.id, targets: ['a', 'b'] })
+    expect(r.body.response.results).toEqual([
+      { server: 'a', ok: true, message: 'Applied and verified.' },
+      { server: 'b', ok: false, message: 'Disk full.' },
+    ])
+    expect(a.apps.Blocker.config).toBe(V1)
+    expect(b.apps.Blocker.config).toBe(V2)
+    // The config rolled away from is kept too.
+    const files = readdirSync(historyDir()).filter((f) => f.endsWith('__a__before-apply.txt'))
+    expect(files).toHaveLength(2)
+    expect(writes(a).at(-1)?.token).toBe('admin-a')
+  })
+
+  it('refuses a rollback when the config changed since the preview, and for an old baseline entry', async () => {
+    const applied = await twoVersions()
+    const entry = applied[0]
+    const stale = await post('apps-admin', 'hub/apps/history/rollback', { name: 'Blocker', id: entry.id, targets: ['a'], expect: { a: 'deadbeefdeadbeef' } })
+    expect(stale.body.response.results[0].message).toMatch(/changed on the server since the preview/)
+    const baselineEntry = (await list()).find((e) => e.kind === 'before-save') as Record<string, any>
+    const refused = await post('apps-admin', 'hub/apps/history/rollback', { name: 'Blocker', id: baselineEntry.id, targets: ['a'] })
+    expect(refused.code).toBe(400)
+    expect(refused.body.errorMessage).toMatch(/Restore it as the baseline/)
+  })
+
+  it('restores an old baseline as the baseline, keeping the current one in the history', async () => {
+    await importFrom('a')
+    await saveBaseline(V2)
+    const old = (await list())[0]
+    const r = await post('apps-admin', 'hub/apps/baseline/restore', { name: 'Blocker', id: old.id })
+    expect(r.body.response.baseline.text).toBe(V1)
+    const entries = await list()
+    expect(entries[0]).toMatchObject({ kind: 'before-save', size: V2.length })
+    expect((await post('apps-admin', 'hub/apps/baseline/restore', { name: 'Blocker', id: 'nope' })).code).toBe(400)
+    // A server entry is not a baseline.
+    b.apps.Blocker.config = V2
+    await post('apps-admin', 'hub/apps/config/apply', { name: 'Blocker', targets: ['b'] })
+    const serverEntry = (await list()).find((e) => e.kind === 'before-apply') as Record<string, any>
+    expect((await post('apps-admin', 'hub/apps/baseline/restore', { name: 'Blocker', id: serverEntry.id })).code).toBe(400)
+  })
+
+  it('lets a viewer look but not roll back or restore', async () => {
+    await twoVersions()
+    const [entry] = await list()
+    expect((await post('viewer', 'hub/apps/history/rollback', { name: 'Blocker', id: entry.id, targets: ['a'] })).code).toBe(403)
+    expect((await post('viewer', 'hub/apps/baseline/restore', { name: 'Blocker', id: entry.id })).code).toBe(403)
+    expect((await api('viewer', 'hub/apps/history/get', { params: { name: 'Blocker', id: entry.id } })).code).toBe(200)
+  })
+
+  it('audits rollbacks and restores without any config text', async () => {
+    const applied = await twoVersions()
+    await post('apps-admin', 'hub/apps/history/rollback', { name: 'Blocker', id: applied[0].id, targets: ['a'] })
+    const text = readFileSync(path.join(dataDir, 'audit.log'), 'utf8')
+    expect(text).toContain('apps.history-rollback')
+    for (const secret of ['/base.db', 'admin-a', 'admin-b']) expect(text).not.toContain(secret)
+  })
+})
+
+describe('drift over the API', () => {
+  it('counts a drifted server in the session, shows it in drift/get and clears it when fixed', async () => {
+    await importFrom('a')
+    expect((await api('viewer', 'user/session/get')).body.info.hubDrift).toBe(0)
+    b.apps.Blocker.config = json({ ttl: 99, db: '/base.db' })
+
+    expect((await post('viewer', 'hub/drift/check', {})).code).toBe(403)
+    const checked = await post('apps-admin', 'hub/drift/check', {})
+    expect(checked.body.response.apps).toMatchObject([{ kind: 'app', app: 'Blocker', name: 'Blocker', server: 'b', status: 'differs' }])
+    expect((await api('viewer', 'user/session/get')).body.info.hubDrift).toBe(1)
+    const got = await api('viewer', 'hub/drift/get')
+    expect(got.body.response.apps).toHaveLength(1)
+    expect(got.body.response.checking).toBe(false)
+
+    // Revert to the baseline is the sync apply on that server.
+    await post('apps-admin', 'hub/apps/config/apply', { name: 'Blocker', targets: ['b'] })
+    await post('apps-admin', 'hub/drift/check', {})
+    expect((await api('viewer', 'user/session/get')).body.info.hubDrift).toBe(0)
+    expect((await api('viewer', 'hub/drift/get')).body.response.apps).toEqual([])
+    expect(readFileSync(path.join(dataDir, 'drift.json'), 'utf8')).toContain('"apps": []')
+  })
+
+  it('accepts a drifted config as the baseline', async () => {
+    await importFrom('a')
+    b.apps.Blocker.config = json({ ttl: 99, db: '/base.db' })
+    const r = await post('apps-admin', 'hub/apps/drift/accept', { name: 'Blocker', server: 'b' })
+    expect(JSON.parse(r.body.response.baseline.text)).toEqual({ ttl: 99, db: '/base.db' })
+    expect(r.body.response.baseline.importedFrom).toBe('b')
+    expect((await list())[0]).toMatchObject({ server: 'baseline', kind: 'before-save' })
+    expect((await api('viewer', 'user/session/get')).body.info.hubDrift).toBe(1)
+    expect((await post('viewer', 'hub/apps/drift/accept', { name: 'Blocker', server: 'b' })).code).toBe(403)
+  })
+
+  it('on accept keeps local fields out of the baseline and moves the server values into the overrides', () => {
+    const out = acceptServerConfig(
+      json({ ttl: 99, db: '/b.db' }),
+      json({ ttl: 30, db: '/base.db' }),
+      { paths: ['db'], values: { a: { db: '/a.db' } } },
+      'b',
+    )
+    expect(JSON.parse(out.baselineText)).toEqual({ ttl: 99, db: '/base.db' })
+    expect(out.overrides.values).toEqual({ a: { db: '/a.db' }, b: { db: '/b.db' } })
+    expect(() => acceptServerConfig('plain', null, { paths: ['db'], values: {} }, 'b')).toThrow(/plain text/)
+  })
+})
+
+describe('export and import of baselines', () => {
+  async function setup() {
+    await importFrom('a')
+    await post('apps-admin', 'hub/apps/overrides/save', { name: 'Blocker', paths: ['db'], values: JSON.stringify({ a: { db: '/a.db' }, b: { db: '/b.db' } }) })
+  }
+
+  it('exports one file with every baseline and its local fields, as a download', async () => {
+    await setup()
+    const res = await raw('apps-admin', 'hub/apps/export')
+    expect(res.headers.get('content-disposition')).toMatch(/^attachment; filename="hub-baselines-\d{4}-\d\d-\d\d\.json"$/)
+    const file = (await res.json()) as Record<string, any>
+    expect(file).toMatchObject({ hub: 'technitium-hub', version: 1 })
+    expect(file.apps).toHaveLength(1)
+    expect(file.apps[0]).toMatchObject({ app: 'Blocker', baseline: { text: V1 }, overrides: { paths: ['db'], values: { a: { db: '/a.db' } } } })
+    expect((await api('viewer', 'hub/apps/export')).code).toBe(403)
+    expect(readFileSync(path.join(dataDir, 'audit.log'), 'utf8')).toContain('apps.export')
+  })
+
+  it('imports into another hub: a dry run says what is new, applying writes it, and no server is touched', async () => {
+    await setup()
+    const file = await (await raw('apps-admin', 'hub/apps/export')).text()
+    const other = mkdtempSync(path.join(tmpdir(), 'hub-hist-other-'))
+    writeUsersFile(other, [...(await import('./users.ts')).readUsersFile(dataDir)!])
+    try {
+      await start(other)
+      const before = a.calls.length + b.calls.length
+      const dry = await post('apps-admin', 'hub/apps/import', { data: file })
+      expect(dry.body.response).toEqual({ apps: [{ app: 'Blocker', status: 'new', ignoredServers: [] }], applied: false })
+      expect(readdirSync(other)).not.toContain('apps')
+      const done = await post('apps-admin', 'hub/apps/import', { data: file, apply: true })
+      expect(done.body.response.applied).toBe(true)
+      expect(JSON.parse(readFileSync(path.join(dirOf(other), 'baseline.json'), 'utf8')).text).toBe(V1)
+      expect(JSON.parse(readFileSync(path.join(dirOf(other), 'overrides.json'), 'utf8')).values.b).toEqual({ db: '/b.db' })
+      expect(a.calls.length + b.calls.length).toBe(before)
+      // The same file again changes nothing.
+      expect((await post('apps-admin', 'hub/apps/import', { data: file })).body.response.apps[0].status).toBe('same')
+    } finally {
+      rmSync(other, { recursive: true, force: true })
+    }
+  })
+
+  it('says changed, and keeps the replaced baseline in the history when applying', async () => {
+    await setup()
+    const file = JSON.parse(await (await raw('apps-admin', 'hub/apps/export')).text())
+    file.apps[0].baseline.text = V2
+    await saveBaseline('{"x":1}')
+    const dry = await post('apps-admin', 'hub/apps/import', { data: JSON.stringify(file) })
+    expect(dry.body.response.apps[0].status).toBe('changed')
+    await post('apps-admin', 'hub/apps/import', { data: JSON.stringify(file), apply: true })
+    expect(JSON.parse(readFileSync(path.join(dirOf(), 'baseline.json'), 'utf8')).text).toBe(V2)
+    expect((await list()).some((e) => e.kind === 'before-save' && e.size === '{"x":1}'.length)).toBe(true)
+  })
+
+  it('ignores local values of servers this hub does not have, and says so', async () => {
+    await setup()
+    const file = JSON.parse(await (await raw('apps-admin', 'hub/apps/export')).text())
+    file.apps[0].overrides.values.zzz = { db: '/z.db' }
+    const r = await post('apps-admin', 'hub/apps/import', { data: JSON.stringify(file) })
+    expect(r.body.response.apps[0].ignoredServers).toEqual(['zzz'])
+  })
+
+  it('refuses a file that is not valid, writing nothing', async () => {
+    await setup()
+    const good = JSON.parse(await (await raw('apps-admin', 'hub/apps/export')).text())
+    const cases: [string, unknown][] = [
+      ['not json', 'this is not json'],
+      ['another file', { hello: 1 }],
+      ['wrong version', { ...good, version: 2 }],
+      ['no apps', { ...good, apps: 'x' }],
+      ['bad json baseline', { ...good, apps: [{ app: 'X', baseline: { text: '{ nope' }, overrides: { paths: [], values: {} } }] }],
+      ['no text', { ...good, apps: [{ app: 'X', baseline: {}, overrides: {} }] }],
+      ['bad path', { ...good, apps: [{ app: 'X', baseline: { text: '{}' }, overrides: { paths: ['a b'], values: {} } }] }],
+      ['duplicate app', { ...good, apps: [good.apps[0], good.apps[0]] }],
+      ['text with local fields', { ...good, apps: [{ app: 'X', baseline: { text: 'plain' }, overrides: { paths: ['a'], values: {} } }] }],
+    ]
+    for (const [label, content] of cases) {
+      const r = await post('apps-admin', 'hub/apps/import', { data: typeof content === 'string' ? content : JSON.stringify(content), apply: true })
+      expect([label, r.code]).toEqual([label, 400])
+    }
+    expect(readdirSync(path.join(dataDir, 'apps'))).toEqual(['Blocker'])
+  })
+
+  it('is for writers only, and accepts a file up to 5 MB from a signed-in caller', async () => {
+    const big = JSON.stringify({ hub: 'technitium-hub', version: 1, apps: [{ app: 'Big', baseline: { text: JSON.stringify({ pad: 'x'.repeat(900_000) }) }, overrides: { paths: [], values: {} } }, ...[1, 2, 3].map((n) => ({ app: `Big${n}`, baseline: { text: JSON.stringify({ pad: 'y'.repeat(900_000) }) }, overrides: { paths: [], values: {} } }))] })
+    expect(big.length).toBeGreaterThan(2 * 1024 * 1024)
+    expect((await post('viewer', 'hub/apps/import', { data: big })).code).toBe(403)
+    const r = await post('apps-admin', 'hub/apps/import', { data: big })
+    expect(r.body.response.apps).toHaveLength(4)
+  })
+})

hub/apps-matrix.test.ts

@@ -0,0 +1,280 @@
+// @vitest-environment node
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { countDifferences } from './apps-matrix.ts'
+import { jsonDiff } from './json-diff.ts'
+
+interface FakeApp {
+  version: string
+  config: string | null
+  updateAvailable?: boolean
+  updateVersion?: string
+}
+
+interface Fake {
+  server: http.Server
+  url: string
+  apps: Record<string, FakeApp>
+  down: boolean
+  /** Config reads with the read token are refused with a permission error. */
+  denyRead: boolean
+  tokens: string[]
+}
+
+function fake(apps: Fake['apps']): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = { server: http.createServer(), url: '', apps, down: false, denyRead: false, tokens: [] }
+    f.server.on('request', (req, res) => {
+      if (f.down) return void req.socket.destroy()
+      const url = new URL(req.url ?? '/', 'http://fake')
+      const apiPath = url.pathname.replace(/^\/api\//, '')
+      const token = url.searchParams.get('token') ?? ''
+      res.setHeader('content-type', 'application/json')
+      if (apiPath === 'apps/list') {
+        const list = Object.entries(f.apps).map(([name, a]) => ({
+          name,
+          description: `${name} description`,
+          version: a.version,
+          ...(a.updateAvailable ? { updateAvailable: true, updateVersion: a.updateVersion } : {}),
+        }))
+        return void res.end(JSON.stringify({ status: 'ok', response: { apps: list } }))
+      }
+      if (apiPath === 'apps/config/get') {
+        f.tokens.push(token)
+        if (f.denyRead && !token.startsWith('admin-')) {
+          return void res.end(JSON.stringify({ status: 'error', errorMessage: "Access was denied to user 'reader' for apps." }))
+        }
+        const app = f.apps[url.searchParams.get('name') ?? '']
+        return void res.end(JSON.stringify({ status: 'ok', response: { config: app?.config ?? null } }))
+      }
+      res.end(JSON.stringify({ status: 'ok', response: {} }))
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+const json = (o: unknown): string => JSON.stringify(o, null, 2)
+
+let a: Fake
+let b: Fake
+let dataDir: string
+let staticDir: string
+let hub: http.Server
+let base: string
+let token: string
+
+function reset(): void {
+  a.down = b.down = false
+  a.denyRead = b.denyRead = false
+  a.tokens = []
+  b.tokens = []
+  a.apps = {
+    Same: { version: '1.0', config: json({ x: 1 }) },
+    Differs: { version: '2.0', config: json({ x: 1, y: { z: 2 }, list: [1, 2] }) },
+    'Only A': { version: '3.0', config: json({ secret: 'top-secret-value' }) },
+    Plain: { version: '4.0', config: 'line one\nline two\nline three' },
+    Empty: { version: '5.0', config: null },
+  }
+  b.apps = {
+    Same: { version: '1.0', config: json({ x: 1 }) },
+    Differs: { version: '2.1', updateAvailable: true, updateVersion: '2.2', config: json({ x: 2, y: { z: 3 }, list: [1, 2, 3] }) },
+    Plain: { version: '4.0', config: 'line one\nline TWO\nline three' },
+    Empty: { version: '5.0', config: null },
+  }
+}
+
+async function start(extra: Record<string, string> = {}): Promise<void> {
+  hub?.close()
+  hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      HUB_USER: 'admin',
+      HUB_PASSWORD: 'secret',
+      HUB_DATA: dataDir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      ...extra,
+    }),
+  )
+  await new Promise<void>((resolve) => hub.listen(0, '127.0.0.1', resolve))
+  base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  const res = await fetch(`${base}/api/user/login`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    body: 'user=admin&pass=secret',
+  })
+  token = ((await res.json()) as { token: string }).token
+}
+
+async function matrix() {
+  const res = await fetch(`${base}/api/hub/apps/matrix`, { headers: { authorization: `Bearer ${token}` } })
+  const text = await res.text()
+  return { text, body: JSON.parse(text) as Record<string, any> }
+}
+
+const byName = (body: Record<string, any>, name: string) => (body.response.apps as any[]).find((x) => x.name === name)
+
+beforeAll(async () => {
+  a = await fake({})
+  b = await fake({})
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-matrix-data-'))
+  staticDir = mkdtempSync(path.join(tmpdir(), 'hub-matrix-static-'))
+})
+
+afterAll(() => {
+  hub?.close()
+  a.server.close()
+  b.server.close()
+  rmSync(dataDir, { recursive: true, force: true })
+  rmSync(staticDir, { recursive: true, force: true })
+})
+
+beforeEach(async () => {
+  reset()
+  await start()
+})
+
+describe('jsonDiff', () => {
+  it('compares objects by key and arrays by index', () => {
+    expect(jsonDiff({ a: 1 }, { a: 1 })).toEqual([])
+    expect(jsonDiff({ a: 1, b: { c: 2 } }, { a: 2, b: { c: 3 } })).toEqual([
+      { path: 'a', a: 1, b: 2 },
+      { path: 'b.c', a: 2, b: 3 },
+    ])
+    expect(jsonDiff([1, 2], [1, 3, 4])).toEqual([
+      { path: '[1]', a: 2, b: 3 },
+      { path: '[2]', a: undefined, b: 4 },
+    ])
+  })
+
+  it('reports type changes once and missing keys on either side', () => {
+    expect(jsonDiff({ a: { x: 1 } }, { a: [1] })).toEqual([{ path: 'a', a: { x: 1 }, b: [1] }])
+    expect(jsonDiff({ a: null }, { a: 0 })).toEqual([{ path: 'a', a: null, b: 0 }])
+    expect(jsonDiff({ a: 1 }, { b: 1 })).toEqual([
+      { path: 'a', a: 1, b: undefined },
+      { path: 'b', a: undefined, b: 1 },
+    ])
+  })
+})
+
+describe('countDifferences', () => {
+  it('counts leaf paths for JSON and lines otherwise', () => {
+    expect(countDifferences([json({ a: 1, b: 2 }), json({ a: 2, b: 3 })])).toBe(2)
+    expect(countDifferences(['one\ntwo', 'one\nTWO'])).toBe(2)
+    expect(countDifferences([json({ a: 1 }), 'not json'])).toBeGreaterThan(0)
+    expect(countDifferences([json({ a: 1 })])).toBe(0)
+  })
+})
+
+describe('hub/apps/matrix', () => {
+  it('has the documented shape and classifies each app', async () => {
+    const { body } = await matrix()
+    expect(body.status).toBe('ok')
+    expect(body.hubWarnings).toEqual([])
+    expect(body.response.servers).toEqual([{ id: 'a', ok: true, writable: false }, { id: 'b', ok: true, writable: false }])
+    expect((body.response.apps as any[]).map((x) => x.name)).toEqual(['Differs', 'Empty', 'Only A', 'Plain', 'Same'])
+
+    const same = byName(body, 'Same')
+    expect(same).toMatchObject({ configs: 'same', versions: 'same', description: 'Same description' })
+    expect(same.diffCount).toBeUndefined()
+    expect(same.servers.a).toMatchObject({ installed: true, version: '1.0', configStatus: 'read' })
+    expect(same.servers.a.configHash).toBe(same.servers.b.configHash)
+    expect(same.servers.a.configHash).toMatch(/^[0-9a-f]{16}$/)
+
+    const differs = byName(body, 'Differs')
+    expect(differs).toMatchObject({ configs: 'different', versions: 'different', diffCount: 3 })
+    expect(differs.servers.b).toMatchObject({ updateAvailable: true, updateVersion: '2.2', version: '2.1' })
+    expect(differs.servers.a.updateAvailable).toBeUndefined()
+
+    expect(byName(body, 'Plain')).toMatchObject({ configs: 'different', diffCount: 2 })
+
+    const only = byName(body, 'Only A')
+    expect(only).toMatchObject({ configs: 'single', versions: 'single' })
+    expect(only.servers.b).toEqual({ installed: false, configStatus: 'none' })
+
+    const empty = byName(body, 'Empty')
+    expect(empty).toMatchObject({ configs: 'same' })
+    expect(empty.servers.a).toMatchObject({ configStatus: 'none' })
+    expect(empty.servers.a.configHash).toBeUndefined()
+  })
+
+  it('never returns a config text', async () => {
+    const { text } = await matrix()
+    expect(text).not.toContain('top-secret-value')
+    expect(text).not.toContain('line one')
+    expect(text).not.toContain('read-a')
+  })
+
+  it('marks a config unreadable when the read token is refused and there is no admin token', async () => {
+    a.denyRead = true
+    const { body } = await matrix()
+    const same = byName(body, 'Same')
+    expect(same.servers.a).toMatchObject({ installed: true, configStatus: 'unreadable' })
+    expect(same.servers.a.configError).toMatch(/denied/)
+    expect(same.servers.a.configHash).toBeUndefined()
+    expect(same.configs).toBe('unknown')
+    expect(byName(body, 'Only A').configs).toBe('unknown')
+    // Versions are still known.
+    expect(same.versions).toBe('same')
+  })
+
+  it('retries with the admin token when the read token is refused', async () => {
+    a.denyRead = true
+    await start({ A_ADMIN_TOKEN: 'admin-a' })
+    const { body, text } = await matrix()
+    expect(byName(body, 'Same')).toMatchObject({ configs: 'same' })
+    expect(byName(body, 'Same').servers.a.configStatus).toBe('read')
+    expect(a.tokens).toContain('read-a')
+    expect(a.tokens).toContain('admin-a')
+    expect(b.tokens.every((t) => t === 'read-b')).toBe(true)
+    expect(text).not.toContain('admin-a')
+  })
+
+  it('shows a server that is down and still renders the rest', async () => {
+    b.down = true
+    const { body } = await matrix()
+    expect(body.status).toBe('ok')
+    expect(body.response.servers[0]).toEqual({ id: 'a', ok: true, writable: false })
+    expect(body.response.servers[1]).toMatchObject({ id: 'b', ok: false })
+    expect(body.response.servers[1].error).toBeTruthy()
+    expect(body.hubWarnings).toHaveLength(1)
+    expect(body.hubWarnings[0]).toMatch(/^b:/)
+    const same = byName(body, 'Same')
+    expect(same.servers.a.installed).toBe(true)
+    expect(same.servers.b).toMatchObject({ installed: false, configStatus: 'unreadable' })
+    expect(same.configs).toBe('unknown')
+  })
+
+  it('fails when no server answers', async () => {
+    a.down = true
+    b.down = true
+    const { body } = await matrix()
+    expect(body.status).toBe('error')
+    expect(body.errorMessage).toMatch(/No server answered/)
+  })
+
+  it('is open to every role that can read', async () => {
+    const res = await fetch(`${base}/api/hub/apps/matrix`)
+    expect(((await res.json()) as { status: string }).status).toBe('invalid-token')
+  })
+})
+
+describe('session permissions', () => {
+  it('lets the admin view and modify Apps (the other roles are in apps-write.test.ts)', async () => {
+    const res = await fetch(`${base}/api/user/session/get`, { headers: { authorization: `Bearer ${token}` } })
+    const info = ((await res.json()) as any).info
+    expect(info.permissions.Apps).toEqual({ canView: true, canModify: true, canDelete: true })
+  })
+})

hub/apps-matrix.ts

@@ -0,0 +1,172 @@
+/*
+`hub/apps/matrix`: which DNS apps are installed on which server, with what version, and
+whether their configs agree. Read-only. The config texts are compared here and never
+leave: the answer carries a short hash per copy and the comparison, because a config may
+hold secrets.
+*/
+import { createHash } from 'node:crypto'
+import type { HubConfig, HubServer } from './config.ts'
+import { comparable } from './apps-sync.ts'
+import { readOverrides } from './store.ts'
+import { jsonDiff } from './json-diff.ts'
+import { Upstream, UpstreamError } from './upstream.ts'
+
+export interface ServerCell {
+  installed: boolean
+  version?: string
+  updateAvailable?: boolean
+  updateVersion?: string
+  configStatus: 'none' | 'read' | 'unreadable'
+  configHash?: string
+  configError?: string
+}
+
+export interface MatrixApp {
+  name: string
+  description: string
+  servers: Record<string, ServerCell>
+  configs: 'same' | 'different' | 'single' | 'unknown'
+  diffCount?: number
+  versions: 'same' | 'different' | 'single'
+}
+
+export interface Matrix {
+  servers: { id: string; ok: boolean; writable: boolean; error?: string }[]
+  apps: MatrixApp[]
+}
+
+interface Listed {
+  name: string
+  description?: string
+  version?: string
+  updateAvailable?: boolean
+  updateVersion?: string
+}
+
+const hash = (text: string): string => createHash('sha256').update(text).digest('hex').slice(0, 16)
+
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+
+/* The read token first; when the server refuses it, once more with the admin token if there is one. */
+async function readConfig(upstream: Upstream, server: HubServer, name: string): Promise<string | null> {
+  const params = { name }
+  let response: unknown
+  try {
+    response = await upstream.call(server, 'apps/config/get', params)
+  } catch (e) {
+    if (!(e instanceof UpstreamError) || e.kind !== 'rejected' || server.adminToken === undefined) throw e
+    response = await upstream.call(server, 'apps/config/get', params, server.adminToken)
+  }
+  const config = (response as { config?: unknown } | null)?.config
+  return typeof config === 'string' ? config : null
+}
+
+/* How many places differ between the readable configs. JSON: differing leaf paths; else differing lines. */
+export function countDifferences(texts: string[]): number {
+  if (texts.length < 2) return 0
+  let parsed: unknown[] | null = []
+  for (const text of texts) {
+    try {
+      parsed.push(JSON.parse(text))
+    } catch {
+      parsed = null
+      break
+    }
+  }
+  if (parsed !== null) {
+    const paths = new Set<string>()
+    for (let i = 1; i < parsed.length; i++) for (const d of jsonDiff(parsed[0], parsed[i])) paths.add(d.path)
+    return paths.size
+  }
+  const lines = texts.map((t) => t.split(/\r?\n/).map((l) => l.trim()))
+  let most = 0
+  for (let i = 1; i < lines.length; i++) {
+    const [a, b] = [new Set(lines[0]), new Set(lines[i])]
+    let n = 0
+    for (const l of a) if (!b.has(l)) n++
+    for (const l of b) if (!a.has(l)) n++
+    most = Math.max(most, n)
+  }
+  return most
+}
+
+/* The matrix and the servers that did not answer, as `hubWarnings` lines. */
+export async function buildMatrix(config: HubConfig, upstream: Upstream): Promise<{ matrix: Matrix; warnings: string[] }> {
+  type Copy = { cell: ServerCell; text: string | null }
+  const perServer = await Promise.all(
+    config.servers.map(async (server) => {
+      let list: Listed[]
+      try {
+        const response = (await upstream.call(server, 'apps/list')) as { apps?: Listed[] } | null
+        list = response?.apps ?? []
+      } catch (e) {
+        return { server, error: reason(e), apps: new Map<string, { listed: Listed } & Copy>() }
+      }
+      const entries = await Promise.all(
+        list.map(async (listed): Promise<[string, { listed: Listed } & Copy]> => {
+          const cell: ServerCell = {
+            installed: true,
+            ...(listed.version === undefined ? {} : { version: listed.version }),
+            ...(listed.updateAvailable === undefined ? {} : { updateAvailable: listed.updateAvailable }),
+            ...(listed.updateVersion === undefined ? {} : { updateVersion: listed.updateVersion }),
+            configStatus: 'none',
+          }
+          let text: string | null = null
+          try {
+            text = await readConfig(upstream, server, listed.name)
+            if (text !== null && text !== '') {
+              cell.configStatus = 'read'
+              cell.configHash = hash(comparable(text, readOverrides(config.dataDir, listed.name).paths))
+            }
+          } catch (e) {
+            cell.configStatus = 'unreadable'
+            cell.configError = reason(e)
+            text = null
+          }
+          return [listed.name, { listed, cell, text }]
+        }),
+      )
+      return { server, error: undefined, apps: new Map(entries) }
+    }),
+  )
+
+  const servers = perServer.map((p) => (p.error === undefined
+      ? { id: p.server.id, ok: true, writable: p.server.adminToken !== undefined }
+      : { id: p.server.id, ok: false, writable: p.server.adminToken !== undefined, error: p.error }))
+  const warnings = perServer.filter((p) => p.error !== undefined).map((p) => `${p.server.id}: ${p.error}`)
+  if (warnings.length === perServer.length) throw new Error(`No server answered. ${warnings.join('; ')}`.trim())
+
+  const names = [...new Set(perServer.flatMap((p) => [...p.apps.keys()]))].sort((a, b) => a.localeCompare(b))
+  const apps: MatrixApp[] = names.map((name) => {
+    const cells: Record<string, ServerCell> = {}
+    const copies: { id: string; cell: ServerCell; text: string | null }[] = []
+    let description = ''
+    for (const p of perServer) {
+      const found = p.apps.get(name)
+      if (found !== undefined) {
+        cells[p.server.id] = found.cell
+        copies.push({ id: p.server.id, cell: found.cell, text: found.text })
+        if (description === '' && found.listed.description) description = found.listed.description
+      } else if (p.error !== undefined) {
+        cells[p.server.id] = { installed: false, configStatus: 'unreadable', configError: `${p.server.id} did not answer` }
+      } else {
+        cells[p.server.id] = { installed: false, configStatus: 'none' }
+      }
+    }
+
+    const versionSet = new Set(copies.map((c) => c.cell.version ?? ''))
+    const versions = copies.length < 2 ? 'single' : versionSet.size === 1 ? 'same' : 'different'
+
+    // Fields marked local do not count: two servers that differ only there are the same.
+    const localPaths = readOverrides(config.dataDir, name).paths
+    const readable = copies.filter((c) => c.cell.configStatus !== 'unreadable').map((c) => comparable(c.text ?? '', localPaths))
+    const hasUnknown = copies.some((c) => c.cell.configStatus === 'unreadable') || (warnings.length > 0 && copies.length < 2)
+    const differs = new Set(readable).size > 1
+    const configs: MatrixApp['configs'] = differs ? 'different' : hasUnknown ? 'unknown' : copies.length < 2 ? 'single' : 'same'
+    const app: MatrixApp = { name, description, servers: cells, configs, versions }
+    if (differs) app.diffCount = countDifferences(readable)
+    return app
+  })
+
+  return { matrix: { servers, apps }, warnings }
+}

hub/apps-sync.test.ts

@@ -0,0 +1,529 @@
+// @vitest-environment node
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import {
+  comparable,
+  diffConfigs,
+  effectiveConfig,
+  importLocalValues,
+  lineDiff,
+  parsePath,
+  sameConfig,
+  stripLocal,
+  validatePaths,
+} from './apps-sync.ts'
+import { addUser, writeUsersFile } from './users.ts'
+
+const json = (o: unknown): string => JSON.stringify(o, null, 2)
+
+describe('effectiveConfig', () => {
+  const baseline = json({ enable: true, db: { path: '/base.db', keep: 30 }, nets: [{ name: 'lan', cidr: '10.0.0.0/8' }, { name: 'wan' }] })
+
+  it('puts the server local values over the baseline, nested and in arrays', () => {
+    const e = effectiveConfig(baseline, { paths: ['db.path', 'nets.0.cidr'], values: { a: { 'db.path': '/a.db', 'nets.0.cidr': '192.168.0.0/16' } } }, 'a')
+    expect(JSON.parse(e.text)).toEqual({ enable: true, db: { path: '/a.db', keep: 30 }, nets: [{ name: 'lan', cidr: '192.168.0.0/16' }, { name: 'wan' }] })
+    expect(e.warnings).toEqual([])
+    expect(e.text).toBe(JSON.stringify(JSON.parse(e.text), null, 2))
+  })
+
+  it('keeps the baseline value and warns when a server has no local value', () => {
+    const e = effectiveConfig(baseline, { paths: ['db.path'], values: { a: {} } }, 'b')
+    expect(JSON.parse(e.text).db.path).toBe('/base.db')
+    expect(e.warnings[0]).toMatch(/No local value set for "db.path" on b/)
+  })
+
+  it('leaves a field absent when it is local, not in the baseline and has no value', () => {
+    const e = effectiveConfig(baseline, { paths: ['extra.deep'], values: {} }, 'a')
+    expect(JSON.parse(e.text).extra).toBeUndefined()
+    expect(e.warnings).toEqual([])
+  })
+
+  it('creates the path when the local value is for a field the baseline lacks', () => {
+    const e = effectiveConfig(baseline, { paths: ['extra.deep'], values: { a: { 'extra.deep': 5 } } }, 'a')
+    expect(JSON.parse(e.text).extra).toEqual({ deep: 5 })
+  })
+
+  it('uses a text baseline as it is', () => {
+    const e = effectiveConfig('plain = 1\nother = 2', { paths: ['a'], values: {} }, 'a')
+    expect(e.text).toBe('plain = 1\nother = 2')
+    expect(e.warnings[0]).toMatch(/plain text/)
+    expect(effectiveConfig('plain', { paths: [], values: {} }, 'a').warnings).toEqual([])
+  })
+
+  it('never writes through __proto__', () => {
+    const e = effectiveConfig('{}', { paths: ['__proto__.polluted'], values: { a: { '__proto__.polluted': true } } }, 'a')
+    expect(({} as Record<string, unknown>).polluted).toBeUndefined()
+    expect(e.warnings[0]).toMatch(/does not fit/)
+  })
+})
+
+describe('paths', () => {
+  it('parses dot paths and refuses odd ones', () => {
+    expect(parsePath('a.b.0.c')).toEqual(['a', 'b', '0', 'c'])
+    expect(parsePath('0.0.0.0/0')).toEqual(['0', '0', '0', '0/0'])
+    for (const bad of ['', '.a', 'a..b', 'a b', 'a[0]', 'a,b']) expect(parsePath(bad)).toBeNull()
+    expect(() => validatePaths(['a', 'a'])).toThrow(/twice/)
+    expect(() => validatePaths(Array.from({ length: 51 }, (_, i) => `p${i}`))).toThrow(/50/)
+    expect(() => validatePaths('a')).toThrow()
+  })
+})
+
+describe('comparisons that ignore local fields', () => {
+  const a = json({ shared: 1, db: { path: '/a' }, list: [1, 2] })
+  const b = json({ db: { path: '/b' }, list: [1, 2], shared: 1 })
+
+  it('treats two configs that differ only in local fields as the same', () => {
+    expect(sameConfig(a, b)).toBe(false)
+    expect(sameConfig(a, b, ['db.path'])).toBe(true)
+    expect(comparable(a, ['db.path'])).toBe(comparable(b, ['db.path']))
+    expect(comparable(a, [])).toBe(a)
+  })
+
+  it('removes array items from the end first', () => {
+    expect(stripLocal([1, 2, 3], ['0', '2'])).toEqual([2])
+  })
+
+  it('diffs JSON by field and leaves local fields out', () => {
+    const d = diffConfigs(json({ a: 1, b: 2, c: { d: 1 }, local: 1 }), json({ a: 1, b: 3, c: { d: 1, e: 2 }, f: 1, local: 2 }), ['local'])
+    expect(d).toEqual({
+      kind: 'json',
+      changes: [
+        { path: 'b', type: 'changed', a: 2, b: 3 },
+        { path: 'c.e', type: 'added', a: undefined, b: 2 },
+        { path: 'f', type: 'added', a: undefined, b: 1 },
+      ],
+    })
+    expect(diffConfigs(json({ x: 1 }), json({}), []).kind).toBe('json')
+    expect((diffConfigs(json({ x: 1 }), json({}), []) as { changes: { type: string }[] }).changes[0].type).toBe('removed')
+  })
+
+  it('diffs text by line', () => {
+    const d = diffConfigs('one\ntwo\nthree', 'one\n2\nthree\nfour')
+    expect(d.kind).toBe('text')
+    expect(d).toMatchObject({ changed: 3, truncated: false })
+    expect((d as { lines: { op: string; text: string }[] }).lines).toEqual([
+      { op: 'same', text: 'one' },
+      { op: 'remove', text: 'two' },
+      { op: 'add', text: '2' },
+      { op: 'same', text: 'three' },
+      { op: 'add', text: 'four' },
+    ])
+  })
+
+  it('does not build a table for a huge text', () => {
+    const big = Array.from({ length: 2500 }, (_, i) => `line ${i}`).join('\n')
+    expect(lineDiff(big, big + '\nx')).toMatchObject({ truncated: true, lines: [] })
+  })
+})
+
+describe('importLocalValues', () => {
+  it('reads each server own value of each path', () => {
+    const values = importLocalValues({ a: json({ db: { path: '/a' } }), b: json({ db: { path: '/b' } }), c: 'not json', d: null }, ['db.path', 'missing'])
+    expect(values).toEqual({ a: { 'db.path': '/a' }, b: { 'db.path': '/b' }, c: {}, d: {} })
+  })
+})
+
+/* ---- With fake servers ---- */
+
+interface Fake {
+  server: http.Server
+  url: string
+  apps: Record<string, { version: string; config: string | null }>
+  calls: { path: string; token: string; method: string; form: URLSearchParams }[]
+  rejectSet: string | null
+  mangleSet: boolean
+  denyRead: boolean
+}
+
+function fake(): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = { server: http.createServer(), url: '', apps: {}, calls: [], rejectSet: null, mangleSet: false, denyRead: false }
+    f.server.on('request', (req, res) => {
+      const chunks: Buffer[] = []
+      req.on('data', (c: Buffer) => chunks.push(c))
+      req.on('end', () => {
+        const url = new URL(req.url ?? '/', 'http://fake')
+        const apiPath = url.pathname.replace(/^\/api\//, '')
+        const token = url.searchParams.get('token') ?? ''
+        const form = new URLSearchParams(Buffer.concat(chunks).toString('utf8'))
+        f.calls.push({ path: apiPath, token, method: req.method ?? '', form })
+        const name = url.searchParams.get('name') ?? form.get('name') ?? ''
+        res.setHeader('content-type', 'application/json')
+        const ok = (response: unknown = {}) => res.end(JSON.stringify({ status: 'ok', response }))
+        const error = (m: string) => res.end(JSON.stringify({ status: 'error', errorMessage: m }))
+        if (apiPath === 'apps/list') return ok({ apps: Object.entries(f.apps).map(([n, a]) => ({ name: n, version: a.version })) })
+        if (apiPath === 'apps/config/get') {
+          if (f.denyRead && !token.startsWith('admin-')) return error('Access was denied.')
+          return f.apps[name] ? ok({ config: f.apps[name].config }) : error('not found')
+        }
+        if (apiPath === 'apps/config/set') {
+          if (!token.startsWith('admin-')) return error('Access was denied.')
+          if (f.rejectSet !== null) return error(f.rejectSet)
+          f.apps[name].config = f.mangleSet ? '{"mangled":true}' : form.get('config')
+          return ok()
+        }
+        ok()
+      })
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+let a: Fake
+let b: Fake
+let dataDir: string
+let staticDir: string
+let hub: http.Server | undefined
+let base: string
+const tokens: Record<string, string> = {}
+
+const CFG_A = json({ enable: true, ttl: 30, dbPath: '/data/a.db' })
+const CFG_B = json({ enable: true, ttl: 60, dbPath: '/data/b.db' })
+
+function reset(): void {
+  a.calls = []
+  b.calls = []
+  for (const f of [a, b]) {
+    f.rejectSet = null
+    f.mangleSet = false
+    f.denyRead = false
+  }
+  a.apps = { Blocker: { version: '1.0', config: CFG_A } }
+  b.apps = { Blocker: { version: '1.0', config: CFG_B } }
+}
+
+async function start(extra: Record<string, string> = {}): Promise<void> {
+  hub?.close()
+  hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      A_ADMIN_TOKEN: 'admin-a',
+      B_ADMIN_TOKEN: 'admin-b',
+      HUB_DATA: dataDir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      ...extra,
+    }),
+  )
+  await new Promise<void>((resolve) => hub!.listen(0, '127.0.0.1', resolve))
+  base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  for (const role of ['viewer', 'apps-admin', 'admin']) {
+    const res = await fetch(`${base}/api/user/login`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: `user=${role}&pass=password-${role}`,
+    })
+    tokens[role] = ((await res.json()) as { token: string }).token
+  }
+}
+
+async function api(role: string, apiPath: string, init: { params?: Record<string, string>; json?: unknown; method?: string } = {}) {
+  const headers: Record<string, string> = { authorization: `Bearer ${tokens[role]}` }
+  let body: string | undefined
+  if (init.json !== undefined) {
+    headers['content-type'] = 'application/json'
+    body = JSON.stringify(init.json)
+  }
+  const res = await fetch(`${base}/api/${apiPath}?${new URLSearchParams(init.params ?? {})}`, { method: init.method ?? (body === undefined ? 'GET' : 'POST'), headers, body })
+  return { code: res.status, body: (await res.json()) as Record<string, any> }
+}
+
+const dirOf = () => path.join(dataDir, 'apps', 'Blocker')
+const history = () => readdirSync(path.join(dirOf(), 'history'))
+const writes = (f: Fake) => f.calls.filter((c) => c.path === 'apps/config/set')
+
+beforeAll(async () => {
+  a = await fake()
+  b = await fake()
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-sync-data-'))
+  staticDir = mkdtempSync(path.join(tmpdir(), 'hub-sync-static-'))
+  let users = addUser([], 'viewer', 'viewer', 'password-viewer')
+  users = addUser(users, 'apps-admin', 'apps-admin', 'password-apps-admin')
+  users = addUser(users, 'admin', 'admin', 'password-admin')
+  writeUsersFile(dataDir, users)
+})
+
+afterAll(() => {
+  hub?.close()
+  a.server.close()
+  b.server.close()
+  rmSync(dataDir, { recursive: true, force: true })
+  rmSync(staticDir, { recursive: true, force: true })
+})
+
+beforeEach(async () => {
+  reset()
+  rmSync(path.join(dataDir, 'apps'), { recursive: true, force: true })
+  rmSync(path.join(dataDir, 'audit.log'), { force: true })
+  await start()
+})
+
+afterEach(() => {
+  hub?.close()
+})
+
+const importFrom = (server: string) => api('apps-admin', 'hub/apps/baseline/import', { json: { name: 'Blocker', server } })
+const saveOverrides = (paths: string[], values: Record<string, Record<string, unknown>>) => api('apps-admin', 'hub/apps/overrides/save', { json: { name: 'Blocker', paths, values } })
+
+describe('baseline and local fields', () => {
+  it('has no baseline at first, and compare shows each server live config', async () => {
+    const r = await api('viewer', 'hub/apps/config/compare', { params: { name: 'Blocker' } })
+    expect(r.body.response.baseline).toBeNull()
+    const [sa, sb] = r.body.response.servers
+    expect(sa).toMatchObject({ id: 'a', installed: true, writable: true, version: '1.0', inSync: null, text: CFG_A })
+    expect(sb.text).toBe(CFG_B)
+    expect(sa.vsOthers.b.changes.map((c: any) => c.path)).toEqual(['ttl', 'dbPath'])
+  })
+
+  it('imports a baseline from a server and stores it as text', async () => {
+    const r = await importFrom('a')
+    expect(r.body.response.baseline).toMatchObject({ app: 'Blocker', text: CFG_A, importedFrom: 'a', updatedBy: 'apps-admin' })
+    const file = JSON.parse(readFileSync(path.join(dirOf(), 'baseline.json'), 'utf8'))
+    expect(file.text).toBe(CFG_A)
+    const cmp = (await api('viewer', 'hub/apps/config/compare', { params: { name: 'Blocker' } })).body.response
+    expect(cmp.servers[0].inSync).toBe(true)
+    expect(cmp.servers[1].inSync).toBe(false)
+    expect(cmp.servers[1].vsBaseline.changes.map((c: any) => c.path)).toEqual(['ttl', 'dbPath'])
+  })
+
+  it('refuses an import from a server with no config', async () => {
+    a.apps.Blocker.config = null
+    const r = await importFrom('a')
+    expect(r.code).toBe(400)
+    expect(r.body.errorMessage).toMatch(/no config/)
+  })
+
+  it('validates a JSON baseline and reports the parse error', async () => {
+    const bad = await api('apps-admin', 'hub/apps/baseline/save', { json: { name: 'Blocker', text: '{ "a": 1,, }' } })
+    expect(bad.code).toBe(400)
+    expect(bad.body.errorMessage).toMatch(/not valid JSON: .*position/)
+    const text = await api('apps-admin', 'hub/apps/baseline/save', { json: { name: 'Blocker', text: 'plain = text' } })
+    expect(text.body.response.baseline.text).toBe('plain = text')
+  })
+
+  it('limits the baseline to 1 MB', async () => {
+    const r = await api('apps-admin', 'hub/apps/baseline/save', { json: { name: 'Blocker', text: 'x'.repeat(1024 * 1024 + 1) } })
+    expect(r.code).toBe(400)
+    expect(r.body.errorMessage).toMatch(/1 MB/)
+  })
+
+  it('saves local fields, rejects bad paths and a text baseline', async () => {
+    await importFrom('a')
+    const ok = await saveOverrides(['dbPath'], { a: { dbPath: '/data/a.db' }, b: { dbPath: '/data/b.db' } })
+    expect(ok.body.response.overrides).toEqual({ paths: ['dbPath'], values: { a: { dbPath: '/data/a.db' }, b: { dbPath: '/data/b.db' } } })
+    expect((await saveOverrides(['a b'], {})).code).toBe(400)
+    expect((await saveOverrides(['dbPath'], { zzz: {} })).code).toBe(400)
+    await api('apps-admin', 'hub/apps/baseline/save', { json: { name: 'Blocker', text: 'plain' } })
+    const refused = await saveOverrides(['dbPath'], {})
+    expect(refused.body.errorMessage).toMatch(/plain text/)
+  })
+
+  it('fills local values from the current server configs', async () => {
+    const r = await api('apps-admin', 'hub/apps/overrides/current', { json: { name: 'Blocker', paths: ['dbPath', 'ttl'] } })
+    expect(r.body.response.values).toEqual({ a: { dbPath: '/data/a.db', ttl: 30 }, b: { dbPath: '/data/b.db', ttl: 60 } })
+  })
+
+  it('makes the matrix say Same when servers differ only in local fields', async () => {
+    const before = (await api('viewer', 'hub/apps/matrix')).body.response.apps[0]
+    expect(before.configs).toBe('different')
+    expect(before.diffCount).toBe(2)
+    await saveOverrides(['ttl', 'dbPath'], {})
+    const after = (await api('viewer', 'hub/apps/matrix')).body.response.apps[0]
+    expect(after.configs).toBe('same')
+    expect(after.servers.a.configHash).toBe(after.servers.b.configHash)
+    await saveOverrides(['dbPath'], {})
+    const some = (await api('viewer', 'hub/apps/matrix')).body.response.apps[0]
+    expect(some).toMatchObject({ configs: 'different', diffCount: 1 })
+  })
+
+  it('refuses writers-only calls to a viewer', async () => {
+    for (const [p, json] of [
+      ['hub/apps/baseline/import', { name: 'Blocker', server: 'a' }],
+      ['hub/apps/baseline/save', { name: 'Blocker', text: '{}' }],
+      ['hub/apps/overrides/save', { name: 'Blocker', paths: [], values: {} }],
+      ['hub/apps/config/preview', { name: 'Blocker', targets: ['a'] }],
+      ['hub/apps/config/apply', { name: 'Blocker', targets: ['a'] }],
+    ] as const) {
+      const r = await api('viewer', p, { json })
+      expect([p, r.code]).toEqual([p, 403])
+    }
+    expect(writes(a)).toHaveLength(0)
+  })
+})
+
+describe('preview and apply', () => {
+  async function setup() {
+    await importFrom('a')
+    await saveOverrides(['dbPath'], { a: { dbPath: '/data/a.db' }, b: { dbPath: '/data/b.db' } })
+  }
+
+  it('previews what would change on each target, with the local values in place', async () => {
+    await setup()
+    const r = await api('apps-admin', 'hub/apps/config/preview', { json: { name: 'Blocker', targets: ['a', 'b'] } })
+    const [pa, pb] = r.body.response.targets
+    expect(pa).toMatchObject({ server: 'a', ok: true, unchanged: true, writable: true })
+    expect(pb).toMatchObject({ server: 'b', ok: true, unchanged: false, current: CFG_B })
+    expect(pb.changes.changes.map((c: any) => [c.path, c.a, c.b])).toEqual([['ttl', 60, 30]])
+    expect(JSON.parse(pb.effective)).toEqual({ enable: true, ttl: 30, dbPath: '/data/b.db' })
+    expect(pb.currentHash).toMatch(/^[0-9a-f]{16}$/)
+  })
+
+  it('warns about a missing local value, a version difference and a server without a token', async () => {
+    await importFrom('a')
+    await saveOverrides(['dbPath'], { a: { dbPath: '/data/a.db' } })
+    b.apps.Blocker.version = '0.9'
+    await start({ B_ADMIN_TOKEN: '' })
+    const r = await api('apps-admin', 'hub/apps/config/preview', { json: { name: 'Blocker', targets: ['b'] } })
+    const warnings: string[] = r.body.response.targets[0].warnings
+    expect(r.body.response.targets[0].writable).toBe(false)
+    expect(warnings.some((w) => /No local value set for "dbPath" on b/.test(w))).toBe(true)
+    expect(warnings.some((w) => /align versions first/.test(w))).toBe(true)
+    expect(warnings.some((w) => /no admin token/.test(w))).toBe(true)
+  })
+
+  it('needs a baseline', async () => {
+    const r = await api('apps-admin', 'hub/apps/config/preview', { json: { name: 'Blocker', targets: ['a'] } })
+    expect(r.code).toBe(400)
+    expect(r.body.errorMessage).toMatch(/no baseline/)
+  })
+
+  it('applies: saves the old config, writes with the admin token, verifies', async () => {
+    await setup()
+    const preview = (await api('apps-admin', 'hub/apps/config/preview', { json: { name: 'Blocker', targets: ['a', 'b'] } })).body.response.targets
+    const expectHashes = Object.fromEntries(preview.map((t: any) => [t.server, t.currentHash]))
+    const r = await api('apps-admin', 'hub/apps/config/apply', { json: { name: 'Blocker', targets: ['a', 'b'], expect: expectHashes } })
+    const results = r.body.response.results
+    expect(results.find((x: any) => x.server === 'a')).toMatchObject({ ok: true, message: expect.stringMatching(/Already in sync/) })
+    expect(results.find((x: any) => x.server === 'b')).toEqual({ server: 'b', ok: true, message: 'Applied and verified.' })
+    expect(JSON.parse(b.apps.Blocker.config as string)).toEqual({ enable: true, ttl: 30, dbPath: '/data/b.db' })
+    expect(writes(a)).toHaveLength(0)
+    const [w] = writes(b)
+    expect(w.token).toBe('admin-b')
+    expect(w.method).toBe('POST')
+    expect(w.form.get('name')).toBe('Blocker')
+    const files = history()
+    expect(files).toHaveLength(1)
+    expect(files[0]).toMatch(/__b__before-apply\.txt$/)
+    expect(readFileSync(path.join(dirOf(), 'history', files[0]), 'utf8')).toBe(CFG_B)
+    expect((await api('viewer', 'hub/apps/matrix')).body.response.apps[0].configs).toBe('same')
+  })
+
+  it('refuses a target whose config changed since the preview', async () => {
+    await setup()
+    const preview = (await api('apps-admin', 'hub/apps/config/preview', { json: { name: 'Blocker', targets: ['b'] } })).body.response.targets
+    b.apps.Blocker.config = json({ enable: false })
+    const r = await api('apps-admin', 'hub/apps/config/apply', { json: { name: 'Blocker', targets: ['b'], expect: { b: preview[0].currentHash } } })
+    expect(r.body.response.results[0]).toMatchObject({ server: 'b', ok: false })
+    expect(r.body.response.results[0].message).toMatch(/changed on the server since the preview/)
+    expect(writes(b)).toHaveLength(0)
+    expect(b.apps.Blocker.config).toBe(json({ enable: false }))
+  })
+
+  it('reports a server that refuses the config and leaves it as it was', async () => {
+    await setup()
+    b.rejectSet = 'The config is invalid.'
+    const r = await api('apps-admin', 'hub/apps/config/apply', { json: { name: 'Blocker', targets: ['b'] } })
+    expect(r.body.response.results[0]).toEqual({ server: 'b', ok: false, message: 'The config is invalid.' })
+    expect(b.apps.Blocker.config).toBe(CFG_B)
+  })
+
+  it('handles one target failing and the other succeeding', async () => {
+    await setup()
+    a.apps.Blocker.config = json({ enable: false })
+    b.rejectSet = 'Disk full.'
+    const r = await api('admin', 'hub/apps/config/apply', { json: { name: 'Blocker', targets: ['a', 'b'] } })
+    expect(r.body.response.results.map((x: any) => [x.server, x.ok])).toEqual([['a', true], ['b', false]])
+    expect(JSON.parse(a.apps.Blocker.config as string).ttl).toBe(30)
+  })
+
+  it('says so when the server reads back something else', async () => {
+    await setup()
+    b.mangleSet = true
+    const r = await api('apps-admin', 'hub/apps/config/apply', { json: { name: 'Blocker', targets: ['b'] } })
+    expect(r.body.response.results[0].ok).toBe(false)
+    expect(r.body.response.results[0].message).toMatch(/reads back a different one/)
+  })
+
+  it('does not write to a server without an admin token', async () => {
+    await setup()
+    await start({ B_ADMIN_TOKEN: '' })
+    const r = await api('apps-admin', 'hub/apps/config/apply', { json: { name: 'Blocker', targets: ['b'] } })
+    expect(r.body.response.results[0]).toEqual({ server: 'b', ok: false, message: 'no admin token configured' })
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('keeps config texts out of the audit log', async () => {
+    await setup()
+    await api('apps-admin', 'hub/apps/baseline/save', { json: { name: 'Blocker', text: '{ "bad": ,, "secret-baseline-text": 1 }' } })
+    await api('apps-admin', 'hub/apps/config/apply', { json: { name: 'Blocker', targets: ['a', 'b'] } })
+    const raw = readFileSync(path.join(dataDir, 'audit.log'), 'utf8')
+    for (const text of ['/data/a.db', '/data/b.db', 'secret-baseline-text', 'admin-a', 'admin-b']) expect(raw).not.toContain(text)
+    const entries = (await api('admin', 'hub/audit/list', { params: { limit: '50' } })).body.response.entries as Record<string, string>[]
+    const applied = entries.filter((e) => e.action === 'apps.config-apply')
+    expect(applied).toHaveLength(2)
+    expect(applied.find((e) => e.server === 'b')).toMatchObject({ user: 'apps-admin', target: 'Blocker', outcome: 'ok' })
+    expect(applied.find((e) => e.server === 'b')?.detail).toMatch(/[0-9a-f]{16} to [0-9a-f]{16}/)
+    expect(entries.some((e) => e.action === 'apps.baseline-import' && e.outcome === 'ok')).toBe(true)
+    expect(entries.some((e) => e.action === 'apps.baseline-save' && e.outcome === 'refused' && e.detail === 'invalid baseline')).toBe(true)
+  })
+
+  it('keeps at most 50 history files', async () => {
+    await setup()
+    mkdirSync(path.join(dirOf(), 'history'), { recursive: true })
+    for (let i = 0; i < 55; i++) writeFileSync(path.join(dirOf(), 'history', `2020-01-01T00-00-${String(i).padStart(2, '0')}.000Z__b__before-apply.txt`), 'old')
+    await api('apps-admin', 'hub/apps/config/apply', { json: { name: 'Blocker', targets: ['b'] } })
+    expect(history()).toHaveLength(50)
+  })
+})
+
+describe('single-server config', () => {
+  it('reads a config from one node with the read token and needs a node', async () => {
+    const r = await api('viewer', 'apps/config/get', { params: { node: 'a', name: 'Blocker' } })
+    expect(r.body.response.config).toBe(CFG_A)
+    expect(a.calls.at(-1)?.token).toBe('read-a')
+    expect((await api('viewer', 'apps/config/get', { params: { name: 'Blocker' } })).code).toBe(400)
+    expect((await api('viewer', 'apps/config/get', { params: { node: 'cluster', name: 'Blocker' } })).code).toBe(400)
+  })
+
+  it('sets a config on one node with the admin token, saving the old one first', async () => {
+    const r = await api('apps-admin', 'apps/config/set', {
+      params: { node: 'b' },
+      method: 'POST',
+      json: { name: 'Blocker', config: '{"enable":false}' },
+    })
+    expect(r.body.status).toBe('ok')
+    const [w] = writes(b)
+    expect(w.token).toBe('admin-b')
+    expect(b.apps.Blocker.config).toBe('{"enable":false}')
+    expect(history()).toHaveLength(1)
+    expect(readFileSync(path.join(dirOf(), 'history', history()[0]), 'utf8')).toBe(CFG_B)
+    const raw = readFileSync(path.join(dataDir, 'audit.log'), 'utf8')
+    expect(raw).toContain('apps.config-set')
+    expect(raw).not.toContain('"enable":false')
+  })
+
+  it('refuses the write without a node, to a viewer, and when the old config cannot be saved', async () => {
+    expect((await api('apps-admin', 'apps/config/set', { json: { name: 'Blocker', config: '{}' } })).code).toBe(400)
+    expect((await api('viewer', 'apps/config/set', { params: { node: 'a' }, json: { name: 'Blocker', config: '{}' } })).code).toBe(403)
+    a.denyRead = true
+    a.apps.Blocker.config = 'unchanged'
+    // The admin token can still read, so make the read fail by removing the app.
+    delete a.apps.Blocker
+    const r = await api('apps-admin', 'apps/config/set', { params: { node: 'a' }, json: { name: 'Blocker', config: '{}' } })
+    expect(r.body.status).toBe('error')
+    expect(r.body.errorMessage).toMatch(/nothing was changed/)
+    expect(writes(a)).toHaveLength(0)
+  })
+})

hub/apps-sync.ts

@@ -0,0 +1,304 @@
+/*
+Pure functions behind the config sync: the effective config of a server (the baseline plus
+that server's local values), comparisons that leave the local fields out, and the diffs the
+console shows. No I/O here.
+*/
+import { createHash } from 'node:crypto'
+import { jsonDiff } from './json-diff.ts'
+import type { Overrides } from './store.ts'
+
+export const MAX_CONFIG = 1024 * 1024
+export const MAX_PATHS = 50
+const MAX_LINES = 2000
+
+type Json = unknown
+
+export type Parsed = { ok: true; value: Json } | { ok: false; error: string }
+
+export function parseJson(text: string): Parsed {
+  try {
+    return { ok: true, value: JSON.parse(text) }
+  } catch (e) {
+    return { ok: false, error: e instanceof Error ? e.message : 'invalid JSON' }
+  }
+}
+
+/** A config that starts like JSON must be JSON; anything else is treated as plain text. */
+export const looksLikeJson = (text: string): boolean => /^\s*[{[]/.test(text)
+
+/* ---- Paths: `a.b.0.c`; a numeric segment indexes an array. ---- */
+
+const SEGMENT = /^[^.\s,[\]]+$/
+
+export function parsePath(path: string): string[] | null {
+  if (path.length === 0 || path.length > 200) return null
+  const segments = path.split('.')
+  return segments.every((s) => SEGMENT.test(s)) ? segments : null
+}
+
+export function validatePaths(paths: unknown): string[] {
+  if (!Array.isArray(paths)) throw new Error('paths must be a list.')
+  if (paths.length > MAX_PATHS) throw new Error(`At most ${MAX_PATHS} local fields.`)
+  const seen = new Set<string>()
+  for (const p of paths) {
+    if (typeof p !== 'string' || parsePath(p) === null) throw new Error(`"${String(p)}" is not a field path such as a.b.0.c.`)
+    if (seen.has(p)) throw new Error(`"${p}" is listed twice.`)
+    seen.add(p)
+  }
+  return paths as string[]
+}
+
+const isContainer = (v: Json): v is Record<string, Json> | Json[] => v !== null && typeof v === 'object'
+const isIndex = (s: string): boolean => /^\d+$/.test(s)
+
+export function getAt(root: Json, segments: string[]): { found: boolean; value?: Json } {
+  let at: Json = root
+  for (const s of segments) {
+    if (!isContainer(at)) return { found: false }
+    if (Array.isArray(at)) {
+      if (!isIndex(s) || Number(s) >= at.length) return { found: false }
+      at = at[Number(s)]
+    } else {
+      if (!Object.hasOwn(at, s)) return { found: false }
+      at = at[s]
+    }
+  }
+  return { found: true, value: at }
+}
+
+/* Sets a value, making the containers on the way when they are missing. Returns false if the path cannot be set. */
+export function setAt(root: Json, segments: string[], value: Json): boolean {
+  let at: Json = root
+  for (let i = 0; i < segments.length; i++) {
+    const s = segments[i]
+    const last = i === segments.length - 1
+    if (!isContainer(at)) return false
+    if (Array.isArray(at)) {
+      if (!isIndex(s)) return false
+      const index = Number(s)
+      if (index > at.length) return false
+      if (last) at[index] = value
+      else {
+        if (at[index] === undefined) at[index] = isIndex(segments[i + 1]) ? [] : {}
+        at = at[index]
+      }
+    } else {
+      if (s === '__proto__') return false
+      if (last) at[s] = value
+      else {
+        if (!Object.hasOwn(at, s) || !isContainer(at[s])) at[s] = isIndex(segments[i + 1]) ? [] : {}
+        at = at[s]
+      }
+    }
+  }
+  return true
+}
+
+export function removeAt(root: Json, segments: string[]): void {
+  const parent = getAt(root, segments.slice(0, -1))
+  const last = segments[segments.length - 1]
+  if (!parent.found || !isContainer(parent.value)) return
+  if (Array.isArray(parent.value)) {
+    if (isIndex(last) && Number(last) < parent.value.length) parent.value.splice(Number(last), 1)
+  } else delete parent.value[last]
+}
+
+const clone = <T>(value: T): T => structuredClone(value)
+
+/* A copy of the value without the local fields. Removing array items shifts the rest, so the deepest and highest go first. */
+export function stripLocal(value: Json, paths: string[]): Json {
+  const copy = clone(value)
+  const sorted = paths
+    .map((p) => parsePath(p))
+    .filter((s): s is string[] => s !== null)
+    .sort((a, b) => b.length - a.length || b.join('.').localeCompare(a.join('.'), undefined, { numeric: true }))
+  for (const segments of sorted) removeAt(copy, segments)
+  return copy
+}
+
+function sortKeys(value: Json): Json {
+  if (Array.isArray(value)) return value.map(sortKeys)
+  if (isContainer(value)) return Object.fromEntries(Object.keys(value).sort().map((k) => [k, sortKeys((value as Record<string, Json>)[k])]))
+  return value
+}
+
+/*
+A string two configs can be compared by: for JSON, the config without the local fields with its keys
+sorted; for anything else, the text. Without local fields it is the text itself, as before.
+*/
+export function comparable(text: string, paths: string[]): string {
+  if (paths.length === 0) return text
+  const parsed = parseJson(text)
+  if (!parsed.ok) return text
+  return JSON.stringify(sortKeys(stripLocal(parsed.value, paths)))
+}
+
+export const textHash = (text: string): string => createHash('sha256').update(text).digest('hex').slice(0, 16)
+
+/* ---- The effective config of one server ---- */
+
+export interface Effective {
+  text: string
+  warnings: string[]
+}
+
+export function effectiveConfig(baselineText: string, overrides: Overrides, server: string): Effective {
+  const parsed = parseJson(baselineText)
+  if (!parsed.ok || !looksLikeJson(baselineText)) {
+    const warnings = overrides.paths.length > 0 ? ['The baseline is plain text, so local fields cannot be applied; the baseline is used as it is.'] : []
+    return { text: baselineText, warnings }
+  }
+  const value = clone(parsed.value)
+  const warnings: string[] = []
+  const mine = overrides.values[server] ?? {}
+  for (const path of overrides.paths) {
+    const segments = parsePath(path)
+    if (segments === null) continue
+    if (Object.hasOwn(mine, path)) {
+      if (!setAt(value, segments, clone(mine[path]))) warnings.push(`Local field "${path}" does not fit the baseline structure and was skipped.`)
+    } else if (getAt(value, segments).found) {
+      warnings.push(`No local value set for "${path}" on ${server}; the baseline value is used.`)
+    }
+    // No local value and not in the baseline: nothing to remove, the field stays absent.
+  }
+  return { text: JSON.stringify(value, null, 2), warnings }
+}
+
+/* The same config? JSON is compared by meaning (key order and spacing do not count), anything else by text. */
+export function sameConfig(a: string, b: string, paths: string[] = []): boolean {
+  const [pa, pb] = [parseJson(a), parseJson(b)]
+  if (pa.ok && pb.ok && looksLikeJson(a) && looksLikeJson(b)) return jsonDiff(stripLocal(pa.value, paths), stripLocal(pb.value, paths)).length === 0
+  return a.replace(/\r\n/g, '\n').trimEnd() === b.replace(/\r\n/g, '\n').trimEnd()
+}
+
+/* ---- Diffs ---- */
+
+export interface FieldChange {
+  path: string
+  type: 'added' | 'removed' | 'changed'
+  a?: Json
+  b?: Json
+}
+
+export interface LineOp {
+  op: 'same' | 'add' | 'remove'
+  text: string
+}
+
+export type Diff =
+  | { kind: 'json'; changes: FieldChange[] }
+  | { kind: 'text'; lines: LineOp[]; truncated: boolean; changed: number }
+
+/* A to B: `added` means B has it and A does not. Local fields are left out of both. */
+export function diffConfigs(a: string, b: string, localPaths: string[] = []): Diff {
+  const [pa, pb] = [parseJson(a), parseJson(b)]
+  if (pa.ok && pb.ok && looksLikeJson(a) && looksLikeJson(b)) {
+    const changes = jsonDiff(stripLocal(pa.value, localPaths), stripLocal(pb.value, localPaths)).map((d): FieldChange => {
+      const type = d.a === undefined ? 'added' : d.b === undefined ? 'removed' : 'changed'
+      return { path: d.path === '' ? '(whole config)' : d.path, type, a: d.a, b: d.b }
+    })
+    return { kind: 'json', changes }
+  }
+  return { kind: 'text', ...lineDiff(a, b) }
+}
+
+export function lineDiff(a: string, b: string): { lines: LineOp[]; truncated: boolean; changed: number } {
+  const [x, y] = [a.replace(/\r\n/g, '\n').split('\n'), b.replace(/\r\n/g, '\n').split('\n')]
+  if (x.length > MAX_LINES || y.length > MAX_LINES) {
+    const set = new Set(x)
+    const changed = y.filter((l) => !set.has(l)).length + x.filter((l) => !new Set(y).has(l)).length
+    return { lines: [], truncated: true, changed }
+  }
+  // Longest common subsequence table, filled from the end.
+  const w = y.length + 1
+  const table = new Uint16Array((x.length + 1) * w)
+  for (let i = x.length - 1; i >= 0; i--) {
+    for (let j = y.length - 1; j >= 0; j--) {
+      table[i * w + j] = x[i] === y[j] ? table[(i + 1) * w + j + 1] + 1 : Math.max(table[(i + 1) * w + j], table[i * w + j + 1])
+    }
+  }
+  const lines: LineOp[] = []
+  let i = 0
+  let j = 0
+  while (i < x.length || j < y.length) {
+    if (i < x.length && j < y.length && x[i] === y[j]) {
+      lines.push({ op: 'same', text: x[i] })
+      i++
+      j++
+    } else if (j < y.length && (i === x.length || table[i * w + j + 1] > table[(i + 1) * w + j])) {
+      lines.push({ op: 'add', text: y[j++] })
+    } else lines.push({ op: 'remove', text: x[i++] })
+  }
+  return { lines, truncated: false, changed: lines.filter((l) => l.op !== 'same').length }
+}
+
+export const diffIsEmpty = (d: Diff): boolean => (d.kind === 'json' ? d.changes.length === 0 : d.changed === 0)
+
+/* ---- Local values ---- */
+
+/*
+Fills each server's current value of each local path from its live config. A server whose config is
+not JSON, or that has no such field, gets no entry for it.
+*/
+export function importLocalValues(configs: Record<string, string | null>, paths: string[]): Overrides['values'] {
+  const out: Overrides['values'] = {}
+  for (const [server, text] of Object.entries(configs)) {
+    out[server] = {}
+    if (text === null) continue
+    const parsed = parseJson(text)
+    if (!parsed.ok) continue
+    for (const path of paths) {
+      const segments = parsePath(path)
+      if (segments === null) continue
+      const found = getAt(parsed.value, segments)
+      if (found.found) out[server][path] = clone(found.value)
+    }
+  }
+  return out
+}
+
+/*
+"Accept as baseline": the baseline becomes this server's config, except that the local fields keep
+the baseline's old values, and this server's own values for them move into the overrides.
+*/
+export function acceptServerConfig(
+  serverText: string,
+  oldBaselineText: string | null,
+  overrides: Overrides,
+  server: string,
+): { baselineText: string; overrides: Overrides } {
+  if (overrides.paths.length === 0) return { baselineText: serverText, overrides }
+  const parsed = parseJson(serverText)
+  if (!parsed.ok || !looksLikeJson(serverText)) throw new Error('The config is plain text, so its local fields cannot be separated out.')
+  const value = clone(parsed.value)
+  const old = oldBaselineText === null ? null : parseJson(oldBaselineText)
+  const mine: Record<string, Json> = {}
+  for (const path of overrides.paths) {
+    const segments = parsePath(path)
+    if (segments === null) continue
+    const here = getAt(value, segments)
+    if (here.found) mine[path] = clone(here.value)
+    const before = old !== null && old.ok ? getAt(old.value, segments) : { found: false }
+    if (before.found) setAt(value, segments, clone(before.value))
+    else removeAt(value, segments)
+  }
+  return {
+    baselineText: JSON.stringify(value, null, 2),
+    overrides: { paths: overrides.paths, values: { ...overrides.values, [server]: { ...(overrides.values[server] ?? {}), ...mine } } },
+  }
+}
+
+/* Local values for paths that are listed, from servers that exist; anything else is dropped. */
+export function cleanValues(values: unknown, paths: string[], servers: string[]): Overrides['values'] {
+  if (values === null || typeof values !== 'object' || Array.isArray(values)) throw new Error('values must be an object of servers.')
+  const out: Overrides['values'] = {}
+  for (const [server, mine] of Object.entries(values as Record<string, unknown>)) {
+    if (!servers.includes(server)) throw new Error(`Unknown server: ${server}.`)
+    if (mine === null || typeof mine !== 'object' || Array.isArray(mine)) throw new Error(`The values of ${server} must be an object of paths.`)
+    out[server] = {}
+    for (const [path, value] of Object.entries(mine as Record<string, unknown>)) {
+      if (paths.includes(path)) out[server][path] = value
+    }
+  }
+  return out
+}

hub/apps-write.test.ts

@@ -0,0 +1,517 @@
+// @vitest-environment node
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { boundaryOf, parseMultipart } from './multipart.ts'
+import { safeName } from './apps-write.ts'
+import { addUser, writeUsersFile } from './users.ts'
+
+interface Call {
+  path: string
+  token: string
+  method: string
+  query: URLSearchParams
+  body: Buffer
+  contentType: string
+}
+
+interface Fake {
+  server: http.Server
+  url: string
+  apps: Record<string, { version: string; config: string | null }>
+  store: { name: string; version: string; url: string; description: string; size: string }[]
+  calls: Call[]
+  failWrites: boolean
+  denyConfig: boolean
+}
+
+function fake(): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = { server: http.createServer(), url: '', apps: {}, store: [], calls: [], failWrites: false, denyConfig: false }
+    f.server.on('request', (req, res) => {
+      const chunks: Buffer[] = []
+      req.on('data', (c: Buffer) => chunks.push(c))
+      req.on('end', () => {
+        const url = new URL(req.url ?? '/', 'http://fake')
+        const apiPath = url.pathname.replace(/^\/api\//, '')
+        const token = url.searchParams.get('token') ?? ''
+        const contentType = String(req.headers['content-type'] ?? '')
+        // Technitium reads query or form alike; `query` here is both (`token` is the only one the hub leaves in the URL).
+        const query = new URLSearchParams(url.searchParams)
+        if (contentType.includes('application/x-www-form-urlencoded')) for (const [k, v] of new URLSearchParams(Buffer.concat(chunks).toString('utf8'))) query.set(k, v)
+        const name = query.get('name') ?? ''
+        f.calls.push({ path: apiPath, token, method: req.method ?? '', query, body: Buffer.concat(chunks), contentType })
+        res.setHeader('content-type', 'application/json')
+        const ok = (response: unknown = {}) => res.end(JSON.stringify({ status: 'ok', response }))
+        const error = (m: string) => res.end(JSON.stringify({ status: 'error', errorMessage: m }))
+        const isWrite = ['apps/downloadAndInstall', 'apps/downloadAndUpdate', 'apps/uninstall', 'apps/install', 'apps/update'].includes(apiPath)
+        if (isWrite && !token.startsWith('admin-')) return error("Access was denied to user 'reader' for apps.")
+        if (isWrite && f.failWrites) return error('Disk full.')
+        switch (apiPath) {
+          case 'apps/list':
+            return ok({ apps: Object.entries(f.apps).map(([n, a]) => ({ name: n, description: `${n} d`, version: a.version })) })
+          case 'apps/listStoreApps':
+            return ok({ storeApps: f.store })
+          case 'apps/config/get':
+            if (f.denyConfig) return error('Access was denied.')
+            return f.apps[name] ? ok({ config: f.apps[name].config }) : error('not found')
+          case 'apps/downloadAndInstall':
+          case 'apps/downloadAndUpdate': {
+            const entry = f.store.find((s) => s.name === name && s.url === query.get('url'))
+            if (!entry) return error('not in store')
+            f.apps[name] = { version: entry.version, config: f.apps[name]?.config ?? null }
+            return ok()
+          }
+          case 'apps/uninstall':
+            delete f.apps[name]
+            return ok()
+          case 'apps/install':
+          case 'apps/update':
+            f.apps[name] = { version: 'zip', config: null }
+            return ok()
+          default:
+            return ok()
+        }
+      })
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+let a: Fake
+let b: Fake
+let dataDir: string
+let staticDir: string
+let hub: http.Server | undefined
+let base: string
+const tokens: Record<string, string> = {}
+
+const STORE = (v: string) => [{ name: 'Blocker', version: v, url: `https://store.example/Blocker-v${v}.zip`, description: 'd', size: '1 KB' }]
+
+function reset(): void {
+  for (const f of [a, b]) {
+    f.calls = []
+    f.failWrites = false
+    f.denyConfig = false
+    f.store = STORE('2.0')
+    f.apps = { Blocker: { version: '1.0', config: '{"secret":"cfg"}' } }
+  }
+}
+
+async function start(extra: Record<string, string> = {}): Promise<void> {
+  hub?.close()
+  hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      A_ADMIN_TOKEN: 'admin-a',
+      B_ADMIN_TOKEN: 'admin-b',
+      HUB_DATA: dataDir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      ...extra,
+    }),
+  )
+  await new Promise<void>((resolve) => hub!.listen(0, '127.0.0.1', resolve))
+  base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  for (const role of ['viewer', 'operator', 'apps-admin', 'admin']) {
+    const res = await fetch(`${base}/api/user/login`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: `user=${role}&pass=password-${role}`,
+    })
+    tokens[role] = ((await res.json()) as { token: string }).token
+  }
+}
+
+async function api(role: string, apiPath: string, init: { params?: Record<string, string>; method?: string; json?: unknown; body?: string | FormData; headers?: Record<string, string> } = {}) {
+  const query = new URLSearchParams(init.params ?? {})
+  const headers: Record<string, string> = { authorization: `Bearer ${tokens[role]}`, ...(init.headers ?? {}) }
+  let body = init.body
+  if (init.json !== undefined) {
+    headers['content-type'] = 'application/json'
+    body = JSON.stringify(init.json)
+  }
+  const res = await fetch(`${base}/api/${apiPath}?${query}`, { method: init.method ?? (body === undefined ? 'GET' : 'POST'), headers, body })
+  return { code: res.status, body: (await res.json()) as Record<string, any> }
+}
+
+const run = (role: string, json: Record<string, unknown>) => api(role, 'hub/apps/run', { json })
+
+function zipForm(extra: Record<string, string>, bytes: Uint8Array): FormData {
+  const form = new FormData()
+  for (const [k, v] of Object.entries(extra)) form.append(k, v)
+  form.append('fileAppZip', new Blob([bytes], { type: 'application/zip' }), 'Blocker.zip')
+  return form
+}
+
+const writes = (f: Fake) => f.calls.filter((c) => c.path.startsWith('apps/') && c.method === 'POST')
+
+beforeAll(async () => {
+  a = await fake()
+  b = await fake()
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-write-data-'))
+  staticDir = mkdtempSync(path.join(tmpdir(), 'hub-write-static-'))
+  let users = addUser([], 'viewer', 'viewer', 'password-viewer')
+  users = addUser(users, 'operator', 'operator', 'password-operator')
+  users = addUser(users, 'apps-admin', 'apps-admin', 'password-apps-admin')
+  users = addUser(users, 'admin', 'admin', 'password-admin')
+  writeUsersFile(dataDir, users)
+})
+
+afterAll(() => {
+  hub?.close()
+  a.server.close()
+  b.server.close()
+  rmSync(dataDir, { recursive: true, force: true })
+  rmSync(staticDir, { recursive: true, force: true })
+})
+
+beforeEach(async () => {
+  reset()
+  rmSync(path.join(dataDir, 'apps'), { recursive: true, force: true })
+  rmSync(path.join(dataDir, 'audit.log'), { force: true })
+  await start()
+})
+
+afterEach(() => {
+  hub?.close()
+})
+
+describe('multipart parser', () => {
+  const boundary = '----testBoundary123'
+  const type = `multipart/form-data; boundary=${boundary}`
+
+  it('reads fields and a binary file, whatever bytes it holds', () => {
+    const binary = Buffer.alloc(256)
+    for (let i = 0; i < 256; i++) binary[i] = i
+    const tricky = Buffer.concat([binary, Buffer.from(`\r\n--${boundary.slice(0, 5)}\r\n\r\n--`), binary])
+    const body = Buffer.concat([
+      Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="name"\r\n\r\nBlocker\r\n`),
+      Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="fileAppZip"; filename="a b.zip"\r\nContent-Type: application/zip\r\n\r\n`),
+      tricky,
+      Buffer.from(`\r\n--${boundary}--\r\n`),
+    ])
+    const parsed = parseMultipart(body, type)
+    expect(parsed.fields).toEqual({ name: 'Blocker' })
+    expect(parsed.files).toHaveLength(1)
+    expect(parsed.files[0]).toMatchObject({ field: 'fileAppZip', filename: 'a b.zip', type: 'application/zip' })
+    expect(parsed.files[0].data.equals(tricky)).toBe(true)
+  })
+
+  it('reads a quoted boundary and an empty file', () => {
+    expect(boundaryOf('multipart/form-data; boundary="x y"')).toBe('x y')
+    const body = Buffer.from(`--q\r\nContent-Disposition: form-data; name="f"; filename="e.zip"\r\n\r\n\r\n--q--`)
+    expect(parseMultipart(body, 'multipart/form-data; boundary=q').files[0].data.length).toBe(0)
+  })
+
+  it('refuses what is not multipart', () => {
+    expect(() => parseMultipart(Buffer.from('x'), 'text/plain')).toThrow()
+    expect(() => parseMultipart(Buffer.from('nothing here'), type)).toThrow(/boundary/)
+    expect(() => parseMultipart(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="a"\r\n\r\nno end`), type)).toThrow(/terminated/)
+  })
+
+  it('names folders safely', () => {
+    expect(safeName('Blocker')).toBe('Blocker')
+    expect(safeName('Query Logs (Sqlite)')).toMatch(/^Query_Logs__Sqlite_-[0-9a-f]{6}$/)
+    expect(safeName('../../etc')).not.toContain('/')
+    expect(safeName('..')).not.toMatch(/^\./)
+  })
+})
+
+describe('roles', () => {
+  it('refuses viewer and operator on every write and audits it', async () => {
+    for (const role of ['viewer', 'operator']) {
+      const r = await run(role, { action: 'uninstall', name: 'Blocker', targets: ['a'] })
+      expect(r.code).toBe(403)
+      expect(r.body).toEqual({ status: 'error', errorMessage: 'Your role does not allow this.' })
+      expect((await api(role, 'apps/uninstall', { params: { name: 'Blocker', node: 'a' } })).code).toBe(403)
+      expect((await api(role, 'hub/apps/upload', { body: zipForm({ action: 'install', name: 'X', targets: 'a' }, new Uint8Array([1])) })).code).toBe(403)
+    }
+    expect(writes(a)).toHaveLength(0)
+    const entries = (await api('admin', 'hub/audit/list', { params: { limit: '100' } })).body.response.entries as Record<string, string>[]
+    expect(entries.filter((e) => e.action === 'denied-write' && e.user === 'viewer').length).toBeGreaterThanOrEqual(3)
+    expect(entries.some((e) => e.action === 'denied-write' && e.user === 'operator' && e.target === 'hub/apps/run')).toBe(true)
+  })
+
+  it('lets apps-admin and admin write', async () => {
+    for (const role of ['apps-admin', 'admin']) {
+      b.apps = a.apps = { Blocker: { version: '1.0', config: null } }
+      const r = await run(role, { action: 'update-store', name: 'Blocker', targets: ['a'] })
+      expect(r.body.status).toBe('ok')
+      expect(r.body.response.results).toEqual([{ server: 'a', ok: true, message: 'Updated from the store.' }])
+    }
+  })
+
+  it('does not let a write endpoint be called with GET', async () => {
+    expect((await api('admin', 'hub/apps/run', { method: 'GET', params: { action: 'uninstall', name: 'Blocker', targets: 'a' } })).code).toBe(405)
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('tells the session who may write what', async () => {
+    const viewer = (await api('viewer', 'user/session/get')).body.info
+    const admin = (await api('apps-admin', 'user/session/get')).body.info
+    expect(viewer.permissions.Apps).toEqual({ canView: true, canModify: false, canDelete: false })
+    expect(admin.permissions.Apps).toEqual({ canView: true, canModify: true, canDelete: true })
+    expect(admin.permissions.Zones.canModify).toBe(false)
+    expect(admin.hubWritable).toEqual(['a', 'b'])
+    await start({ B_ADMIN_TOKEN: '' })
+    expect((await api('viewer', 'user/session/get')).body.info.hubWritable).toEqual(['a'])
+  })
+})
+
+describe('hub/apps/run', () => {
+  it('uses only the admin token and the URL of the target own store', async () => {
+    a.store = STORE('2.0')
+    b.store = [{ ...STORE('2.5')[0], url: 'https://other.example/b.zip' }]
+    const r = await run('admin', { action: 'update-store', name: 'Blocker', targets: ['a', 'b'] })
+    expect(r.body.response.results.map((x: any) => x.ok)).toEqual([true, true])
+    const [wa] = writes(a)
+    const [wb] = writes(b)
+    expect(wa.token).toBe('admin-a')
+    expect(wa.query.get('url')).toBe('https://store.example/Blocker-v2.0.zip')
+    expect(wb.token).toBe('admin-b')
+    expect(wb.query.get('url')).toBe('https://other.example/b.zip')
+    expect(a.calls.filter((c) => c.token === 'read-a').every((c) => !c.path.includes('downloadAnd'))).toBe(true)
+  })
+
+  it('refuses a url that is not from the store listing, and an app the store does not have', async () => {
+    const r = await run('admin', { action: 'install-store', name: 'Blocker', url: 'https://evil.example/x.zip', targets: ['a'] })
+    expect(r.code).toBe(400)
+    expect(r.body.errorMessage).toMatch(/store listing/)
+    expect(writes(a)).toHaveLength(0)
+
+    const same = await run('admin', { action: 'update-store', name: 'Blocker', url: STORE('2.0')[0].url, targets: ['a'] })
+    expect(same.body.status).toBe('ok')
+
+    const missing = await run('admin', { action: 'install-store', name: 'Nope', targets: ['a'] })
+    expect(missing.body.response.results[0]).toMatchObject({ server: 'a', ok: false })
+    expect(missing.body.response.results[0].message).toMatch(/not in this server/)
+  })
+
+  it('refuses unknown servers, no targets and bad actions', async () => {
+    expect((await run('admin', { action: 'update-store', name: 'Blocker', targets: ['zzz'] })).code).toBe(400)
+    expect((await run('admin', { action: 'update-store', name: 'Blocker', targets: [] })).code).toBe(400)
+    expect((await run('admin', { action: 'format', name: 'Blocker', targets: ['a'] })).code).toBe(400)
+    expect((await run('admin', { action: 'uninstall', name: '', targets: ['a'] })).code).toBe(400)
+  })
+
+  it('accepts a form body with comma separated targets', async () => {
+    const r = await api('admin', 'hub/apps/run', {
+      body: new URLSearchParams({ action: 'update-store', name: 'Blocker', targets: 'a,b' }).toString(),
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    })
+    expect(r.body.response.results).toHaveLength(2)
+  })
+
+  it('refuses a server without an admin token, without calling it', async () => {
+    await start({ B_ADMIN_TOKEN: '' })
+    const r = await run('admin', { action: 'update-store', name: 'Blocker', targets: ['a', 'b'] })
+    const results = r.body.response.results
+    expect(results.find((x: any) => x.server === 'a').ok).toBe(true)
+    expect(results.find((x: any) => x.server === 'b')).toEqual({ server: 'b', ok: false, message: 'no admin token configured' })
+    expect(writes(b)).toHaveLength(0)
+    expect(b.calls.every((c) => c.token === 'read-b')).toBe(true)
+    const m = await api('admin', 'hub/apps/matrix')
+    expect(m.body.response.servers).toEqual([{ id: 'a', ok: true, writable: true }, { id: 'b', ok: true, writable: false }])
+  })
+
+  it('reports each target on its own when one fails, and audits every result', async () => {
+    b.failWrites = true
+    const r = await run('apps-admin', { action: 'update-store', name: 'Blocker', targets: ['a', 'b'] })
+    expect(r.body.response.results).toEqual([
+      { server: 'a', ok: true, message: 'Updated from the store.' },
+      { server: 'b', ok: false, message: 'Disk full.' },
+    ])
+    expect(a.apps.Blocker.version).toBe('2.0')
+    expect(b.apps.Blocker.version).toBe('1.0')
+    const entries = (await api('admin', 'hub/audit/list', { params: { limit: '20' } })).body.response.entries as Record<string, string>[]
+    const mine = entries.filter((e) => e.action === 'apps.update-store')
+    expect(mine).toHaveLength(2)
+    expect(mine.find((e) => e.server === 'a')).toMatchObject({ user: 'apps-admin', role: 'apps-admin', target: 'Blocker', outcome: 'ok' })
+    expect(mine.find((e) => e.server === 'b')).toMatchObject({ outcome: 'failed', detail: 'Disk full.' })
+    const raw = readFileSync(path.join(dataDir, 'audit.log'), 'utf8')
+    for (const secret of ['admin-a', 'admin-b', 'read-a', 'password-apps-admin', '"secret":"cfg"']) expect(raw).not.toContain(secret)
+  })
+
+  it('shows the new version in the matrix after an update (the cache is dropped)', async () => {
+    await start({ HUB_CACHE_SECONDS: '60' })
+    const before = (await api('viewer', 'hub/apps/matrix')).body.response.apps[0]
+    expect(before.servers.a.version).toBe('1.0')
+    await run('admin', { action: 'update-store', name: 'Blocker', targets: ['a'] })
+    const after = (await api('viewer', 'hub/apps/matrix')).body.response.apps[0]
+    expect(after.servers.a.version).toBe('2.0')
+    expect(after.servers.b.version).toBe('1.0')
+    expect(after.versions).toBe('different')
+  })
+})
+
+describe('uninstall and its history', () => {
+  const history = () => path.join(dataDir, 'apps', 'Blocker', 'history')
+
+  it('saves the config first, then uninstalls', async () => {
+    const r = await run('admin', { action: 'uninstall', name: 'Blocker', targets: ['a', 'b'] })
+    expect(r.body.response.results.map((x: any) => x.ok)).toEqual([true, true])
+    expect(a.apps.Blocker).toBeUndefined()
+    const files = readdirSync(history()).sort()
+    expect(files).toHaveLength(2)
+    expect(files.some((f) => /^\d{4}-\d\d-\d\dT\d\d-\d\d-\d\d\.\d{3}Z__a__before-uninstall\.txt$/.test(f))).toBe(true)
+    expect(readFileSync(path.join(history(), files[0]), 'utf8')).toBe('{"secret":"cfg"}')
+    expect(readdirSync(history()).some((f) => f.endsWith('.tmp'))).toBe(false)
+  })
+
+  it('keeps at most 50 files per app, dropping the oldest', async () => {
+    mkdirSync(history(), { recursive: true })
+    for (let i = 0; i < 55; i++) writeFileSync(path.join(history(), `2020-01-01T00-00-${String(i).padStart(2, '0')}.000Z__a__before-uninstall.txt`), 'old')
+    await run('admin', { action: 'uninstall', name: 'Blocker', targets: ['a'] })
+    const files = readdirSync(history()).sort()
+    expect(files).toHaveLength(50)
+    expect(files.some((f) => f.startsWith('2020-01-01T00-00-00.'))).toBe(false)
+    expect(files.some((f) => f.includes('__a__') && f.startsWith('20') && !f.startsWith('2020-'))).toBe(true)
+  })
+
+  it('does not uninstall when the config cannot be saved', async () => {
+    a.denyConfig = true
+    const r = await run('admin', { action: 'uninstall', name: 'Blocker', targets: ['a'] })
+    expect(r.body.response.results[0].ok).toBe(false)
+    expect(r.body.response.results[0].message).toMatch(/nothing was uninstalled/)
+    expect(a.apps.Blocker).toBeDefined()
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('keeps an odd app name inside the data folder', async () => {
+    a.apps['../../evil'] = { version: '1', config: 'x' }
+    await run('admin', { action: 'uninstall', name: '../../evil', targets: ['a'] })
+    expect(existsSync(path.join(dataDir, '..', 'evil'))).toBe(false)
+    const dirs = readdirSync(path.join(dataDir, 'apps'))
+    expect(dirs.every((d) => d !== '.' && d !== '..' && !/[\/]/.test(d) && !d.includes('..'))).toBe(true)
+  })
+})
+
+describe('hub/apps/upload', () => {
+  const bytes = new Uint8Array(70_000).map((_, i) => (i * 7 + 3) % 256)
+
+  it('forwards the same zip to every target, byte for byte', async () => {
+    b.apps = {}
+    a.apps = {}
+    const r = await api('apps-admin', 'hub/apps/upload', { body: zipForm({ action: 'install', name: 'Blocker', targets: 'a,b' }, bytes) })
+    expect(r.body.response.results).toEqual([
+      { server: 'a', ok: true, message: 'Installed from the file.' },
+      { server: 'b', ok: true, message: 'Installed from the file.' },
+    ])
+    for (const [f, admin] of [[a, 'admin-a'], [b, 'admin-b']] as const) {
+      const [w] = writes(f)
+      expect(w.path).toBe('apps/install')
+      expect(w.token).toBe(admin)
+      expect(w.query.get('name')).toBe('Blocker')
+      const parsed = parseMultipart(w.body, w.contentType)
+      expect(Buffer.from(parsed.files[0].data).equals(Buffer.from(bytes))).toBe(true)
+    }
+    const audit = (await api('admin', 'hub/audit/list', { params: { limit: '10' } })).body.response.entries as Record<string, string>[]
+    expect(audit.filter((e) => e.action === 'apps.install-file' && e.outcome === 'ok')).toHaveLength(2)
+  })
+
+  it('updates, and reports an app that is not there as that target failure', async () => {
+    delete b.apps.Blocker
+    const r = await api('admin', 'hub/apps/upload', { body: zipForm({ action: 'update', name: 'Blocker', targets: 'a,b' }, bytes) })
+    expect(writes(a)[0].path).toBe('apps/update')
+    expect(r.body.response.results.map((x: any) => x.ok)).toEqual([true, true])
+  })
+
+  it('refuses a missing file, a bad action and a body above 20 MB', async () => {
+    const noFile = new FormData()
+    noFile.append('action', 'install')
+    noFile.append('name', 'X')
+    noFile.append('targets', 'a')
+    expect((await api('admin', 'hub/apps/upload', { body: noFile })).code).toBe(400)
+    expect((await api('admin', 'hub/apps/upload', { body: zipForm({ action: 'wipe', name: 'X', targets: 'a' }, bytes) })).code).toBe(400)
+
+    const big = new Uint8Array(21 * 1024 * 1024)
+    const r = await api('admin', 'hub/apps/upload', { body: zipForm({ action: 'install', name: 'X', targets: 'a' }, big) })
+    expect(r.code).toBe(413)
+    expect(r.body.errorMessage).toMatch(/20 MB/)
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('does not read a big body from someone who is not allowed to upload', async () => {
+    const r = await api('viewer', 'hub/apps/upload', { body: zipForm({ action: 'install', name: 'X', targets: 'a' }, bytes) })
+    expect(r.code).toBe(403)
+  })
+})
+
+describe('single-server writes', () => {
+  it('forwards with the admin token', async () => {
+    const r = await api('apps-admin', 'apps/downloadAndUpdate', { params: { node: 'a', name: 'Blocker', url: STORE('2.0')[0].url } })
+    expect(r.body.status).toBe('ok')
+    const [w] = writes(a)
+    expect(w).toMatchObject({ path: 'apps/downloadAndUpdate', token: 'admin-a' })
+    expect(w.query.get('node')).toBeNull()
+    expect(writes(b)).toHaveLength(0)
+    expect(a.apps.Blocker.version).toBe('2.0')
+  })
+
+  it('works with a form POST too, and uninstalls with the history', async () => {
+    const r = await api('admin', 'apps/uninstall', {
+      body: new URLSearchParams({ node: 'b', name: 'Blocker' }).toString(),
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    })
+    expect(r.body.status).toBe('ok')
+    expect(b.apps.Blocker).toBeUndefined()
+    expect(readdirSync(path.join(dataDir, 'apps', 'Blocker', 'history'))).toHaveLength(1)
+  })
+
+  it('is refused without a node or with the aggregate', async () => {
+    for (const params of [{ name: 'Blocker' } as Record<string, string>, { name: 'Blocker', node: 'cluster' }, { name: 'Blocker', node: 'zzz' }]) {
+      const r = await api('admin', 'apps/uninstall', { params })
+      expect(r.code).toBe(400)
+    }
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('does not forward a download url the store did not offer', async () => {
+    const r = await api('admin', 'apps/downloadAndInstall', { params: { node: 'a', name: 'Blocker', url: 'https://evil.example/x.zip' } })
+    expect(r.body.status).toBe('error')
+    expect(r.body.errorMessage).toMatch(/store listing/)
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('reports a server without an admin token', async () => {
+    await start({ A_ADMIN_TOKEN: '' })
+    const r = await api('admin', 'apps/uninstall', { params: { node: 'a', name: 'Blocker' } })
+    expect(r.body).toEqual({ status: 'error', errorMessage: 'a: no admin token configured' })
+    expect(a.calls.filter((c) => c.method === 'POST')).toHaveLength(0)
+  })
+
+  it('forwards a multipart install to the node', async () => {
+    delete a.apps.Blocker
+    const bytes = new Uint8Array([80, 75, 3, 4, 13, 10, 45, 45, 0, 255])
+    const r = await api('admin', 'apps/install', { params: { node: 'a', name: 'Blocker' }, body: zipForm({}, bytes) })
+    expect(r.body.status).toBe('ok')
+    const [w] = writes(a)
+    expect(w.token).toBe('admin-a')
+    expect(Buffer.from(parseMultipart(w.body, w.contentType).files[0].data).equals(Buffer.from(bytes))).toBe(true)
+  })
+
+  it('needs a node for a config read or write', async () => {
+    for (const p of ['apps/config/get', 'apps/config/set']) {
+      expect((await api('admin', p, { params: { name: 'Blocker' } })).code).toBe(400)
+    }
+  })
+
+  it('lists the store with the servers each app is installed on', async () => {
+    delete b.apps.Blocker
+    const r = await api('viewer', 'apps/listStoreApps')
+    expect(r.body.response.storeApps[0]).toMatchObject({ name: 'Blocker', installedOn: ['a'] })
+  })
+})

hub/apps-write.ts

@@ -0,0 +1,178 @@
+/*
+App writes: install, update and uninstall on one or several servers. The first part of the
+hub that changes anything, so the rules are strict:
+
+  * only a server's ADMIN token writes (`<ID>_ADMIN_TOKEN`); a server without one is refused;
+  * a store download URL is never taken from the client: it is looked up in the target
+    server's own `apps/listStoreApps`, and a URL the client sends must match it;
+  * before an uninstall the app's config is saved to `${HUB_DATA}/apps/<app>/history/`, and
+    the uninstall is refused if that cannot be done;
+  * every target's result is audited (the caller does that with `Audit`).
+*/
+import type { HubConfig, HubServer } from './config.ts'
+import type { UploadedFile } from './multipart.ts'
+import { MAX_CONFIG } from './apps-sync.ts'
+import { safeName, writeHistory } from './store.ts'
+import { Upstream } from './upstream.ts'
+
+export { safeName }
+
+export type StoreAction = 'install-store' | 'update-store' | 'uninstall'
+export type FileAction = 'install-file' | 'update-file'
+export type AppAction = StoreAction | FileAction | 'config-set'
+
+export interface TargetResult {
+  server: string
+  ok: boolean
+  message?: string
+}
+
+const API: Record<AppAction, string> = {
+  'install-store': 'apps/downloadAndInstall',
+  'update-store': 'apps/downloadAndUpdate',
+  uninstall: 'apps/uninstall',
+  'install-file': 'apps/install',
+  'update-file': 'apps/update',
+  'config-set': 'apps/config/set',
+}
+
+const DONE: Record<AppAction, string> = {
+  'install-store': 'Installed from the store.',
+  'update-store': 'Updated from the store.',
+  uninstall: 'Uninstalled.',
+  'install-file': 'Installed from the file.',
+  'update-file': 'Updated from the file.',
+  'config-set': 'Config saved.',
+}
+
+/** The audit/api action for an upstream write path, or null when it is not an app write. */
+export function actionForPath(apiPath: string): AppAction | null {
+  for (const [action, p] of Object.entries(API)) if (p === apiPath) return action as AppAction
+  return null
+}
+
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+
+export class AppsWriter {
+  #config: HubConfig
+  #upstream: Upstream
+  #now: () => number
+
+  constructor(config: HubConfig, upstream: Upstream, now: () => number = Date.now) {
+    this.#config = config
+    this.#upstream = upstream
+    this.#now = now
+  }
+
+  /** The ids of the servers that have an admin token. */
+  writableIds(): string[] {
+    return this.#config.servers.filter((s) => s.adminToken !== undefined).map((s) => s.id)
+  }
+
+  /* The URL the server's own store gives for this app, or null. */
+  async storeUrl(server: HubServer, name: string): Promise<string | null> {
+    const response = (await this.#upstream.call(server, 'apps/listStoreApps')) as { storeApps?: { name: string; url?: string }[] } | null
+    const found = (response?.storeApps ?? []).find((a) => a.name === name)
+    return typeof found?.url === 'string' && found.url !== '' ? found.url : null
+  }
+
+  /* Saves the app's current config next to the hub's data; throws if it cannot. */
+  async saveHistory(server: HubServer, name: string, kind: 'before-uninstall' | 'before-set' = 'before-uninstall'): Promise<string> {
+    if (server.adminToken === undefined) throw new Error('no admin token configured')
+    this.#upstream.invalidate(server.id)
+    const response = (await this.#upstream.call(server, 'apps/config/get', { name }, server.adminToken)) as { config?: unknown } | null
+    const text = typeof response?.config === 'string' ? response.config : ''
+    return writeHistory(this.#config.dataDir, name, server.id, text, kind, this.#now())
+  }
+
+  /*
+  One write on one server. `url` must already be checked against the store; `file` is the zip
+  for the two file actions. Returns the server's response, or throws.
+  */
+  async perform(server: HubServer, action: AppAction, name: string, url?: string, file?: UploadedFile, config?: string): Promise<unknown> {
+    if (server.adminToken === undefined) throw new Error('no admin token configured')
+    switch (action) {
+      case 'install-store':
+      case 'update-store':
+        return await this.#upstream.write(server, API[action], { name, url: url ?? '' })
+      case 'uninstall':
+        try {
+          await this.saveHistory(server, name)
+        } catch (e) {
+          throw new Error(`the config could not be saved before the uninstall (${reason(e)}); nothing was uninstalled`)
+        }
+        return await this.#upstream.write(server, API[action], { name })
+      case 'config-set': {
+        if (config === undefined || config.length > MAX_CONFIG) throw new Error('the config is missing or larger than 1 MB')
+        try {
+          await this.saveHistory(server, name, 'before-set')
+        } catch (e) {
+          throw new Error(`the old config could not be saved first (${reason(e)}); nothing was changed`)
+        }
+        return await this.#upstream.write(server, API[action], {}, new URLSearchParams({ name, config }))
+      }
+      case 'install-file':
+      case 'update-file': {
+        if (file === undefined) throw new Error('no zip file')
+        const form = new FormData()
+        form.append('fileAppZip', new Blob([file.data], { type: file.type }), file.filename)
+        return await this.#upstream.write(server, API[action], { name }, form)
+      }
+    }
+  }
+
+  /*
+  The same write on several servers, in parallel. A store action first resolves each target's
+  URL from its own store; `suppliedUrl` (if the client sent one) has to match every one of them.
+  Returns `{ error }` when the request itself is refused, else one result per target.
+  */
+  async runMany(
+    action: AppAction,
+    name: string,
+    targets: string[],
+    options: { suppliedUrl?: string; file?: UploadedFile } = {},
+  ): Promise<{ error: string } | { results: TargetResult[] }> {
+    const known = new Map(this.#config.servers.map((s) => [s.id, s]))
+    const unique = [...new Set(targets)]
+    if (name === '' || name.length > 200) return { error: 'An app name is required.' }
+    if (unique.length === 0) return { error: 'Choose at least one server.' }
+    const unknown = unique.filter((t) => !known.has(t))
+    if (unknown.length > 0) return { error: `Unknown server: ${unknown.join(', ')}.` }
+
+    const urls = new Map<string, string>()
+    const early = new Map<string, string>()
+    if (action === 'install-store' || action === 'update-store') {
+      await Promise.all(
+        unique.map(async (id) => {
+          const server = known.get(id) as HubServer
+          if (server.adminToken === undefined) return early.set(id, 'no admin token configured')
+          try {
+            const url = await this.storeUrl(server, name)
+            if (url === null) early.set(id, `"${name}" is not in this server's store listing`)
+            else urls.set(id, url)
+          } catch (e) {
+            early.set(id, `the store listing could not be read (${reason(e)})`)
+          }
+        }),
+      )
+      if (options.suppliedUrl !== undefined && options.suppliedUrl !== '' && [...urls.values()].some((u) => u !== options.suppliedUrl)) {
+        return { error: 'The url is not from the store listing; the hub only installs what a server offers.' }
+      }
+    }
+
+    const results = await Promise.all(
+      unique.map(async (id): Promise<TargetResult> => {
+        const server = known.get(id) as HubServer
+        const refused = early.get(id) ?? (server.adminToken === undefined ? 'no admin token configured' : undefined)
+        if (refused !== undefined) return { server: id, ok: false, message: refused }
+        try {
+          await this.perform(server, action, name, urls.get(id), options.file)
+          return { server: id, ok: true, message: DONE[action] }
+        } catch (e) {
+          return { server: id, ok: false, message: reason(e) }
+        }
+      }),
+    )
+    return { results }
+  }
+}

hub/audit.ts

@@ -0,0 +1,136 @@
+/*
+The audit log: JSON Lines in `${HUB_DATA}/audit.log`, one entry per line, newest last on
+disk and newest first from `list`. Rotated at 10 MB to audit.log.1 (one kept). If the
+folder is not writable, one line goes to stderr and the log lives in memory (last 1000).
+
+Entries carry fixed fields only. Callers never pass passwords, tokens or request bodies.
+*/
+import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync } from 'node:fs'
+import path from 'node:path'
+
+export const MAX_BYTES = 10 * 1024 * 1024
+const RING = 1000
+const DETAIL_MAX = 300
+
+export interface AuditEntry {
+  ts: string
+  user: string
+  role?: string
+  ip: string
+  action: string
+  server?: string
+  target?: string
+  outcome: 'ok' | 'failed' | 'refused'
+  detail?: string
+}
+
+export type AuditInput = Omit<AuditEntry, 'ts'>
+
+const cut = (value: string, max: number): string => value.slice(0, max)
+
+export class Audit {
+  #file: string
+  #maxBytes: number
+  #now: () => number
+  #ring: AuditEntry[] = []
+  #size = 0
+  #memoryOnly = false
+  #listener: ((entry: AuditEntry) => void) | null = null
+
+  constructor(dir: string, options: { maxBytes?: number; now?: () => number } = {}) {
+    this.#maxBytes = options.maxBytes ?? MAX_BYTES
+    this.#now = options.now ?? Date.now
+    this.#file = path.join(dir, 'audit.log')
+    try {
+      mkdirSync(dir, { recursive: true })
+      this.#size = existsSync(this.#file) ? statSync(this.#file).size : 0
+      appendFileSync(this.#file, '')
+    } catch (e) {
+      this.#fallBack(e)
+    }
+  }
+
+  get inMemory(): boolean {
+    return this.#memoryOnly
+  }
+
+  #fallBack(e: unknown): void {
+    if (!this.#memoryOnly) {
+      console.error(
+        `hub: audit log is not writable (${e instanceof Error ? e.message : String(e)}); keeping the last ${RING} entries in memory.`,
+      )
+    }
+    this.#memoryOnly = true
+  }
+
+  /** One function is told about every entry as it is written (the webhook uses it for failed writes). */
+  onEntry(listener: (entry: AuditEntry) => void): void {
+    this.#listener = listener
+  }
+
+  log(input: AuditInput): void {
+    const entry: AuditEntry = {
+      ts: new Date(this.#now()).toISOString(),
+      user: cut(input.user, 64),
+      ...(input.role === undefined ? {} : { role: input.role }),
+      ip: cut(input.ip, 64),
+      action: input.action,
+      ...(input.server === undefined ? {} : { server: input.server }),
+      ...(input.target === undefined ? {} : { target: cut(input.target, 200) }),
+      outcome: input.outcome,
+      ...(input.detail === undefined ? {} : { detail: cut(input.detail, DETAIL_MAX) }),
+    }
+    this.#ring.push(entry)
+    try {
+      this.#listener?.(entry)
+    } catch {
+      // A listener never stops the log.
+    }
+    if (this.#ring.length > RING) this.#ring.shift()
+    if (this.#memoryOnly) return
+    const line = JSON.stringify(entry) + '\n'
+    try {
+      if (this.#size + Buffer.byteLength(line) > this.#maxBytes) this.#rotate()
+      appendFileSync(this.#file, line)
+      this.#size += Buffer.byteLength(line)
+    } catch (e) {
+      this.#fallBack(e)
+    }
+  }
+
+  #rotate(): void {
+    rmSync(`${this.#file}.1`, { force: true })
+    renameSync(this.#file, `${this.#file}.1`)
+    this.#size = 0
+  }
+
+  #read(file: string): AuditEntry[] {
+    if (!existsSync(file)) return []
+    const out: AuditEntry[] = []
+    for (const line of readFileSync(file, 'utf8').split('\n')) {
+      if (line === '') continue
+      try {
+        out.push(JSON.parse(line) as AuditEntry)
+      } catch {
+        // A torn line is skipped.
+      }
+    }
+    return out
+  }
+
+  /** Newest first. `before` is an ISO time or epoch milliseconds: only older entries are returned. */
+  list(limit = 200, before?: string): AuditEntry[] {
+    const max = Math.min(Math.max(Math.trunc(limit) || 200, 1), 1000)
+    let cutoff = Infinity
+    if (before !== undefined && before !== '') {
+      const t = /^\d+$/.test(before) ? Number(before) : Date.parse(before)
+      if (Number.isFinite(t)) cutoff = t
+    }
+    const all = this.#memoryOnly ? this.#ring : [...this.#read(`${this.#file}.1`), ...this.#read(this.#file)]
+    const result: AuditEntry[] = []
+    for (let i = all.length - 1; i >= 0 && result.length < max; i--) {
+      if (Date.parse(all[i].ts) < cutoff) result.push(all[i])
+    }
+    return result
+  }
+}

hub/config.ts

@@ -9,15 +9,33 @@ export interface HubServer {
   id: string
   url: string
   token: string
+  /** Optional write-capable token (`<ID>_ADMIN_TOKEN`). Not used yet; never logged. */
+  adminToken?: string
+}
+
+/** The single admin from HUB_USER and HUB_PASSWORD, used only while there is no users file. */
+export interface Bootstrap {
+  user: string
+  password: string
 }
 
 export interface HubConfig {
   servers: HubServer[]
-  user: string
-  password: string
+  /** From HUB_USER and HUB_PASSWORD; null when they are not set. Whether it is used depends on the users file. */
+  bootstrap: Bootstrap | null
+  /** Where users.json and audit.log live (HUB_DATA). */
+  dataDir: string
+  /** Believe the first X-Forwarded-For entry as the client address (HUB_TRUST_PROXY=1). */
+  trustProxy: boolean
   port: number
   cacheSeconds: number
+  /** How often the drift check runs (HUB_DRIFT_MINUTES); 0 turns it off. */
+  driftMinutes: number
   timeoutMs: number
+  /** Where drift and failed-write notices are POSTed (HUB_WEBHOOK_URL); null when not set. */
+  webhookUrl: string | null
+  /** Signs the notices with HMAC-SHA256 (HUB_WEBHOOK_SECRET); null when not set. Never logged. */
+  webhookSecret: string | null
   /** Shown as `dnsServerDomain` (and `clusterDomain`) in the session. */
   domain: string
   /** The console's static build. */
@@ -31,6 +49,11 @@ export function tokenVariable(id: string): string {
   return `${id.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_TOKEN`
 }
 
+/* `ndns` -> `NDNS_ADMIN_TOKEN`. */
+export function adminTokenVariable(id: string): string {
+  return `${id.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_ADMIN_TOKEN`
+}
+
 function number(env: Env, name: string, fallback: number, min: number): number {
   const raw = env[name]
   if (raw === undefined || raw.trim() === '') return fallback
@@ -40,9 +63,10 @@ function number(env: Env, name: string, fallback: number, min: number): number {
 }
 
 export function parseConfig(env: Env): HubConfig {
+  // The users file decides whether these are needed; the config does not read the file.
   const user = env.HUB_USER ?? ''
   const password = env.HUB_PASSWORD ?? ''
-  if (user === '' || password === '') throw new Error('HUB_USER and HUB_PASSWORD are required.')
+  if ((user === '') !== (password === '')) throw new Error('HUB_USER and HUB_PASSWORD must be set together.')
 
   const servers: HubServer[] = []
   for (const part of (env.HUB_SERVERS ?? '').split(',')) {
@@ -58,19 +82,35 @@ export function parseConfig(env: Env): HubConfig {
     const variable = tokenVariable(id)
     const token = env[variable] ?? ''
     if (token === '') throw new Error(`${variable} is required for the server "${id}".`)
-    servers.push({ id, url, token })
+    const adminToken = env[adminTokenVariable(id)] ?? ''
+    servers.push(adminToken === '' ? { id, url, token } : { id, url, token, adminToken })
   }
   if (servers.length === 0) {
     throw new Error('HUB_SERVERS is required, e.g. dns=https://dns.example.net,ndns=https://ndns.example.net.')
   }
 
+  const webhookUrl = (env.HUB_WEBHOOK_URL ?? '').trim()
+  if (webhookUrl !== '') {
+    let ok = /^https?:\/\/[^/\s]/i.test(webhookUrl)
+    try {
+      new URL(webhookUrl)
+    } catch {
+      ok = false
+    }
+    if (!ok) throw new Error('HUB_WEBHOOK_URL must be an http or https address.')
+  }
+
   return {
     servers,
-    user,
-    password,
+    bootstrap: user === '' ? null : { user, password },
+    dataDir: env.HUB_DATA?.trim() || '/data',
+    trustProxy: ['1', 'true'].includes((env.HUB_TRUST_PROXY ?? '').trim().toLowerCase()),
     port: number(env, 'HUB_PORT', 8080, 0),
     cacheSeconds: number(env, 'HUB_CACHE_SECONDS', 15, 0),
+    driftMinutes: number(env, 'HUB_DRIFT_MINUTES', 15, 0),
     timeoutMs: number(env, 'HUB_TIMEOUT_MS', 8000, 1),
+    webhookUrl: webhookUrl === '' ? null : webhookUrl,
+    webhookSecret: (env.HUB_WEBHOOK_SECRET ?? '') === '' ? null : (env.HUB_WEBHOOK_SECRET as string),
     domain: env.HUB_DOMAIN?.trim() || 'hub',
     staticDir: env.HUB_STATIC?.trim() || './dist',
   }

hub/drift.test.ts

@@ -0,0 +1,343 @@
+// @vitest-environment node
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { Drift } from './drift.ts'
+import { writeBaseline, writeListsBaseline, writeOverrides, writeSettingsBaseline } from './store.ts'
+import { Webhook } from './webhook.ts'
+import { Upstream } from './upstream.ts'
+
+const json = (o: unknown): string => JSON.stringify(o, null, 2)
+
+interface FakeServer {
+  apps: Record<string, string>
+  down: boolean
+  denyConfig: boolean
+  settings: Record<string, unknown>
+  blocked: string[]
+  allowed: string[]
+}
+
+let dir: string
+let servers: Record<string, FakeServer>
+let calls: string[]
+let gate: Promise<void> | null
+let clock: number
+
+/* A fetch that answers from memory: no sockets, no timers. */
+const fakeFetch = (async (input: string | URL | Request) => {
+  const url = new URL(String(input))
+  const id = url.port === '1' ? 'a' : 'b'
+  const server = servers[id]
+  const apiPath = url.pathname.replace(/^\/api\//, '')
+  calls.push(`${id}:${apiPath}`)
+  if (gate !== null) await gate
+  if (server.down) throw new TypeError('fetch failed')
+  const name = url.searchParams.get('name') ?? ''
+  const reply = (response: unknown) => Response.json({ status: 'ok', response })
+  if (apiPath === 'apps/list') return reply({ apps: Object.keys(server.apps).map((n) => ({ name: n, version: '1.0' })) })
+  if (apiPath === 'apps/config/get') {
+    if (server.denyConfig) return Response.json({ status: 'error', errorMessage: 'Access was denied.' })
+    return reply({ config: server.apps[name] ?? null })
+  }
+  if (apiPath === 'settings/get') return reply(server.settings)
+  if (apiPath === 'blocked/export' || apiPath === 'allowed/export') {
+    return new Response((apiPath === 'blocked/export' ? server.blocked : server.allowed).join('\n') + '\n', { headers: { 'content-type': 'text/plain' } })
+  }
+  return reply({})
+}) as typeof fetch
+
+function make(webhook?: Webhook) {
+  const config = parseConfig({
+    HUB_SERVERS: 'a=http://127.0.0.1:1,b=http://127.0.0.1:2',
+    A_TOKEN: 'ra',
+    B_TOKEN: 'rb',
+    HUB_DATA: dir,
+    HUB_CACHE_SECONDS: '0',
+  })
+  const drift = new Drift(config, new Upstream(config, fakeFetch, () => clock), () => clock, { webhook })
+  return { config, drift }
+}
+
+const baseline = (app: string, text: string) =>
+  writeBaseline(dir, { app, text, updatedAt: '2026-01-01T00:00:00Z', updatedBy: 'test' })
+
+beforeEach(() => {
+  dir = mkdtempSync(path.join(tmpdir(), 'hub-drift-'))
+  const blank = (): FakeServer => ({ apps: {}, down: false, denyConfig: false, settings: {}, blocked: [], allowed: [] })
+  servers = { a: blank(), b: blank() }
+  calls = []
+  gate = null
+  clock = Date.parse('2026-10-31T10:00:00Z')
+})
+
+afterEach(() => rmSync(dir, { recursive: true, force: true }))
+
+describe('drift check', () => {
+  it('finds nothing when no app has a baseline, and calls no server', async () => {
+    servers.a.apps.X = json({ a: 1 })
+    const { drift } = make()
+    const state = await drift.check()
+    expect(state.apps).toEqual([])
+    expect(state.checkedAt).toBe('2026-10-31T10:00:00.000Z')
+    expect(calls).toEqual([])
+    expect(drift.count()).toBe(0)
+  })
+
+  it('reports a server that differs from the baseline and not one that matches', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    servers.a.apps.Blocker = json({ ttl: 30 })
+    servers.b.apps.Blocker = json({ ttl: 99 })
+    const { drift } = make()
+    const state = await drift.check()
+    expect(state.apps).toEqual([{ kind: 'app', name: 'Blocker', app: 'Blocker', server: 'b', since: '2026-10-31T10:00:00.000Z', status: 'differs' }])
+    expect(drift.count()).toBe(1)
+  })
+
+  it('compares by meaning, not by spacing or key order', async () => {
+    baseline('Blocker', json({ a: 1, b: 2 }))
+    servers.a.apps.Blocker = '{"b":2,"a":1}'
+    servers.b.apps.Blocker = json({ a: 1, b: 2 })
+    expect((await make().drift.check()).apps).toEqual([])
+  })
+
+  it('uses each server own local values, so a local difference is not drift', async () => {
+    baseline('Blocker', json({ ttl: 30, db: '/base' }))
+    writeOverrides(dir, 'Blocker', { paths: ['db'], values: { a: { db: '/a.db' }, b: { db: '/b.db' } } })
+    servers.a.apps.Blocker = json({ ttl: 30, db: '/a.db' })
+    servers.b.apps.Blocker = json({ ttl: 30, db: '/b.db' })
+    const { drift } = make()
+    expect((await drift.check()).apps).toEqual([])
+    // A wrong local value is drift.
+    servers.b.apps.Blocker = json({ ttl: 30, db: '/elsewhere.db' })
+    expect((await drift.check()).apps.map((e) => [e.server, e.status])).toEqual([['b', 'differs']])
+  })
+
+  it('lists a server that does not answer as unreadable, and does not count it', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    servers.a.apps.Blocker = json({ ttl: 30 })
+    servers.b.down = true
+    const { drift } = make()
+    const state = await drift.check()
+    expect(state.apps).toEqual([{ kind: 'app', name: 'Blocker', app: 'Blocker', server: 'b', since: '2026-10-31T10:00:00.000Z', status: 'unreadable' }])
+    expect(drift.count()).toBe(0)
+  })
+
+  it('lists a refused config read as unreadable and an app that is missing as not-installed', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    servers.a.apps.Blocker = json({ ttl: 30 })
+    servers.a.denyConfig = true
+    const { drift } = make()
+    const state = await drift.check()
+    expect(state.apps.map((e) => [e.server, e.status])).toEqual([['a', 'unreadable'], ['b', 'not-installed']])
+  })
+
+  it('keeps `since` across checks and drops the entry once the problem is fixed', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    servers.a.apps.Blocker = json({ ttl: 30 })
+    servers.b.apps.Blocker = json({ ttl: 99 })
+    const { drift } = make()
+    await drift.check()
+    clock += 15 * 60_000
+    const second = await drift.check()
+    expect(second.apps[0].since).toBe('2026-10-31T10:00:00.000Z')
+    expect(second.checkedAt).toBe('2026-10-31T10:15:00.000Z')
+
+    servers.b.apps.Blocker = json({ ttl: 30 })
+    clock += 15 * 60_000
+    expect((await drift.check()).apps).toEqual([])
+    expect(drift.count()).toBe(0)
+
+    // It drifts again: a new `since`.
+    servers.b.apps.Blocker = json({ ttl: 5 })
+    clock += 15 * 60_000
+    expect((await drift.check()).apps[0].since).toBe('2026-10-31T10:45:00.000Z')
+  })
+
+  it('changes the status (and `since`) when the problem changes', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    servers.a.apps.Blocker = json({ ttl: 30 })
+    servers.b.apps.Blocker = json({ ttl: 99 })
+    const { drift } = make()
+    await drift.check()
+    servers.b.down = true
+    clock += 60_000
+    const state = await drift.check()
+    expect(state.apps[0]).toMatchObject({ status: 'unreadable', since: '2026-10-31T10:01:00.000Z' })
+  })
+
+  it('never runs two checks at once', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    servers.a.apps.Blocker = json({ ttl: 30 })
+    servers.b.apps.Blocker = json({ ttl: 30 })
+    let open!: () => void
+    gate = new Promise<void>((resolve) => (open = resolve))
+    const { drift } = make()
+    const first = drift.check()
+    const second = drift.check()
+    expect(drift.get().checking).toBe(true)
+    open()
+    expect(await second).toBe(await first)
+    // Two servers, two app lists and two config reads: one check's worth of calls.
+    expect(calls).toHaveLength(4)
+    expect(drift.get().checking).toBe(false)
+    gate = null
+    await drift.check()
+    expect(calls).toHaveLength(8)
+  })
+
+  it('keeps the last result in drift.json and loads it again', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    servers.a.apps.Blocker = json({ ttl: 30 })
+    servers.b.apps.Blocker = json({ ttl: 99 })
+    await make().drift.check()
+    const file = JSON.parse(readFileSync(path.join(dir, 'drift.json'), 'utf8'))
+    expect(file).toEqual({
+      checkedAt: '2026-10-31T10:00:00.000Z',
+      apps: [{ kind: 'app', name: 'Blocker', app: 'Blocker', server: 'b', since: '2026-10-31T10:00:00.000Z', status: 'differs' }],
+    })
+    const again = make().drift
+    expect(again.get()).toMatchObject({ checkedAt: '2026-10-31T10:00:00.000Z', checking: false })
+    expect(again.count()).toBe(1)
+  })
+})
+
+const baselineSettings = (settings: Record<string, unknown>) => writeSettingsBaseline(dir, { settings, updatedAt: '2026-01-01T00:00:00Z', updatedBy: 'test' })
+const baselineLists = (blocked: string[], allowed: string[]) => writeListsBaseline(dir, { blocked, allowed, updatedAt: '2026-01-01T00:00:00Z', updatedBy: 'test' })
+
+describe('drift of settings and lists', () => {
+  it('reports shared settings fields that differ from the baseline, by field name only', async () => {
+    baselineSettings({ blockingType: 'NxDomain', defaultRecordTtl: 3600, dnsServerDomain: 'x.example' })
+    servers.a.settings = { blockingType: 'NxDomain', defaultRecordTtl: 3600, dnsServerDomain: 'a.example' }
+    servers.b.settings = { blockingType: 'AnyAddress', defaultRecordTtl: 60, dnsServerDomain: 'b.example' }
+    const { drift } = make()
+    const state = await drift.check()
+    // dnsServerDomain is local: server a is fine even though its value differs.
+    expect(state.apps).toEqual([{ kind: 'settings', server: 'b', since: '2026-10-31T10:00:00.000Z', status: 'differs', fields: ['blockingType', 'defaultRecordTtl'] }])
+    expect(JSON.stringify(state)).not.toContain('AnyAddress')
+    expect(drift.count()).toBe(1)
+  })
+
+  it('reports Blocked and Allowed lists that differ, without a domain', async () => {
+    baselineLists(['ads.example'], ['ok.example'])
+    servers.a.blocked = ['ads.example']
+    servers.a.allowed = ['ok.example']
+    servers.b.blocked = ['ads.example', 'extra.example']
+    servers.b.allowed = ['ok.example']
+    const { drift } = make()
+    const state = await drift.check()
+    expect(state.apps).toEqual([{ kind: 'lists', server: 'b', since: '2026-10-31T10:00:00.000Z', status: 'differs', fields: ['blocked'] }])
+    expect(JSON.stringify(state)).not.toContain('extra.example')
+  })
+
+  it('reads no settings and no lists when there is no baseline for them', async () => {
+    const { drift } = make()
+    await drift.check()
+    expect(calls).toEqual([])
+  })
+
+  it('lists a server that does not answer as unreadable for each baseline, and does not count it', async () => {
+    baselineSettings({ blockingType: 'NxDomain' })
+    baselineLists([], [])
+    servers.b.down = true
+    const { drift } = make()
+    const state = await drift.check()
+    expect(state.apps.map((e) => [e.kind, e.server, e.status])).toEqual([['lists', 'b', 'unreadable'], ['settings', 'b', 'unreadable']])
+    expect(drift.count()).toBe(0)
+  })
+
+  it('counts every kind in one number', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    baselineSettings({ blockingType: 'NxDomain' })
+    baselineLists(['x.example'], [])
+    servers.a.apps.Blocker = json({ ttl: 1 })
+    servers.b.apps.Blocker = json({ ttl: 30 })
+    servers.a.settings = { blockingType: 'AnyAddress' }
+    servers.b.settings = { blockingType: 'NxDomain' }
+    servers.a.blocked = ['x.example']
+    const { drift } = make()
+    await drift.check()
+    // a: app and settings; b: lists (the baseline has x.example, b has none).
+    expect(drift.get().apps.map((e) => `${e.kind}:${e.server}`).sort()).toEqual(['app:a', 'lists:b', 'settings:a'])
+    expect(drift.count()).toBe(3)
+  })
+
+  it('loads a drift.json written before stage 8, where `kind` held the status', () => {
+    writeFileSync(
+      path.join(dir, 'drift.json'),
+      JSON.stringify({
+        checkedAt: '2026-10-31T09:00:00.000Z',
+        apps: [
+          { app: 'Blocker', server: 'b', since: '2026-10-31T08:00:00.000Z', kind: 'differs' },
+          { app: 'Other', server: 'a', since: '2026-10-31T08:00:00.000Z', kind: 'not-installed' },
+          { server: 'a', since: 'x', kind: 'bogus', app: 'Z' },
+        ],
+      }),
+    )
+    const { drift } = make()
+    expect(drift.get().apps).toEqual([
+      { kind: 'app', name: 'Blocker', app: 'Blocker', server: 'b', since: '2026-10-31T08:00:00.000Z', status: 'differs' },
+      { kind: 'app', name: 'Other', app: 'Other', server: 'a', since: '2026-10-31T08:00:00.000Z', status: 'not-installed' },
+    ])
+    expect(drift.count()).toBe(1)
+  })
+})
+
+describe('drift notices', () => {
+  const sent: { event: string; items: Record<string, unknown>[] }[] = []
+  const hook = () =>
+    new Webhook(
+      { webhookUrl: 'https://hooks.example/x?token=SECRET', webhookSecret: null, domain: 'hub' },
+      {
+        retryDelayMs: 0,
+        fetch: (async (_url: unknown, init: RequestInit) => {
+          sent.push(JSON.parse(String(init.body)))
+          return new Response('ok')
+        }) as typeof fetch,
+      },
+    )
+
+  beforeEach(() => {
+    sent.length = 0
+  })
+
+  it('tells what newly differs once, and what is cleared, and nothing when nothing changed', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    servers.a.apps.Blocker = json({ ttl: 30 })
+    servers.b.apps.Blocker = json({ ttl: 99 })
+    const webhook = hook()
+    const { drift } = make(webhook)
+    await drift.check()
+    await webhook.settled()
+    expect(sent.map((s) => s.event)).toEqual(['drift.found'])
+    expect(sent[0].items).toMatchObject([{ kind: 'app', name: 'Blocker', server: 'b' }])
+
+    clock += 15 * 60_000
+    await drift.check()
+    await webhook.settled()
+    expect(sent).toHaveLength(1)
+
+    servers.b.apps.Blocker = json({ ttl: 30 })
+    await drift.check()
+    await webhook.settled()
+    expect(sent.map((s) => s.event)).toEqual(['drift.found', 'drift.cleared'])
+  })
+
+  it('does not repeat a notice after a restart, and does not call an unreadable server cleared', async () => {
+    baseline('Blocker', json({ ttl: 30 }))
+    servers.a.apps.Blocker = json({ ttl: 30 })
+    servers.b.apps.Blocker = json({ ttl: 99 })
+    await make().drift.check()
+    const webhook = hook()
+    const restarted = make(webhook).drift
+    await restarted.check()
+    await webhook.settled()
+    expect(sent).toEqual([])
+    servers.b.down = true
+    await restarted.check()
+    await webhook.settled()
+    expect(sent).toEqual([])
+  })
+})

hub/drift.ts

@@ -0,0 +1,209 @@
+/*
+Drift: a server whose config is no longer what the hub says it should be, usually because someone
+changed it in that server's own console. Three kinds:
+
+  app       an app's config is not the baseline plus the server's own local values
+  settings  the shared settings fields differ from the settings baseline
+  lists     the custom Blocked or Allowed list differs from the lists baseline
+
+`check` reads every server with the same rules as the compare views and keeps the result in memory
+and in `${HUB_DATA}/drift.json`; `since` is the first check that saw the same problem, and it is
+dropped once the problem is gone. An entry carries names and field names only, never a value.
+Two checks never run at once. The timer lives in server.ts, so the tests drive `check` themselves.
+
+When a webhook is given, a CHANGE of state is sent to it: `drift.found` for what newly differs and
+`drift.cleared` for what no longer does (an entry that became unreadable is not cleared: it is not
+known). The first state is the one loaded from drift.json, so a restart does not repeat notices.
+*/
+import { existsSync, readFileSync } from 'node:fs'
+import path from 'node:path'
+import type { HubConfig } from './config.ts'
+import { effectiveConfig, sameConfig } from './apps-sync.ts'
+import { readConfigText } from './apps-config.ts'
+import { SettingsConfig } from './settings-config.ts'
+import { listBaselineApps, readBaseline, readOverrides, writeAtomic } from './store.ts'
+import type { Upstream } from './upstream.ts'
+import type { Webhook } from './webhook.ts'
+
+export type DriftStatus = 'differs' | 'unreadable' | 'not-installed'
+export type DriftType = 'app' | 'settings' | 'lists'
+
+export interface DriftEntry {
+  kind: DriftType
+  /** The app's name; absent for settings and lists. */
+  name?: string
+  /** Same as `name` for an app (the console's matrix reads it). */
+  app?: string
+  server: string
+  since: string
+  status: DriftStatus
+  /** Setting names, or `blocked` / `allowed`. Never values. */
+  fields?: string[]
+}
+
+export interface DriftState {
+  checkedAt: string | null
+  /** Every entry, whatever its kind (the key is kept from stage 4). */
+  apps: DriftEntry[]
+}
+
+const STATUSES: DriftStatus[] = ['differs', 'unreadable', 'not-installed']
+const TYPES: DriftType[] = ['app', 'settings', 'lists']
+
+export const keyOf = (e: Pick<DriftEntry, 'kind' | 'name' | 'server'>): string => `${e.kind}\n${e.name ?? ''}\n${e.server}`
+
+/* An entry from drift.json. Before stage 8 `kind` held the status and `app` the name. */
+export function readEntry(raw: unknown): DriftEntry | null {
+  if (raw === null || typeof raw !== 'object') return null
+  const e = raw as Record<string, unknown>
+  if (typeof e.server !== 'string' || typeof e.since !== 'string') return null
+  const oldKind = typeof e.kind === 'string' && STATUSES.includes(e.kind as DriftStatus)
+  const status = (oldKind ? e.kind : e.status) as DriftStatus
+  if (!STATUSES.includes(status)) return null
+  const kind = (oldKind || e.kind === undefined ? 'app' : e.kind) as DriftType
+  if (!TYPES.includes(kind)) return null
+  const name = typeof e.name === 'string' ? e.name : typeof e.app === 'string' ? e.app : undefined
+  if (kind === 'app' && name === undefined) return null
+  const fields = Array.isArray(e.fields) ? e.fields.filter((f): f is string => typeof f === 'string').slice(0, 100) : undefined
+  return {
+    kind,
+    ...(name === undefined ? {} : { name }),
+    ...(kind === 'app' ? { app: name } : {}),
+    server: e.server,
+    since: e.since,
+    status,
+    ...(fields === undefined || fields.length === 0 ? {} : { fields }),
+  }
+}
+
+export class Drift {
+  #config: HubConfig
+  #upstream: Upstream
+  #now: () => number
+  #settings: SettingsConfig
+  #webhook: Webhook | null
+  #state: DriftState
+  #running: Promise<DriftState> | null = null
+
+  constructor(config: HubConfig, upstream: Upstream, now: () => number = Date.now, options: { webhook?: Webhook } = {}) {
+    this.#config = config
+    this.#upstream = upstream
+    this.#now = now
+    this.#settings = new SettingsConfig(config, upstream, now)
+    this.#webhook = options.webhook ?? null
+    this.#state = this.#load()
+  }
+
+  #file(): string {
+    return path.join(this.#config.dataDir, 'drift.json')
+  }
+
+  #load(): DriftState {
+    try {
+      if (!existsSync(this.#file())) return { checkedAt: null, apps: [] }
+      const data = JSON.parse(readFileSync(this.#file(), 'utf8')) as Partial<DriftState>
+      const apps = (Array.isArray(data.apps) ? data.apps : []).map(readEntry).filter((e): e is DriftEntry => e !== null)
+      return { checkedAt: typeof data.checkedAt === 'string' ? data.checkedAt : null, apps }
+    } catch {
+      return { checkedAt: null, apps: [] }
+    }
+  }
+
+  get(): DriftState & { checking: boolean } {
+    return { ...this.#state, checking: this.#running !== null }
+  }
+
+  /** How many entries differ from their baseline, of any kind (unreadable ones are listed, not counted). */
+  count(): number {
+    return this.#state.apps.filter((e) => e.status === 'differs').length
+  }
+
+  /* Runs a check, or joins the one that is already running. */
+  check(): Promise<DriftState> {
+    if (this.#running !== null) return this.#running
+    const run = this.#run().finally(() => {
+      this.#running = null
+    })
+    this.#running = run
+    return run
+  }
+
+  async #run(): Promise<DriftState> {
+    const apps = listBaselineApps(this.#config.dataDir).map((app) => ({
+      app,
+      baseline: readBaseline(this.#config.dataDir, app),
+      overrides: readOverrides(this.#config.dataDir, app),
+    }))
+    const found: Omit<DriftEntry, 'since'>[] = []
+
+    const appChecks = Promise.all(
+      this.#config.servers.map(async (server) => {
+        if (apps.length === 0) return
+        this.#upstream.invalidate(server.id)
+        let installed: Set<string>
+        try {
+          const list = (await this.#upstream.call(server, 'apps/list')) as { apps?: { name: string }[] } | null
+          installed = new Set((list?.apps ?? []).map((a) => a.name))
+        } catch {
+          for (const a of apps) found.push({ kind: 'app', name: a.app, app: a.app, server: server.id, status: 'unreadable' })
+          return
+        }
+        await Promise.all(
+          apps.map(async (a) => {
+            if (a.baseline === null) return
+            const entry = { kind: 'app' as const, name: a.app, app: a.app, server: server.id }
+            if (!installed.has(a.app)) return void found.push({ ...entry, status: 'not-installed' })
+            try {
+              const text = (await readConfigText(this.#upstream, server, a.app)) ?? ''
+              const effective = effectiveConfig(a.baseline.text, a.overrides, server.id).text
+              if (!sameConfig(text, effective)) found.push({ ...entry, status: 'differs' })
+            } catch {
+              found.push({ ...entry, status: 'unreadable' })
+            }
+          }),
+        )
+      }),
+    )
+    const otherChecks = this.#settings.driftFindings().then(
+      (list) => {
+        for (const f of list) found.push({ kind: f.kind, server: f.server, status: f.status, ...(f.fields.length > 0 ? { fields: f.fields } : {}) })
+      },
+      () => undefined,
+    )
+    await Promise.all([appChecks, otherChecks])
+
+    const at = new Date(this.#now()).toISOString()
+    const before = new Map(this.#state.apps.map((e) => [`${keyOf(e)}\n${e.status}`, e.since]))
+    const entries = found
+      .map((f): DriftEntry => ({ ...f, since: before.get(`${keyOf(f)}\n${f.status}`) ?? at }))
+      .sort((x, y) => x.kind.localeCompare(y.kind) || (x.name ?? '').localeCompare(y.name ?? '') || x.server.localeCompare(y.server))
+    const previous = this.#state.apps
+    this.#state = { checkedAt: at, apps: entries }
+    try {
+      writeAtomic(this.#file(), JSON.stringify(this.#state, null, 2) + '\n')
+    } catch (e) {
+      console.error(`hub: drift.json could not be written (${e instanceof Error ? e.message : String(e)}).`)
+    }
+    this.#notify(previous, entries)
+    return this.#state
+  }
+
+  /* A notice for what newly differs and for what is back in line. Names and field names only. */
+  #notify(previous: DriftEntry[], now: DriftEntry[]): void {
+    if (this.#webhook === null || !this.#webhook.configured) return
+    const item = (e: DriftEntry): Record<string, unknown> => ({
+      kind: e.kind,
+      ...(e.name === undefined ? {} : { name: e.name }),
+      server: e.server,
+      since: e.since,
+      ...(e.fields === undefined ? {} : { fields: e.fields }),
+    })
+    const was = new Set(previous.filter((e) => e.status === 'differs').map(keyOf))
+    const nowDiffers = new Set(now.filter((e) => e.status === 'differs').map(keyOf))
+    const unknown = new Set(now.filter((e) => e.status === 'unreadable').map(keyOf))
+    const newly = now.filter((e) => e.status === 'differs' && !was.has(keyOf(e)))
+    const cleared = previous.filter((e) => e.status === 'differs' && !nowDiffers.has(keyOf(e)) && !unknown.has(keyOf(e)))
+    if (newly.length > 0) this.#webhook.send('drift.found', newly.map(item))
+    if (cleared.length > 0) this.#webhook.send('drift.cleared', cleared.map(item))
+  }
+}

hub/fake-servers.ts

@@ -0,0 +1,152 @@
+/*
+Test helpers for the stage 7 suites: fake Technitium servers that answer by path and record every
+call, and a hub started against two of them with one user per role. Not used by the hub itself.
+*/
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { addUser, writeUsersFile } from './users.ts'
+
+export interface Call {
+  path: string
+  method: string
+  token: string
+  query: URLSearchParams
+  form: URLSearchParams
+  /* A text/plain body (a zone file). */
+  text: string
+}
+
+/* What a handler answers: the envelope's `response`, an error message, or a file. */
+export type Reply = { response?: unknown } | { error: string } | { text: string }
+
+export interface Fake {
+  server: http.Server
+  url: string
+  calls: Call[]
+  down: boolean
+  handlers: Map<string, (call: Call) => Reply>
+  /** Answers one path; later calls replace earlier ones. */
+  on: (apiPath: string, handler: (call: Call) => Reply) => void
+}
+
+export function fake(): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = {
+      server: http.createServer(),
+      url: '',
+      calls: [],
+      down: false,
+      handlers: new Map(),
+      on(apiPath, handler) {
+        f.handlers.set(apiPath, handler)
+      },
+    }
+    f.server.on('request', (req, res) => {
+      const chunks: Buffer[] = []
+      req.on('data', (c: Buffer) => chunks.push(c))
+      req.on('end', () => {
+        if (f.down) return req.socket.destroy()
+        const url = new URL(req.url ?? '/', 'http://fake')
+        const type = String(req.headers['content-type'] ?? '')
+        const raw = Buffer.concat(chunks).toString('utf8')
+        const call: Call = {
+          path: url.pathname.replace(/^\/api\//, ''),
+          method: req.method ?? '',
+          token: url.searchParams.get('token') ?? '',
+          query: url.searchParams,
+          form: new URLSearchParams(type.includes('x-www-form-urlencoded') ? raw : ''),
+          text: type.includes('text/plain') ? raw : '',
+        }
+        f.calls.push(call)
+        const reply: Reply = f.handlers.get(call.path)?.(call) ?? { response: {} }
+        if ('text' in reply) {
+          res.setHeader('content-type', 'text/plain')
+          return res.end(reply.text)
+        }
+        res.setHeader('content-type', 'application/json')
+        res.end(JSON.stringify('error' in reply ? { status: 'error', errorMessage: reply.error } : { status: 'ok', response: reply.response ?? {} }))
+      })
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+export const ROLES = ['viewer', 'operator', 'apps-admin', 'admin'] as const
+
+export interface TestHub {
+  base: string
+  tokens: Record<string, string>
+  dataDir: string
+  /** Calls the hub with a role's session. */
+  call: (role: string, apiPath: string, params?: Record<string, string>, method?: 'GET' | 'POST') => Promise<{ code: number; body: Record<string, any> }>
+  audit: () => Promise<Record<string, string>[]>
+  close: () => void
+}
+
+export async function startHub(a: Fake, b: Fake, extra: Record<string, string> = {}): Promise<TestHub> {
+  const dataDir = mkdtempSync(path.join(tmpdir(), 'hub-s7-data-'))
+  const staticDir = mkdtempSync(path.join(tmpdir(), 'hub-s7-static-'))
+  let users = addUser([], 'viewer', 'viewer', 'password-viewer')
+  users = addUser(users, 'operator', 'operator', 'password-operator')
+  users = addUser(users, 'apps-admin', 'apps-admin', 'password-apps-admin')
+  users = addUser(users, 'admin', 'admin', 'password-admin')
+  writeUsersFile(dataDir, users)
+  const hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      A_ADMIN_TOKEN: 'admin-a',
+      B_ADMIN_TOKEN: 'admin-b',
+      HUB_DATA: dataDir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      HUB_DRIFT_MINUTES: '0',
+      ...extra,
+    }),
+  )
+  await new Promise<void>((resolve) => hub.listen(0, '127.0.0.1', resolve))
+  const base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  const tokens: Record<string, string> = {}
+  for (const role of ROLES) {
+    const res = await fetch(`${base}/api/user/login`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: `user=${role}&pass=password-${role}`,
+    })
+    tokens[role] = ((await res.json()) as { token: string }).token
+  }
+  const call: TestHub['call'] = async (role, apiPath, params = {}, method = 'POST') => {
+    const query = new URLSearchParams(method === 'GET' ? params : {})
+    const res = await fetch(`${base}/api/${apiPath}${query.size > 0 ? `?${query}` : ''}`, {
+      method,
+      headers: { authorization: `Bearer ${tokens[role]}`, ...(method === 'POST' ? { 'content-type': 'application/x-www-form-urlencoded' } : {}) },
+      ...(method === 'POST' ? { body: new URLSearchParams(params).toString() } : {}),
+    })
+    return { code: res.status, body: (await res.json()) as Record<string, any> }
+  }
+  return {
+    base,
+    tokens,
+    dataDir,
+    call,
+    audit: async () => {
+      const res = await fetch(`${base}/api/hub/audit/list?limit=500`, { headers: { authorization: `Bearer ${tokens.admin}` } })
+      return ((await res.json()) as { response: { entries: Record<string, string>[] } }).response.entries
+    },
+    close: () => {
+      hub.close()
+      rmSync(dataDir, { recursive: true, force: true })
+      rmSync(staticDir, { recursive: true, force: true })
+    },
+  }
+}

hub/guards.ts

@@ -0,0 +1,96 @@
+/*
+Checks the hub makes on what passes through it to a server (stage 5): which `settings/set`
+fields it will not forward, which values of `settings/get` a role may see, and which single
+identifier of a write may go into the audit log.
+
+Pure functions, so the tests feed them any input.
+*/
+
+/*
+`settings/set` and the hub's own way in.
+
+The hub reaches a server through that server's web service, with an API token. Changing the
+service's bind addresses, ports, TLS or reverse-proxy settings from here can cut the hub off with
+no way back except the server's own console, so the hub refuses a change to any `webService*`
+field. The console sends the whole Web Service block with every save, unchanged values included,
+so a field is refused only when its value DIFFERS from what the server holds now (see
+`webServiceChanges`). Users, SSO and LDAP are not part of `settings/set`; a field that looks like
+one (`user*`, `sso*`, `ldap*`, `auth*`) is refused outright, in case a server version adds it.
+*/
+export const WEB_SERVICE_MESSAGE = "Change the web service settings in that server's own console."
+
+const FORBIDDEN_FIELD = /^(user|sso|ldap|auth)/i
+const WEB_FIELD = /^webService/
+
+/** True for a field the hub never forwards, whatever its value. */
+export const isForbiddenSettingsField = (name: string): boolean => FORBIDDEN_FIELD.test(name)
+
+/* A list as the console sends it (`a,b`, one per line, or `false` for none) or as the server holds it. */
+function listOf(value: unknown): string[] {
+  const items = Array.isArray(value) ? value.map(String) : String(value ?? '').split(/[,\n]/)
+  return items.map((v) => v.trim()).filter((v) => v !== '' && v !== 'false')
+}
+
+/* Whether a submitted text means what the server holds. Booleans and numbers by value, lists as sets, text trimmed. */
+function sameSetting(submitted: string, current: unknown): boolean {
+  if (typeof current === 'boolean') return submitted.trim().toLowerCase() === String(current)
+  if (typeof current === 'number') return submitted.trim() !== '' && Number(submitted) === current
+  if (Array.isArray(current)) {
+    const a = listOf(submitted).sort()
+    const b = listOf(current).sort()
+    return a.length === b.length && a.every((v, i) => v === b[i])
+  }
+  return submitted.trim() === String(current ?? '').trim()
+}
+
+/*
+The `webService*` fields of a `settings/set` whose value differs from the server's current
+settings (`settings/get`). A field the server does not report is not compared: it would be ignored
+there too.
+*/
+export function webServiceChanges(submitted: Record<string, string>, current: unknown): string[] {
+  const now = (current !== null && typeof current === 'object' ? current : {}) as Record<string, unknown>
+  const changed: string[] = []
+  for (const [name, value] of Object.entries(submitted)) {
+    if (!WEB_FIELD.test(name) || !Object.hasOwn(now, name)) continue
+    if (!sameSetting(value, now[name])) changed.push(name)
+  }
+  return changed
+}
+
+/*
+`settings/get` carries secrets: the TSIG shared secrets, the forward proxy password and the two
+certificate passwords. A role without `admin` gets a copy with those values blanked (a viewer
+needs the screen, not the keys); an admin gets them, since saving the form sends them back. The
+blanked copy is for display only and `settings/set` is `admin`-only, so a blank cannot be saved
+over a real secret by someone who may not read it.
+*/
+export function redactSettings(response: unknown): unknown {
+  if (response === null || typeof response !== 'object') return response
+  const copy = { ...(response as Record<string, unknown>) }
+  for (const key of ['webServiceTlsCertificatePassword', 'dnsTlsCertificatePassword']) {
+    if (typeof copy[key] === 'string' && copy[key] !== '') copy[key] = ''
+  }
+  if (Array.isArray(copy.tsigKeys)) {
+    copy.tsigKeys = copy.tsigKeys.map((k: unknown) => (k !== null && typeof k === 'object' ? { ...k, sharedSecret: '' } : k))
+  }
+  const proxy = copy.proxy
+  if (proxy !== null && typeof proxy === 'object' && typeof (proxy as { password?: unknown }).password === 'string') {
+    copy.proxy = { ...proxy, password: '' }
+  }
+  return copy
+}
+
+/*
+The one identifier an audit line may carry for a write: a zone, a domain, a name or a scope. Never
+the other parameters (records carry data, settings carry passwords and TSIG secrets). Cut short.
+*/
+const IDENTIFIERS = ['zone', 'domain', 'name', 'scope', 'zones']
+
+export function safeTarget(params: Record<string, string>): string | undefined {
+  for (const key of IDENTIFIERS) {
+    const value = params[key]
+    if (typeof value === 'string' && value.trim() !== '') return value.trim().slice(0, 120)
+  }
+  return undefined
+}

hub/json-diff.ts

@@ -0,0 +1,38 @@
+/*
+A small structural diff of two JSON values: objects by key, arrays by index. A leaf that
+differs, a key present on one side only, and a change of type each give one entry.
+*/
+export interface JsonDifference {
+  path: string
+  a: unknown
+  b: unknown
+}
+
+const kind = (v: unknown): string => (v === null ? 'null' : Array.isArray(v) ? 'array' : typeof v)
+
+export function jsonDiff(a: unknown, b: unknown, path = ''): JsonDifference[] {
+  const ka = kind(a)
+  if (ka !== kind(b)) return [{ path, a, b }]
+  if (ka === 'array') {
+    const [xs, ys] = [a as unknown[], b as unknown[]]
+    const out: JsonDifference[] = []
+    for (let i = 0; i < Math.max(xs.length, ys.length); i++) {
+      const here = `${path}[${i}]`
+      if (i >= xs.length || i >= ys.length) out.push({ path: here, a: xs[i], b: ys[i] })
+      else out.push(...jsonDiff(xs[i], ys[i], here))
+    }
+    return out
+  }
+  if (ka === 'object') {
+    const [x, y] = [a as Record<string, unknown>, b as Record<string, unknown>]
+    const out: JsonDifference[] = []
+    for (const key of new Set([...Object.keys(x), ...Object.keys(y)])) {
+      const here = path === '' ? key : `${path}.${key}`
+      const [inX, inY] = [Object.hasOwn(x, key), Object.hasOwn(y, key)]
+      if (inX && inY) out.push(...jsonDiff(x[key], y[key], here))
+      else out.push({ path: here, a: x[key], b: y[key] })
+    }
+    return out
+  }
+  return a === b || (Number.isNaN(a) && Number.isNaN(b)) ? [] : [{ path, a, b }]
+}

hub/multipart.ts

@@ -0,0 +1,91 @@
+/*
+A small multipart/form-data reader for the app zip uploads, and a body reader with a size
+limit. No dependencies; the whole body stays in memory (the limit is 20 MB).
+*/
+import type http from 'node:http'
+
+export const MAX_UPLOAD = 20 * 1024 * 1024
+
+export class TooLarge extends Error {
+  constructor(limit: number) {
+    super(`The upload is larger than ${Math.floor(limit / (1024 * 1024))} MB.`)
+    this.name = 'TooLarge'
+  }
+}
+
+/*
+Reads the request body, counting bytes as they arrive; throws `TooLarge` above `limit`.
+Past the limit nothing more is kept, but the rest is still read (up to DRAIN more bytes), so the
+client gets its 413 instead of a reset connection.
+*/
+const DRAIN = 32 * 1024 * 1024
+
+export async function readLimited(req: http.IncomingMessage, limit: number = MAX_UPLOAD): Promise<Buffer> {
+  const declared = Number(req.headers['content-length'])
+  if (Number.isFinite(declared) && declared > limit + DRAIN) throw new TooLarge(limit)
+  const chunks: Buffer[] = []
+  let size = 0
+  for await (const chunk of req) {
+    size += (chunk as Buffer).length
+    if (size > limit + DRAIN) throw new TooLarge(limit)
+    if (size <= limit) chunks.push(chunk as Buffer)
+  }
+  if (size > limit) throw new TooLarge(limit)
+  return Buffer.concat(chunks)
+}
+
+export interface UploadedFile {
+  field: string
+  filename: string
+  type: string
+  data: Buffer
+}
+
+export interface Multipart {
+  fields: Record<string, string>
+  files: UploadedFile[]
+}
+
+export function boundaryOf(contentType: string): string | null {
+  const match = /boundary=(?:"([^"]+)"|([^;\s]+))/i.exec(contentType)
+  return match === null ? null : (match[1] ?? match[2])
+}
+
+const CRLF2 = Buffer.from('\r\n\r\n')
+
+/* Throws on anything that is not a well-formed multipart body. */
+export function parseMultipart(body: Buffer, contentType: string): Multipart {
+  const boundary = boundaryOf(contentType)
+  if (boundary === null || !/^multipart\/form-data/i.test(contentType)) throw new Error('Expected multipart/form-data with a boundary.')
+  const delimiter = Buffer.from(`--${boundary}`)
+  const result: Multipart = { fields: {}, files: [] }
+
+  let at = body.indexOf(delimiter)
+  if (at < 0) throw new Error('The multipart body has no boundary.')
+  for (;;) {
+    at += delimiter.length
+    // "--" after a delimiter closes the body.
+    if (body[at] === 0x2d && body[at + 1] === 0x2d) return result
+    if (body[at] !== 0x0d || body[at + 1] !== 0x0a) throw new Error('Malformed multipart body.')
+    at += 2
+    const headerEnd = body.indexOf(CRLF2, at)
+    if (headerEnd < 0) throw new Error('Malformed multipart part.')
+    const headers = body.subarray(at, headerEnd).toString('utf8')
+    const start = headerEnd + CRLF2.length
+    // The part ends at the next CRLF + delimiter, so binary content can hold anything else.
+    const next = body.indexOf(Buffer.concat([Buffer.from('\r\n'), delimiter]), start)
+    if (next < 0) throw new Error('The multipart body is not terminated.')
+    const data = body.subarray(start, next)
+
+    const disposition = /^content-disposition:\s*form-data;(.*)$/im.exec(headers)?.[1] ?? ''
+    const name = /\bname="([^"]*)"/i.exec(disposition)?.[1]
+    const filename = /\bfilename="([^"]*)"/i.exec(disposition)?.[1]
+    if (name === undefined) throw new Error('A multipart part has no name.')
+    if (filename === undefined) result.fields[name] = data.toString('utf8')
+    else {
+      const type = /^content-type:\s*(.+)$/im.exec(headers)?.[1].trim() ?? 'application/octet-stream'
+      result.files.push({ field: name, filename, type, data: Buffer.from(data) })
+    }
+    at = next + 2
+  }
+}

hub/profiles.test.ts

@@ -0,0 +1,372 @@
+// @vitest-environment node
+/*
+Stage 8, profiles: making one from the hub's baselines, what a viewer of it may see, export and
+import, the preview, and the order and the stop of an apply, on two fake servers that keep their
+apps, configs, settings and lists.
+*/
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import path from 'node:path'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { fake, startHub, ROLES, type Call, type Fake, type TestHub } from './fake-servers.ts'
+import { cleanProfile } from './profiles.ts'
+import { writeBaseline, writeListsBaseline, writeSettingsBaseline } from './store.ts'
+
+interface Box {
+  apps: Record<string, string>
+  configs: Record<string, string>
+  store: { name: string; version: string }[]
+  settings: Record<string, unknown>
+  blocked: string[]
+  allowed: string[]
+  failSettings: boolean
+}
+
+const boxes = new Map<Fake, Box>()
+const urlOf = (f: Fake, name: string): string => `${f.url}/store/${name}.zip`
+
+function install(f: Fake, box: Box): void {
+  boxes.set(f, box)
+  f.on('apps/list', () => ({ response: { apps: Object.entries(box.apps).map(([name, version]) => ({ name, version })) } }))
+  f.on('apps/listStoreApps', () => ({ response: { storeApps: box.store.map((a) => ({ ...a, url: urlOf(f, a.name) })) } }))
+  f.on('apps/downloadAndInstall', (c: Call) => {
+    const name = c.form.get('name') ?? c.query.get('name') ?? ''
+    const url = c.form.get('url') ?? c.query.get('url') ?? ''
+    const entry = box.store.find((a) => a.name === name)
+    if (entry === undefined || url !== urlOf(f, name)) return { error: 'The app was not found in the store.' }
+    box.apps[name] = entry.version
+    return { response: {} }
+  })
+  f.on('apps/config/get', (c: Call) => ({ response: { config: box.configs[c.query.get('name') ?? ''] ?? null } }))
+  f.on('apps/config/set', (c: Call) => {
+    box.configs[c.form.get('name') ?? c.query.get('name') ?? ''] = c.form.get('config') ?? ''
+    return { response: {} }
+  })
+  f.on('settings/get', () => ({ response: box.settings }))
+  f.on('settings/set', (c: Call) => {
+    if (box.failSettings) return { error: 'The settings were refused.' }
+    for (const [k, v] of c.form) {
+      if (k === 'tsigKeys') {
+        const cells = v.split('|')
+        box.settings.tsigKeys = v === 'false' ? [] : [{ keyName: cells[0], sharedSecret: cells[1], algorithmName: cells[2] }]
+      } else if (k in box.settings) box.settings[k] = v
+    }
+    return { response: {} }
+  })
+  for (const list of ['blocked', 'allowed'] as const) {
+    f.on(`${list}/export`, () => ({ text: box[list].join('\n') + '\n' }))
+    f.on(`${list}/add`, (c: Call) => {
+      box[list].push(c.form.get('domain') ?? '')
+      return { response: {} }
+    })
+    f.on(`${list}/delete`, (c: Call) => {
+      box[list] = box[list].filter((d) => d !== c.form.get('domain'))
+      return { response: {} }
+    })
+  }
+}
+
+let a: Fake
+let b: Fake
+let hub: TestHub
+const boxA = (): Box => boxes.get(a)!
+const boxB = (): Box => boxes.get(b)!
+const order = (f: Fake): string[] => f.calls.filter((c) => c.method === 'POST').map((c) => c.path)
+
+const CONFIG = JSON.stringify({ ttl: 30 })
+const settingsOf = (over: Record<string, unknown> = {}): Record<string, unknown> => ({
+  dnsServerDomain: 'a.example',
+  blockingType: 'NxDomain',
+  tsigKeys: [{ keyName: 'k1', sharedSecret: 'REALSECRET', algorithmName: 'hmac-sha256' }],
+  ...over,
+})
+
+function baselines(): void {
+  writeBaseline(hub.dataDir, { app: 'Blocker', text: CONFIG, updatedAt: '2026-01-01T00:00:00Z', updatedBy: 't' })
+  writeSettingsBaseline(hub.dataDir, { settings: settingsOf(), updatedAt: '2026-01-01T00:00:00Z', updatedBy: 't' })
+  writeListsBaseline(hub.dataDir, { blocked: ['ads.example'], allowed: ['ok.example'], updatedAt: '2026-01-01T00:00:00Z', updatedBy: 't' })
+}
+
+beforeAll(async () => {
+  a = await fake()
+  b = await fake()
+  hub = await startHub(a, b)
+})
+
+afterAll(() => {
+  hub.close()
+  a.server.close()
+  b.server.close()
+})
+
+beforeEach(async () => {
+  a.calls = []
+  b.calls = []
+  install(a, {
+    apps: { Blocker: '2.0', Zipped: '1.0' },
+    configs: { Blocker: CONFIG },
+    store: [{ name: 'Blocker', version: '2.0' }],
+    settings: settingsOf(),
+    blocked: ['ads.example'],
+    allowed: ['ok.example'],
+    failSettings: false,
+  })
+  // b is a fresh server: no apps, different settings, a missing domain, an extra one.
+  install(b, {
+    apps: {},
+    configs: {},
+    store: [{ name: 'Blocker', version: '2.0' }],
+    settings: settingsOf({ dnsServerDomain: 'b.example', blockingType: 'AnyAddress', tsigKeys: [] }),
+    blocked: ['mine.example'],
+    allowed: ['ok.example'],
+    failSettings: false,
+  })
+  baselines()
+  for (const f of readdirSync(path.join(hub.dataDir)).filter((n) => n === 'profiles')) void f
+  await hub.call('admin', 'hub/profiles/delete', { name: 'Fresh' })
+})
+
+const snapshot = (over: Record<string, string> = {}) =>
+  hub.call('admin', 'hub/profiles/snapshot', { name: 'Fresh', apps: 'Blocker', settings: 'true', lists: 'true', ...over })
+
+describe('making a profile', () => {
+  it('copies the baselines at that moment: store apps with version and config, shared settings fields, lists', async () => {
+    const done = await snapshot()
+    expect(done.code).toBe(200)
+    expect(done.body.status).toBe('ok')
+    const file = JSON.parse(readFileSync(path.join(hub.dataDir, 'profiles', 'Fresh.json'), 'utf8'))
+    expect(file.apps).toEqual([{ name: 'Blocker', version: '2.0', config: CONFIG }])
+    // The host field stays out; the real secret is kept, as the baseline file has it.
+    expect(file.settings.dnsServerDomain).toBeUndefined()
+    expect(file.settings.tsigKeys[0].sharedSecret).toBe('REALSECRET')
+    expect(file.lists).toEqual({ blocked: ['ads.example'], allowed: ['ok.example'] })
+    expect(file.updatedBy).toBe('admin')
+    // Later changes of the baselines do not change the profile.
+    writeBaseline(hub.dataDir, { app: 'Blocker', text: '{"ttl":1}', updatedAt: 'x', updatedBy: 't' })
+    const again = JSON.parse(readFileSync(path.join(hub.dataDir, 'profiles', 'Fresh.json'), 'utf8'))
+    expect(again.apps[0].config).toBe(CONFIG)
+  })
+
+  it('lets the admin choose the parts, and refuses an empty choice', async () => {
+    expect((await snapshot({ settings: 'false', lists: 'false' })).body.status).toBe('ok')
+    const file = JSON.parse(readFileSync(path.join(hub.dataDir, 'profiles', 'Fresh.json'), 'utf8'))
+    expect(file.settings).toBeNull()
+    expect(file.lists).toBeNull()
+    const none = await snapshot({ apps: '', settings: 'false', lists: 'false' })
+    expect(none.code).toBe(400)
+    expect(none.body.errorMessage).toMatch(/at least one part/)
+  })
+
+  it('refuses an app that is a zip, one that is installed nowhere, and a bad name', async () => {
+    const zip = await snapshot({ apps: 'Zipped' })
+    expect(zip.code).toBe(400)
+    expect(zip.body.errorMessage).toMatch(/store listing/)
+    expect((await snapshot({ apps: 'Ghost' })).body.errorMessage).toMatch(/not installed/)
+    expect((await snapshot({ name: '../evil' })).code).toBe(400)
+    expect(existsSync(path.join(hub.dataDir, 'profiles', 'Fresh.json'))).toBe(false)
+  })
+
+  it('replaces a profile of the same name and lists it with counts only', async () => {
+    await snapshot()
+    await snapshot({ lists: 'false' })
+    const list = await hub.call('admin', 'hub/profiles/list', {}, 'GET')
+    expect(list.body.response.profiles).toMatchObject([{ name: 'Fresh', apps: 1, configs: 1, lists: null }])
+    expect(list.body.response.profiles[0].settings).toBeGreaterThan(0)
+  })
+})
+
+describe('viewing a profile', () => {
+  it('masks secrets and never returns an app config', async () => {
+    await snapshot()
+    const got = await hub.call('admin', 'hub/profiles/get', { name: 'Fresh' }, 'GET')
+    const text = JSON.stringify(got.body)
+    expect(text).not.toContain('REALSECRET')
+    expect(text).toContain('<secret>')
+    expect(text).not.toContain('"ttl"')
+    expect(got.body.response.apps).toEqual([{ name: 'Blocker', version: '2.0', hasConfig: true, bytes: CONFIG.length }])
+    expect(got.body.response.lists).toEqual({ blocked: 1, allowed: 1 })
+  })
+
+  it('answers an unknown profile with an error', async () => {
+    expect((await hub.call('admin', 'hub/profiles/get', { name: 'Nope' }, 'GET')).code).toBe(400)
+  })
+})
+
+describe('roles', () => {
+  it('refuses every profile endpoint to every role but admin, and answers none of them from a server', async () => {
+    const reads = ['list', 'get', 'export', 'preview']
+    const writes = ['snapshot', 'delete', 'import', 'apply']
+    for (const role of ROLES.filter((r) => r !== 'admin')) {
+      for (const p of reads) expect((await hub.call(role, `hub/profiles/${p}`, { name: 'Fresh', targets: 'b' }, 'GET')).code).toBe(403)
+      for (const p of writes) expect((await hub.call(role, `hub/profiles/${p}`, { name: 'Fresh', targets: 'b', data: '{}' })).code).toBe(403)
+    }
+    // An unlisted profile path is refused, not forwarded.
+    expect((await hub.call('admin', 'hub/profiles/nothing', {}, 'GET')).code).toBe(403)
+    expect(a.calls).toEqual([])
+    expect(b.calls).toEqual([])
+  })
+})
+
+describe('export and import', () => {
+  it('exports with the real secret, and imports it dry first, then for real', async () => {
+    await snapshot()
+    const res = await fetch(`${hub.base}/api/hub/profiles/export`, { headers: { authorization: `Bearer ${hub.tokens.admin}` } })
+    expect(res.headers.get('content-disposition')).toMatch(/hub-profiles-/)
+    const text = await res.text()
+    expect(text).toContain('REALSECRET')
+    const file = JSON.parse(text)
+    expect(file).toMatchObject({ hub: 'technitium-hub', type: 'profiles', version: 1 })
+
+    await hub.call('admin', 'hub/profiles/delete', { name: 'Fresh' })
+    const dry = await hub.call('admin', 'hub/profiles/import', { data: text, apply: 'false' })
+    expect(dry.body.response).toMatchObject({ applied: false, profiles: [{ name: 'Fresh', status: 'new', apps: 1, settings: true, lists: true }] })
+    expect(existsSync(path.join(hub.dataDir, 'profiles', 'Fresh.json'))).toBe(false)
+    const done = await hub.call('admin', 'hub/profiles/import', { data: text, apply: 'true' })
+    expect(done.body.response.applied).toBe(true)
+    const again = await hub.call('admin', 'hub/profiles/import', { data: text, apply: 'true' })
+    expect(again.body.response.profiles[0].status).toBe('same')
+    expect(JSON.stringify(await hub.audit())).not.toContain('REALSECRET')
+  })
+
+  it('refuses a file of another kind, a placeholder where a secret should be, and local fields are dropped', async () => {
+    const bad = await hub.call('admin', 'hub/profiles/import', { data: JSON.stringify({ hub: 'technitium-hub', version: 1, apps: [] }) })
+    expect(bad.code).toBe(400)
+    const masked = { hub: 'technitium-hub', type: 'profiles', version: 1, profiles: [{ name: 'M', apps: [], settings: { tsigKeys: [{ keyName: 'k', sharedSecret: '<secret>' }] }, lists: null }] }
+    const refused = await hub.call('admin', 'hub/profiles/import', { data: JSON.stringify(masked) })
+    expect(refused.code).toBe(400)
+    expect(refused.body.errorMessage).toMatch(/<secret>/)
+    expect(() => cleanProfile({ name: 'x y', apps: [], settings: { dnsServerDomain: 'h', blockingType: 'NxDomain' }, lists: null }).settings).not.toThrow()
+    expect(cleanProfile({ name: 'x y', apps: [], settings: { dnsServerDomain: 'h', blockingType: 'NxDomain' }, lists: null }).settings).toEqual({ blockingType: 'NxDomain' })
+  })
+
+  it('accepts a body above the usual limit for an admin', async () => {
+    const big = JSON.stringify({ hub: 'technitium-hub', type: 'profiles', version: 1, profiles: [{ name: 'Big', apps: [{ name: 'X', version: '1', config: JSON.stringify({ pad: 'x'.repeat(200_000) }) }], settings: null, lists: null }] })
+    expect(big.length).toBeGreaterThan(64 * 1024)
+    const done = await hub.call('admin', 'hub/profiles/import', { data: big, apply: 'false' })
+    expect(done.body.response.profiles[0]).toMatchObject({ name: 'Big', status: 'new' })
+  })
+})
+
+describe('preview', () => {
+  it('says what each part would change on a target, and writes nothing', async () => {
+    await snapshot()
+    const got = await hub.call('admin', 'hub/profiles/preview', { name: 'Fresh', targets: 'b,a' })
+    expect(got.code).toBe(200)
+    const [tb, ta] = got.body.response.targets
+    expect(tb.server).toBe('b')
+    expect(tb.apps).toEqual([{ name: 'Blocker', state: 'install', profileVersion: '2.0', storeVersion: '2.0' }])
+    expect(tb.configs).toEqual([{ name: 'Blocker', state: 'after-install' }])
+    expect(tb.settings.fields).toContain('blockingType')
+    expect(tb.settings.secretFields).toContain('tsigKeys')
+    expect(JSON.stringify(got.body)).not.toContain('REALSECRET')
+    expect(tb.lists.blocked.missing).toBe(1)
+    expect(ta.apps).toEqual([{ name: 'Blocker', state: 'present', profileVersion: '2.0', installedVersion: '2.0' }])
+    expect(ta.configs).toEqual([{ name: 'Blocker', state: 'same' }])
+    expect(ta.settings.unchanged).toBe(true)
+    expect(order(a).concat(order(b))).toEqual([])
+  })
+
+  it('marks an installed app whose version differs, and an app the store does not offer', async () => {
+    await snapshot()
+    boxB().apps.Blocker = '1.0'
+    const one = await hub.call('admin', 'hub/profiles/preview', { name: 'Fresh', targets: 'b' })
+    expect(one.body.response.targets[0].apps[0]).toMatchObject({ state: 'version-differs', installedVersion: '1.0', profileVersion: '2.0' })
+    boxB().apps = {}
+    boxB().store = []
+    const two = await hub.call('admin', 'hub/profiles/preview', { name: 'Fresh', targets: 'b' })
+    expect(two.body.response.targets[0].apps[0].state).toBe('not-in-store')
+  })
+})
+
+describe('apply', () => {
+  it('runs the steps in order on a fresh server: install, config, settings, lists', async () => {
+    await snapshot()
+    const done = await hub.call('admin', 'hub/profiles/apply', { name: 'Fresh', targets: 'b' })
+    expect(done.body.response.results[0]).toMatchObject({ server: 'b', ok: true })
+    expect(done.body.response.results[0].steps.map((s: Record<string, string>) => [s.step, s.status])).toEqual([
+      ['apps', 'ok'],
+      ['configs', 'ok'],
+      ['settings', 'ok'],
+      ['lists', 'ok'],
+    ])
+    expect(order(b)).toEqual(['apps/downloadAndInstall', 'apps/config/set', 'settings/set', 'blocked/add'])
+    expect(boxB().apps.Blocker).toBe('2.0')
+    expect(JSON.parse(boxB().configs.Blocker)).toEqual({ ttl: 30 })
+    expect(boxB().settings.blockingType).toBe('NxDomain')
+    // The host field was left alone, and the extra domain stays unless asked.
+    expect(boxB().settings.dnsServerDomain).toBe('b.example')
+    expect(boxB().blocked).toEqual(['mine.example', 'ads.example'])
+    // The old config of a server that had one goes to the history; b had none, so nothing is claimed.
+    expect(order(a)).toEqual([])
+  })
+
+  it('takes the install URL from the target own store, never from the request', async () => {
+    await snapshot()
+    await hub.call('admin', 'hub/profiles/apply', { name: 'Fresh', targets: 'b', url: 'https://evil.example/x.zip' })
+    const install = b.calls.find((c) => c.path === 'apps/downloadAndInstall')!
+    expect(install.query.get('url') ?? install.form.get('url')).toBe(urlOf(b, 'Blocker'))
+    expect(a.calls.some((c) => c.path === 'apps/downloadAndInstall')).toBe(false)
+    // Each server is asked for its own listing, with its own read token.
+    expect(b.calls.find((c) => c.path === 'apps/listStoreApps')!.token).toBe('read-b')
+  })
+
+  it('removes extra domains only when asked', async () => {
+    await snapshot()
+    await hub.call('admin', 'hub/profiles/apply', { name: 'Fresh', targets: 'b', removeExtras: 'true' })
+    expect(boxB().blocked).toEqual(['ads.example'])
+  })
+
+  it('stops the later steps of a server when one fails, says so, and leaves the other server alone', async () => {
+    await snapshot()
+    boxB().failSettings = true
+    boxA().settings.blockingType = 'AnyAddress'
+    const done = await hub.call('admin', 'hub/profiles/apply', { name: 'Fresh', targets: 'a,b' })
+    const [ra, rb] = done.body.response.results
+    expect(ra).toMatchObject({ server: 'a', ok: true })
+    expect(rb).toMatchObject({ server: 'b', ok: false })
+    expect(rb.message).toMatch(/Stopped at settings/)
+    expect(rb.steps.map((s: Record<string, string>) => [s.step, s.status])).toEqual([
+      ['apps', 'ok'],
+      ['configs', 'ok'],
+      ['settings', 'failed'],
+      ['lists', 'not-run'],
+    ])
+    expect(order(b)).not.toContain('blocked/add')
+    const audit = await hub.audit()
+    const lines = audit.filter((e) => e.action === 'profiles.apply' && e.server === 'b').slice(0, 3)
+    expect(lines.map((e) => [e.outcome, e.detail?.split(':')[0]])).toEqual([['failed', 'settings'], ['ok', 'configs'], ['ok', 'apps']])
+    expect(JSON.stringify(audit)).not.toContain('REALSECRET')
+  })
+
+  it('retries from the failed step without redoing the earlier ones', async () => {
+    await snapshot()
+    boxB().failSettings = true
+    await hub.call('admin', 'hub/profiles/apply', { name: 'Fresh', targets: 'b' })
+    b.calls = []
+    boxB().failSettings = false
+    const again = await hub.call('admin', 'hub/profiles/apply', { name: 'Fresh', targets: 'b', from: JSON.stringify({ b: 'settings' }) })
+    const r = again.body.response.results[0]
+    expect(r.ok).toBe(true)
+    expect(r.steps.map((s: Record<string, string>) => [s.step, s.status])).toEqual([
+      ['apps', 'skipped'],
+      ['configs', 'skipped'],
+      ['settings', 'ok'],
+      ['lists', 'ok'],
+    ])
+    expect(order(b)).toEqual(['settings/set', 'blocked/add'])
+  })
+
+  it('fails a server with no admin token on its first step, and refuses an unknown server or profile', async () => {
+    await snapshot()
+    const unknown = await hub.call('admin', 'hub/profiles/apply', { name: 'Fresh', targets: 'zzz' })
+    expect(unknown.code).toBe(400)
+    expect((await hub.call('admin', 'hub/profiles/apply', { name: 'Nope', targets: 'b' })).code).toBe(400)
+    expect((await hub.call('admin', 'hub/profiles/apply', { name: 'Fresh', targets: '' })).code).toBe(400)
+  })
+
+  it('deletes a profile', async () => {
+    await snapshot()
+    expect((await hub.call('admin', 'hub/profiles/delete', { name: 'Fresh' })).body.response).toEqual({ deleted: true })
+    expect((await hub.call('admin', 'hub/profiles/delete', { name: 'Fresh' })).code).toBe(400)
+    expect((await hub.call('admin', 'hub/profiles/list', {}, 'GET')).body.response.profiles).toEqual([])
+  })
+})

hub/profiles.ts

@@ -0,0 +1,510 @@
+/*
+Profiles (stage 8): a named bundle of what a server should have, kept in `${HUB_DATA}/profiles/`,
+to bring a new or reset server in line in one go.
+
+A profile holds, as the admin chose when it was made from the hub's current baselines:
+
+  apps      store apps only (never a zip): the name and the version they had, and for each the hub's
+            baseline config at that moment
+  settings  the SHARED fields of the settings baseline (host fields never travel), secrets as the
+            baseline file keeps them
+  lists     the custom Blocked and Allowed lists baseline
+
+Applying a profile to a server runs four steps in order, and a failed step stops the later ones for
+that server (the other servers go on):
+
+  apps      install the apps that are missing, from THAT server's own store listing (apps-write.ts)
+  configs   the profile's config of each app plus that server's own local values; the old config
+            goes to the history first (apps-config.ts)
+  settings  the changed shared fields, through the same checks as a per-server settings/set
+  lists     the missing domains; extras only when asked
+
+`from` restarts a server at a given step (the retry of a failed run), so what already worked is not
+done again. Everything here is for admins. A view never carries a secret and never an app config.
+*/
+import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs'
+import path from 'node:path'
+import type { HubConfig, HubServer } from './config.ts'
+import { MAX_CONFIG, looksLikeJson, parseJson, type Diff } from './apps-sync.ts'
+import { UserError, compareVersions, type AppsConfig } from './apps-config.ts'
+import type { AppsWriter } from './apps-write.ts'
+import { isFieldName, isBuiltInLocal, MAX_LIST, SECRET_PLACEHOLDER, maskSettings } from './settings-sync.ts'
+import type { SettingsConfig } from './settings-config.ts'
+import { listBaselineApps, readBaseline, readListsBaseline, readSettingsBaseline, safeName, writeAtomic } from './store.ts'
+import type { Upstream } from './upstream.ts'
+
+export const STEPS = ['apps', 'configs', 'settings', 'lists'] as const
+export type Step = (typeof STEPS)[number]
+
+export interface ProfileApp {
+  name: string
+  /** The version the app had when the profile was made. */
+  version: string
+  /** The hub's baseline config of the app then; null when it had none. */
+  config: string | null
+}
+
+export interface Profile {
+  name: string
+  createdAt: string
+  updatedAt: string
+  updatedBy: string
+  apps: ProfileApp[]
+  settings: Record<string, unknown> | null
+  lists: { blocked: string[]; allowed: string[] } | null
+}
+
+export interface StepResult {
+  step: Step
+  /* `skipped`: done by an earlier run. `not-run`: an earlier step failed. */
+  status: 'ok' | 'failed' | 'skipped' | 'not-run'
+  message: string
+}
+
+export interface ApplyResult {
+  server: string
+  ok: boolean
+  message: string
+  steps: StepResult[]
+}
+
+const NAME = /^[A-Za-z0-9][A-Za-z0-9 ._-]{0,59}$/
+const MAX_PROFILES = 50
+const MAX_APPS = 100
+const MAX_SETTINGS_BYTES = 1024 * 1024
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+const isObject = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === 'object' && !Array.isArray(v)
+
+const hasPlaceholder = (value: unknown): boolean => JSON.stringify(value).includes(JSON.stringify(SECRET_PLACEHOLDER))
+
+/* The stored shape, checked field by field. Used for files on disk and for imports. */
+export function cleanProfile(raw: unknown, fallbackName?: string): Profile {
+  if (!isObject(raw)) throw new UserError('A profile must be an object.')
+  const name = typeof raw.name === 'string' ? raw.name : (fallbackName ?? '')
+  if (!NAME.test(name)) throw new UserError('A profile name uses letters, digits, spaces and . _ - (up to 60, starting with a letter or digit).')
+  if (!Array.isArray(raw.apps) || raw.apps.length > MAX_APPS) throw new UserError(`"${name}": the apps must be a list of at most ${MAX_APPS}.`)
+  const seen = new Set<string>()
+  const apps = raw.apps.map((a): ProfileApp => {
+    if (!isObject(a) || typeof a.name !== 'string' || a.name === '' || a.name.length > 200 || seen.has(a.name)) throw new UserError(`"${name}": an app has no valid, unique name.`)
+    seen.add(a.name)
+    const version = typeof a.version === 'string' ? a.version.slice(0, 40) : ''
+    const config = typeof a.config === 'string' ? a.config : null
+    if (config !== null) {
+      if (config.length > MAX_CONFIG) throw new UserError(`"${name}": the config of "${a.name}" is larger than 1 MB.`)
+      if (looksLikeJson(config) && !parseJson(config).ok) throw new UserError(`"${name}": the config of "${a.name}" is not valid JSON.`)
+    }
+    return { name: a.name, version, config }
+  })
+  let settings: Profile['settings'] = null
+  if (raw.settings !== null && raw.settings !== undefined) {
+    if (!isObject(raw.settings)) throw new UserError(`"${name}": the settings must be an object.`)
+    if (JSON.stringify(raw.settings).length > MAX_SETTINGS_BYTES) throw new UserError(`"${name}": the settings are larger than 1 MB.`)
+    if (hasPlaceholder(raw.settings)) throw new UserError(`"${name}": the settings hold ${SECRET_PLACEHOLDER} where a secret should be. Export the profile from the hub that has it.`)
+    settings = {}
+    for (const [k, v] of Object.entries(raw.settings)) {
+      if (!isFieldName(k)) throw new UserError(`"${name}": "${k.slice(0, 40)}" is not a setting name.`)
+      // Host fields never travel.
+      if (!isBuiltInLocal(k)) settings[k] = v
+    }
+  }
+  let lists: Profile['lists'] = null
+  if (raw.lists !== null && raw.lists !== undefined) {
+    const l = raw.lists as { blocked?: unknown; allowed?: unknown }
+    if (!isObject(raw.lists) || !Array.isArray(l.blocked) || !Array.isArray(l.allowed)) throw new UserError(`"${name}": the lists need a blocked and an allowed array.`)
+    const clean = (v: unknown[]): string[] => {
+      if (v.length > MAX_LIST || v.some((d) => typeof d !== 'string' || d.length > 253)) throw new UserError(`"${name}": a list is longer than ${MAX_LIST} domains or holds a bad entry.`)
+      return [...new Set(v as string[])].sort()
+    }
+    lists = { blocked: clean(l.blocked), allowed: clean(l.allowed) }
+  }
+  const text = (v: unknown): string => (typeof v === 'string' ? v : '')
+  return { name, createdAt: text(raw.createdAt), updatedAt: text(raw.updatedAt), updatedBy: text(raw.updatedBy), apps, settings, lists }
+}
+
+const shape = (p: Profile): string => JSON.stringify({ apps: p.apps, settings: p.settings, lists: p.lists })
+
+export class Profiles {
+  #config: HubConfig
+  #upstream: Upstream
+  #appsConfig: AppsConfig
+  #settings: SettingsConfig
+  #writer: AppsWriter
+  #now: () => number
+
+  constructor(config: HubConfig, upstream: Upstream, appsConfig: AppsConfig, settings: SettingsConfig, writer: AppsWriter, now: () => number = Date.now) {
+    this.#config = config
+    this.#upstream = upstream
+    this.#appsConfig = appsConfig
+    this.#settings = settings
+    this.#writer = writer
+    this.#now = now
+  }
+
+  #dir(): string {
+    return path.join(this.#config.dataDir, 'profiles')
+  }
+
+  #file(name: string): string {
+    return path.join(this.#dir(), `${safeName(name)}.json`)
+  }
+
+  #read(name: string): Profile | null {
+    const file = this.#file(name)
+    if (!existsSync(file)) return null
+    try {
+      const p = cleanProfile(JSON.parse(readFileSync(file, 'utf8')))
+      return p.name === name ? p : null
+    } catch {
+      return null
+    }
+  }
+
+  #all(): Profile[] {
+    if (!existsSync(this.#dir())) return []
+    const out: Profile[] = []
+    for (const f of readdirSync(this.#dir())) {
+      if (!f.endsWith('.json')) continue
+      try {
+        out.push(cleanProfile(JSON.parse(readFileSync(path.join(this.#dir(), f), 'utf8'))))
+      } catch {
+        // A file that is not a profile is left alone.
+      }
+    }
+    return out.sort((a, b) => a.name.localeCompare(b.name))
+  }
+
+  #require(name: string): Profile {
+    const found = this.#read(name)
+    if (found === null) throw new UserError(`There is no profile named "${name.slice(0, 60)}".`)
+    return found
+  }
+
+  #write(p: Profile): void {
+    writeAtomic(this.#file(p.name), JSON.stringify(p, null, 2) + '\n')
+  }
+
+  /* What the console lists: counts, no config and no value. */
+  list(): { name: string; apps: number; configs: number; settings: number | null; lists: { blocked: number; allowed: number } | null; updatedAt: string; updatedBy: string }[] {
+    return this.#all().map((p) => ({
+      name: p.name,
+      apps: p.apps.length,
+      configs: p.apps.filter((a) => a.config !== null).length,
+      settings: p.settings === null ? null : Object.keys(p.settings).length,
+      lists: p.lists === null ? null : { blocked: p.lists.blocked.length, allowed: p.lists.allowed.length },
+      updatedAt: p.updatedAt,
+      updatedBy: p.updatedBy,
+    }))
+  }
+
+  /* One profile for the console: secrets masked, no app config text. */
+  get(name: string): unknown {
+    const p = this.#require(name)
+    return {
+      name: p.name,
+      createdAt: p.createdAt,
+      updatedAt: p.updatedAt,
+      updatedBy: p.updatedBy,
+      apps: p.apps.map((a) => ({ name: a.name, version: a.version, hasConfig: a.config !== null, bytes: a.config === null ? 0 : a.config.length })),
+      settings: p.settings === null ? null : maskSettings(p.settings),
+      lists: p.lists === null ? null : { blocked: p.lists.blocked.length, allowed: p.lists.allowed.length },
+    }
+  }
+
+  delete(name: string): void {
+    this.#require(name)
+    rmSync(this.#file(name), { force: true })
+  }
+
+  /* ---- Making one from the hub's current baselines ---- */
+
+  async #installedVersions(): Promise<Map<string, string>> {
+    const versions = new Map<string, string>()
+    let answered = 0
+    await Promise.all(
+      this.#config.servers.map(async (s) => {
+        try {
+          this.#upstream.invalidate(s.id)
+          const list = (await this.#upstream.call(s, 'apps/list')) as { apps?: { name: string; version?: string }[] } | null
+          answered++
+          for (const a of list?.apps ?? []) {
+            const have = versions.get(a.name)
+            const v = typeof a.version === 'string' ? a.version : ''
+            if (have === undefined || compareVersions(v, have) > 0) versions.set(a.name, v)
+          }
+        } catch {
+          // A server that does not answer adds nothing.
+        }
+      }),
+    )
+    if (answered === 0) throw new UserError('No server answered, so the installed apps cannot be read.')
+    return versions
+  }
+
+  async #storeNames(): Promise<Set<string>> {
+    const names = new Set<string>()
+    let answered = 0
+    await Promise.all(
+      this.#config.servers.map(async (s) => {
+        try {
+          const r = (await this.#upstream.call(s, 'apps/listStoreApps')) as { storeApps?: { name: string }[] } | null
+          answered++
+          for (const a of r?.storeApps ?? []) names.add(a.name)
+        } catch {
+          // Same.
+        }
+      }),
+    )
+    if (answered === 0) throw new UserError('No server answered, so the store listing cannot be read.')
+    return names
+  }
+
+  /*
+  Makes or replaces a profile from the hub's current baselines. `apps`: the names to take (undefined
+  takes every app that has a baseline). Settings and lists are taken whole when asked.
+  */
+  async snapshot(name: string, parts: { apps?: string[]; settings: boolean; lists: boolean }, user: string): Promise<{ replaced: boolean; detail: string }> {
+    if (!NAME.test(name)) throw new UserError('A profile name uses letters, digits, spaces and . _ - (up to 60, starting with a letter or digit).')
+    const appNames = parts.apps ?? listBaselineApps(this.#config.dataDir)
+    if (appNames.length === 0 && !parts.settings && !parts.lists) throw new UserError('Choose at least one part: apps, settings or lists.')
+    if (appNames.length > MAX_APPS) throw new UserError(`At most ${MAX_APPS} apps in a profile.`)
+    const existing = this.#read(name)
+    if (existing === null && this.#all().length >= MAX_PROFILES) throw new UserError(`At most ${MAX_PROFILES} profiles.`)
+
+    let apps: ProfileApp[] = []
+    if (appNames.length > 0) {
+      const [versions, store] = await Promise.all([this.#installedVersions(), this.#storeNames()])
+      apps = [...new Set(appNames)].map((app) => {
+        const version = versions.get(app)
+        if (version === undefined) throw new UserError(`"${app}" is not installed on any server, so its version is not known.`)
+        if (!store.has(app)) throw new UserError(`"${app}" is not in the store listing. A profile holds store apps only, not apps installed from a file.`)
+        return { name: app, version, config: readBaseline(this.#config.dataDir, app)?.text ?? null }
+      })
+    }
+    let settings: Profile['settings'] = null
+    if (parts.settings) {
+      const baseline = readSettingsBaseline(this.#config.dataDir)
+      if (baseline === null) throw new UserError('There is no settings baseline yet. Accept one from a server first.')
+      const extra = this.#settings.extraLocal()
+      settings = Object.fromEntries(Object.entries(baseline.settings).filter(([k]) => !isBuiltInLocal(k) && !extra.includes(k)))
+    }
+    let lists: Profile['lists'] = null
+    if (parts.lists) {
+      const baseline = readListsBaseline(this.#config.dataDir)
+      if (baseline === null) throw new UserError('There is no lists baseline yet. Accept one from a server first.')
+      lists = { blocked: [...baseline.blocked], allowed: [...baseline.allowed] }
+    }
+    const at = new Date(this.#now()).toISOString()
+    this.#write({ name, createdAt: existing?.createdAt || at, updatedAt: at, updatedBy: user, apps, settings, lists })
+    return {
+      replaced: existing !== null,
+      detail: `${existing === null ? 'created' : 'replaced'}: ${apps.length} app(s), settings ${settings === null ? 'no' : Object.keys(settings).length + ' fields'}, lists ${lists === null ? 'no' : 'yes'}`,
+    }
+  }
+
+  /* ---- Export and import ---- */
+
+  exportFile(name?: string): { hub: string; type: string; version: number; exportedAt: string; profiles: Profile[] } {
+    const profiles = name === undefined || name === '' ? this.#all() : [this.#require(name)]
+    return { hub: 'technitium-hub', type: 'profiles', version: 1, exportedAt: new Date(this.#now()).toISOString(), profiles }
+  }
+
+  importFile(raw: string, apply: boolean, user: string): { profiles: { name: string; status: 'new' | 'changed' | 'same'; apps: number; settings: boolean; lists: boolean }[]; applied: boolean } {
+    const parsed = parseJson(raw)
+    if (!parsed.ok) throw new UserError('The file is not valid JSON.')
+    const file = parsed.value as { hub?: unknown; type?: unknown; version?: unknown; profiles?: unknown }
+    if (file === null || typeof file !== 'object' || file.hub !== 'technitium-hub' || file.type !== 'profiles') throw new UserError('This is not a profiles file of the hub.')
+    if (file.version !== 1) throw new UserError('This file has a version the hub does not know.')
+    if (!Array.isArray(file.profiles) || file.profiles.length > MAX_PROFILES) throw new UserError(`The file has no list of profiles, or has more than ${MAX_PROFILES}.`)
+    const plan: { profile: Profile; status: 'new' | 'changed' | 'same' }[] = []
+    const seen = new Set<string>()
+    for (const item of file.profiles) {
+      const profile = cleanProfile(item)
+      if (seen.has(profile.name)) throw new UserError(`"${profile.name}" is in the file twice.`)
+      seen.add(profile.name)
+      const old = this.#read(profile.name)
+      plan.push({ profile, status: old === null ? 'new' : shape(old) === shape(profile) ? 'same' : 'changed' })
+    }
+    const total = this.#all().filter((p) => !seen.has(p.name)).length + plan.length
+    if (total > MAX_PROFILES) throw new UserError(`That would make more than ${MAX_PROFILES} profiles.`)
+    if (apply) {
+      const at = new Date(this.#now()).toISOString()
+      for (const p of plan) {
+        if (p.status === 'same') continue
+        const old = this.#read(p.profile.name)
+        this.#write({ ...p.profile, createdAt: old?.createdAt || at, updatedAt: at, updatedBy: user })
+      }
+    }
+    return { profiles: plan.map((p) => ({ name: p.profile.name, status: p.status, apps: p.profile.apps.length, settings: p.profile.settings !== null, lists: p.profile.lists !== null })), applied: apply }
+  }
+
+  /* ---- Preview ---- */
+
+  #server(id: string): HubServer {
+    const found = this.#config.servers.find((s) => s.id === id)
+    if (found === undefined) throw new UserError(`Unknown server: ${id}.`)
+    return found
+  }
+
+  #targets(ids: string[]): HubServer[] {
+    const unique = [...new Set(ids)]
+    if (unique.length === 0) throw new UserError('Choose at least one server.')
+    return unique.map((id) => this.#server(id))
+  }
+
+  async #look(server: HubServer): Promise<{ installed: Map<string, string>; store: Map<string, string> }> {
+    this.#upstream.invalidate(server.id)
+    const list = (await this.#upstream.call(server, 'apps/list')) as { apps?: { name: string; version?: string }[] } | null
+    const store = (await this.#upstream.call(server, 'apps/listStoreApps')) as { storeApps?: { name: string; version?: string }[] } | null
+    return {
+      installed: new Map((list?.apps ?? []).map((a) => [a.name, typeof a.version === 'string' ? a.version : ''])),
+      store: new Map((store?.storeApps ?? []).map((a) => [a.name, typeof a.version === 'string' ? a.version : ''])),
+    }
+  }
+
+  /* What applying the profile would change on each target. Nothing is written. */
+  async preview(name: string, ids: string[], removeExtras: boolean): Promise<{ profile: string; targets: unknown[] }> {
+    const profile = this.#require(name)
+    const targets = this.#targets(ids)
+    const out = await Promise.all(
+      targets.map(async (s) => {
+        const writable = s.adminToken !== undefined
+        const base = { server: s.id, writable }
+        try {
+          const { installed, store } = await this.#look(s)
+          const apps = profile.apps.map((a) => {
+            const have = installed.get(a.name)
+            if (have === undefined) {
+              const offered = store.get(a.name)
+              return offered === undefined
+                ? { name: a.name, state: 'not-in-store' as const, profileVersion: a.version }
+                : { name: a.name, state: 'install' as const, profileVersion: a.version, storeVersion: offered }
+            }
+            return { name: a.name, state: have === a.version ? ('present' as const) : ('version-differs' as const), profileVersion: a.version, installedVersion: have }
+          })
+          const configs = await Promise.all(
+            profile.apps
+              .filter((a) => a.config !== null)
+              .map(async (a) => {
+                if (!installed.has(a.name)) return { name: a.name, state: 'after-install' as const }
+                try {
+                  const p = (await this.#appsConfig.preview(a.name, [s.id], a.config as string)).targets[0]
+                  if (!p.ok) return { name: a.name, state: 'unreadable' as const, error: p.error }
+                  return p.unchanged ? { name: a.name, state: 'same' as const } : { name: a.name, state: 'differs' as const, changes: p.changes as Diff }
+                } catch (e) {
+                  return { name: a.name, state: 'unreadable' as const, error: reason(e) }
+                }
+              }),
+          )
+          let settings: unknown = null
+          if (profile.settings !== null) {
+            const p = (await this.#settings.preview([s.id], undefined, { settings: profile.settings, updatedAt: '', updatedBy: '' })).targets[0]
+            settings = p.ok ? { fields: p.changes.changes.map((c) => c.path), secretFields: p.secretFields, unchanged: p.unchanged, changes: p.changes, warnings: p.warnings } : { error: p.error }
+          }
+          let lists: unknown = null
+          if (profile.lists !== null) {
+            const p = (await this.#settings.listsPreview([s.id], removeExtras, { ...profile.lists, updatedAt: '', updatedBy: '' })).targets[0]
+            lists = p.ok ? { blocked: p.blocked, allowed: p.allowed, status: p.status, tooMany: p.tooMany } : { error: p.error }
+          }
+          return { ...base, ok: true, apps, configs, settings, lists, warnings: writable ? [] : ['The hub has no admin token for this server, so it cannot write to it.'] }
+        } catch (e) {
+          return { ...base, ok: false, error: reason(e), apps: [], configs: [], settings: null, lists: null, warnings: [] }
+        }
+      }),
+    )
+    return { profile: profile.name, targets: out }
+  }
+
+  /* ---- Apply ---- */
+
+  async #run(profile: Profile, s: HubServer, step: Step, removeExtras: boolean): Promise<{ ok: boolean; message: string }> {
+    const failed = (message: string) => ({ ok: false, message })
+    if (step === 'apps') {
+      this.#upstream.invalidate(s.id)
+      const list = (await this.#upstream.call(s, 'apps/list')) as { apps?: { name: string }[] } | null
+      const have = new Set((list?.apps ?? []).map((a) => a.name))
+      const installed: string[] = []
+      for (const a of profile.apps) {
+        if (have.has(a.name)) continue
+        const outcome = await this.#writer.runMany('install-store', a.name, [s.id])
+        if ('error' in outcome) return failed(`${a.name}: ${outcome.error}`)
+        const r = outcome.results[0]
+        if (!r.ok) return failed(`${a.name}: ${r.message ?? 'could not be installed'}`)
+        installed.push(a.name)
+      }
+      return { ok: true, message: installed.length === 0 ? 'Every app is already installed.' : `Installed ${installed.join(', ')}.` }
+    }
+    if (step === 'configs') {
+      let written = 0
+      let same = 0
+      for (const a of profile.apps) {
+        if (a.config === null) continue
+        const done = await this.#appsConfig.apply(a.name, [s.id], {}, a.config)
+        const r = done.results[0]
+        if (!r.ok) return failed(`${a.name}: ${r.message ?? 'failed'}`)
+        if (/nothing was written/.test(r.message ?? '')) same++
+        else written++
+      }
+      return { ok: true, message: `${written} config(s) applied, ${same} already in sync.` }
+    }
+    if (step === 'settings') {
+      const done = await this.#settings.apply([s.id], {}, undefined, { settings: profile.settings ?? {}, updatedAt: '', updatedBy: '' })
+      const r = done.results[0]
+      return { ok: r.ok, message: r.message ?? '' }
+    }
+    const done = await this.#settings.listsApply([s.id], removeExtras, { ...(profile.lists ?? { blocked: [], allowed: [] }), updatedAt: '', updatedBy: '' })
+    const r = done.results[0]
+    return { ok: r.ok, message: r.message ?? '' }
+  }
+
+  #stepsOf(profile: Profile): Step[] {
+    return STEPS.filter((step) => (step === 'apps' ? profile.apps.length > 0 : step === 'configs' ? profile.apps.some((a) => a.config !== null) : step === 'settings' ? profile.settings !== null : profile.lists !== null))
+  }
+
+  async #applyOne(profile: Profile, s: HubServer, from: Step | undefined, removeExtras: boolean): Promise<ApplyResult> {
+    const steps: StepResult[] = []
+    const plan = this.#stepsOf(profile)
+    if (plan.length === 0) return { server: s.id, ok: true, message: 'The profile has nothing to apply.', steps }
+    if (s.adminToken === undefined) {
+      return { server: s.id, ok: false, message: 'no admin token configured', steps: plan.map((step, i) => ({ step, status: i === 0 ? ('failed' as const) : ('not-run' as const), message: i === 0 ? 'no admin token configured' : 'An earlier step failed.' })) }
+    }
+    const start = from === undefined ? 0 : Math.max(0, plan.indexOf(from))
+    let stopped: Step | null = null
+    for (const [i, step] of plan.entries()) {
+      if (stopped !== null) {
+        steps.push({ step, status: 'not-run', message: 'An earlier step failed.' })
+        continue
+      }
+      if (i < start) {
+        steps.push({ step, status: 'skipped', message: 'Done by an earlier run.' })
+        continue
+      }
+      let outcome: { ok: boolean; message: string }
+      try {
+        outcome = await this.#run(profile, s, step, removeExtras)
+      } catch (e) {
+        outcome = { ok: false, message: reason(e) }
+      }
+      steps.push({ step, status: outcome.ok ? 'ok' : 'failed', message: outcome.message })
+      if (!outcome.ok) stopped = step
+    }
+    return {
+      server: s.id,
+      ok: stopped === null,
+      message: stopped === null ? 'Profile applied.' : `Stopped at ${stopped}: ${steps.find((x) => x.step === stopped)?.message ?? 'failed'}`.slice(0, 300),
+      steps,
+    }
+  }
+
+  /* Several servers go on in parallel; the steps of one server go one after the other. */
+  async apply(name: string, ids: string[], options: { removeExtras: boolean; from: Record<string, string> }): Promise<{ results: ApplyResult[] }> {
+    const profile = this.#require(name)
+    const targets = this.#targets(ids)
+    const results = await Promise.all(
+      targets.map((s) => {
+        const from = (STEPS as readonly string[]).includes(options.from[s.id]) ? (options.from[s.id] as Step) : undefined
+        return this.#applyOne(profile, s, from, options.removeExtras)
+      }),
+    )
+    return { results }
+  }
+}

hub/ratelimit.ts

@@ -0,0 +1,71 @@
+/*
+Login rate limiting. Failures are counted per client address and per user name: after
+MAX_FAILURES within WINDOW_MS, further attempts are refused for LOCK_MS. A success clears
+the user's counter. In memory; `sweep` drops what has expired.
+*/
+export const MAX_FAILURES = 5
+export const WINDOW_MS = 15 * 60_000
+export const LOCK_MS = 15 * 60_000
+
+interface Entry {
+  failures: number[]
+  lockedUntil: number
+}
+
+const userKey = (user: string): string => user.trim().toLowerCase().slice(0, 128)
+
+export class LoginLimiter {
+  #now: () => number
+  #ips = new Map<string, Entry>()
+  #users = new Map<string, Entry>()
+
+  constructor(now: () => number = Date.now) {
+    this.#now = now
+  }
+
+  /** Milliseconds left of a lock on this address or this user, or 0 when attempts are allowed. */
+  lockedFor(ip: string, user: string): number {
+    const now = this.#now()
+    const until = Math.max(this.#ips.get(ip)?.lockedUntil ?? 0, this.#users.get(userKey(user))?.lockedUntil ?? 0)
+    return until > now ? until - now : 0
+  }
+
+  failure(ip: string, user: string): void {
+    this.#record(this.#ips, ip)
+    this.#record(this.#users, userKey(user))
+    this.sweep()
+  }
+
+  /** A good login clears the user's counter (the address keeps its own). */
+  success(user: string): void {
+    this.#users.delete(userKey(user))
+  }
+
+  #record(map: Map<string, Entry>, key: string): void {
+    const now = this.#now()
+    const entry = map.get(key) ?? { failures: [], lockedUntil: 0 }
+    entry.failures = entry.failures.filter((t) => now - t < WINDOW_MS)
+    entry.failures.push(now)
+    if (entry.failures.length >= MAX_FAILURES) {
+      entry.lockedUntil = now + LOCK_MS
+      entry.failures = []
+    }
+    map.set(key, entry)
+  }
+
+  sweep(): void {
+    const now = this.#now()
+    for (const map of [this.#ips, this.#users]) {
+      for (const [key, entry] of map) {
+        const recent = entry.failures.some((t) => now - t < WINDOW_MS)
+        if (entry.lockedUntil <= now && !recent) map.delete(key)
+      }
+    }
+  }
+}
+
+/** Technitium-like message for a locked attempt. */
+export const lockedMessage = (ms: number): string => {
+  const minutes = Math.max(1, Math.ceil(ms / 60_000))
+  return `Too many failed logins. Try again in ${minutes} minute${minutes === 1 ? '' : 's'}.`
+}

hub/routes.ts

@@ -0,0 +1,242 @@
+/*
+What each `/api/<path>` does and who may call it. `kind` says what it touches:
+`read` (no change), `write-one` (changes one server) and `write-all` (changes every server).
+A path that is not listed is refused. The app writes need `apps.write`. `hub/servers/run` (blocking, Blocked/Allowed and cache on several
+servers at once) needs `operate`.
+
+The sections a server is managed from (stage 5) are listed by what the console calls. A
+write there goes to ONE server (`node=<id>`) with that server's admin token. Capabilities:
+cache and Blocked/Allowed writes, the temporary blocking switch and the list update need
+`operate`; zones (records and DNSSEC too), settings and DHCP need `admin`.
+
+Left out on purpose, so they are refused: everything under `admin/` and `user/` (users,
+groups, sessions, SSO, LDAP, tokens: those stay in each server's own console), `logs/delete`,
+`logs/deleteAll`, `dashboard/stats/deleteAll`, `settings/backup` and `settings/restore` (a
+backup holds the users and their tokens), and `zones/permissions/*` (they name users).
+
+Not in the table: `status`, `user/login`, `user/session/get`, `user/logout` and
+`user/createSingleUseToken` (the hub's own, for downloads), which the server answers itself
+before the table is consulted.
+*/
+import type { Capability } from './users.ts'
+
+export type RouteKind = 'read' | 'write-one' | 'write-all'
+
+export interface Route {
+  kind: RouteKind
+  capability: Capability
+  /* A read whose answer is a file, not JSON: it is passed through (see `Upstream.raw`). */
+  raw?: boolean
+  /* A read that has no merger: without `node=<id>` it is refused, not answered by the first server. */
+  perServer?: boolean
+  /* A write that may carry a file or a text body (a zone import). */
+  upload?: boolean
+}
+
+const read = (capability: Capability = 'read'): Route => ({ kind: 'read', capability })
+const writeOne = (capability: Capability): Route => ({ kind: 'write-one', capability })
+const writeAll = (capability: Capability): Route => ({ kind: 'write-all', capability })
+/* A read that only makes sense for one server. */
+const readOne = (): Route => ({ kind: 'read', capability: 'read', perServer: true })
+/* A download: one server's file. */
+const download = (): Route => ({ kind: 'read', capability: 'read', perServer: true, raw: true })
+const upload = (capability: Capability): Route => ({ ...writeOne(capability), upload: true })
+
+export const ROUTES: Record<string, Route> = {
+  'dashboard/stats/get': read(),
+  'dashboard/stats/getTop': read(),
+  'dashboard/metrics/json': read(),
+  'apps/list': read(),
+  'logs/query': read(),
+  'logs/list': readOne(),
+  'settings/get': read(),
+  'user/checkForUpdate': read(),
+  'admin/cluster/state': read(),
+  // Zones. Reads need a server; writes need `admin`.
+  'zones/list': readOne(),
+  'zones/catalogs/list': readOne(),
+  'zones/options/get': readOne(),
+  'zones/records/get': readOne(),
+  'zones/dnssec/properties/get': readOne(),
+  'zones/dnssec/viewDS': readOne(),
+  'zones/export': download(),
+  'zones/create': upload('admin'),
+  'zones/import': upload('admin'),
+  'zones/delete': writeOne('admin'),
+  'zones/enable': writeOne('admin'),
+  'zones/disable': writeOne('admin'),
+  'zones/resync': writeOne('admin'),
+  'zones/clone': writeOne('admin'),
+  'zones/convert': writeOne('admin'),
+  'zones/options/set': writeOne('admin'),
+  'zones/records/add': writeOne('admin'),
+  'zones/records/update': writeOne('admin'),
+  'zones/records/delete': writeOne('admin'),
+  'zones/dnssec/sign': writeOne('admin'),
+  'zones/dnssec/unsign': writeOne('admin'),
+  'zones/dnssec/properties/addPrivateKey': writeOne('admin'),
+  'zones/dnssec/properties/updatePrivateKey': writeOne('admin'),
+  'zones/dnssec/properties/deletePrivateKey': writeOne('admin'),
+  'zones/dnssec/properties/publishAllPrivateKeys': writeOne('admin'),
+  'zones/dnssec/properties/activateKskDnsKey': writeOne('admin'),
+  'zones/dnssec/properties/rolloverDnsKey': writeOne('admin'),
+  'zones/dnssec/properties/retireDnsKey': writeOne('admin'),
+  'zones/dnssec/properties/convertToNSEC': writeOne('admin'),
+  'zones/dnssec/properties/convertToNSEC3': writeOne('admin'),
+  'zones/dnssec/properties/updateNSEC3Params': writeOne('admin'),
+  'zones/dnssec/properties/updateDnsKeyTtl': writeOne('admin'),
+  // Cache.
+  'cache/list': readOne(),
+  'cache/flush': writeOne('operate'),
+  'cache/delete': writeOne('operate'),
+  // Blocked and Allowed.
+  'allowed/list': readOne(),
+  'blocked/list': readOne(),
+  'allowed/export': download(),
+  'blocked/export': download(),
+  'allowed/add': writeOne('operate'),
+  'allowed/delete': writeOne('operate'),
+  'allowed/flush': writeOne('operate'),
+  'allowed/import': writeOne('operate'),
+  'blocked/add': writeOne('operate'),
+  'blocked/delete': writeOne('operate'),
+  'blocked/flush': writeOne('operate'),
+  'blocked/import': writeOne('operate'),
+  // DHCP.
+  'dhcp/leases/list': readOne(),
+  'dhcp/scopes/list': readOne(),
+  'dhcp/scopes/get': readOne(),
+  'dhcp/scopes/set': writeOne('admin'),
+  'dhcp/scopes/enable': writeOne('admin'),
+  'dhcp/scopes/disable': writeOne('admin'),
+  'dhcp/scopes/delete': writeOne('admin'),
+  'dhcp/leases/remove': writeOne('admin'),
+  'dhcp/leases/convertToReserved': writeOne('admin'),
+  'dhcp/leases/convertToDynamic': writeOne('admin'),
+  // DNS Client. `import=true` turns a resolve into a zone write: see `effectiveRoute`.
+  'dnsClient/resolve': readOne(),
+  // Settings. `settings/get` (above) is not `perServer`: the header's blocking mark reads it without a node.
+  'settings/getTsigKeyNames': readOne(),
+  'settings/set': writeOne('admin'), // see `effectiveRoute` for the blocking switch
+  'settings/temporaryDisableBlocking': writeOne('operate'),
+  'settings/forceUpdateBlockLists': writeOne('operate'),
+  // Logs (no delete).
+  'logs/download': download(),
+  'logs/export': download(),
+  // The hub's own namespace; never forwarded to a server.
+  'hub/audit/list': read('audit.read'),
+  'hub/apps/matrix': read(),
+  'apps/listStoreApps': read(),
+  // Writes. They go to a server's ADMIN token only; see apps-write.ts.
+  'apps/downloadAndInstall': writeOne('apps.write'),
+  'apps/downloadAndUpdate': writeOne('apps.write'),
+  'apps/uninstall': writeOne('apps.write'),
+  'apps/install': writeOne('apps.write'),
+  'apps/update': writeOne('apps.write'),
+  // Config sync (apps-config.ts). Compare shows live configs to anyone who can read; the rest needs apps.write.
+  'apps/config/get': read(),
+  'apps/config/set': writeOne('apps.write'),
+  'hub/apps/config/compare': read(),
+  'hub/apps/config/preview': read('apps.write'),
+  'hub/apps/overrides/current': read('apps.write'),
+  'hub/apps/baseline/import': writeAll('apps.write'),
+  'hub/apps/baseline/save': writeAll('apps.write'),
+  'hub/apps/overrides/save': writeAll('apps.write'),
+  'hub/apps/config/apply': writeAll('apps.write'),
+  // History, drift, and moving baselines between hubs (apps-config.ts, drift.ts).
+  'hub/apps/history/list': read(),
+  'hub/apps/history/get': read(),
+  'hub/apps/history/diff': read(),
+  'hub/apps/history/rollback': writeAll('apps.write'),
+  'hub/apps/baseline/restore': writeAll('apps.write'),
+  'hub/apps/drift/accept': writeAll('apps.write'),
+  // Profiles (profiles.ts): bundles of apps, settings and lists. They hold settings secrets, so everything needs admin.
+  'hub/profiles/list': read('admin'),
+  'hub/profiles/get': read('admin'),
+  'hub/profiles/export': read('admin'),
+  'hub/profiles/preview': read('admin'),
+  'hub/profiles/snapshot': writeAll('admin'),
+  'hub/profiles/delete': writeAll('admin'),
+  'hub/profiles/import': writeAll('admin'),
+  'hub/profiles/apply': writeAll('admin'),
+  'hub/webhook/status': read('admin'),
+  'hub/webhook/test': writeAll('admin'),
+  'hub/drift/get': read(),
+  'hub/drift/check': writeAll('apps.write'),
+  // Configs can hold secrets, so the file of all baselines needs apps.write even to read.
+  'hub/apps/export': read('apps.write'),
+  'hub/apps/import': writeAll('apps.write'),
+  'hub/apps/run': writeAll('apps.write'),
+  'hub/apps/upload': writeAll('apps.write'),
+  // The same change on several servers, outside Apps (servers-run.ts). One endpoint with a fixed list of actions.
+  'hub/servers/run': writeAll('operate'),
+  // Zones across the servers (zones-sync.ts). The merged list is zone names and serials only.
+  'hub/zones/matrix': read(),
+  'hub/zones/copy/preview': read('admin'),
+  'hub/zones/copy': writeAll('admin'),
+  'hub/zones/secondary/preview': read('admin'),
+  'hub/zones/secondary': writeAll('admin'),
+  // Settings and blocking lists across the servers (settings-config.ts). Settings carry secrets, so even reads need admin.
+  'hub/settings/compare': read('admin'),
+  'hub/settings/preview': read('admin'),
+  'hub/settings/history/list': read('admin'),
+  'hub/settings/history/get': read('admin'),
+  'hub/settings/baseline/import': writeAll('admin'),
+  'hub/settings/baseline/save': writeAll('admin'),
+  'hub/settings/overrides/save': writeAll('admin'),
+  'hub/settings/apply': writeAll('admin'),
+  'hub/blocking/lists/compare': read('admin'),
+  'hub/blocking/lists/preview': read('admin'),
+  'hub/blocking/lists/baseline/import': writeAll('admin'),
+  'hub/blocking/lists/apply': writeAll('admin'),
+}
+
+export function routeFor(apiPath: string): Route | undefined {
+  return Object.hasOwn(ROUTES, apiPath) ? ROUTES[apiPath] : undefined
+}
+
+/*
+The route for THIS call. `dnsClient/resolve` reads, but with `import=true` it also imports the
+answer into a zone of the server: that variant is a zone write (`admin`, one server). Any value
+but an absent or `false` one counts, so a spelling the server might still accept does not slip
+through as a read.
+*/
+export function effectiveRoute(apiPath: string, params: Record<string, string>): Route | undefined {
+  if (apiPath === 'dnsClient/resolve') {
+    const flag = (params.import ?? '').trim().toLowerCase()
+    if (flag !== '' && flag !== 'false') return writeOne('admin')
+  }
+  /*
+  The blocking switch is blocking, not server settings: a `settings/set` that carries nothing but
+  `enableBlocking` needs `operate`, as the temporary switch does. Any other field in the call
+  makes it the usual `admin` write. (`node` and `token` are the hub's own.)
+  */
+  if (apiPath === 'settings/set') {
+    const fields = Object.keys(params).filter((k) => k !== 'node' && k !== 'token')
+    if (fields.length === 1 && fields[0] === 'enableBlocking') return writeOne('operate')
+  }
+  return routeFor(apiPath)
+}
+
+/*
+The largest body a signed-in caller may send to a path outside `hub/apps/`. The servers read the
+rest with the small default, so a long TXT record or a block list import has to be listed here.
+*/
+const MB = 1024 * 1024
+const LARGE_BODIES: Record<string, number> = {
+  'settings/set': 2 * MB,
+  'zones/records/add': 2 * MB,
+  'zones/records/update': 2 * MB,
+  'zones/options/set': 2 * MB,
+  'dhcp/scopes/set': 2 * MB,
+  'hub/settings/baseline/save': 2 * MB,
+  'hub/settings/overrides/save': 2 * MB,
+  'hub/profiles/import': 5 * MB,
+  'allowed/import': 5 * MB,
+  'blocked/import': 5 * MB,
+}
+
+export const largeBodyLimit = (apiPath: string): number | undefined =>
+  Object.hasOwn(LARGE_BODIES, apiPath) ? LARGE_BODIES[apiPath] : undefined
+
+export const isHubPath = (apiPath: string): boolean => apiPath.startsWith('hub/')

hub/security.test.ts

@@ -0,0 +1,376 @@
+// @vitest-environment node
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { Audit } from './audit.ts'
+import { LoginLimiter, LOCK_MS, lockedMessage } from './ratelimit.ts'
+import { ROUTES, routeFor } from './routes.ts'
+import { run } from './users-cli.ts'
+import {
+  Users,
+  addUser,
+  can,
+  hashPassword,
+  parseUsersFile,
+  readUsersFile,
+  requires,
+  setDisabled,
+  setPassword,
+  setRole,
+  verifyPassword,
+  writeUsersFile,
+  type Role,
+} from './users.ts'
+
+let dir: string
+let servers: http.Server[] = []
+
+beforeEach(() => {
+  dir = mkdtempSync(path.join(tmpdir(), 'hub-sec-'))
+})
+
+afterEach(() => {
+  for (const s of servers) s.close()
+  servers = []
+  rmSync(dir, { recursive: true, force: true })
+})
+
+const env = (over: Record<string, string> = {}) => ({
+  HUB_SERVERS: 'a=http://127.0.0.1:1',
+  A_TOKEN: 'token-a',
+  A_ADMIN_TOKEN: 'admin-secret-a',
+  HUB_DATA: dir,
+  HUB_STATIC: dir,
+  ...over,
+})
+
+async function start(over: Record<string, string> = {}): Promise<string> {
+  const hub = createHub(parseConfig(env(over)))
+  servers.push(hub)
+  await new Promise<void>((resolve) => hub.listen(0, '127.0.0.1', resolve))
+  return `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+}
+
+async function call(base: string, apiPath: string, params: Record<string, string> = {}, token?: string, headers: Record<string, string> = {}) {
+  const headerSet: Record<string, string> = { ...headers }
+  if (token) headerSet.authorization = `Bearer ${token}`
+  const isLogin = apiPath === 'user/login'
+  const res = await fetch(`${base}/api/${apiPath}${isLogin ? '' : `?${new URLSearchParams(params)}`}`, {
+    method: isLogin ? 'POST' : 'GET',
+    headers: isLogin ? { ...headerSet, 'content-type': 'application/x-www-form-urlencoded' } : headerSet,
+    body: isLogin ? new URLSearchParams(params).toString() : undefined,
+  })
+  return { code: res.status, body: (await res.json()) as Record<string, any> }
+}
+
+const login = (base: string, user: string, pass: string, headers: Record<string, string> = {}) =>
+  call(base, 'user/login', { user, pass }, undefined, headers)
+
+function seed(...rows: [string, Role, string][]): void {
+  let users = readUsersFile(dir) ?? []
+  for (const [name, role, pass] of rows) users = addUser(users, name, role, pass)
+  writeUsersFile(dir, users)
+}
+
+describe('roles and capabilities', () => {
+  it('ranks the roles and answers capabilities', () => {
+    expect(requires('admin', 'operator')).toBe(true)
+    expect(requires('viewer', 'operator')).toBe(false)
+    expect(requires('apps-admin', 'operator')).toBe(true)
+    expect(can('viewer', 'read')).toBe(true)
+    expect(can('viewer', 'operate')).toBe(false)
+    expect(can('operator', 'operate')).toBe(true)
+    expect(can('operator', 'apps.write')).toBe(false)
+    expect(can('apps-admin', 'apps.write')).toBe(true)
+    expect(can('apps-admin', 'audit.read')).toBe(false)
+    for (const c of ['read', 'apps.write', 'operate', 'audit.read', 'admin'] as const) expect(can('admin', c)).toBe(true)
+  })
+})
+
+describe('passwords and the users file', () => {
+  it('hashes with scrypt and verifies', async () => {
+    const hash = hashPassword('correct horse')
+    expect(hash).toMatch(/^scrypt\$16384\$8\$1\$[A-Za-z0-9+/=]+\$[A-Za-z0-9+/=]+$/)
+    expect(hash.split('$')[5].length).toBeGreaterThan(80)
+    expect(await verifyPassword('correct horse', hash)).toBe(true)
+    expect(await verifyPassword('wrong', hash)).toBe(false)
+    expect(await verifyPassword('x', 'not a hash')).toBe(false)
+    expect(hashPassword('correct horse')).not.toBe(hash)
+  })
+
+  it('parses and rejects bad files', () => {
+    const hash = hashPassword('longenough')
+    const ok = parseUsersFile(JSON.stringify({ users: [{ name: 'Alice', role: 'admin', hash, disabled: true }] }))
+    expect(ok).toEqual([{ name: 'Alice', role: 'admin', hash, disabled: true }])
+    expect(() => parseUsersFile('nope')).toThrow(/JSON/)
+    expect(() => parseUsersFile('{}')).toThrow(/users/)
+    expect(() => parseUsersFile(JSON.stringify({ users: [{ name: 'a', role: 'root', hash }] }))).toThrow(/role/)
+    expect(() => parseUsersFile(JSON.stringify({ users: [{ name: 'a', role: 'admin', hash: 'x' }] }))).toThrow(/hash/)
+    expect(() =>
+      parseUsersFile(JSON.stringify({ users: [{ name: 'a', role: 'admin', hash }, { name: 'A', role: 'viewer', hash }] })),
+    ).toThrow(/twice/)
+  })
+
+  it('edits users (the functions the CLI calls)', async () => {
+    let users = addUser([], 'bob', 'viewer', 'password1')
+    expect(() => addUser(users, 'BOB', 'viewer', 'password1')).toThrow(/exists/)
+    expect(() => addUser(users, 'x', 'root', 'password1')).toThrow(/role/)
+    expect(() => addUser(users, 'x', 'viewer', 'short')).toThrow(/8/)
+    users = setRole(users, 'Bob', 'operator')
+    expect(users[0].role).toBe('operator')
+    const before = users[0].hash
+    users = setPassword(users, 'bob', 'password2')
+    expect(users[0].hash).not.toBe(before)
+    expect(await verifyPassword('password2', users[0].hash)).toBe(true)
+    users = setDisabled(users, 'bob', true)
+    expect(users[0].disabled).toBe(true)
+    users = setDisabled(users, 'bob', false)
+    expect('disabled' in users[0]).toBe(false)
+    expect(() => setRole(users, 'nobody', 'viewer')).toThrow(/No user/)
+  })
+
+  it('writes atomically, leaving no temp file', () => {
+    writeUsersFile(dir, addUser([], 'bob', 'viewer', 'password1'))
+    const file = path.join(dir, 'users.json')
+    expect(readUsersFile(dir)?.[0].name).toBe('bob')
+    expect(existsSync(`${file}.${process.pid}.tmp`)).toBe(false)
+    if (process.platform !== 'win32') expect(statSync(file).mode & 0o777).toBe(0o600)
+  })
+
+  it('runs CLI commands', async () => {
+    const pw = async () => 'password1'
+    expect(await run(['list'], dir, pw)).toEqual(['No users.'])
+    await run(['add', 'alice', 'apps-admin'], dir, pw)
+    await run(['role', 'alice', 'operator'], dir, pw)
+    await run(['disable', 'alice'], dir, pw)
+    expect(await run(['list'], dir, pw)).toEqual(['alice\toperator\tdisabled'])
+    await run(['enable', 'alice'], dir, pw)
+    await run(['passwd', 'alice'], dir, async () => 'password2')
+    expect(await verifyPassword('password2', readUsersFile(dir)![0].hash)).toBe(true)
+    await expect(run(['bogus'], dir, pw)).rejects.toThrow(/Usage/)
+    await expect(run(['add', 'bob'], dir, pw)).rejects.toThrow(/role/)
+  })
+})
+
+describe('bootstrap', () => {
+  it('uses the environment user as admin when there is no file, and does not write it', async () => {
+    const users = Users.load(dir, { user: 'Root', password: 'secret' })
+    expect(existsSync(path.join(dir, 'users.json'))).toBe(false)
+    expect(users.notice).toMatch(/HUB_USER/)
+    expect((await users.authenticate('root', 'secret'))?.role).toBe('admin')
+    expect(await users.authenticate('root', 'nope')).toBeNull()
+  })
+
+  it('ignores the environment user when the file exists', async () => {
+    seed(['alice', 'viewer', 'password1'])
+    const users = Users.load(dir, { user: 'root', password: 'secret' })
+    expect(users.notice).toMatch(/ignored/)
+    expect(await users.authenticate('root', 'secret')).toBeNull()
+    expect((await users.authenticate('ALICE', 'password1'))?.role).toBe('viewer')
+  })
+
+  it('refuses to start with neither', () => {
+    expect(() => Users.load(dir, null)).toThrow(/No users/)
+    expect(() => createHub(parseConfig(env()))).toThrow(/No users/)
+  })
+
+  it('config reads no file and takes the optional variables', () => {
+    const c = parseConfig(env({ HUB_TRUST_PROXY: '1' }))
+    expect(c.bootstrap).toBeNull()
+    expect(c.dataDir).toBe(dir)
+    expect(c.trustProxy).toBe(true)
+    expect(c.servers[0].adminToken).toBe('admin-secret-a')
+    expect(parseConfig({ ...env(), HUB_DATA: '' }).dataDir).toBe('/data')
+    expect(parseConfig(env()).trustProxy).toBe(false)
+  })
+})
+
+describe('rate limit', () => {
+  it('locks after 5 failures, expires, and a success resets the user', () => {
+    let now = 1_000_000
+    const limiter = new LoginLimiter(() => now)
+    for (let i = 0; i < 4; i++) limiter.failure('1.1.1.1', 'Bob')
+    expect(limiter.lockedFor('1.1.1.1', 'bob')).toBe(0)
+    limiter.success('bob')
+    for (let i = 0; i < 4; i++) limiter.failure('2.2.2.2', 'bob')
+    expect(limiter.lockedFor('2.2.2.2', 'bob')).toBe(0)
+    limiter.failure('2.2.2.2', 'bob')
+    expect(limiter.lockedFor('3.3.3.3', 'BOB')).toBe(LOCK_MS)
+    expect(limiter.lockedFor('2.2.2.2', 'other')).toBe(LOCK_MS)
+    now += LOCK_MS + 1
+    expect(limiter.lockedFor('2.2.2.2', 'bob')).toBe(0)
+    expect(lockedMessage(LOCK_MS)).toBe('Too many failed logins. Try again in 15 minutes.')
+    expect(lockedMessage(1000)).toBe('Too many failed logins. Try again in 1 minute.')
+  })
+
+  it('forgets failures older than the window', () => {
+    let now = 0
+    const limiter = new LoginLimiter(() => now)
+    for (let i = 0; i < 4; i++) limiter.failure('ip', 'u')
+    now += 16 * 60_000
+    limiter.failure('ip', 'u')
+    expect(limiter.lockedFor('ip', 'u')).toBe(0)
+  })
+
+  it('locks the hub login with 429 and the message, even for the right password', async () => {
+    seed(['alice', 'admin', 'password1'])
+    const base = await start()
+    for (let i = 0; i < 5; i++) expect((await login(base, 'alice', 'bad')).body.status).toBe('error')
+    const locked = await login(base, 'alice', 'password1')
+    expect(locked.code).toBe(429)
+    expect(locked.body).toEqual({ status: 'error', errorMessage: 'Too many failed logins. Try again in 15 minutes.' })
+  })
+
+  it('counts X-Forwarded-For only when the proxy is trusted', async () => {
+    seed(['alice', 'admin', 'password1'])
+    // Untrusted: the header is ignored, so five attempts from one socket lock that socket.
+    const plain = await start()
+    for (let i = 0; i < 5; i++) await login(plain, `u${i}`, 'bad', { 'x-forwarded-for': `9.9.9.${i}` })
+    expect((await login(plain, 'alice', 'password1', { 'x-forwarded-for': '8.8.8.8' })).code).toBe(429)
+    // Trusted: each forwarded address has its own count.
+    const trusted = await start({ HUB_TRUST_PROXY: '1' })
+    for (let i = 0; i < 5; i++) await login(trusted, `u${i}`, 'bad', { 'x-forwarded-for': `9.9.9.${i}` })
+    expect((await login(trusted, 'alice', 'password1', { 'x-forwarded-for': '8.8.8.8, 10.0.0.1' })).code).toBe(200)
+  })
+})
+
+describe('audit', () => {
+  it('appends JSON lines, lists newest first and filters with before', () => {
+    let now = Date.parse('2026-01-01T00:00:00Z')
+    const audit = new Audit(dir, { now: () => now })
+    for (const action of ['one', 'two', 'three']) {
+      audit.log({ user: 'u', ip: 'i', action, outcome: 'ok' })
+      now += 1000
+    }
+    const lines = readFileSync(path.join(dir, 'audit.log'), 'utf8').trim().split('\n')
+    expect(lines).toHaveLength(3)
+    expect(JSON.parse(lines[0])).toMatchObject({ user: 'u', ip: 'i', action: 'one', outcome: 'ok' })
+    expect(audit.list().map((e) => e.action)).toEqual(['three', 'two', 'one'])
+    expect(audit.list(1).map((e) => e.action)).toEqual(['three'])
+    expect(audit.list(200, '2026-01-01T00:00:02.000Z').map((e) => e.action)).toEqual(['two', 'one'])
+  })
+
+  it('rotates once the file is too big and keeps one old file', () => {
+    const audit = new Audit(dir, { maxBytes: 600 })
+    for (let i = 0; i < 12; i++) audit.log({ user: 'u', ip: 'i', action: `a${i}`, outcome: 'ok' })
+    expect(existsSync(path.join(dir, 'audit.log.1'))).toBe(true)
+    expect(existsSync(path.join(dir, 'audit.log.2'))).toBe(false)
+    expect(statSync(path.join(dir, 'audit.log')).size).toBeLessThanOrEqual(600)
+    expect(audit.list(1)[0].action).toBe('a11')
+  })
+
+  it('falls back to memory when the folder is not writable', () => {
+    const file = path.join(dir, 'blocker')
+    writeFileSync(file, 'x')
+    const err = vi.spyOn(console, 'error').mockImplementation(() => {})
+    const audit = new Audit(path.join(file, 'sub'))
+    audit.log({ user: 'u', ip: 'i', action: 'x', outcome: 'ok' })
+    audit.log({ user: 'u', ip: 'i', action: 'y', outcome: 'ok' })
+    expect(err).toHaveBeenCalledTimes(1)
+    err.mockRestore()
+    expect(audit.inMemory).toBe(true)
+    expect(audit.list().map((e) => e.action)).toEqual(['y', 'x'])
+  })
+
+  it('records login, logout and refusals without passwords or tokens, and is for audit.read only', async () => {
+    seed(['alice', 'admin', 'password1'], ['vic', 'viewer', 'password2'])
+    const base = await start()
+    await login(base, 'alice', 'wrong-password-xyz')
+    const adminLogin = await login(base, 'alice', 'password1')
+    const admin = adminLogin.body.token as string
+    expect(adminLogin.body.hubRole).toBe('admin')
+    const viewer = (await login(base, 'vic', 'password2')).body.token as string
+
+    expect((await call(base, 'admin/users/delete', { user: 'x', node: 'a' }, admin)).code).toBe(403)
+    expect((await call(base, 'hub/audit/list', {}, viewer)).code).toBe(403)
+    await call(base, 'user/logout', {}, viewer)
+
+    const list = await call(base, 'hub/audit/list', { limit: '50' }, admin)
+    expect(list.body.status).toBe('ok')
+    const entries = list.body.response.entries as Record<string, string>[]
+    const actions = entries.map((e) => `${e.action}:${e.outcome}`)
+    expect(actions).toEqual(expect.arrayContaining(['login:failed', 'login:ok', 'refused-write:refused', 'denied-read:refused', 'logout:ok']))
+    const refused = entries.find((e) => e.action === 'refused-write')
+    expect(refused).toMatchObject({ user: 'alice', role: 'admin', target: 'admin/users/delete', server: 'a' })
+    const ts = entries.map((e) => e.ts)
+    expect([...ts].sort().reverse()).toEqual(ts)
+
+    const raw = readFileSync(path.join(dir, 'audit.log'), 'utf8')
+    for (const secret of ['wrong-password-xyz', 'password1', 'password2', admin, viewer, 'token-a', 'admin-secret-a']) {
+      expect(raw).not.toContain(secret)
+    }
+  })
+
+  it('logs a locked attempt', async () => {
+    seed(['alice', 'admin', 'password1'])
+    const base = await start()
+    for (let i = 0; i < 6; i++) await login(base, 'alice', 'bad')
+    const lines = readFileSync(path.join(dir, 'audit.log'), 'utf8').trim().split('\n').map((l) => JSON.parse(l))
+    expect(lines.filter((l) => l.outcome === 'failed')).toHaveLength(5)
+    expect(lines[5]).toMatchObject({ action: 'login', outcome: 'refused', detail: 'locked' })
+  })
+})
+
+describe('routes table', () => {
+  it('lists every old read endpoint as a read and hub/audit/list as audit.read', () => {
+    for (const p of [
+      'dashboard/stats/get', 'dashboard/stats/getTop', 'dashboard/metrics/json', 'apps/list', 'logs/query',
+      'settings/get', 'user/checkForUpdate', 'admin/cluster/state',
+    ]) expect(ROUTES[p]).toEqual({ kind: 'read', capability: 'read' })
+    expect(routeFor('hub/audit/list')).toEqual({ kind: 'read', capability: 'audit.read' })
+    expect(routeFor('logs/delete')).toBeUndefined()
+    expect(routeFor('constructor')).toBeUndefined()
+    expect(routeFor('hub/nothing')).toBeUndefined()
+  })
+})
+
+describe('sessions by role', () => {
+  it('puts hubRole in the body and in info', async () => {
+    seed(['vic', 'viewer', 'password2'], ['op', 'operator', 'password3'])
+    const base = await start()
+    const a = await login(base, 'vic', 'password2')
+    expect(a.body.hubRole).toBe('viewer')
+    expect(a.body.info.hubRole).toBe('viewer')
+    expect(a.body.info.permissions.Dashboard.canView).toBe(true)
+    const b = await call(base, 'user/session/get', {}, (await login(base, 'op', 'password3')).body.token)
+    expect(b.body.hubRole).toBe('operator')
+    expect(b.body.info.hubRole).toBe('operator')
+  })
+
+  it('drops the sessions of a disabled user and refuses a new login', async () => {
+    seed(['alice', 'admin', 'password1'], ['vic', 'viewer', 'password2'])
+    const base = await start()
+    const vic = (await login(base, 'vic', 'password2')).body.token as string
+    const alice = (await login(base, 'alice', 'password1')).body.token as string
+    expect((await call(base, 'user/session/get', {}, vic)).body.status).toBe('ok')
+
+    // The file changes: vic is disabled, alice is untouched.
+    const users = setDisabled(readUsersFile(dir)!, 'vic', true)
+    writeUsersFile(dir, users)
+    const future = new Date(Date.now() + 5000)
+    const { utimesSync } = await import('node:fs')
+    utimesSync(path.join(dir, 'users.json'), future, future)
+
+    expect((await call(base, 'user/session/get', {}, vic)).body.status).toBe('invalid-token')
+    expect((await call(base, 'user/session/get', {}, alice)).body.status).toBe('ok')
+    expect((await login(base, 'vic', 'password2')).body.status).toBe('error')
+  })
+
+  it('drops sessions when the role or password changes', async () => {
+    seed(['alice', 'admin', 'password1'])
+    const base = await start()
+    const t = (await login(base, 'alice', 'password1')).body.token as string
+    writeUsersFile(dir, setRole(readUsersFile(dir)!, 'alice', 'viewer'))
+    const { utimesSync } = await import('node:fs')
+    const future = new Date(Date.now() + 5000)
+    utimesSync(path.join(dir, 'users.json'), future, future)
+    expect((await call(base, 'user/session/get', {}, t)).body.status).toBe('invalid-token')
+    const again = await login(base, 'ALICE', 'password1')
+    expect(again.body.hubRole).toBe('viewer')
+  })
+})

hub/server-write.test.ts

@@ -0,0 +1,600 @@
+// @vitest-environment node
+/*
+Stage 5: managing one chosen server from the hub (zones, cache, blocking, DNS client, settings,
+DHCP, logs). The servers are fakes that record how they were called.
+*/
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { addUser, writeUsersFile } from './users.ts'
+import { webServiceChanges, redactSettings, safeTarget } from './guards.ts'
+import { effectiveRoute, routeFor } from './routes.ts'
+
+interface Call {
+  path: string
+  method: string
+  token: string
+  /* The URL's query alone. */
+  query: URLSearchParams
+  /* The form body, when it is one. */
+  form: URLSearchParams
+  contentType: string
+  body: Buffer
+}
+
+interface Fake {
+  server: http.Server
+  url: string
+  calls: Call[]
+  settings: Record<string, unknown>
+  down: boolean
+  failWith: string | null
+}
+
+const SETTINGS = {
+  dnsServerDomain: 'a.example',
+  webServiceLocalAddresses: ['0.0.0.0', '[::]'],
+  webServiceHttpPort: 5380,
+  webServiceEnableTls: false,
+  webServiceTlsCertificatePath: null,
+  webServiceTlsCertificatePassword: '************',
+  tsigKeys: [{ keyName: 'k', sharedSecret: 'TOPSECRET', algorithmName: 'hmac-sha256' }],
+  proxy: { type: 'Http', address: 'p', port: 3128, username: 'u', password: 'PROXYPASS', bypass: [] },
+  dnsTlsCertificatePassword: 'CERTPASS',
+  enableBlocking: true,
+}
+
+function fake(): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = { server: http.createServer(), url: '', calls: [], settings: { ...SETTINGS }, down: false, failWith: null }
+    f.server.on('request', (req, res) => {
+      const chunks: Buffer[] = []
+      req.on('data', (c: Buffer) => chunks.push(c))
+      req.on('end', () => {
+        if (f.down) return req.socket.destroy()
+        const url = new URL(req.url ?? '/', 'http://fake')
+        const apiPath = url.pathname.replace(/^\/api\//, '')
+        const body = Buffer.concat(chunks)
+        const contentType = String(req.headers['content-type'] ?? '')
+        const form = new URLSearchParams(contentType.includes('x-www-form-urlencoded') ? body.toString('utf8') : '')
+        f.calls.push({ path: apiPath, method: req.method ?? '', token: url.searchParams.get('token') ?? '', query: url.searchParams, form, contentType, body })
+        if (f.failWith !== null && req.method === 'POST') {
+          res.setHeader('content-type', 'application/json')
+          return res.end(JSON.stringify({ status: 'error', errorMessage: f.failWith }))
+        }
+        if (apiPath === 'zones/export' || apiPath === 'logs/download' || apiPath === 'blocked/export') {
+          if (url.searchParams.get('zone') === 'missing.example') {
+            res.setHeader('content-type', 'application/json')
+            return res.end(JSON.stringify({ status: 'error', errorMessage: 'No such zone: missing.example' }))
+          }
+          res.writeHead(200, { 'content-type': 'text/plain', 'content-disposition': `attachment; filename="${apiPath.replace('/', '-')}.txt"` })
+          return res.end(`line one\nline two (${apiPath})\n`)
+        }
+        res.setHeader('content-type', 'application/json')
+        const ok = (response: unknown = {}) => res.end(JSON.stringify({ status: 'ok', response }))
+        if (apiPath === 'settings/get') return ok(f.settings)
+        if (apiPath === 'zones/list') return ok({ zones: [{ name: 'example.com' }], pageNumber: 1, totalPages: 1 })
+        ok({ done: apiPath })
+      })
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+let a: Fake
+let b: Fake
+let dataDir: string
+let staticDir: string
+let hub: http.Server | undefined
+let base: string
+const tokens: Record<string, string> = {}
+const ROLES = ['viewer', 'operator', 'apps-admin', 'admin']
+
+async function start(extra: Record<string, string> = {}): Promise<void> {
+  hub?.close()
+  hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      A_ADMIN_TOKEN: 'admin-a',
+      B_ADMIN_TOKEN: 'admin-b',
+      HUB_DATA: dataDir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      ...extra,
+    }),
+  )
+  await new Promise<void>((resolve) => hub!.listen(0, '127.0.0.1', resolve))
+  base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  for (const role of ROLES) {
+    const res = await fetch(`${base}/api/user/login`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: `user=${role}&pass=password-${role}`,
+    })
+    tokens[role] = ((await res.json()) as { token: string }).token
+  }
+}
+
+interface Init {
+  params?: Record<string, string>
+  method?: string
+  body?: string | FormData
+  headers?: Record<string, string>
+  auth?: boolean
+}
+
+async function api(role: string, apiPath: string, init: Init = {}) {
+  const query = new URLSearchParams(init.params ?? {})
+  const headers: Record<string, string> = { ...(init.auth === false ? {} : { authorization: `Bearer ${tokens[role]}` }), ...(init.headers ?? {}) }
+  const res = await fetch(`${base}/api/${apiPath}?${query}`, { method: init.method ?? (init.body === undefined ? 'GET' : 'POST'), headers, body: init.body })
+  const text = await res.text()
+  let json: Record<string, any> = {}
+  try {
+    json = JSON.parse(text) as Record<string, any>
+  } catch {
+    // A file.
+  }
+  return { code: res.status, body: json, text, headers: res.headers }
+}
+
+const posts = (f: Fake) => f.calls.filter((c) => c.method === 'POST')
+const form = (params: Record<string, string>) => ({
+  body: new URLSearchParams(params).toString(),
+  headers: { 'content-type': 'application/x-www-form-urlencoded' },
+})
+
+async function auditEntries(): Promise<Record<string, string>[]> {
+  return (await api('admin', 'hub/audit/list', { params: { limit: '200' } })).body.response.entries
+}
+
+beforeAll(async () => {
+  a = await fake()
+  b = await fake()
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-s5-data-'))
+  staticDir = mkdtempSync(path.join(tmpdir(), 'hub-s5-static-'))
+  let users = addUser([], 'viewer', 'viewer', 'password-viewer')
+  users = addUser(users, 'operator', 'operator', 'password-operator')
+  users = addUser(users, 'apps-admin', 'apps-admin', 'password-apps-admin')
+  users = addUser(users, 'admin', 'admin', 'password-admin')
+  writeUsersFile(dataDir, users)
+  await start()
+})
+
+beforeEach(() => {
+  for (const f of [a, b]) {
+    f.calls = []
+    f.down = false
+    f.failWith = null
+    f.settings = { ...SETTINGS }
+  }
+})
+
+afterAll(() => {
+  hub?.close()
+  a.server.close()
+  b.server.close()
+  rmSync(dataDir, { recursive: true, force: true })
+  rmSync(staticDir, { recursive: true, force: true })
+})
+
+/* One write of each area, with the lowest capability that may make it. */
+const OPERATE = [
+  ['cache/flush', {}],
+  ['cache/delete', { domain: 'x.example' }],
+  ['blocked/add', { domain: 'ads.example' }],
+  ['allowed/delete', { domain: 'ok.example' }],
+  ['settings/temporaryDisableBlocking', { minutes: '5' }],
+  ['settings/forceUpdateBlockLists', {}],
+  // The blocking switch alone is blocking, not server settings (routes.ts `effectiveRoute`).
+  ['settings/set', { enableBlocking: 'false' }],
+] as const
+const ADMIN = [
+  ['zones/records/add', { zone: 'example.com', domain: 'www.example.com', type: 'A', ipAddress: '192.0.2.1' }],
+  ['zones/delete', { zone: 'example.com' }],
+  ['zones/dnssec/sign', { zone: 'example.com', algorithm: 'ECDSA' }],
+  ['dhcp/scopes/enable', { name: 'Default' }],
+  ['settings/set', { dnsServerDomain: 'x.example' }],
+  // The switch plus any other field is a settings change.
+  ['settings/set', { enableBlocking: 'true', dnsServerDomain: 'x.example' }],
+] as const
+
+describe('who may write what', () => {
+  it('refuses a viewer on every write, without calling a server', async () => {
+    for (const [p, params] of [...OPERATE, ...ADMIN]) {
+      const r = await api('viewer', p, { params: { node: 'a', ...params } })
+      expect(r.code, p).toBe(403)
+      expect(r.body.errorMessage).toBe('Your role does not allow this.')
+    }
+    expect(posts(a)).toHaveLength(0)
+  })
+
+  it('lets an operator operate (cache, blocking) but not change zones, settings or DHCP', async () => {
+    for (const [p, params] of OPERATE) expect((await api('operator', p, { params: { node: 'a', ...params } })).body.status, p).toBe('ok')
+    for (const [p, params] of ADMIN) expect((await api('operator', p, { params: { node: 'a', ...params } })).code, p).toBe(403)
+    expect(posts(a).map((c) => c.path)).toEqual(OPERATE.map(([p]) => p))
+  })
+
+  it('lets an apps-admin do neither', async () => {
+    for (const [p, params] of [...OPERATE, ...ADMIN]) expect((await api('apps-admin', p, { params: { node: 'a', ...params } })).code, p).toBe(403)
+    expect(posts(a)).toHaveLength(0)
+  })
+
+  it('lets an admin do all of it', async () => {
+    for (const [p, params] of [...OPERATE, ...ADMIN]) expect((await api('admin', p, { params: { node: 'a', ...params } })).body.status, p).toBe('ok')
+    expect(posts(a)).toHaveLength(OPERATE.length + ADMIN.length)
+  })
+
+  it('tells the console what each role may modify', async () => {
+    const info = async (role: string) => (await api(role, 'user/session/get')).body.info.permissions
+    const viewer = await info('viewer')
+    const operator = await info('operator')
+    const admin = await info('admin')
+    expect(viewer.Zones).toEqual({ canView: true, canModify: false, canDelete: false })
+    expect(operator.Zones.canModify).toBe(false)
+    expect(operator.Cache).toEqual({ canView: true, canModify: true, canDelete: true })
+    expect(operator.Blocked.canModify).toBe(true)
+    expect(operator.Settings.canModify).toBe(false)
+    expect(admin.Zones).toEqual({ canView: true, canModify: true, canDelete: true })
+    expect(admin.DhcpServer).toEqual({ canView: true, canModify: true, canDelete: true })
+    // Backup and restore, deleting logs and stats: nobody.
+    for (const p of [viewer, operator, admin]) {
+      expect(p.Logs).toEqual({ canView: true, canModify: false, canDelete: false })
+      expect(p.Dashboard.canDelete).toBe(false)
+      expect(p.Settings.canDelete).toBe(false)
+      expect(p.Administration.canView).toBe(false)
+    }
+  })
+})
+
+describe('what stays refused', () => {
+  const REFUSED = [
+    'admin/users/list', 'admin/users/create', 'admin/sso/set', 'admin/ldap/set', 'admin/groups/list', 'admin/sessions/createToken',
+    'user/createToken', 'user/changePassword', 'user/profile/get',
+    'logs/delete', 'logs/deleteAll', 'dashboard/stats/deleteAll',
+    'settings/restore', 'settings/backup', 'zones/permissions/get', 'zones/permissions/set',
+  ]
+
+  it('answers "read-only" to each, even to an admin, and calls no server', async () => {
+    for (const p of REFUSED) {
+      const r = await api('admin', p, { params: { node: 'a' }, method: 'POST', body: '' })
+      expect(r.code, p).toBe(403)
+      expect(r.body.errorMessage, p).toBe('The hub is read-only.')
+    }
+    expect(a.calls).toHaveLength(0)
+    const refused = (await auditEntries()).filter((e) => e.action === 'refused-write')
+    expect(refused.map((e) => e.target)).toEqual(expect.arrayContaining(['logs/delete', 'settings/backup']))
+  })
+})
+
+describe('choosing the server', () => {
+  it('refuses a write with no node, the aggregate or an unknown node (400) and calls no server', async () => {
+    for (const node of [undefined, '', 'cluster', 'zzz']) {
+      const r = await api('admin', 'cache/flush', { params: node === undefined ? {} : { node } })
+      expect(r.code).toBe(400)
+      expect(r.body.errorMessage).toBe(node === 'zzz' ? 'Unknown node: zzz' : 'Choose a server: this change is made on one server at a time.')
+    }
+    expect(a.calls).toHaveLength(0)
+    expect(b.calls).toHaveLength(0)
+  })
+
+  it('refuses a server the hub has no admin token for (403), and audits it', async () => {
+    await start({ B_ADMIN_TOKEN: '' })
+    const r = await api('admin', 'cache/flush', { params: { node: 'b' } })
+    expect(r.code).toBe(403)
+    expect(r.body.errorMessage).toMatch(/no admin token/)
+    expect(b.calls).toHaveLength(0)
+    expect((await api('admin', 'cache/flush', { params: { node: 'a' } })).body.status).toBe('ok')
+    expect((await auditEntries()).find((e) => e.detail === 'no admin token')).toMatchObject({ server: 'b', outcome: 'refused' })
+    await start()
+  })
+
+  it('writes only to the chosen server', async () => {
+    await api('admin', 'cache/flush', { params: { node: 'b' } })
+    expect(posts(a)).toHaveLength(0)
+    expect(posts(b)).toHaveLength(1)
+  })
+
+  it('refuses a read that has no merger when no server is named', async () => {
+    for (const p of ['zones/list', 'cache/list', 'blocked/list', 'dhcp/scopes/list', 'logs/list', 'dnsClient/resolve', 'zones/export']) {
+      const r = await api('viewer', p)
+      expect(r.code, p).toBe(400)
+      expect(r.body.errorMessage).toBe('Choose a server: this is read from one server at a time.')
+      expect((await api('viewer', p, { params: { node: 'cluster' } })).code, p).toBe(400)
+    }
+    expect(a.calls).toHaveLength(0)
+    const one = await api('viewer', 'zones/list', { params: { node: 'a', pageNumber: '1' } })
+    expect(one.body.response.zones[0].name).toBe('example.com')
+    expect(a.calls[0]).toMatchObject({ method: 'GET', token: 'read-a' })
+  })
+})
+
+describe('how the hub calls the server', () => {
+  it('sends the parameters in a form body and leaves only the token in the query', async () => {
+    const r = await api('admin', 'zones/records/add', {
+      params: { node: 'a', zone: 'example.com' },
+      ...form({ domain: 'www.example.com', type: 'TXT', text: 'v=spf1 secret-value' }),
+    })
+    expect(r.body.status).toBe('ok')
+    const [call] = posts(a)
+    expect(call.path).toBe('zones/records/add')
+    expect(call.token).toBe('admin-a')
+    expect([...call.query.keys()]).toEqual(['token'])
+    expect(call.contentType).toContain('application/x-www-form-urlencoded')
+    expect(Object.fromEntries(call.form)).toEqual({ zone: 'example.com', domain: 'www.example.com', type: 'TXT', text: 'v=spf1 secret-value' })
+  })
+
+  it('forwards a GET the console sent as a POST', async () => {
+    await api('admin', 'zones/enable', { params: { node: 'a', zone: 'example.com' } })
+    expect(posts(a)[0].form.get('zone')).toBe('example.com')
+    expect(posts(a)[0].form.has('node')).toBe(false)
+  })
+
+  it('never forwards the caller token or node, and uses the admin token alone', async () => {
+    await api('admin', 'cache/flush', { params: { node: 'a', token: 'evil' } })
+    const call = posts(a)[0]
+    expect(call.token).toBe('admin-a')
+    expect(call.form.has('token')).toBe(false)
+    expect(call.form.has('node')).toBe(false)
+  })
+
+  it('relays the server refusal as an error naming the server', async () => {
+    a.failWith = 'Zone already exists.'
+    const r = await api('admin', 'zones/create', { params: { node: 'a', zone: 'example.com', type: 'Primary' }, method: 'POST' })
+    expect(r.code).toBe(200)
+    expect(r.body).toEqual({ status: 'error', errorMessage: 'a: Zone already exists.' })
+  })
+
+  it('says so when the server does not answer', async () => {
+    a.down = true
+    const r = await api('admin', 'cache/flush', { params: { node: 'a' } })
+    expect(r.body.status).toBe('error')
+    expect(r.body.errorMessage).toMatch(/^a did not answer/)
+  })
+})
+
+describe('the audit line', () => {
+  it('has the path, the server, one identifier and the outcome, and no value', async () => {
+    await api('operator', 'blocked/add', { params: { node: 'a' }, ...form({ domain: 'ads.example', note: 'PRIVATE-NOTE' }) })
+    await api('admin', 'zones/records/add', { params: { node: 'b' }, ...form({ zone: 'example.com', domain: 'www.example.com', type: 'TXT', text: 'SECRET-TXT' }) })
+    await api('admin', 'settings/set', { params: { node: 'a' }, ...form({ tsigKeys: 'k|TOPSECRET|hmac-sha256', proxyPassword: 'PROXYPASS2' }) })
+    b.failWith = 'Denied for SECRET-TXT.'
+    await api('admin', 'zones/delete', { params: { node: 'b', zone: 'example.com' } })
+    const entries = await auditEntries()
+    const mine = entries.filter((e) => /^(blocked|zones|settings)\./.test(e.action))
+    expect(mine.find((e) => e.action === 'blocked.add')).toMatchObject({ user: 'operator', role: 'operator', server: 'a', target: 'ads.example', outcome: 'ok' })
+    expect(mine.find((e) => e.action === 'zones.records.add')).toMatchObject({ user: 'admin', server: 'b', target: 'example.com', outcome: 'ok' })
+    expect(mine.find((e) => e.action === 'settings.set')).toMatchObject({ server: 'a', outcome: 'ok' })
+    expect(mine.find((e) => e.action === 'zones.delete')).toMatchObject({ outcome: 'failed', detail: 'refused by the server' })
+    expect(mine.find((e) => e.action === 'blocked.add')?.ip).toBeTruthy()
+    const raw = readFileSync(path.join(dataDir, 'audit.log'), 'utf8')
+    for (const value of ['PRIVATE-NOTE', 'SECRET-TXT', 'TOPSECRET', 'PROXYPASS2', 'admin-a', 'admin-b', 'ads.example"}']) expect(raw).not.toContain(value)
+  })
+})
+
+describe('settings/set', () => {
+  const unchanged = {
+    webServiceLocalAddresses: '0.0.0.0,[::]',
+    webServiceHttpPort: '5380',
+    webServiceEnableTls: 'false',
+    webServiceTlsCertificatePath: '',
+    webServiceTlsCertificatePassword: '************',
+  }
+
+  it('passes a save that carries the web service block unchanged, as the console sends it', async () => {
+    const r = await api('admin', 'settings/set', { params: { node: 'a' }, ...form({ ...unchanged, enableBlocking: 'false', dnsServerDomain: 'a2.example' }) })
+    expect(r.body.status).toBe('ok')
+    expect(r.body.server).toBe('a')
+    expect(posts(a)[0].form.get('enableBlocking')).toBe('false')
+  })
+
+  it.each([
+    ['webServiceHttpPort', '5381'],
+    ['webServiceLocalAddresses', '127.0.0.1'],
+    ['webServiceEnableTls', 'true'],
+    ['webServiceTlsCertificatePassword', 'new-pass'],
+  ])('refuses a changed %s and forwards nothing', async (name, value) => {
+    const r = await api('admin', 'settings/set', { params: { node: 'a' }, ...form({ ...unchanged, [name]: value, enableBlocking: 'false' }) })
+    expect(r.code).toBe(400)
+    expect(r.body.errorMessage).toBe("Change the web service settings in that server's own console.")
+    expect(posts(a)).toHaveLength(0)
+    const entry = (await auditEntries()).find((e) => e.action === 'settings.set' && e.outcome === 'refused')
+    expect(entry?.detail).toContain(name)
+    expect(entry?.detail).not.toContain(value)
+  })
+
+  it('refuses user, SSO and LDAP fields whatever their value', async () => {
+    for (const field of ['ssoEnabled', 'ldapServer', 'userName', 'authSessionTimeout']) {
+      const r = await api('admin', 'settings/set', { params: { node: 'a' }, ...form({ [field]: 'x' }) })
+      expect(r.code, field).toBe(400)
+    }
+    expect(posts(a)).toHaveLength(0)
+  })
+
+  it('does not save when the current settings cannot be read first', async () => {
+    a.down = true
+    const r = await api('admin', 'settings/set', { params: { node: 'a' }, ...form({ enableBlocking: 'false' }) })
+    expect(r.body.status).toBe('error')
+    expect(r.body.errorMessage).toMatch(/nothing was changed/)
+  })
+
+  it('compares as the console writes (lists, numbers, booleans)', () => {
+    const current = { webServiceLocalAddresses: ['0.0.0.0', '[::]'], webServiceHttpPort: 5380, webServiceEnableTls: false, webServiceRealIpHeader: 'X-Real-IP', other: 'x' }
+    expect(webServiceChanges({ webServiceLocalAddresses: '[::]\n0.0.0.0', webServiceHttpPort: '5380', webServiceEnableTls: 'False', webServiceRealIpHeader: 'X-Real-IP', other: 'changed' }, current)).toEqual([])
+    expect(webServiceChanges({ webServiceHttpPort: '' }, current)).toEqual(['webServiceHttpPort'])
+    expect(webServiceChanges({ webServiceLocalAddresses: 'false' }, current)).toEqual(['webServiceLocalAddresses'])
+    expect(webServiceChanges({ webServiceUnknownField: 'x' }, current)).toEqual([])
+  })
+})
+
+describe('settings/get', () => {
+  it('blanks the secrets for every role but admin', async () => {
+    for (const role of ['viewer', 'operator', 'apps-admin']) {
+      const r = await api(role, 'settings/get', { params: { node: 'a' } })
+      expect(r.body.status).toBe('ok')
+      const s = r.body.response
+      expect(s.tsigKeys).toEqual([{ keyName: 'k', sharedSecret: '', algorithmName: 'hmac-sha256' }])
+      expect(s.proxy).toMatchObject({ address: 'p', username: 'u', password: '' })
+      expect(s.dnsTlsCertificatePassword).toBe('')
+      expect(s.webServiceTlsCertificatePassword).toBe('')
+      expect(s.enableBlocking).toBe(true)
+      expect(r.text).not.toMatch(/TOPSECRET|PROXYPASS|CERTPASS/)
+    }
+    const admin = (await api('admin', 'settings/get', { params: { node: 'a' } })).body.response
+    expect(admin.tsigKeys[0].sharedSecret).toBe('TOPSECRET')
+    expect(admin.proxy.password).toBe('PROXYPASS')
+  })
+
+  it('also blanks the answer given without a node (the first server)', async () => {
+    const r = await api('viewer', 'settings/get')
+    expect(r.body.response.tsigKeys[0].sharedSecret).toBe('')
+  })
+
+  it('does not change what it is given', () => {
+    const input = { tsigKeys: [{ sharedSecret: 's' }], proxy: null }
+    expect(redactSettings(input)).toEqual({ tsigKeys: [{ sharedSecret: '' }], proxy: null })
+    expect(input.tsigKeys[0].sharedSecret).toBe('s')
+    expect(redactSettings(null)).toBeNull()
+  })
+})
+
+describe('dnsClient/resolve', () => {
+  it('is a plain read, but with import it is a zone write needing admin', async () => {
+    expect(effectiveRoute('dnsClient/resolve', {})).toMatchObject({ kind: 'read', capability: 'read' })
+    expect(effectiveRoute('dnsClient/resolve', { import: 'false' })?.kind).toBe('read')
+    for (const flag of ['true', 'True', ' TRUE ', '1', 'yes']) {
+      expect(effectiveRoute('dnsClient/resolve', { import: flag })).toMatchObject({ kind: 'write-one', capability: 'admin' })
+    }
+
+    expect((await api('viewer', 'dnsClient/resolve', { params: { node: 'a', domain: 'example.com', type: 'A' } })).body.status).toBe('ok')
+    expect(a.calls[0]).toMatchObject({ method: 'GET', token: 'read-a' })
+    for (const role of ['viewer', 'operator', 'apps-admin']) {
+      expect((await api(role, 'dnsClient/resolve', { params: { node: 'a', domain: 'example.com', import: 'true' } })).code, role).toBe(403)
+    }
+    expect(posts(a)).toHaveLength(0)
+    expect((await api('admin', 'dnsClient/resolve', { params: { node: 'a', domain: 'example.com', import: 'true' } })).body.status).toBe('ok')
+    expect(posts(a)[0]).toMatchObject({ token: 'admin-a' })
+    expect(posts(a)[0].form.get('import')).toBe('true')
+  })
+})
+
+describe('downloads', () => {
+  it('passes a file through with the reader token', async () => {
+    const r = await api('viewer', 'zones/export', { params: { node: 'a', zone: 'example.com' } })
+    expect(r.code).toBe(200)
+    expect(r.text).toBe('line one\nline two (zones/export)\n')
+    expect(r.headers.get('content-type')).toBe('text/plain')
+    expect(r.headers.get('content-disposition')).toContain('attachment')
+    expect(r.headers.get('x-content-type-options')).toBe('nosniff')
+    expect(a.calls[0]).toMatchObject({ method: 'GET', token: 'read-a' })
+  })
+
+  it('turns the server error envelope into the hub one', async () => {
+    const r = await api('viewer', 'zones/export', { params: { node: 'a', zone: 'missing.example' } })
+    expect(r.body).toEqual({ status: 'error', errorMessage: 'a: No such zone: missing.example' })
+  })
+
+  it('opens a download with a single-use token, once', async () => {
+    const issued = await api('viewer', 'user/createSingleUseToken')
+    const once = issued.body.response.token as string
+    expect(once).toMatch(/^[0-9a-f]{64}$/)
+    const first = await api('viewer', 'logs/download', { auth: false, params: { node: 'a', fileName: '2026-01-01', token: once } })
+    expect(first.text).toContain('logs/download')
+    const second = await api('viewer', 'logs/download', { auth: false, params: { node: 'a', fileName: '2026-01-01', token: once } })
+    expect(second.body.status).toBe('invalid-token')
+  })
+
+  it('does not accept a single-use token for anything but a download', async () => {
+    const once = (await api('admin', 'user/createSingleUseToken')).body.response.token as string
+    const r = await api('admin', 'cache/flush', { auth: false, params: { node: 'a', token: once } })
+    expect(r.body.status).toBe('invalid-token')
+    expect(posts(a)).toHaveLength(0)
+    expect((await api('admin', 'zones/list', { auth: false, params: { node: 'a', token: once } })).body.status).toBe('invalid-token')
+  })
+
+  it('gives no single-use token without a session', async () => {
+    expect((await api('viewer', 'user/createSingleUseToken', { auth: false })).body.status).toBe('invalid-token')
+  })
+})
+
+describe('uploads and large bodies', () => {
+  it('forwards a pasted zone file as text/plain, with the options in the query', async () => {
+    const r = await api('admin', 'zones/import', {
+      params: { node: 'a', zone: 'example.com', overwrite: 'true' },
+      body: '$ORIGIN example.com.\n@ IN SOA ns hostmaster 1 2 3 4 5\n',
+      headers: { 'content-type': 'text/plain' },
+    })
+    expect(r.body.status).toBe('ok')
+    const call = posts(a)[0]
+    expect(call.token).toBe('admin-a')
+    expect(call.contentType).toContain('text/plain')
+    expect(call.body.toString()).toContain('$ORIGIN example.com.')
+    expect(call.query.get('zone')).toBe('example.com')
+    expect(call.query.get('overwrite')).toBe('true')
+  })
+
+  it('forwards a zone file sent as multipart, and a create with no body', async () => {
+    const data = new FormData()
+    data.append('fileImportZone', new Blob(['zone text'], { type: 'text/plain' }), 'example.com.zone')
+    const r = await api('admin', 'zones/create', { params: { node: 'b', zone: 'example.com', type: 'Primary' }, body: data })
+    expect(r.body.status).toBe('ok')
+    const call = posts(b)[0]
+    expect(call.contentType).toContain('multipart/form-data')
+    expect(call.body.toString()).toContain('zone text')
+    expect(call.body.toString()).toContain('name="fileImportZone"')
+    expect(call.query.get('zone')).toBe('example.com')
+
+    await api('admin', 'zones/create', { params: { node: 'a', zone: 'plain.example', type: 'Primary' }, method: 'POST' })
+    expect(posts(a)[0].form.get('zone')).toBe('plain.example')
+  })
+
+  it('does not take an upload from a role that may not write zones', async () => {
+    const r = await api('operator', 'zones/import', { params: { node: 'a', zone: 'example.com' }, body: 'x', headers: { 'content-type': 'text/plain' } })
+    expect(r.code).toBe(403)
+    expect(posts(a)).toHaveLength(0)
+  })
+
+  it('takes a long TXT record from a signed-in caller, and a small limit from anyone else', async () => {
+    const text = 'v=spf1 ' + 'a'.repeat(200_000)
+    const r = await api('admin', 'zones/records/add', { params: { node: 'a' }, ...form({ zone: 'example.com', domain: 'www.example.com', type: 'TXT', text }) })
+    expect(r.body.status).toBe('ok')
+    expect(posts(a)[0].form.get('text')).toBe(text)
+
+    // Not signed in: the small default applies (the hub drops the connection), and nothing reaches the server.
+    const anon = await api('admin', 'zones/records/add', { auth: false, params: { node: 'a' }, ...form({ zone: 'example.com', text }) }).catch(() => null)
+    expect(anon?.body.status).not.toBe('ok')
+    expect(posts(a)).toHaveLength(1)
+  })
+
+  it('takes a block list import of a few MB', async () => {
+    const list = Array.from({ length: 150_000 }, (_, i) => `d${i}.example`).join(',')
+    const r = await api('operator', 'blocked/import', { params: { node: 'a' }, ...form({ blockedZones: list }) })
+    expect(r.body.status).toBe('ok')
+    expect(posts(a)[0].form.get('blockedZones')).toBe(list)
+  })
+})
+
+describe('the table', () => {
+  it('has every area and none of the refused paths', () => {
+    expect(routeFor('zones/records/add')).toMatchObject({ kind: 'write-one', capability: 'admin' })
+    expect(routeFor('cache/flush')).toMatchObject({ kind: 'write-one', capability: 'operate' })
+    expect(routeFor('settings/temporaryDisableBlocking')).toMatchObject({ kind: 'write-one', capability: 'operate' })
+    expect(routeFor('settings/get')).toEqual({ kind: 'read', capability: 'read' })
+    expect(routeFor('zones/export')).toMatchObject({ kind: 'read', raw: true, perServer: true })
+    for (const p of ['logs/delete', 'logs/deleteAll', 'dashboard/stats/deleteAll', 'settings/backup', 'settings/restore', 'admin/users/list', 'user/createToken']) {
+      expect(routeFor(p), p).toBeUndefined()
+    }
+    expect(safeTarget({ zone: 'example.com', text: 'x' })).toBe('example.com')
+    expect(safeTarget({ text: 'x' })).toBeUndefined()
+  })
+})

hub/server.test.ts

@@ -77,6 +77,7 @@ let b: Fake
 let hub: http.Server
 let base: string
 let staticDir: string
+let dataDir: string
 
 function makeConfig(over: Record<string, string> = {}) {
   return parseConfig({
@@ -86,6 +87,7 @@ function makeConfig(over: Record<string, string> = {}) {
     HUB_USER: 'admin',
     HUB_PASSWORD: 'secret',
     HUB_STATIC: staticDir,
+    HUB_DATA: dataDir,
     HUB_CACHE_SECONDS: '15',
     HUB_TIMEOUT_MS: '2000',
     ...over,
@@ -117,6 +119,7 @@ async function login(): Promise<string> {
 beforeAll(async () => {
   a = await fakeUpstream(canned(100, ['2026-01-01T10:00:04Z', '2026-01-01T10:00:02Z']))
   b = await fakeUpstream(canned(300, ['2026-01-01T10:00:03Z', '2026-01-01T10:00:01Z']))
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-data-'))
   staticDir = mkdtempSync(path.join(tmpdir(), 'hub-static-'))
   writeFileSync(path.join(staticDir, 'index.html'), '<html>root</html>')
   mkdirSync(path.join(staticDir, 'logs', 'query-logs'), { recursive: true })
@@ -129,6 +132,7 @@ afterAll(() => {
   a.server.close()
   b.server.close()
   rmSync(staticDir, { recursive: true, force: true })
+  rmSync(dataDir, { recursive: true, force: true })
 })
 
 beforeEach(async () => {
@@ -142,9 +146,9 @@ beforeEach(async () => {
 })
 
 describe('config', () => {
-  it('refuses to start without credentials or a token', () => {
+  it('refuses to start with half the credentials or without a token', () => {
     const env = { HUB_SERVERS: 'a=http://x', A_TOKEN: 't' }
-    expect(() => parseConfig(env)).toThrow(/HUB_USER/)
+    expect(() => parseConfig({ ...env, HUB_USER: 'u' })).toThrow(/HUB_USER/)
     expect(() => parseConfig({ ...env, HUB_USER: 'u', HUB_PASSWORD: 'p', A_TOKEN: '' })).toThrow(/A_TOKEN/)
   })
 
@@ -166,7 +170,8 @@ describe('session', () => {
     expect(body.info.clusterNodes).toEqual([{ name: 'a', type: 'Primary' }, { name: 'b', type: 'Secondary' }])
     expect(body.info.permissions.Dashboard.canView).toBe(true)
     expect(body.info.permissions.Logs.canView).toBe(true)
-    expect(body.info.permissions.Zones).toEqual({ canView: false, canModify: false, canDelete: false })
+    expect(body.info.permissions.Zones).toEqual({ canView: true, canModify: true, canDelete: true })
+    expect(body.info.permissions.Administration).toEqual({ canView: false, canModify: false, canDelete: false })
   })
 
   it('refuses a wrong password', async () => {
@@ -192,7 +197,7 @@ describe('session', () => {
 describe('read-only', () => {
   it('refuses everything that is not a listed read', async () => {
     const token = await login()
-    const { code, body } = await call('zones/delete', { zone: 'x' }, token, 'POST')
+    const { code, body } = await call('admin/users/delete', { user: 'x' }, token, 'POST')
     expect(code).toBe(403)
     expect(body).toEqual({ status: 'error', errorMessage: 'The hub is read-only.' })
     expect(a.calls).toHaveLength(0)

hub/server.ts

@@ -4,19 +4,39 @@ answers from several real ones.
 
     node --experimental-strip-types hub/server.ts
 
-It serves the console's static build and a read-only, Technitium-compatible `/api/*`.
-A call aimed at one server (`node=<id>`) is forwarded to it with that server's token; a
-call with no node (or `node=cluster`) asks every server in parallel and merges the
-answers (see merge.ts). Anything that would change data is refused.
+It serves the console's static build and a Technitium-compatible `/api/*`. A read aimed at
+one server (`node=<id>`) is forwarded to it with that server's token; a read with no node (or
+`node=cluster`) asks every server in parallel and merges the answers (see merge.ts), and the
+reads that have no merger (`perServer` in routes.ts) are refused without a node. A change
+is made on ONE chosen server with its admin token, or by the app and config endpoints; whatever
+is not in the table of routes.ts is refused.
 */
 import http from 'node:http'
 import { createReadStream } from 'node:fs'
 import { stat } from 'node:fs/promises'
 import path from 'node:path'
+import { Readable } from 'node:stream'
+import type { ReadableStream as WebStream } from 'node:stream/web'
 import { fileURLToPath } from 'node:url'
 import { loadConfig, HUB_VERSION, type HubConfig, type HubServer } from './config.ts'
 import { Upstream, UpstreamError, type Params } from './upstream.ts'
 import { Sessions } from './session.ts'
+import { Users, can } from './users.ts'
+import { LoginLimiter, lockedMessage } from './ratelimit.ts'
+import { Audit } from './audit.ts'
+import { effectiveRoute, largeBodyLimit, routeFor, type Route } from './routes.ts'
+import { WEB_SERVICE_MESSAGE, isForbiddenSettingsField, redactSettings, safeTarget, webServiceChanges } from './guards.ts'
+import { buildMatrix } from './apps-matrix.ts'
+import { ServersRunner, planAction } from './servers-run.ts'
+import { AppsWriter, actionForPath, type FileAction, type TargetResult } from './apps-write.ts'
+import { TooLarge, parseMultipart, readLimited, type UploadedFile } from './multipart.ts'
+import { AppsConfig, UserError } from './apps-config.ts'
+import { Drift } from './drift.ts'
+import { Webhook, type WebhookOptions } from './webhook.ts'
+import { Profiles } from './profiles.ts'
+import { SettingsConfig } from './settings-config.ts'
+import { BLOCKING_FIELDS } from './settings-sync.ts'
+import { ZoneUserError, ZonesSync, buildZoneMatrix } from './zones-sync.ts'
 import {
   mergeQueryLogs,
   mergeSettled,
@@ -31,19 +51,10 @@ import {
 type Json = Record<string, unknown>
 
 const MAX_BODY = 64 * 1024
-
-/* Paths relative to `/api/`. Anything not listed here is refused with 403. */
-const READ_ENDPOINTS = new Set([
-  'dashboard/stats/get',
-  'dashboard/stats/getTop',
-  'dashboard/metrics/json',
-  'apps/list',
-  'logs/query',
-  'logs/list',
-  'settings/get',
-  'user/checkForUpdate',
-  'admin/cluster/state',
-])
+/* The config endpoints carry config texts (up to 1 MB). */
+const MAX_CONFIG_BODY = 2 * 1024 * 1024
+/* A file of all baselines. */
+const MAX_IMPORT_BODY = 5 * 1024 * 1024
 
 const TYPES: Record<string, string> = {
   '.html': 'text/html; charset=utf-8',
@@ -78,20 +89,41 @@ function send(res: http.ServerResponse, code: number, body: unknown): void {
 
 const fail = (errorMessage: string): Json => ({ status: 'error', errorMessage })
 
-async function readForm(req: http.IncomingMessage): Promise<URLSearchParams> {
+async function readForm(req: http.IncomingMessage, limit: number = MAX_BODY): Promise<URLSearchParams> {
   if (req.method !== 'POST') return new URLSearchParams()
   const chunks: Buffer[] = []
   let size = 0
   for await (const chunk of req) {
     size += (chunk as Buffer).length
-    if (size > MAX_BODY) break
+    if (size > limit) break
     chunks.push(chunk as Buffer)
   }
   const type = String(req.headers['content-type'] ?? '')
-  if (size > MAX_BODY || !type.includes('application/x-www-form-urlencoded')) return new URLSearchParams()
-  return new URLSearchParams(Buffer.concat(chunks).toString('utf8'))
+  if (size > limit) return new URLSearchParams()
+  const text = Buffer.concat(chunks).toString('utf8')
+  if (type.includes('application/x-www-form-urlencoded')) return new URLSearchParams(text)
+  if (type.includes('application/json')) {
+    // A flat object; an array becomes a comma-separated list, as the form fields are.
+    const out = new URLSearchParams()
+    try {
+      const data = JSON.parse(text) as Record<string, unknown>
+      for (const [k, v] of Object.entries(data)) {
+        if (Array.isArray(v)) out.set(k, v.some((x) => x !== null && typeof x === 'object') ? JSON.stringify(v) : v.map(String).join(','))
+        else if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') out.set(k, String(v))
+        else if (v !== null && typeof v === 'object') out.set(k, JSON.stringify(v))
+      }
+    } catch {
+      // Not JSON: no parameters.
+    }
+    return out
+  }
+  return new URLSearchParams()
 }
 
+const isMultipart = (req: http.IncomingMessage): boolean => String(req.headers['content-type'] ?? '').toLowerCase().startsWith('multipart/form-data')
+
+const csv = (value: string | undefined): string[] => (value ?? '').split(',').map((v) => v.trim()).filter((v) => v !== '')
+
 function bearer(req: http.IncomingMessage): string | null {
   const header = String(req.headers.authorization ?? '')
   return header.startsWith('Bearer ') ? header.slice(7).trim() : null
@@ -102,9 +134,40 @@ function positive(value: string | undefined, fallback: number): number {
   return Number.isInteger(n) && n > 0 ? n : fallback
 }
 
-export function createHub(config: HubConfig): http.Server {
+/* The client's address: the socket's, or with HUB_TRUST_PROXY the first X-Forwarded-For entry. */
+function clientIp(req: http.IncomingMessage, trustProxy: boolean): string {
+  if (trustProxy) {
+    const forwarded = String(req.headers['x-forwarded-for'] ?? '').split(',')[0].trim()
+    if (forwarded !== '') return forwarded
+  }
+  return req.socket.remoteAddress ?? 'unknown'
+}
+
+export function createHub(config: HubConfig, users: Users = Users.load(config.dataDir, config.bootstrap), options: { webhook?: WebhookOptions } = {}): http.Server {
   const upstream = new Upstream(config)
-  const sessions = new Sessions(config)
+  const webhook = new Webhook(config, options.webhook)
+  const drift = new Drift(config, upstream, Date.now, { webhook })
+  const sessions = new Sessions(config, users, Date.now, () => drift.count())
+  const limiter = new LoginLimiter()
+  const audit = new Audit(config.dataDir)
+  // A failed write is told to the webhook: who, what, where and the audited reason (never a value). Logins and the test itself are left out.
+  audit.onEntry((e) => {
+    if (e.outcome !== 'failed' || e.action === 'login' || e.action.startsWith('webhook.')) return
+    webhook.queueFailed({
+      action: e.action,
+      ...(e.server === undefined ? {} : { server: e.server }),
+      ...(e.target === undefined ? {} : { target: e.target }),
+      user: e.user,
+      ...(e.detail === undefined ? {} : { detail: e.detail.slice(0, 200) }),
+      at: e.ts,
+    })
+  })
+  const writer = new AppsWriter(config, upstream)
+  const runner = new ServersRunner(config, upstream)
+  const appsConfig = new AppsConfig(config, upstream)
+  const zonesSync = new ZonesSync(config, upstream)
+  const settingsConfig = new SettingsConfig(config, upstream)
+  const profiles = new Profiles(config, upstream, appsConfig, settingsConfig, writer)
   const byId = new Map(config.servers.map((s) => [s.id, s]))
 
   /* Ask every server; the ones that fail become warnings (see mergeSettled). */
@@ -159,6 +222,16 @@ export function createHub(config: HubConfig): http.Server {
           const pages = data.map((page, i) => ({ node: answered[i], page }))
           return ok(mergeQueryLogs(pages, { pageNumber, entriesPerPage, descending: params.descendingOrder !== 'false' }), warnings)
         }
+        case 'apps/listStoreApps': {
+          // The first answering server's listing, each app tagged with the servers it is installed on.
+          const { data, warnings } = mergeSettled(await everywhere<{ storeApps?: { name: string }[] }>(apiPath, params))
+          const installedOn = new Map<string, string[]>()
+          for (const r of await everywhere<{ apps?: { name: string }[] }>('apps/list', {})) {
+            if (r.ok) for (const a of r.data.apps ?? []) installedOn.set(a.name, [...(installedOn.get(a.name) ?? []), r.node])
+          }
+          const storeApps = (data[0].storeApps ?? []).map((a) => ({ ...a, installedOn: installedOn.get(a.name) ?? [] }))
+          return ok({ ...data[0], storeApps }, warnings)
+        }
         case 'apps/list': {
           const { data, warnings } = mergeSettled(await everywhere<{ apps?: { name: string }[] }>(apiPath, params))
           const seen = new Map<string, { name: string }>()
@@ -175,8 +248,589 @@ export function createHub(config: HubConfig): http.Server {
     }
   }
 
+  type Who = { user: string; role: string }
+
+  /* One audit line per target: who, which server, which app, what happened. */
+  function auditResults(who: Who, ip: string, action: string, name: string, results: TargetResult[]): void {
+    for (const r of results) {
+      audit.log({
+        user: who.user, role: who.role, ip, action: `apps.${action}`, server: r.server, target: name,
+        outcome: r.ok ? 'ok' : 'failed', detail: r.message,
+      })
+    }
+  }
+
+  async function uploadOf(req: http.IncomingMessage, res: http.ServerResponse, params: Params): Promise<UploadedFile | null> {
+    let parsed
+    try {
+      parsed = parseMultipart(await readLimited(req), String(req.headers['content-type'] ?? ''))
+    } catch (e) {
+      if (e instanceof TooLarge) {
+        res.setHeader('connection', 'close')
+        send(res, 413, fail(e.message))
+      } else send(res, 400, fail('The upload could not be read.'))
+      return null
+    }
+    Object.assign(params, parsed.fields)
+    const file = parsed.files[0]
+    if (file === undefined || file.data.length === 0) {
+      send(res, 400, fail('The zip file is missing.'))
+      return null
+    }
+    return file
+  }
+
+  /* A value from a JSON-encoded parameter, or the fallback. */
+  function jsonParam(value: string | undefined, fallback: unknown): unknown {
+    if (value === undefined || value === '') return fallback
+    try {
+      return JSON.parse(value)
+    } catch {
+      throw new UserError('A parameter is not valid JSON.')
+    }
+  }
+
+  function userFailure(res: http.ServerResponse, e: unknown): void {
+    if (e instanceof UserError) send(res, 400, fail(e.message))
+    else throw e
+  }
+
+  /* compare, preview and the live local values: no change is made. */
+  async function configRead(res: http.ServerResponse, apiPath: string, params: Params, who: Who, ip: string): Promise<void> {
+    const name = params.name ?? ''
+    try {
+      if (apiPath === 'hub/apps/export') {
+        const file = appsConfig.exportAll()
+        audit.log({ user: who.user, role: who.role, ip, action: 'apps.export', outcome: 'ok', detail: `${file.apps.length} app(s)` })
+        const text = JSON.stringify(file, null, 2) + '\n'
+        res.writeHead(200, {
+          'content-type': 'application/json; charset=utf-8',
+          'content-disposition': `attachment; filename="hub-baselines-${file.exportedAt.slice(0, 10)}.json"`,
+          'cache-control': 'no-store',
+          'content-length': Buffer.byteLength(text),
+        })
+        res.end(text)
+        return
+      }
+      if (name === '') throw new UserError('An app name is required.')
+      if (apiPath === 'hub/apps/config/compare') send(res, 200, ok(await appsConfig.compare(name)))
+      else if (apiPath === 'hub/apps/config/preview') {
+        const preview = await appsConfig.preview(name, csv(params.targets))
+        audit.log({ user: who.user, role: who.role, ip, action: 'apps.config-preview', target: name, outcome: 'ok', detail: csv(params.targets).join(',') })
+        send(res, 200, ok(preview))
+      } else if (apiPath === 'hub/apps/history/list') {
+        send(res, 200, ok({ entries: appsConfig.historyList(name) }))
+      } else if (apiPath === 'hub/apps/history/get') {
+        send(res, 200, ok(appsConfig.historyGet(name, params.id ?? '')))
+      } else if (apiPath === 'hub/apps/history/diff') {
+        send(res, 200, ok(await appsConfig.historyDiff(name, params.id ?? '', params.server ?? '')))
+      } else if (apiPath === 'hub/apps/overrides/current') {
+        send(res, 200, ok({ values: await appsConfig.currentLocalValues(name, csv(params.paths)) }))
+      } else send(res, 403, fail('The hub is read-only.'))
+    } catch (e) {
+      userFailure(res, e)
+    }
+  }
+
+  /* baseline import and save, local fields save, apply. */
+  async function configWrite(res: http.ServerResponse, apiPath: string, params: Params, who: Who, ip: string): Promise<void> {
+    const name = params.name ?? ''
+    const base = { user: who.user, role: who.role, ip, target: name === '' ? undefined : name }
+    const action = apiPath.replace('hub/apps/', 'apps.').replace(/\//g, '-')
+    try {
+      if (apiPath === 'hub/apps/import') {
+        const done = appsConfig.importAll(params.data ?? '', params.apply === 'true', who.user)
+        const count = (status: string) => done.apps.filter((a) => a.status === status).length
+        audit.log({ ...base, action: 'apps.import', outcome: 'ok', detail: `${done.applied ? 'applied' : 'dry run'}: ${count('new')} new, ${count('changed')} changed, ${count('same')} same` })
+        send(res, 200, ok(done))
+        return
+      }
+      if (name === '' || name.length > 200) throw new UserError('An app name is required.')
+      if (apiPath === 'hub/apps/baseline/import') {
+        const done = await appsConfig.importBaseline(name, params.server ?? '', who.user)
+        audit.log({ ...base, action, server: params.server, outcome: 'ok', detail: done.detail })
+        send(res, 200, ok({ baseline: appsConfig.getBaseline(name) }))
+      } else if (apiPath === 'hub/apps/baseline/save') {
+        const done = appsConfig.saveBaseline(name, params.text ?? '', who.user)
+        audit.log({ ...base, action, outcome: 'ok', detail: done.detail })
+        send(res, 200, ok({ baseline: appsConfig.getBaseline(name) }))
+      } else if (apiPath === 'hub/apps/baseline/restore') {
+        const done = appsConfig.restoreBaseline(name, params.id ?? '', who.user)
+        audit.log({ ...base, action, outcome: 'ok', detail: done.detail })
+        send(res, 200, ok({ baseline: appsConfig.getBaseline(name) }))
+      } else if (apiPath === 'hub/apps/drift/accept') {
+        const done = await appsConfig.acceptDrift(name, params.server ?? '', who.user)
+        audit.log({ ...base, action: 'apps.drift-accept', server: params.server, outcome: 'ok', detail: done.detail })
+        await drift.check()
+        send(res, 200, ok({ baseline: appsConfig.getBaseline(name), overrides: appsConfig.getOverrides(name) }))
+      } else if (apiPath === 'hub/apps/history/rollback') {
+        const expect = jsonParam(params.expect, {}) as Record<string, string>
+        const done = await appsConfig.rollback(name, params.id ?? '', csv(params.targets), expect)
+        for (const r of done.results) {
+          audit.log({ ...base, action: 'apps.history-rollback', server: r.server, outcome: r.ok ? 'ok' : 'failed', detail: [r.message, done.details[r.server]].filter(Boolean).join(' | ') })
+        }
+        send(res, 200, ok({ results: done.results }))
+      } else if (apiPath === 'hub/apps/overrides/save') {
+        const done = appsConfig.saveOverrides(name, csv(params.paths), jsonParam(params.values, {}))
+        audit.log({ ...base, action, outcome: 'ok', detail: done.detail })
+        send(res, 200, ok({ overrides: appsConfig.getOverrides(name) }))
+      } else if (apiPath === 'hub/apps/config/apply') {
+        const expect = jsonParam(params.expect, {}) as Record<string, string>
+        const done = await appsConfig.apply(name, csv(params.targets), expect)
+        for (const r of done.results) {
+          audit.log({ ...base, action: 'apps.config-apply', server: r.server, outcome: r.ok ? 'ok' : 'failed', detail: [r.message, done.details[r.server]].filter(Boolean).join(' | ') })
+        }
+        send(res, 200, ok({ results: done.results }))
+      } else send(res, 403, fail('The hub is read-only.'))
+    } catch (e) {
+      if (e instanceof UserError) {
+        audit.log({ ...base, action, outcome: 'refused', detail: apiPath.endsWith('baseline/save') ? 'invalid baseline' : e.message.slice(0, 120) })
+      }
+      userFailure(res, e)
+    }
+  }
+
+  /* The merged zone list and the previews of copying a zone and of making it a Secondary: no change is made. */
+  async function zonesRead(res: http.ServerResponse, apiPath: string, params: Params): Promise<void> {
+    try {
+      if (apiPath === 'hub/zones/matrix') {
+        const { matrix, warnings } = await buildZoneMatrix(config, upstream)
+        send(res, 200, ok(matrix, warnings))
+      } else if (apiPath === 'hub/zones/copy/preview') {
+        send(res, 200, ok(await zonesSync.copyPreview(params.source ?? '', params.zone ?? '', csv(params.targets))))
+      } else if (apiPath === 'hub/zones/secondary/preview') {
+        send(res, 200, ok(await zonesSync.secondaryPreview(secondaryInput(params))))
+      } else send(res, 403, fail('The hub is read-only.'))
+    } catch (e) {
+      if (e instanceof ZoneUserError) send(res, 400, fail(e.message))
+      else if (e instanceof Error && !(e instanceof UpstreamError)) send(res, 200, fail(e.message))
+      else throw e
+    }
+  }
+
+  const secondaryInput = (params: Params) => ({
+    source: params.source ?? '',
+    target: params.target ?? '',
+    zone: params.zone ?? '',
+    primaryAddress: params.primaryAddress ?? '',
+    targetAddress: params.targetAddress ?? '',
+    tsigKeyName: params.tsigKeyName,
+  })
+
+  /* Copy a zone and make a Secondary. One audit line per server: zone, source, target, outcome. */
+  async function zonesWrite(res: http.ServerResponse, apiPath: string, params: Params, who: Who, ip: string): Promise<void> {
+    const copy = apiPath === 'hub/zones/copy'
+    const line = { user: who.user, role: who.role, ip, action: copy ? 'zones.copy' : 'zones.secondary', target: (params.zone ?? '').slice(0, 120) || undefined }
+    try {
+      const done = copy
+        ? await zonesSync.copy(params.source ?? '', params.zone ?? '', csv(params.targets), csv(params.overwrite))
+        : await zonesSync.makeSecondary(secondaryInput(params))
+      for (const r of done.results) {
+        audit.log({ ...line, server: r.server, outcome: r.ok ? 'ok' : 'failed', detail: `from ${params.source ?? ''}${copy ? '' : ` to ${params.target ?? ''}`}: ${r.message ?? ''}`.slice(0, 300) })
+      }
+      send(res, 200, ok({ results: done.results }))
+    } catch (e) {
+      if (!(e instanceof ZoneUserError)) throw e
+      audit.log({ ...line, outcome: 'refused', detail: e.message.slice(0, 200) })
+      send(res, 400, fail(e.message))
+    }
+  }
+
+  /* `group=blocking` limits a settings preview or apply to the blocking fields. */
+  const onlyOf = (params: Params): readonly string[] | undefined => (params.group === 'blocking' ? BLOCKING_FIELDS : undefined)
+
+  const userMessage = (e: unknown): string | null => (e instanceof UserError || e instanceof ZoneUserError ? e.message : null)
+
+  /* Settings and blocking lists: compare, preview, history. No change is made. */
+  async function settingsRead(res: http.ServerResponse, apiPath: string, params: Params, who: Who, ip: string): Promise<void> {
+    try {
+      if (apiPath === 'hub/settings/compare') send(res, 200, ok(await settingsConfig.compare()))
+      else if (apiPath === 'hub/settings/preview') {
+        const preview = await settingsConfig.preview(csv(params.targets), onlyOf(params))
+        audit.log({ user: who.user, role: who.role, ip, action: 'settings.preview', outcome: 'ok', detail: csv(params.targets).join(',') })
+        send(res, 200, ok(preview))
+      } else if (apiPath === 'hub/settings/history/list') send(res, 200, ok({ entries: settingsConfig.historyList() }))
+      else if (apiPath === 'hub/settings/history/get') send(res, 200, ok(settingsConfig.historyGet(params.id ?? '')))
+      else if (apiPath === 'hub/blocking/lists/compare') send(res, 200, ok(await settingsConfig.listsCompare()))
+      else if (apiPath === 'hub/blocking/lists/preview') send(res, 200, ok(await settingsConfig.listsPreview(csv(params.targets), params.removeExtras === 'true')))
+      else send(res, 403, fail('The hub is read-only.'))
+    } catch (e) {
+      const message = userMessage(e)
+      if (message === null) throw e
+      send(res, 400, fail(message))
+    }
+  }
+
+  /* Baseline, local fields and apply for settings; baseline and apply for the lists. Audit lines carry field names, counts and outcomes, never a value. */
+  async function settingsWrite(res: http.ServerResponse, apiPath: string, params: Params, who: Who, ip: string): Promise<void> {
+    const action = apiPath.replace('hub/', '').replace(/\//g, '.')
+    const base = { user: who.user, role: who.role, ip, action }
+    try {
+      if (apiPath === 'hub/settings/baseline/import') {
+        const done = await settingsConfig.importBaseline(params.server ?? '', params.group, who.user)
+        audit.log({ ...base, server: params.server, outcome: 'ok', detail: done.detail })
+        send(res, 200, ok({ baseline: settingsConfig.baselineView() }))
+      } else if (apiPath === 'hub/settings/baseline/save') {
+        const done = settingsConfig.saveBaseline(params.text ?? '', who.user)
+        audit.log({ ...base, outcome: 'ok', detail: done.detail })
+        send(res, 200, ok({ baseline: settingsConfig.baselineView() }))
+      } else if (apiPath === 'hub/settings/overrides/save') {
+        const done = settingsConfig.saveOverrides(csv(params.paths))
+        audit.log({ ...base, outcome: 'ok', detail: done.detail })
+        send(res, 200, ok({ overrides: { paths: csv(params.paths) } }))
+      } else if (apiPath === 'hub/settings/apply') {
+        const expect = jsonParam(params.expect, {}) as Record<string, string>
+        const done = await settingsConfig.apply(csv(params.targets), expect, onlyOf(params))
+        for (const r of done.results) {
+          audit.log({ ...base, server: r.server, outcome: r.ok ? 'ok' : 'failed', detail: [r.message, done.details[r.server] === undefined ? '' : `fields: ${done.details[r.server]}`].filter(Boolean).join(' | ').slice(0, 400) })
+        }
+        send(res, 200, ok({ results: done.results }))
+      } else if (apiPath === 'hub/blocking/lists/baseline/import') {
+        const done = await settingsConfig.listsImport(params.server ?? '', who.user)
+        audit.log({ ...base, server: params.server, outcome: 'ok', detail: done.detail })
+        send(res, 200, ok({ imported: true }))
+      } else if (apiPath === 'hub/blocking/lists/apply') {
+        const done = await settingsConfig.listsApply(csv(params.targets), params.removeExtras === 'true')
+        for (const r of done.results) {
+          audit.log({ ...base, server: r.server, outcome: r.ok ? 'ok' : 'failed', detail: [r.message, done.details[r.server]].filter(Boolean).join(' | ').slice(0, 400) })
+        }
+        send(res, 200, ok({ results: done.results }))
+      } else send(res, 403, fail('The hub is read-only.'))
+    } catch (e) {
+      const message = userMessage(e)
+      if (message === null) throw e
+      audit.log({ ...base, outcome: 'refused', detail: apiPath.endsWith('baseline/save') ? 'invalid baseline' : message.slice(0, 120) })
+      send(res, 400, fail(message))
+    }
+  }
+
+  /* Profiles (profiles.ts): list, view, export and the preview. No change is made. */
+  async function profilesRead(res: http.ServerResponse, apiPath: string, params: Params, who: Who, ip: string): Promise<void> {
+    const line = { user: who.user, role: who.role, ip, action: apiPath.replace('hub/', '').replace(/\//g, '.') }
+    try {
+      if (apiPath === 'hub/profiles/list') send(res, 200, ok({ profiles: profiles.list() }))
+      else if (apiPath === 'hub/profiles/get') send(res, 200, ok(profiles.get(params.name ?? '')))
+      else if (apiPath === 'hub/profiles/preview') {
+        const preview = await profiles.preview(params.name ?? '', csv(params.targets), params.removeExtras === 'true')
+        audit.log({ ...line, target: params.name, outcome: 'ok', detail: csv(params.targets).join(',') })
+        send(res, 200, ok(preview))
+      } else if (apiPath === 'hub/profiles/export') {
+        const file = profiles.exportFile(params.name)
+        audit.log({ ...line, target: params.name || undefined, outcome: 'ok', detail: `${file.profiles.length} profile(s)` })
+        const text = JSON.stringify(file, null, 2) + '\n'
+        res.writeHead(200, {
+          'content-type': 'application/json; charset=utf-8',
+          'content-disposition': `attachment; filename="hub-profiles-${file.exportedAt.slice(0, 10)}.json"`,
+          'cache-control': 'no-store',
+          'content-length': Buffer.byteLength(text),
+        })
+        res.end(text)
+      } else send(res, 403, fail('The hub is read-only.'))
+    } catch (e) {
+      const message = userMessage(e)
+      if (message === null) throw e
+      send(res, 400, fail(message))
+    }
+  }
+
+  /* Make, delete, import and apply a profile. Audit lines carry names, counts and the outcome of each step, never a value. */
+  async function profilesWrite(res: http.ServerResponse, apiPath: string, params: Params, who: Who, ip: string): Promise<void> {
+    const line = { user: who.user, role: who.role, ip, action: apiPath.replace('hub/', '').replace(/\//g, '.'), target: (params.name ?? '').slice(0, 60) || undefined }
+    try {
+      if (apiPath === 'hub/profiles/snapshot') {
+        const done = await profiles.snapshot(params.name ?? '', { apps: params.apps === undefined ? undefined : csv(params.apps), settings: params.settings === 'true', lists: params.lists === 'true' }, who.user)
+        audit.log({ ...line, outcome: 'ok', detail: done.detail })
+        send(res, 200, ok({ profile: profiles.get(params.name ?? ''), replaced: done.replaced }))
+      } else if (apiPath === 'hub/profiles/delete') {
+        profiles.delete(params.name ?? '')
+        audit.log({ ...line, outcome: 'ok' })
+        send(res, 200, ok({ deleted: true }))
+      } else if (apiPath === 'hub/profiles/import') {
+        const done = profiles.importFile(params.data ?? '', params.apply === 'true', who.user)
+        const count = (status: string) => done.profiles.filter((p) => p.status === status).length
+        audit.log({ ...line, target: undefined, outcome: 'ok', detail: `${done.applied ? 'applied' : 'dry run'}: ${count('new')} new, ${count('changed')} changed, ${count('same')} same` })
+        send(res, 200, ok(done))
+      } else if (apiPath === 'hub/profiles/apply') {
+        const from = jsonParam(params.from, {}) as Record<string, string>
+        const done = await profiles.apply(params.name ?? '', csv(params.targets), { removeExtras: params.removeExtras === 'true', from: from !== null && typeof from === 'object' ? from : {} })
+        for (const r of done.results) {
+          for (const s of r.steps.length > 0 ? r.steps : [{ step: 'apps', status: r.ok ? 'ok' : 'failed', message: r.message }]) {
+            if (s.status === 'not-run' || s.status === 'skipped') continue
+            audit.log({ ...line, server: r.server, outcome: s.status === 'ok' ? 'ok' : 'failed', detail: `${s.step}: ${s.message}`.slice(0, 300) })
+          }
+        }
+        send(res, 200, ok({ results: done.results }))
+      } else send(res, 403, fail('The hub is read-only.'))
+    } catch (e) {
+      const message = userMessage(e)
+      if (message === null) throw e
+      audit.log({ ...line, outcome: 'refused', detail: message.slice(0, 120) })
+      send(res, 400, fail(message))
+    }
+  }
+
+  /* The write endpoints. The role has already been checked against the route's capability. */
+  async function write(req: http.IncomingMessage, res: http.ServerResponse, apiPath: string, route: Route, params: Params, who: Who, ip: string): Promise<void> {
+    if (route.kind === 'write-all') {
+      if (req.method !== 'POST') {
+        send(res, 405, fail('Use POST.'))
+        return
+      }
+      if (apiPath === 'hub/webhook/test') {
+        const sent = await webhook.deliver('webhook.test', [{ message: 'A test notice from the hub.', by: who.user }])
+        audit.log({ user: who.user, role: who.role, ip, action: 'webhook.test', outcome: sent.ok ? 'ok' : 'failed', detail: sent.ok ? undefined : sent.error })
+        send(res, 200, ok(webhook.status()))
+        return
+      }
+      if (apiPath === 'hub/drift/check') {
+        const state = await drift.check()
+        audit.log({ user: who.user, role: who.role, ip, action: 'drift.check', outcome: 'ok', detail: `${state.apps.length} finding(s), ${drift.count()} differ` })
+        send(res, 200, ok(drift.get()))
+        return
+      }
+      if (apiPath.startsWith('hub/apps/baseline/') || apiPath.startsWith('hub/apps/overrides/') || apiPath.startsWith('hub/apps/history/') || apiPath.startsWith('hub/apps/drift/') || apiPath === 'hub/apps/import' || apiPath === 'hub/apps/config/apply') {
+        await configWrite(res, apiPath, params, who, ip)
+        return
+      }
+      if (apiPath.startsWith('hub/profiles/')) {
+        await profilesWrite(res, apiPath, params, who, ip)
+        return
+      }
+      if (apiPath.startsWith('hub/settings/') || apiPath.startsWith('hub/blocking/')) {
+        await settingsWrite(res, apiPath, params, who, ip)
+        return
+      }
+      if (apiPath === 'hub/zones/copy' || apiPath === 'hub/zones/secondary') {
+        await zonesWrite(res, apiPath, params, who, ip)
+        return
+      }
+      if (apiPath === 'hub/servers/run') {
+        const planned = planAction(params.action, params)
+        const line = { user: who.user, role: who.role, ip, action: 'error' in planned ? 'servers.run' : `servers.${planned.plan.action}` }
+        if ('error' in planned) {
+          audit.log({ ...line, outcome: 'refused', detail: planned.error })
+          send(res, 400, fail(planned.error))
+          return
+        }
+        const outcome = await runner.run(planned.plan, csv(params.targets))
+        if ('error' in outcome) {
+          audit.log({ ...line, target: planned.plan.target, outcome: 'refused', detail: outcome.error })
+          send(res, 400, fail(outcome.error))
+          return
+        }
+        // One line per server. The target is a domain or a count, never the list of URLs.
+        for (const r of outcome.results) {
+          audit.log({ ...line, server: r.server, target: planned.plan.target, outcome: r.ok ? 'ok' : 'failed', detail: r.message })
+        }
+        send(res, 200, ok({ results: outcome.results }))
+        return
+      }
+      if (apiPath === 'hub/apps/run') {
+        const action = params.action
+        if (action !== 'install-store' && action !== 'update-store' && action !== 'uninstall') {
+          send(res, 400, fail('action must be install-store, update-store or uninstall.'))
+          return
+        }
+        const outcome = await writer.runMany(action, params.name ?? '', csv(params.targets), { suppliedUrl: params.url })
+        if ('error' in outcome) {
+          audit.log({ user: who.user, role: who.role, ip, action: `apps.${action}`, target: params.name, outcome: 'refused', detail: outcome.error })
+          send(res, 400, fail(outcome.error))
+          return
+        }
+        auditResults(who, ip, action, params.name ?? '', outcome.results)
+        send(res, 200, ok({ results: outcome.results }))
+        return
+      }
+      if (apiPath === 'hub/apps/upload') {
+        if (!isMultipart(req)) {
+          send(res, 400, fail('Send multipart/form-data.'))
+          return
+        }
+        const file = await uploadOf(req, res, params)
+        if (file === null) return
+        if (params.action !== 'install' && params.action !== 'update') {
+          send(res, 400, fail('action must be install or update.'))
+          return
+        }
+        const action: FileAction = params.action === 'install' ? 'install-file' : 'update-file'
+        const outcome = await writer.runMany(action, params.name ?? '', csv(params.targets), { file })
+        if ('error' in outcome) {
+          audit.log({ user: who.user, role: who.role, ip, action: `apps.${action}`, target: params.name, outcome: 'refused', detail: outcome.error })
+          send(res, 400, fail(outcome.error))
+          return
+        }
+        auditResults(who, ip, action, params.name ?? '', outcome.results)
+        send(res, 200, ok({ results: outcome.results }))
+        return
+      }
+    }
+
+    // write-one: an upstream endpoint aimed at ONE server (`node=<id>`).
+    const action = actionForPath(apiPath)
+    if (action === null) {
+      await serverWrite(req, res, apiPath, route, params, who, ip)
+      return
+    }
+    const node = params.node ?? ''
+    const server = byId.get(node)
+    if (server === undefined) {
+      send(res, 400, fail(node === '' || node === 'cluster' ? 'Choose one server: this change cannot be made on every server at once here.' : `Unknown node: ${node}`))
+      return
+    }
+    const name = params.name ?? ''
+    let file: UploadedFile | undefined
+    if (action === 'install-file' || action === 'update-file') {
+      if (!isMultipart(req)) {
+        send(res, 400, fail('Send multipart/form-data.'))
+        return
+      }
+      const uploaded = await uploadOf(req, res, params)
+      if (uploaded === null) return
+      file = uploaded
+    }
+    let result: TargetResult
+    let response: unknown = {}
+    try {
+      let url: string | undefined
+      if (action === 'install-store' || action === 'update-store') {
+        const fromStore = server.adminToken === undefined ? null : await writer.storeUrl(server, name).catch(() => null)
+        if (server.adminToken !== undefined && (fromStore === null || params.url !== fromStore)) {
+          throw new Error("the url is not from this server's store listing")
+        }
+        url = fromStore ?? undefined
+      }
+      response = await writer.perform(server, action, name, url, file, params.config)
+      result = { server: server.id, ok: true }
+    } catch (e) {
+      result = { server: server.id, ok: false, message: e instanceof Error ? e.message : 'failed' }
+    }
+    auditResults(who, ip, action, name, [result])
+    send(res, 200, result.ok ? ok(response) : fail(`${server.id}: ${result.message}`))
+  }
+
+  const textBody = (req: http.IncomingMessage): boolean => /^text\/plain/i.test(String(req.headers['content-type'] ?? ''))
+
+  /*
+  A change to ONE server from the sections it is managed from (zones, cache, blocking, DNS client,
+  settings, DHCP). The role has been checked against the route. The server is the one named by
+  `node`, and its admin token does the work. What is forwarded is exactly the parameters that were
+  looked at here, in a form body (see `Upstream.write`), minus `node` and `token`. The audit line
+  holds the path, the server, one identifier and the outcome, never a value.
+  */
+  async function serverWrite(req: http.IncomingMessage, res: http.ServerResponse, apiPath: string, route: Route, params: Params, who: Who, ip: string): Promise<void> {
+    const node = params.node ?? ''
+    const server = byId.get(node)
+    const line = { user: who.user, role: who.role, ip, action: apiPath.replace(/\//g, '.') }
+    if (server === undefined) {
+      send(res, 400, fail(node === '' || node === 'cluster' ? 'Choose a server: this change is made on one server at a time.' : `Unknown node: ${node}`))
+      return
+    }
+    if (server.adminToken === undefined) {
+      audit.log({ ...line, server: server.id, target: safeTarget(params), outcome: 'refused', detail: 'no admin token' })
+      send(res, 403, fail(`${server.id}: the hub has no admin token for this server, so it cannot change it.`))
+      return
+    }
+
+    const forward: Params = {}
+    for (const [k, v] of Object.entries(params)) if (k !== 'node' && k !== 'token') forward[k] = v
+
+    // A file or a text body (a zone import). The parameters may also arrive as multipart fields.
+    let body: FormData | string | undefined
+    if (route.upload === true && (isMultipart(req) || textBody(req))) {
+      let data: Buffer
+      try {
+        data = await readLimited(req)
+      } catch (e) {
+        if (e instanceof TooLarge) {
+          res.setHeader('connection', 'close')
+          send(res, 413, fail(e.message))
+        } else send(res, 400, fail('The upload could not be read.'))
+        return
+      }
+      if (isMultipart(req)) {
+        let parsed
+        try {
+          parsed = parseMultipart(data, String(req.headers['content-type'] ?? ''))
+        } catch {
+          send(res, 400, fail('The upload could not be read.'))
+          return
+        }
+        for (const [k, v] of Object.entries(parsed.fields)) if (k !== 'node' && k !== 'token') forward[k] = v
+        const file = parsed.files[0]
+        if (file !== undefined) {
+          const form = new FormData()
+          form.append(file.field, new Blob([file.data], { type: file.type }), file.filename)
+          body = form
+        }
+      } else body = data.toString('utf8')
+    }
+
+    // settings/set: no change to the web service the hub itself uses, and nothing about users (guards.ts).
+    if (apiPath === 'settings/set') {
+      if (Object.keys(forward).some(isForbiddenSettingsField)) {
+        audit.log({ ...line, server: server.id, outcome: 'refused', detail: 'user, SSO or LDAP field' })
+        send(res, 400, fail("User, SSO and LDAP settings are changed in that server's own console."))
+        return
+      }
+      try {
+        upstream.invalidate(server.id)
+        const changed = webServiceChanges(forward, await upstream.call(server, 'settings/get', {}, server.adminToken))
+        if (changed.length > 0) {
+          audit.log({ ...line, server: server.id, outcome: 'refused', detail: `web service: ${changed.join(', ').slice(0, 200)}` })
+          send(res, 400, fail(WEB_SERVICE_MESSAGE))
+          return
+        }
+      } catch (e) {
+        if (!(e instanceof UpstreamError)) throw e
+        audit.log({ ...line, server: server.id, outcome: 'failed', detail: 'could not read the current settings' })
+        send(res, 200, fail(`${server.id}: the web service settings could not be checked first (${e.message}); nothing was changed.`))
+        return
+      }
+    }
+
+    try {
+      const response = await upstream.write(server, apiPath, forward, body)
+      audit.log({ ...line, server: server.id, target: safeTarget(forward), outcome: 'ok' })
+      send(res, 200, apiPath === 'settings/set' ? { ...ok(response), server: server.id } : ok(response))
+    } catch (e) {
+      if (!(e instanceof UpstreamError)) throw e
+      audit.log({ ...line, server: server.id, target: safeTarget(forward), outcome: 'failed', detail: e.kind === 'unreachable' ? 'no answer' : 'refused by the server' })
+      send(res, 200, fail(e.kind === 'unreachable' ? `${server.id} did not answer: ${e.message}` : `${server.id}: ${e.message}`))
+    }
+  }
+
+  /* A file from one server (a zone, a log, a block list), passed on as it comes. */
+  async function download(res: http.ServerResponse, server: HubServer, apiPath: string, params: Params): Promise<void> {
+    try {
+      const file = await upstream.raw(server, apiPath, params)
+      res.writeHead(200, {
+        'content-type': file.contentType,
+        ...(file.disposition === undefined ? {} : { 'content-disposition': file.disposition }),
+        'cache-control': 'no-store',
+        'x-content-type-options': 'nosniff',
+        'content-security-policy': "default-src 'none'; sandbox",
+      })
+      if (file.body === null) {
+        res.end()
+        return
+      }
+      res.on('close', () => file.abort())
+      Readable.fromWeb(file.body as WebStream).on('error', () => res.destroy()).pipe(res)
+    } catch (e) {
+      if (!(e instanceof UpstreamError)) throw e
+      send(res, 200, fail(e.kind === 'unreachable' ? `${server.id} did not answer: ${e.message}` : `${server.id}: ${e.message}`))
+    }
+  }
+
   async function api(req: http.IncomingMessage, res: http.ServerResponse, apiPath: string, url: URL): Promise<void> {
-    const form = await readForm(req)
+    // A multipart body (an app zip, a zone file) and a text body (a pasted zone file) are read later,
+    // once the caller is known to be allowed to send them.
+    // A config body may be large, but only for a signed-in caller (token in the header or the query).
+    const own = apiPath.startsWith('hub/apps/') || apiPath === 'apps/config/set'
+    const wanted = own ? (apiPath === 'hub/apps/import' ? MAX_IMPORT_BODY : MAX_CONFIG_BODY) : largeBodyLimit(apiPath)
+    const large = wanted !== undefined && sessions.touch(bearer(req) ?? url.searchParams.get('token')) !== null
+    const deferred = isMultipart(req) || (routeFor(apiPath)?.upload === true && textBody(req))
+    const form = deferred ? new URLSearchParams() : await readForm(req, large ? (wanted as number) : MAX_BODY)
     const params: Params = {}
     for (const [k, v] of url.searchParams) params[k] = v
     for (const [k, v] of form) params[k] = v
@@ -186,30 +840,112 @@ export function createHub(config: HubConfig): http.Server {
       send(res, 200, { status: 'ok', hasDefaultCredentials: false, ssoEnabled: false })
       return
     }
+    const ip = clientIp(req, config.trustProxy)
     if (apiPath === 'user/login') {
-      const token = sessions.login(params.user ?? '', params.pass ?? '')
-      send(res, 200, token === null ? fail(`Invalid username or password for user: ${params.user ?? ''}`) : sessions.body(token))
+      const name = params.user ?? ''
+      const locked = limiter.lockedFor(ip, name)
+      if (locked > 0) {
+        audit.log({ user: name, ip, action: 'login', outcome: 'refused', detail: 'locked' })
+        send(res, 429, fail(lockedMessage(locked)))
+        return
+      }
+      const user = await users.authenticate(name, params.pass ?? '')
+      if (user === null) {
+        limiter.failure(ip, name)
+        audit.log({ user: name, ip, action: 'login', outcome: 'failed', detail: users.isDisabled(name) ? 'disabled' : undefined })
+        send(res, 200, fail(`Invalid username or password for user: ${name}`))
+        return
+      }
+      limiter.success(name)
+      audit.log({ user: user.name, role: user.role, ip, action: 'login', outcome: 'ok' })
+      const token = sessions.open(user)
+      send(res, 200, sessions.body(token, { user: user.name, role: user.role }))
       return
     }
 
     const token = bearer(req) ?? params.token ?? null
-    if (!sessions.touch(token)) {
+    // A download opened in a new tab carries a single-use token instead of the session's.
+    const who = sessions.touch(token) ?? (req.method === 'GET' && routeFor(apiPath)?.raw === true ? sessions.takeSingleUse(token) : null)
+    if (who === null) {
       send(res, 200, { status: 'invalid-token', errorMessage: 'Invalid token or session expired.' })
       return
     }
     if (apiPath === 'user/session/get') {
-      send(res, 200, sessions.body(token as string))
+      send(res, 200, sessions.body(token as string, who))
       return
     }
     if (apiPath === 'user/logout') {
       sessions.logout(token as string)
+      audit.log({ user: who.user, role: who.role, ip, action: 'logout', outcome: 'ok' })
       send(res, 200, { status: 'ok' })
       return
     }
-    if (!READ_ENDPOINTS.has(apiPath)) {
+    if (apiPath === 'user/createSingleUseToken') {
+      send(res, 200, ok({ token: sessions.openSingleUse(token as string) }))
+      return
+    }
+    const route = effectiveRoute(apiPath, params)
+    if (route === undefined) {
+      audit.log({
+        user: who.user, role: who.role, ip, action: 'refused-write', target: apiPath,
+        server: params.node || undefined, outcome: 'refused', detail: 'not in the permission table',
+      })
       send(res, 403, fail('The hub is read-only.'))
       return
     }
+    if (!can(who.role, route.capability)) {
+      audit.log({
+        user: who.user, role: who.role, ip, action: route.kind === 'read' ? 'denied-read' : 'denied-write',
+        target: apiPath, outcome: 'refused', detail: `needs ${route.capability}`,
+      })
+      send(res, 403, fail('Your role does not allow this.'))
+      return
+    }
+    if (apiPath === 'hub/audit/list') {
+      send(res, 200, ok({ entries: audit.list(Number(params.limit ?? 200), params.before) }))
+      return
+    }
+    if (route.kind !== 'read') {
+      await write(req, res, apiPath, route, params, who, ip)
+      return
+    }
+    if (apiPath.startsWith('hub/apps/config/') || apiPath.startsWith('hub/apps/history/') || apiPath === 'hub/apps/overrides/current' || apiPath === 'hub/apps/export') {
+      await configRead(res, apiPath, params, who, ip)
+      return
+    }
+    if (apiPath.startsWith('hub/profiles/')) {
+      await profilesRead(res, apiPath, params, who, ip)
+      return
+    }
+    if (apiPath.startsWith('hub/settings/') || apiPath.startsWith('hub/blocking/')) {
+      await settingsRead(res, apiPath, params, who, ip)
+      return
+    }
+    if (apiPath.startsWith('hub/zones/')) {
+      await zonesRead(res, apiPath, params)
+      return
+    }
+    if (apiPath === 'hub/webhook/status') {
+      send(res, 200, ok(webhook.status()))
+      return
+    }
+    if (apiPath === 'hub/drift/get') {
+      send(res, 200, ok(drift.get()))
+      return
+    }
+    if (apiPath === 'apps/config/get' && !byId.has(params.node ?? '')) {
+      send(res, 400, fail('Choose one server: a config is read from a single server.'))
+      return
+    }
+    if (apiPath === 'hub/apps/matrix') {
+      try {
+        const { matrix, warnings } = await buildMatrix(config, upstream)
+        send(res, 200, ok(matrix, warnings))
+      } catch (e) {
+        send(res, 200, fail(e instanceof Error ? e.message : 'failed'))
+      }
+      return
+    }
     if (apiPath === 'user/checkForUpdate') {
       send(res, 200, ok({ updateAvailable: false }))
       return
@@ -239,15 +975,22 @@ export function createHub(config: HubConfig): http.Server {
     // What the real servers get: the caller's params minus the hub's own.
     const { node = '', token: _token, ...forward } = params
     const server = byId.get(node)
-    if (server !== undefined) {
-      send(res, 200, await single(server, apiPath, forward))
+    // A read with no merger is never answered by "the first server" for the aggregate: it needs a server.
+    if (route.perServer === true && server === undefined) {
+      send(res, 400, fail(node === '' || node === 'cluster' ? 'Choose a server: this is read from one server at a time.' : `Unknown node: ${node}`))
       return
     }
-    if (node !== '' && node !== 'cluster') {
-      send(res, 200, fail(`Unknown node: ${node}`))
+    if (route.raw === true && server !== undefined) {
+      await download(res, server, apiPath, forward)
       return
     }
-    send(res, 200, await aggregate(apiPath, forward))
+    let answer: Json
+    if (server !== undefined) answer = await single(server, apiPath, forward)
+    else if (node !== '' && node !== 'cluster') answer = fail(`Unknown node: ${node}`)
+    else answer = await aggregate(apiPath, forward)
+    // The settings hold secrets (TSIG keys, passwords): only an admin sees them (guards.ts).
+    if (apiPath === 'settings/get' && !can(who.role, 'admin') && answer.status === 'ok') answer = { ...answer, response: redactSettings(answer.response) }
+    send(res, 200, answer)
   }
 
   async function serveStatic(req: http.IncomingMessage, res: http.ServerResponse, url: URL): Promise<void> {
@@ -302,7 +1045,10 @@ export function createHub(config: HubConfig): http.Server {
     createReadStream(file).on('error', () => res.destroy()).pipe(res)
   }
 
-  return http.createServer((req, res) => {
+  const sweeper = setInterval(() => limiter.sweep(), 5 * 60_000)
+  sweeper.unref()
+
+  const server = http.createServer((req, res) => {
     void (async () => {
       try {
         const url = new URL(req.url ?? '/', 'http://hub')
@@ -317,6 +1063,18 @@ export function createHub(config: HubConfig): http.Server {
       }
     })()
   })
+  // The drift check: once shortly after start, then every HUB_DRIFT_MINUTES. Off when that is 0.
+  const timers: NodeJS.Timeout[] = []
+  if (config.driftMinutes > 0) {
+    const run = () => void drift.check().catch((e: unknown) => console.error(`hub: drift check failed (${e instanceof Error ? e.message : String(e)}).`))
+    timers.push(setTimeout(run, 5000), setInterval(run, config.driftMinutes * 60_000))
+    for (const t of timers) t.unref()
+  }
+  server.on('close', () => {
+    clearInterval(sweeper)
+    for (const t of timers) clearTimeout(t)
+  })
+  return server
 }
 
 /* Run directly (not imported by the tests). */
@@ -328,7 +1086,16 @@ if (process.argv[1] !== undefined && path.resolve(process.argv[1]) === fileURLTo
     console.error(`hub: ${e instanceof Error ? e.message : String(e)}`)
     process.exit(1)
   }
-  createHub(config).listen(config.port, () => {
+  let hub: http.Server
+  try {
+    const users = Users.load(config.dataDir, config.bootstrap)
+    if (users.notice !== null) console.log(`hub: ${users.notice}`)
+    hub = createHub(config, users)
+  } catch (e) {
+    console.error(`hub: ${e instanceof Error ? e.message : String(e)}`)
+    process.exit(1)
+  }
+  hub.listen(config.port, () => {
     console.log(`Hub ${HUB_VERSION} on :${config.port}, servers: ${config.servers.map((s) => s.id).join(', ')}`)
   })
 }

hub/servers-run.test.ts

@@ -0,0 +1,354 @@
+// @vitest-environment node
+/*
+Stage 6: the same change on several servers at once, outside Apps (`hub/servers/run`). The servers
+are fakes that record how they were called.
+*/
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { addUser, writeUsersFile } from './users.ts'
+import { MAX_MINUTES, MAX_LINES, SERVER_ACTIONS, cleanDomain, cleanUrls, planAction } from './servers-run.ts'
+import { routeFor } from './routes.ts'
+
+interface Call {
+  path: string
+  method: string
+  token: string
+  query: URLSearchParams
+  form: URLSearchParams
+}
+
+interface Fake {
+  server: http.Server
+  url: string
+  calls: Call[]
+  failWith: string | null
+  down: boolean
+}
+
+function fake(): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = { server: http.createServer(), url: '', calls: [], failWith: null, down: false }
+    f.server.on('request', (req, res) => {
+      const chunks: Buffer[] = []
+      req.on('data', (c: Buffer) => chunks.push(c))
+      req.on('end', () => {
+        if (f.down) return req.socket.destroy()
+        const url = new URL(req.url ?? '/', 'http://fake')
+        const contentType = String(req.headers['content-type'] ?? '')
+        const form = new URLSearchParams(contentType.includes('x-www-form-urlencoded') ? Buffer.concat(chunks).toString('utf8') : '')
+        f.calls.push({ path: url.pathname.replace(/^\/api\//, ''), method: req.method ?? '', token: url.searchParams.get('token') ?? '', query: url.searchParams, form })
+        res.setHeader('content-type', 'application/json')
+        if (f.failWith !== null && req.method === 'POST') return res.end(JSON.stringify({ status: 'error', errorMessage: f.failWith }))
+        res.end(JSON.stringify({ status: 'ok', response: {} }))
+      })
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+let a: Fake
+let b: Fake
+let dataDir: string
+let staticDir: string
+let hub: http.Server | undefined
+let base: string
+const tokens: Record<string, string> = {}
+const ROLES = ['viewer', 'operator', 'apps-admin', 'admin']
+
+async function start(extra: Record<string, string> = {}): Promise<void> {
+  hub?.close()
+  hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      A_ADMIN_TOKEN: 'admin-a',
+      B_ADMIN_TOKEN: 'admin-b',
+      HUB_DATA: dataDir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      ...extra,
+    }),
+  )
+  await new Promise<void>((resolve) => hub!.listen(0, '127.0.0.1', resolve))
+  base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  for (const role of ROLES) {
+    const res = await fetch(`${base}/api/user/login`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: `user=${role}&pass=password-${role}`,
+    })
+    tokens[role] = ((await res.json()) as { token: string }).token
+  }
+}
+
+async function run(role: string, params: Record<string, string>) {
+  const res = await fetch(`${base}/api/hub/servers/run`, {
+    method: 'POST',
+    headers: { authorization: `Bearer ${tokens[role]}`, 'content-type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams(params).toString(),
+  })
+  return { code: res.status, body: (await res.json()) as Record<string, any> }
+}
+
+const writes = (f: Fake) => f.calls.filter((c) => c.method === 'POST')
+
+async function auditEntries(): Promise<Record<string, string>[]> {
+  const res = await fetch(`${base}/api/hub/audit/list?limit=200`, { headers: { authorization: `Bearer ${tokens.admin}` } })
+  return ((await res.json()) as { response: { entries: Record<string, string>[] } }).response.entries
+}
+
+beforeAll(async () => {
+  a = await fake()
+  b = await fake()
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-s6-data-'))
+  staticDir = mkdtempSync(path.join(tmpdir(), 'hub-s6-static-'))
+  let users = addUser([], 'viewer', 'viewer', 'password-viewer')
+  users = addUser(users, 'operator', 'operator', 'password-operator')
+  users = addUser(users, 'apps-admin', 'apps-admin', 'password-apps-admin')
+  users = addUser(users, 'admin', 'admin', 'password-admin')
+  writeUsersFile(dataDir, users)
+  await start()
+})
+
+beforeEach(() => {
+  for (const f of [a, b]) {
+    f.calls = []
+    f.failWith = null
+    f.down = false
+  }
+})
+
+afterAll(() => {
+  hub?.close()
+  a.server.close()
+  b.server.close()
+  rmSync(dataDir, { recursive: true, force: true })
+  rmSync(staticDir, { recursive: true, force: true })
+})
+
+describe('the plan of an action', () => {
+  const plan = (action: string, raw: Record<string, string> = {}) => {
+    const out = planAction(action, raw)
+    if ('error' in out) throw new Error(out.error)
+    return out.plan
+  }
+
+  it('maps each action to its Technitium call and sends only what it needs', () => {
+    expect(plan('blocked.add', { domain: 'Ads.Example.' })).toMatchObject({ path: 'blocked/add', params: { domain: 'Ads.Example' } })
+    expect(plan('blocked.delete', { domain: 'ads.example' })).toMatchObject({ path: 'blocked/delete', params: { domain: 'ads.example' } })
+    expect(plan('allowed.add', { domain: 'ok.example' })).toMatchObject({ path: 'allowed/add' })
+    expect(plan('allowed.delete', { domain: 'ok.example' })).toMatchObject({ path: 'allowed/delete' })
+    expect(plan('blocking.disable', { minutes: '15' })).toMatchObject({ path: 'settings/temporaryDisableBlocking', params: { minutes: '15' }, target: 'minutes=15' })
+    expect(plan('blocking.enable')).toMatchObject({ path: 'settings/set', params: { enableBlocking: 'true' } })
+    expect(plan('blocking.updateLists')).toMatchObject({ path: 'settings/forceUpdateBlockLists', params: {} })
+    expect(plan('blocking.setListUrls', { urls: 'https://a.example/x\nhttp://b.example/y' })).toMatchObject({
+      path: 'settings/set',
+      params: { blockListUrls: 'https://a.example/x,http://b.example/y' },
+      target: 'urls=2',
+    })
+    expect(plan('cache.flush')).toMatchObject({ path: 'cache/flush', params: {} })
+    expect(plan('cache.delete', { domain: 'x.example' })).toMatchObject({ path: 'cache/delete', params: { domain: 'x.example' } })
+    expect(SERVER_ACTIONS).toHaveLength(10)
+  })
+
+  it('refuses an action that is not on the list', () => {
+    for (const bad of [undefined, '', 'settings.set', 'zones/delete', 'cache.flush ', '__proto__']) expect(planAction(bad, {})).toHaveProperty('error')
+  })
+
+  it('checks the domain', () => {
+    expect(cleanDomain('example.com')).toBe('example.com')
+    expect(cleanDomain('*.example.com')).toBe('*.example.com')
+    for (const bad of ['', ' ', 'a b.example', 'a..example', '-a.example', 'a.example/x', 'a.example?x=1', 'a,b.example', 'a'.repeat(64) + '.example', ('a.'.repeat(130)) + 'com']) {
+      expect(cleanDomain(bad)).toBeNull()
+      expect(planAction('blocked.add', { domain: bad })).toHaveProperty('error')
+    }
+  })
+
+  it('checks the minutes', () => {
+    for (const bad of [undefined, '', '0', '-5', '1.5', 'abc', '5m', String(MAX_MINUTES + 1), '99999999']) {
+      expect(planAction('blocking.disable', { minutes: bad })).toHaveProperty('error')
+    }
+    expect(planAction('blocking.disable', { minutes: String(MAX_MINUTES) })).toHaveProperty('plan')
+  })
+
+  it('checks the lines', () => {
+    for (const bad of ['', '   \n  ', 'ftp://a.example/x', '!ftp://a.example/x', '! javascript:alert(1)', 'not a url', 'https://a.example/x,y', '# a, b', 'https://a.example/a b', '#' + 'x'.repeat(300)]) {
+      expect(cleanUrls(bad), bad).toHaveProperty('error')
+    }
+    expect(cleanUrls(Array.from({ length: MAX_LINES + 1 }, (_, i) => `https://a.example/${i}`).join('\n'))).toHaveProperty('error')
+  })
+
+  it('takes the grammar of the lists: allow lines, switched-off lines, headings, file addresses', () => {
+    const text = '# Privacy\nhttps://a.example/hosts\n!https://b.example/allow.txt\n#https://c.example/off\n\nfile:///etc/extra.txt\r\n  http://d.example/x  \n'
+    expect(cleanUrls(text)).toEqual({
+      urls: ['# Privacy', 'https://a.example/hosts', '!https://b.example/allow.txt', '#https://c.example/off', 'file:///etc/extra.txt', 'http://d.example/x'],
+    })
+  })
+
+  it('sends the same blockListUrls the console Settings save sends (the lines joined with commas, in order)', () => {
+    const lines = ['# Privacy', 'https://a.example/hosts', '!https://b.example/allow.txt', '#https://c.example/off', '# Ads', 'https://d.example/x']
+    const out = planAction('blocking.setListUrls', { urls: lines.join('\n') })
+    if ('error' in out) throw new Error(out.error)
+    // cleanList in src/screens/settings/model.ts: newlines to commas, repeated commas collapsed, none at the ends.
+    expect(out.plan.params).toEqual({ blockListUrls: lines.join(',') })
+    expect(out.plan.target).toBe('urls=6')
+  })
+})
+
+describe('who may run it', () => {
+  it('is a write-all route for operate', () => {
+    expect(routeFor('hub/servers/run')).toMatchObject({ kind: 'write-all', capability: 'operate' })
+  })
+
+  it('refuses a viewer and an apps admin, calling no server', async () => {
+    for (const role of ['viewer', 'apps-admin']) {
+      const r = await run(role, { action: 'cache.flush', targets: 'a,b' })
+      expect(r.code).toBe(403)
+    }
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('lets an operator and an admin run it', async () => {
+    for (const role of ['operator', 'admin']) {
+      const r = await run(role, { action: 'cache.flush', targets: 'a,b' })
+      expect(r.code).toBe(200)
+      expect(r.body.response.results).toHaveLength(2)
+    }
+  })
+
+  it('needs a POST', async () => {
+    const res = await fetch(`${base}/api/hub/servers/run?action=cache.flush&targets=a`, { headers: { authorization: `Bearer ${tokens.operator}` } })
+    expect(res.status).toBe(405)
+    expect(writes(a)).toHaveLength(0)
+  })
+})
+
+describe('running it', () => {
+  it('sends each action to each server with its admin token, parameters in the body', async () => {
+    const cases: [Record<string, string>, string, Record<string, string>][] = [
+      [{ action: 'blocked.add', domain: 'ads.example' }, 'blocked/add', { domain: 'ads.example' }],
+      [{ action: 'blocked.delete', domain: 'ads.example' }, 'blocked/delete', { domain: 'ads.example' }],
+      [{ action: 'allowed.add', domain: 'ok.example' }, 'allowed/add', { domain: 'ok.example' }],
+      [{ action: 'allowed.delete', domain: 'ok.example' }, 'allowed/delete', { domain: 'ok.example' }],
+      [{ action: 'blocking.disable', minutes: '15' }, 'settings/temporaryDisableBlocking', { minutes: '15' }],
+      [{ action: 'blocking.enable' }, 'settings/set', { enableBlocking: 'true' }],
+      [{ action: 'blocking.updateLists' }, 'settings/forceUpdateBlockLists', {}],
+      [{ action: 'blocking.setListUrls', urls: 'https://a.example/x\nhttps://b.example/y' }, 'settings/set', { blockListUrls: 'https://a.example/x,https://b.example/y' }],
+      [{ action: 'cache.flush' }, 'cache/flush', {}],
+      [{ action: 'cache.delete', domain: 'x.example' }, 'cache/delete', { domain: 'x.example' }],
+    ]
+    for (const [params, upstreamPath, sent] of cases) {
+      a.calls = []
+      b.calls = []
+      const r = await run('operator', { ...params, targets: 'a,b' })
+      expect(r.code, params.action).toBe(200)
+      expect(r.body.response.results.map((x: { server: string; ok: boolean }) => [x.server, x.ok])).toEqual([['a', true], ['b', true]])
+      for (const [f, token] of [[a, 'admin-a'], [b, 'admin-b']] as const) {
+        const call = writes(f).filter((c) => c.path !== 'settings/get')
+        expect(call, params.action).toHaveLength(1)
+        expect(call[0].path).toBe(upstreamPath)
+        expect(call[0].token).toBe(token)
+        expect(Object.fromEntries(call[0].form)).toEqual(sent)
+        // Nothing but the token in the URL.
+        expect([...call[0].query.keys()]).toEqual(['token'])
+      }
+    }
+  })
+
+  it('sends only the one setting, and never reads or writes anything else', async () => {
+    await run('operator', { action: 'blocking.setListUrls', urls: 'https://a.example/x', targets: 'a' })
+    expect(a.calls.map((c) => c.path)).toEqual(['settings/set'])
+    expect([...(a.calls[0].form.keys())]).toEqual(['blockListUrls'])
+  })
+
+  it('shows a partial failure per server, never a bare ok', async () => {
+    b.failWith = 'Access was denied to user x for Blocked.'
+    const r = await run('operator', { action: 'blocked.add', domain: 'ads.example', targets: 'a,b' })
+    expect(r.code).toBe(200)
+    const results = r.body.response.results as { server: string; ok: boolean; message: string }[]
+    expect(results.find((x) => x.server === 'a')).toMatchObject({ ok: true })
+    expect(results.find((x) => x.server === 'b')).toMatchObject({ ok: false })
+    expect(results.find((x) => x.server === 'b')?.message).toContain('Access was denied')
+    expect(results.every((x) => typeof x.message === 'string' && x.message !== '')).toBe(true)
+  })
+
+  it('reports a server that does not answer, and still runs the others', async () => {
+    b.down = true
+    const r = await run('operator', { action: 'cache.flush', targets: 'a,b' })
+    const results = r.body.response.results as { server: string; ok: boolean }[]
+    expect(results.map((x) => [x.server, x.ok])).toEqual([['a', true], ['b', false]])
+    expect(writes(a)).toHaveLength(1)
+  })
+
+  it('fails only the target that has no admin token, and one that is unknown refuses the request', async () => {
+    await start({ A_ADMIN_TOKEN: '' })
+    for (const f of [a, b]) f.calls = []
+    const r = await run('operator', { action: 'cache.flush', targets: 'a,b' })
+    expect(r.code).toBe(200)
+    const results = r.body.response.results as { server: string; ok: boolean; message: string }[]
+    expect(results.find((x) => x.server === 'a')).toMatchObject({ ok: false, message: 'no admin token configured' })
+    expect(results.find((x) => x.server === 'b')).toMatchObject({ ok: true })
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b)).toHaveLength(1)
+    const unknown = await run('operator', { action: 'cache.flush', targets: 'a,nope' })
+    expect(unknown.code).toBe(400)
+    expect(writes(a)).toHaveLength(0)
+    await start()
+  })
+
+  it('refuses empty targets and a bad parameter, calling no server', async () => {
+    const bad: Record<string, string>[] = [
+      { action: 'cache.flush', targets: '' },
+      { action: 'cache.flush' },
+      { action: 'cache.flush', targets: ' , ' },
+      { action: 'blocked.add', domain: 'a b', targets: 'a' },
+      { action: 'blocking.disable', minutes: '0', targets: 'a' },
+      { action: 'blocking.setListUrls', urls: 'ftp://x.example', targets: 'a' },
+      { action: 'settings.set', targets: 'a' },
+    ]
+    for (const params of bad) {
+      const r = await run('operator', params)
+      expect(r.code, JSON.stringify(params)).toBe(400)
+      expect(r.body.status).toBe('error')
+    }
+    expect(a.calls).toHaveLength(0)
+    expect(b.calls).toHaveLength(0)
+  })
+
+  it('runs on a server once even when it is named twice', async () => {
+    await run('operator', { action: 'cache.flush', targets: 'a,a' })
+    expect(writes(a)).toHaveLength(1)
+  })
+})
+
+describe('the audit log', () => {
+  it('has one line per server with the target, and never the URL list', async () => {
+    b.failWith = 'Disk full.'
+    await run('operator', { action: 'blocking.setListUrls', urls: 'https://secret.example/token123/list.txt\nhttps://x.example/y', targets: 'a,b' })
+    await run('operator', { action: 'blocked.add', domain: 'ads.example', targets: 'a,b' })
+    await run('operator', { action: 'blocking.disable', minutes: '15', targets: 'a' })
+    await run('operator', { action: 'cache.flush', targets: '' })
+    const entries = (await auditEntries()).filter((e) => e.action?.startsWith('servers.'))
+    const lists = entries.filter((e) => e.action === 'servers.blocking.setListUrls')
+    expect(lists.map((e) => [e.server, e.outcome, e.target])).toEqual(expect.arrayContaining([['a', 'ok', 'urls=2'], ['b', 'failed', 'urls=2']]))
+    expect(entries.filter((e) => e.action === 'servers.blocked.add').map((e) => [e.server, e.target])).toEqual(expect.arrayContaining([['a', 'ads.example'], ['b', 'ads.example']]))
+    expect(entries.find((e) => e.action === 'servers.blocking.disable')).toMatchObject({ server: 'a', target: 'minutes=15', outcome: 'ok', user: 'operator' })
+    expect(entries.find((e) => e.action === 'servers.cache.flush')).toMatchObject({ outcome: 'refused' })
+    const text = JSON.stringify(entries)
+    expect(text).not.toContain('secret.example')
+    expect(text).not.toContain('token123')
+    expect(text).not.toContain('admin-a')
+  })
+})

hub/servers-run.ts

@@ -0,0 +1,183 @@
+/*
+Actions on several servers at once, outside Apps (stage 6): Blocked and Allowed, the temporary
+blocking switch, the block lists, and the cache. One endpoint, `hub/servers/run`, takes a fixed
+`action` and the action's own parameters, checks them HERE, and runs the one matching Technitium
+call on every chosen server, in parallel, with that server's admin token.
+
+  * `planAction` is pure: the allowlist, the validation, and the path and parameters to send.
+  * `ServersRunner` runs a plan on targets. A server that is unknown or has no admin token fails
+    on its own; the others still run. The answer is one result per server, never a bare "ok".
+
+The block list URLs are part of blocking, so `blocking.setListUrls` needs `operate` like the rest,
+not `admin` like the per-server `settings/set`. Only `blockListUrls` is sent, so nothing else of
+the server's settings is touched, and the plan still goes through the checks of guards.ts.
+*/
+import type { HubConfig } from './config.ts'
+import { isForbiddenSettingsField } from './guards.ts'
+import { Upstream } from './upstream.ts'
+
+export const SERVER_ACTIONS = [
+  'blocked.add',
+  'blocked.delete',
+  'allowed.add',
+  'allowed.delete',
+  'blocking.disable',
+  'blocking.enable',
+  'blocking.updateLists',
+  'blocking.setListUrls',
+  'cache.flush',
+  'cache.delete',
+] as const
+
+export type ServerAction = (typeof SERVER_ACTIONS)[number]
+
+export interface TargetResult {
+  server: string
+  ok: boolean
+  message?: string
+}
+
+export interface Plan {
+  action: ServerAction
+  /* The Technitium endpoint. */
+  path: string
+  /* Exactly what is sent (a form body). */
+  params: Record<string, string>
+  /* What the audit line may say about it: a domain, `minutes=15`, `urls=3`. Never the URLs. */
+  target?: string
+  /* The sentence for a server where it worked. */
+  done: string
+}
+
+export const MAX_MINUTES = 1440
+const MAX_URL_LENGTH = 2000
+
+const LABEL = '[a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9_])?'
+const DOMAIN = new RegExp(`^(?:\\*\\.)?(?:${LABEL}\\.)*${LABEL}$`, 'i')
+
+/* A domain as the hub accepts it (a trailing dot is dropped), or null. */
+export function cleanDomain(value: string | undefined): string | null {
+  const domain = (value ?? '').trim().replace(/\.$/, '')
+  return domain.length > 0 && domain.length <= 253 && DOMAIN.test(domain) ? domain : null
+}
+
+/*
+The lines of a block list request, one per line, in the grammar of the servers and of the console's
+Lists tab (src/screens/blocking/list-lines.ts): `<url>` a block list, `!<url>` an allow list,
+`#...` a switched-off list or a comment (a group heading), free text. Each URL is http, https or
+file. Blank lines go. They travel as the console's own Settings save sends them: joined with a
+comma. A comma cannot be told from the separator, so a line that has one is refused, not split.
+*/
+export const MAX_LINES = 300
+export const MAX_COMMENT = 200
+const SCHEMES = new Set(['http:', 'https:', 'file:'])
+
+export function cleanUrls(value: string | undefined): { urls: string[] } | { error: string } {
+  const lines = (value ?? '')
+    .split(/\r?\n/)
+    .map((l) => l.trim())
+    .filter((l) => l !== '')
+  if (lines.length === 0) return { error: 'Give at least one block list URL.' }
+  if (lines.length > MAX_LINES) return { error: `At most ${MAX_LINES} lines.` }
+  for (const line of lines) {
+    if (line.includes(',')) return { error: `A line cannot contain a comma: ${line.slice(0, 60)}` }
+    if (line.startsWith('#')) {
+      if (line.length > MAX_COMMENT) return { error: 'A comment line is too long.' }
+      continue
+    }
+    if (line.length > MAX_URL_LENGTH) return { error: 'A block list URL is too long.' }
+    const text = line.startsWith('!') ? line.slice(1).trim() : line
+    let url: URL
+    try {
+      url = new URL(text)
+    } catch {
+      return { error: `Not a valid http, https or file address: ${line.slice(0, 60)}` }
+    }
+    if (!SCHEMES.has(url.protocol) || /\s/.test(text)) return { error: `Not a valid http, https or file address: ${line.slice(0, 60)}` }
+  }
+  return { urls: lines }
+}
+
+export function planAction(action: string | undefined, raw: Record<string, string | undefined>): { error: string } | { plan: Plan } {
+  if (action === undefined || !(SERVER_ACTIONS as readonly string[]).includes(action)) {
+    return { error: `action must be one of: ${SERVER_ACTIONS.join(', ')}.` }
+  }
+  const kind = action as ServerAction
+  switch (kind) {
+    case 'blocked.add':
+    case 'blocked.delete':
+    case 'allowed.add':
+    case 'allowed.delete':
+    case 'cache.delete': {
+      const domain = cleanDomain(raw.domain)
+      if (domain === null) return { error: 'A valid domain name is required.' }
+      const [list, verb] = kind.split('.')
+      const path = list === 'cache' ? 'cache/delete' : `${list}/${verb}`
+      const done =
+        kind === 'cache.delete' ? 'Deleted from the cache.' : `${verb === 'add' ? 'Added to' : 'Removed from'} ${list === 'blocked' ? 'Blocked' : 'Allowed'}.`
+      return { plan: { action: kind, path, params: { domain }, target: domain, done } }
+    }
+    case 'blocking.disable': {
+      const text = (raw.minutes ?? '').trim()
+      const minutes = /^\d{1,5}$/.test(text) ? Number(text) : 0
+      if (minutes < 1 || minutes > MAX_MINUTES) return { error: `minutes must be a whole number from 1 to ${MAX_MINUTES}.` }
+      return {
+        plan: { action: kind, path: 'settings/temporaryDisableBlocking', params: { minutes: String(minutes) }, target: `minutes=${minutes}`, done: `Blocking disabled for ${minutes} minute(s).` },
+      }
+    }
+    case 'blocking.enable':
+      return settingsPlan(kind, { enableBlocking: 'true' }, undefined, 'Blocking enabled.')
+    case 'blocking.setListUrls': {
+      const cleaned = cleanUrls(raw.urls)
+      if ('error' in cleaned) return cleaned
+      return settingsPlan(kind, { blockListUrls: cleaned.urls.join(',') }, `urls=${cleaned.urls.length}`, `Block list URLs replaced (${cleaned.urls.length} line(s)).`)
+    }
+    case 'blocking.updateLists':
+      return { plan: { action: kind, path: 'settings/forceUpdateBlockLists', params: {}, done: 'Block list update started.' } }
+    case 'cache.flush':
+      return { plan: { action: kind, path: 'cache/flush', params: {}, done: 'Cache flushed.' } }
+  }
+}
+
+/* A `settings/set` of one field. The same checks as the per-server route, applied to what the hub itself builds. */
+function settingsPlan(action: ServerAction, params: Record<string, string>, target: string | undefined, done: string): { error: string } | { plan: Plan } {
+  if (Object.keys(params).some((k) => isForbiddenSettingsField(k) || k.startsWith('webService'))) return { error: 'That setting is not changed from the hub.' }
+  return { plan: { action, path: 'settings/set', params, target, done } }
+}
+
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+
+export class ServersRunner {
+  #config: HubConfig
+  #upstream: Upstream
+
+  constructor(config: HubConfig, upstream: Upstream) {
+    this.#config = config
+    this.#upstream = upstream
+  }
+
+  /*
+  The plan on every target, in parallel. Returns `{ error }` when the request itself is refused
+  (no targets, an unknown server), else one result per target.
+  */
+  async run(plan: Plan, targets: string[]): Promise<{ error: string } | { results: TargetResult[] }> {
+    const known = new Map(this.#config.servers.map((s) => [s.id, s]))
+    const unique = [...new Set(targets)]
+    if (unique.length === 0) return { error: 'Choose at least one server.' }
+    const unknown = unique.filter((t) => !known.has(t))
+    if (unknown.length > 0) return { error: `Unknown server: ${unknown.join(', ')}.` }
+    const results = await Promise.all(
+      unique.map(async (id): Promise<TargetResult> => {
+        const server = known.get(id)!
+        if (server.adminToken === undefined) return { server: id, ok: false, message: 'no admin token configured' }
+        try {
+          await this.#upstream.write(server, plan.path, plan.params)
+          return { server: id, ok: true, message: plan.done }
+        } catch (e) {
+          return { server: id, ok: false, message: reason(e) }
+        }
+      }),
+    )
+    return { results }
+  }
+}

hub/session.ts

@@ -1,85 +1,143 @@
 /*
-The hub's own sign-in. One user from the environment, sessions kept in memory with an
-idle expiry, and the `info` object the console reads from `user/login` and
-`user/session/get` (flat, as the real server sends it).
+The hub's own sign-in. Sessions are kept in memory with an idle expiry; each remembers who
+signed in and what that user looked like (role, hash, disabled flag) at that moment, so a
+change to the users file ends the sessions of the users it touched. `info` is the object
+the console reads from `user/login` and `user/session/get` (flat, as the real server sends it).
 */
-import { createHash, randomBytes, timingSafeEqual } from 'node:crypto'
+import { randomBytes } from 'node:crypto'
 import { HUB_VERSION, type HubConfig } from './config.ts'
+import { can, signature, type Capability, type Role, type UserRecord, type Users } from './users.ts'
 
 export const IDLE_MS = 30 * 60_000
+/* A download link carries a token that works once, within this time. */
+export const SINGLE_USE_MS = 60_000
 
-/* Compared as hashes so the lengths always match and the time does not depend on the content. */
-function sameSecret(a: string, b: string): boolean {
-  const ha = createHash('sha256').update(a).digest()
-  const hb = createHash('sha256').update(b).digest()
-  return timingSafeEqual(ha, hb)
+export interface Identity {
+  user: string
+  role: Role
+}
+
+interface Session extends Identity {
+  last: number
+  sig: string
+}
+
+interface SingleUse extends Identity {
+  until: number
+  sig: string
 }
 
 /*
 Every key `src/app/sections.ts` reads, spelled out: the console hides a section only
-when `canView === false`, so a missing key would leave it visible. The hub shows the
-Dashboard and the Logs; About has no permission and is always there. The hub is
-read-only, so nothing can be modified or deleted.
+when `canView === false`, so a missing key would leave it visible. About has no permission
+and is always there. Administration (users, SSO, LDAP of each server) is not offered.
+
+What a role may change comes from its capabilities, per key. Settings never get `canDelete`
+(that is the backup and restore, which the hub does not forward), nor do the Logs and the
+Dashboard (deleting logs and stats stays in each server's own console). DNS Client needs
+`admin` to modify because its Import writes a zone. The blocking switch and the block list update are blocking,
+so behind a hub the console asks for Blocked/Allowed Modify there (which an operator has), not for Settings Modify (which
+only an admin has). Saving the list URLs of one server stays a settings change.
 */
 const PERMISSION_KEYS = [
   'Dashboard', 'Zones', 'Cache', 'Allowed', 'Blocked', 'Apps', 'DnsClient', 'Settings', 'DhcpServer', 'Administration', 'Logs',
 ]
-const VIEWABLE = new Set(['Dashboard', 'Logs'])
+const VIEWABLE = new Set(['Dashboard', 'Logs', 'Apps', 'Zones', 'Cache', 'Allowed', 'Blocked', 'DnsClient', 'Settings', 'DhcpServer'])
+const MODIFIES: Record<string, Capability> = {
+  Apps: 'apps.write', Zones: 'admin', Cache: 'operate', Allowed: 'operate', Blocked: 'operate',
+  DnsClient: 'admin', Settings: 'admin', DhcpServer: 'admin',
+}
+const DELETES: Record<string, Capability> = {
+  Apps: 'apps.write', Zones: 'admin', Cache: 'operate', Allowed: 'operate', Blocked: 'operate', DhcpServer: 'admin',
+}
 
 export class Sessions {
   #config: HubConfig
+  #users: Users
   #now: () => number
   #started: string
-  #seen = new Map<string, number>()
+  #seen = new Map<string, Session>()
+  #single = new Map<string, SingleUse>()
 
-  constructor(config: HubConfig, now: () => number = Date.now) {
+  #drift: () => number
+
+  constructor(config: HubConfig, users: Users, now: () => number = Date.now, drift: () => number = () => 0) {
+    this.#drift = drift
     this.#config = config
+    this.#users = users
     this.#now = now
     this.#started = new Date(now()).toISOString()
   }
 
-  /** A new session token, or null when the credentials are wrong. */
-  login(user: string, password: string): string | null {
-    // Both are always compared, so a wrong user and a wrong password cost the same.
-    const userOk = sameSecret(user.trim().toLowerCase(), this.#config.user.toLowerCase())
-    const passOk = sameSecret(password, this.#config.password)
-    if (!(userOk && passOk)) return null
+  /** Opens a session for a user already authenticated by `Users.authenticate`. */
+  open(user: UserRecord): string {
     this.#sweep()
     const token = randomBytes(32).toString('hex')
-    this.#seen.set(token, this.#now())
+    this.#seen.set(token, { user: user.name, role: user.role, last: this.#now(), sig: signature(user) })
     return token
   }
 
-  /** True while the token is valid; each use extends it. */
-  touch(token: string | null | undefined): boolean {
-    if (!token) return false
-    const last = this.#seen.get(token)
-    if (last === undefined) return false
-    if (this.#now() - last > IDLE_MS) {
+  /* The session's identity while it is valid (extending it), else null. A user who was removed,
+     disabled or edited since sign-in ends the session. */
+  touch(token: string | null | undefined): Identity | null {
+    if (!token) return null
+    const session = this.#seen.get(token)
+    if (session === undefined) return null
+    const record = this.#users.find(session.user)
+    if (this.#now() - session.last > IDLE_MS || record === undefined || signature(record) !== session.sig) {
       this.#seen.delete(token)
-      return false
+      return null
     }
-    this.#seen.set(token, this.#now())
-    return true
+    session.last = this.#now()
+    return { user: session.user, role: session.role }
   }
 
   logout(token: string): void {
     this.#seen.delete(token)
   }
 
-  #sweep(): void {
-    const now = this.#now()
-    for (const [token, last] of this.#seen) if (now - last > IDLE_MS) this.#seen.delete(token)
+  /* A token for one download (the console opens it in a new tab, which has no Authorization header). */
+  openSingleUse(token: string): string | null {
+    const session = this.#seen.get(token)
+    if (session === undefined) return null
+    const once = randomBytes(32).toString('hex')
+    this.#single.set(once, { user: session.user, role: session.role, sig: session.sig, until: this.#now() + SINGLE_USE_MS })
+    return once
   }
 
-  info() {
+  /* The identity behind a single-use token, which is spent by asking; null when it is unknown, used,
+     expired, or its user was removed or edited since. */
+  takeSingleUse(token: string | null | undefined): Identity | null {
+    if (!token) return null
+    const once = this.#single.get(token)
+    if (once === undefined) return null
+    this.#single.delete(token)
+    const record = this.#users.find(once.user)
+    if (this.#now() > once.until || record === undefined || signature(record) !== once.sig) return null
+    return { user: once.user, role: once.role }
+  }
+
+  #sweep(): void {
+    const now = this.#now()
+    for (const [token, session] of this.#seen) if (now - session.last > IDLE_MS) this.#seen.delete(token)
+    for (const [token, once] of this.#single) if (now > once.until) this.#single.delete(token)
+  }
+
+  info(role: Role) {
     const { servers, domain } = this.#config
     return {
       version: HUB_VERSION,
       uptimestamp: this.#started,
       dnsServerDomain: domain,
       permissions: Object.fromEntries(
-        PERMISSION_KEYS.map((key) => [key, { canView: VIEWABLE.has(key), canModify: false, canDelete: false }]),
+        PERMISSION_KEYS.map((key) => [
+          key,
+          {
+            canView: VIEWABLE.has(key),
+            canModify: key in MODIFIES && can(role, MODIFIES[key]),
+            canDelete: key in DELETES && can(role, DELETES[key]),
+          },
+        ]),
       ),
       /* The console offers its node selector when the cluster is initialised. The
          first server plays the Primary (`primaryNodeName` looks for one). */
@@ -92,20 +150,26 @@ export class Sessions {
       useSoaSerialDateScheme: true,
       dnssecValidation: true,
       hub: true,
+      hubRole: role,
+      /* The servers the hub holds an admin token for: the only ones a write can reach. */
+      /* How many servers differ from their baseline now: the console marks the header with it. */
+      hubDrift: this.#drift(),
+      hubWritable: servers.filter((s) => s.adminToken !== undefined).map((s) => s.id),
     }
   }
 
   /** The flat body of `user/login` and `user/session/get`. */
-  body(token: string) {
+  body(token: string, identity: Identity) {
     return {
       status: 'ok',
       token,
-      username: this.#config.user,
-      displayName: this.#config.user,
+      username: identity.user,
+      displayName: identity.user,
       type: 'Local',
       isSsoUser: false,
       totpEnabled: false,
-      info: this.info(),
+      hubRole: identity.role,
+      info: this.info(identity.role),
     }
   }
 }

hub/settings-config.ts

@@ -0,0 +1,586 @@
+/*
+Settings and blocking compare across the servers (stage 7): the baseline kept in the hub, the
+fields that stay local, a preview, an apply with history and read-back, and the custom Blocked and
+Allowed lists compared as sets.
+
+The settings are read with each server's ADMIN token when the hub has one (a read-only token may
+see blanked secrets, and a baseline imported from those would blank them elsewhere). Apply sends
+ONLY the shared fields that differ, through the same checks as a per-server `settings/set`
+(guards.ts), after saving the server's settings to the history, and reads them back.
+
+Nothing here puts a secret value, or a hash of one, in an answer; the audit lines carry field
+names only (see settings-sync.ts).
+*/
+import { createHash } from 'node:crypto'
+import type { HubConfig, HubServer } from './config.ts'
+import { WEB_SERVICE_MESSAGE, isForbiddenSettingsField, webServiceChanges } from './guards.ts'
+import {
+  BLOCKING_FIELDS,
+  LOCAL_DEFAULTS,
+  MAX_DELETES,
+  MAX_LIST,
+  changedFields,
+  compareSettings,
+  diffSets,
+  hasSecret,
+  isBuiltInLocal,
+  isFieldName,
+  maskSettings,
+  maskValue,
+  normalize,
+  parseDomains,
+  prepare,
+  restoreSecrets,
+  sameValue,
+  toParams,
+  type Comparison,
+} from './settings-sync.ts'
+import { cleanDomain } from './servers-run.ts'
+import { UserError } from './apps-config.ts'
+import type { FieldChange } from './apps-sync.ts'
+import { MAX_CONFIG } from './apps-sync.ts'
+import {
+  listHistoryIn,
+  readHistoryIn,
+  readListsBaseline,
+  readSettingsBaseline,
+  readSettingsOverrides,
+  settingsHistoryDir,
+  writeHistoryIn,
+  writeListsBaseline,
+  writeSettingsBaseline,
+  writeSettingsOverrides,
+  type HistoryEntry,
+  type ListsBaseline,
+  type SettingsBaseline,
+} from './store.ts'
+import { Upstream } from './upstream.ts'
+import { readCapped } from './zones-sync.ts'
+
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+const hashOf = (value: unknown): string => createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 16)
+
+export const MAX_ADDS = 1000
+const LIST_TIMEOUT_MS = 30_000
+const LIST_BYTES = 8 * 1024 * 1024
+
+export interface TargetResult {
+  server: string
+  ok: boolean
+  message?: string
+}
+
+export interface SettingsTargetPreview {
+  server: string
+  ok: boolean
+  error?: string
+  writable: boolean
+  /* Tells the apply whether the server changed since this preview. Built from masked values. */
+  currentHash: string
+  /* `{kind:'json'}` so the console's field table draws it as it does a config. */
+  changes: { kind: 'json'; changes: FieldChange[] }
+  /* Fields whose change is a secret: shown as "secret changed", never as values. */
+  secretFields: string[]
+  unchanged: boolean
+  warnings: string[]
+}
+
+export interface ListsServerView {
+  id: string
+  ok: boolean
+  error?: string
+  writable: boolean
+  blocked: { count: number; missing: number; extra: number; missingSample: string[]; extraSample: string[] }
+  allowed: { count: number; missing: number; extra: number; missingSample: string[]; extraSample: string[] }
+  status: 'same' | 'differs' | 'unknown'
+}
+
+const SAMPLE = 100
+
+export class SettingsConfig {
+  #config: HubConfig
+  #upstream: Upstream
+  #now: () => number
+
+  constructor(config: HubConfig, upstream: Upstream, now: () => number = Date.now) {
+    this.#config = config
+    this.#upstream = upstream
+    this.#now = now
+  }
+
+  #server(id: string): HubServer {
+    const found = this.#config.servers.find((s) => s.id === id)
+    if (found === undefined) throw new UserError(`Unknown server: ${id}.`)
+    return found
+  }
+
+  #targets(ids: string[]): HubServer[] {
+    const unique = [...new Set(ids)]
+    if (unique.length === 0) throw new UserError('Choose at least one server.')
+    return unique.map((id) => this.#server(id))
+  }
+
+  /* Fresh `settings/get`, with the admin token when there is one. */
+  async #read(server: HubServer): Promise<unknown> {
+    this.#upstream.invalidate(server.id)
+    return this.#upstream.call(server, 'settings/get', {}, server.adminToken ?? server.token)
+  }
+
+  async #readAll(): Promise<Record<string, { settings: unknown } | { error: string }>> {
+    const entries = await Promise.all(
+      this.#config.servers.map(async (s): Promise<[string, { settings: unknown } | { error: string }]> => {
+        try {
+          return [s.id, { settings: await this.#read(s) }]
+        } catch (e) {
+          return [s.id, { error: reason(e) }]
+        }
+      }),
+    )
+    return Object.fromEntries(entries)
+  }
+
+  getBaseline(): SettingsBaseline | null {
+    return readSettingsBaseline(this.#config.dataDir)
+  }
+
+  #extraLocal(): string[] {
+    return readSettingsOverrides(this.#config.dataDir).paths
+  }
+
+  /* The baseline as the console may see it: secrets masked. */
+  baselineView(): { settings: Record<string, unknown>; importedFrom?: string; updatedAt: string; updatedBy: string } | null {
+    const b = this.getBaseline()
+    return b === null ? null : { ...b, settings: maskSettings(b.settings) }
+  }
+
+  async compare(): Promise<unknown> {
+    const baseline = this.getBaseline()
+    const extra = this.#extraLocal()
+    const live = await this.#readAll()
+    const comparison: Comparison = compareSettings({ baseline: baseline?.settings ?? null, live, extraLocal: extra })
+    return {
+      baseline: this.baselineView(),
+      localFields: { builtIn: [...LOCAL_DEFAULTS], builtInPatterns: ['webService*', 'user*, sso*, ldap*, auth*'], extra },
+      blockingFields: [...BLOCKING_FIELDS],
+      servers: comparison.servers.map((s) => ({ ...s, writable: this.#config.servers.find((x) => x.id === s.id)?.adminToken !== undefined })),
+      fields: comparison.fields,
+      withBaseline: comparison.withBaseline,
+    }
+  }
+
+  /* Writes the baseline; the one it replaces goes to the history first. */
+  #setBaseline(settings: Record<string, unknown>, importedFrom: string | undefined, user: string): void {
+    const old = this.getBaseline()
+    if (old !== null && JSON.stringify(old.settings) !== JSON.stringify(settings)) {
+      try {
+        writeHistoryIn(settingsHistoryDir(this.#config.dataDir), 'baseline', JSON.stringify(old.settings), 'before-save', this.#now())
+      } catch (e) {
+        throw new UserError(`The old baseline could not be saved first (${reason(e)}); nothing was changed.`)
+      }
+    }
+    writeSettingsBaseline(this.#config.dataDir, {
+      settings,
+      ...(importedFrom === undefined ? {} : { importedFrom }),
+      updatedAt: new Date(this.#now()).toISOString(),
+      updatedBy: user,
+    })
+  }
+
+  /* Takes a server's settings as the baseline; with `group=blocking` only its blocking fields replace the baseline's. */
+  async importBaseline(serverId: string, group: string | undefined, user: string): Promise<{ detail: string }> {
+    const server = this.#server(serverId)
+    let raw: unknown
+    try {
+      raw = await this.#read(server)
+    } catch (e) {
+      throw new UserError(`${serverId}: ${reason(e)}`)
+    }
+    const read = normalize(raw)
+    if (Object.keys(read).length === 0) throw new UserError(`${serverId} sent no settings.`)
+    let next = read
+    if (group === 'blocking') {
+      const old = this.getBaseline()?.settings ?? {}
+      const kept: Record<string, unknown> = { ...old }
+      for (const name of BLOCKING_FIELDS) if (Object.hasOwn(read, name)) kept[name] = read[name]
+      next = kept
+    } else if (group !== undefined && group !== '') throw new UserError('group can only be "blocking".')
+    this.#setBaseline(next, serverId, user)
+    return { detail: `from ${serverId}${group === 'blocking' ? ' (blocking fields)' : ''}, ${Object.keys(next).length} fields` }
+  }
+
+  /* The baseline from the editor's JSON. A secret shown as the placeholder keeps the stored one. */
+  saveBaseline(textBody: string, user: string): { detail: string } {
+    if (textBody.length > MAX_CONFIG) throw new UserError('The baseline is larger than 1 MB.')
+    let parsed: unknown
+    try {
+      parsed = JSON.parse(textBody)
+    } catch (e) {
+      throw new UserError(`The baseline is not valid JSON: ${reason(e)}`)
+    }
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) throw new UserError('The baseline must be a JSON object of settings.')
+    const named = Object.keys(parsed as object)
+    const bad = named.find((k) => !isFieldName(k))
+    if (bad !== undefined) throw new UserError(`"${bad.slice(0, 40)}" is not a setting name.`)
+    let settings: Record<string, unknown>
+    try {
+      settings = restoreSecrets(normalize(parsed), this.getBaseline()?.settings ?? null)
+    } catch (e) {
+      throw new UserError(reason(e))
+    }
+    this.#setBaseline(settings, this.getBaseline()?.importedFrom, user)
+    return { detail: `${Object.keys(settings).length} fields` }
+  }
+
+  saveOverrides(pathsRaw: string[]): { detail: string } {
+    const seen = new Set<string>()
+    if (pathsRaw.length > 50) throw new UserError('At most 50 extra local fields.')
+    for (const p of pathsRaw) {
+      if (!isFieldName(p)) throw new UserError(`"${p.slice(0, 40)}" is not a setting name.`)
+      if (seen.has(p)) throw new UserError(`"${p}" is listed twice.`)
+      seen.add(p)
+    }
+    writeSettingsOverrides(this.#config.dataDir, { paths: pathsRaw })
+    return { detail: `${pathsRaw.length} extra local field(s)` }
+  }
+
+  /* ---- Preview and apply ---- */
+
+  #fingerprint(current: unknown, baseline: Record<string, unknown>, extra: string[]): string {
+    const { settings } = prepare(current)
+    const shared = Object.keys(settings)
+      .filter((k) => !isBuiltInLocal(k) && !extra.includes(k) && Object.hasOwn(baseline, k))
+      .sort()
+      .map((k) => [k, maskValue(k, settings[k])])
+    return hashOf(shared)
+  }
+
+  async preview(ids: string[], only?: readonly string[], override?: SettingsBaseline): Promise<{ targets: SettingsTargetPreview[] }> {
+    const targets = this.#targets(ids)
+    const baseline = override ?? this.getBaseline()
+    if (baseline === null) throw new UserError('There is no settings baseline yet. Accept one from a server first.')
+    const extra = this.#extraLocal()
+    const base = normalize(baseline.settings)
+    const out = await Promise.all(
+      targets.map(async (s): Promise<SettingsTargetPreview> => {
+        const empty = { kind: 'json' as const, changes: [] }
+        const writable = s.adminToken !== undefined
+        let current: unknown
+        try {
+          current = await this.#read(s)
+        } catch (e) {
+          return { server: s.id, ok: false, error: reason(e), writable, currentHash: '', changes: empty, secretFields: [], unchanged: false, warnings: [] }
+        }
+        const names = changedFields(baseline.settings, current, extra, only)
+        const live = prepare(current)
+        const warnings: string[] = []
+        if (!writable) warnings.push('The hub has no admin token for this server, so it cannot write to it.')
+        if (live.paused) warnings.push('Blocking is paused for a while on this server; its blocking switch is left alone.')
+        const changes: FieldChange[] = names.map((name) => {
+          const [a, b] = [maskValue(name, live.settings[name]), maskValue(name, base[name])]
+          return { path: name, type: a === undefined ? 'added' : b === undefined ? 'removed' : 'changed', a, b }
+        })
+        return {
+          server: s.id,
+          ok: true,
+          writable,
+          currentHash: this.#fingerprint(current, base, extra),
+          changes: { kind: 'json', changes },
+          secretFields: names.filter(hasSecret),
+          unchanged: names.length === 0,
+          warnings,
+        }
+      }),
+    )
+    return { targets: out }
+  }
+
+  async #applyOne(s: HubServer, baseline: SettingsBaseline, extra: string[], expect: string | undefined, only?: readonly string[]): Promise<{ result: TargetResult; detail?: string }> {
+    const fail = (message: string): { result: TargetResult } => ({ result: { server: s.id, ok: false, message } })
+    if (s.adminToken === undefined) return fail('no admin token configured')
+    try {
+      let current: unknown
+      try {
+        current = await this.#read(s)
+      } catch (e) {
+        return fail(reason(e))
+      }
+      const base = normalize(baseline.settings)
+      if (expect !== undefined && expect !== this.#fingerprint(current, base, extra)) return fail('the settings changed on the server since the preview; run the preview again')
+      const names = changedFields(baseline.settings, current, extra, only)
+      if (names.length === 0) return { result: { server: s.id, ok: true, message: 'Already in sync; nothing was written.' } }
+
+      const params: Record<string, string> = {}
+      for (const name of names) {
+        const encoded = toParams(name, base[name])
+        if ('error' in encoded) return fail(`${name}: ${encoded.error}; nothing was changed`)
+        Object.assign(params, encoded.params)
+      }
+      // The same checks as a per-server settings/set (guards.ts).
+      if (Object.keys(params).some((k) => isForbiddenSettingsField(k) || k.startsWith('webService'))) return fail('a setting the hub does not change was in the list; nothing was changed')
+      const web = webServiceChanges(params, current)
+      if (web.length > 0) return fail(`${WEB_SERVICE_MESSAGE} nothing was changed`)
+
+      try {
+        writeHistoryIn(settingsHistoryDir(this.#config.dataDir), s.id, JSON.stringify(normalize(current)), 'before-apply', this.#now())
+      } catch (e) {
+        return fail(`the old settings could not be saved first (${reason(e)}); nothing was changed`)
+      }
+      await this.#upstream.write(s, 'settings/set', params)
+      const detail = names.join(', ')
+      let after: unknown
+      try {
+        after = prepare(await this.#read(s)).settings
+      } catch (e) {
+        return { result: { server: s.id, ok: false, message: `the server accepted the change but could not be read back (${reason(e)})` }, detail }
+      }
+      const wrong = names.filter((n) => !sameValue(n, base[n], (after as Record<string, unknown>)[n]))
+      if (wrong.length > 0) return { result: { server: s.id, ok: false, message: `the server accepted the change but reads back different values for: ${wrong.join(', ')}` }, detail }
+      return { result: { server: s.id, ok: true, message: `Applied and verified (${names.length} field${names.length === 1 ? '' : 's'}).` }, detail }
+    } catch (e) {
+      return fail(reason(e))
+    }
+  }
+
+  /* `override` is a baseline that is not the hub's own (a profile's settings). */
+  async apply(ids: string[], expect: Record<string, string>, only?: readonly string[], override?: SettingsBaseline): Promise<{ results: TargetResult[]; details: Record<string, string> }> {
+    const targets = this.#targets(ids)
+    const baseline = override ?? this.getBaseline()
+    if (baseline === null) throw new UserError('There is no settings baseline yet. Accept one from a server first.')
+    const extra = this.#extraLocal()
+    const done = await Promise.all(targets.map((s) => this.#applyOne(s, baseline, extra, expect[s.id], only)))
+    const details: Record<string, string> = {}
+    for (const d of done) if (d.detail !== undefined) details[d.result.server] = d.detail
+    return { results: done.map((d) => d.result), details }
+  }
+
+  /* The extra local field names (for the profiles, which keep shared fields only). */
+  extraLocal(): string[] {
+    return this.#extraLocal()
+  }
+
+  /*
+  For the drift check: which servers differ from the settings baseline and from the lists baseline.
+  Nothing is read for a baseline that does not exist. Field names only, never a value.
+  */
+  async driftFindings(): Promise<{ kind: 'settings' | 'lists'; server: string; status: 'differs' | 'unreadable'; fields: string[] }[]> {
+    const out: { kind: 'settings' | 'lists'; server: string; status: 'differs' | 'unreadable'; fields: string[] }[] = []
+    const baseline = this.getBaseline()
+    const lists = this.getListsBaseline()
+    if (baseline === null && lists === null) return out
+    const extra = this.#extraLocal()
+    await Promise.all(
+      this.#config.servers.map(async (s) => {
+        if (baseline !== null) {
+          try {
+            const fields = changedFields(baseline.settings, await this.#read(s), extra)
+            if (fields.length > 0) out.push({ kind: 'settings', server: s.id, status: 'differs', fields })
+          } catch {
+            out.push({ kind: 'settings', server: s.id, status: 'unreadable', fields: [] })
+          }
+        }
+        if (lists !== null) {
+          try {
+            const live = await this.#readLists(s)
+            const fields = (['blocked', 'allowed'] as const).filter((l) => {
+              const d = diffSets(lists[l], live[l])
+              return d.missing.length + d.extra.length > 0
+            })
+            if (fields.length > 0) out.push({ kind: 'lists', server: s.id, status: 'differs', fields })
+          } catch {
+            out.push({ kind: 'lists', server: s.id, status: 'unreadable', fields: [] })
+          }
+        }
+      }),
+    )
+    return out
+  }
+
+  /* ---- History: list and view (no rollback; apply the baseline again, or accept an old one by hand) ---- */
+
+  historyList(): HistoryEntry[] {
+    return listHistoryIn(settingsHistoryDir(this.#config.dataDir))
+  }
+
+  historyGet(id: string): { entry: HistoryEntry; settings?: Record<string, unknown>; lists?: { blocked: number; allowed: number } } {
+    const entry = this.historyList().find((e) => e.id === id)
+    const text = entry === undefined ? null : readHistoryIn(settingsHistoryDir(this.#config.dataDir), id)
+    if (entry === undefined || text === null) throw new UserError('There is no such history entry.')
+    let parsed: unknown
+    try {
+      parsed = JSON.parse(text)
+    } catch {
+      throw new UserError('That history entry cannot be read.')
+    }
+    if (entry.kind === 'before-lists') {
+      const p = parsed as { blocked?: unknown[]; allowed?: unknown[] }
+      return { entry, lists: { blocked: p.blocked?.length ?? 0, allowed: p.allowed?.length ?? 0 } }
+    }
+    return { entry, settings: maskSettings(normalize(parsed)) }
+  }
+
+  /* ---- Blocked and Allowed lists ---- */
+
+  async #readList(server: HubServer, list: 'blocked' | 'allowed'): Promise<string[]> {
+    const file = await this.#upstream.raw(server, `${list}/export`, {})
+    return parseDomains(await readCapped(file, LIST_BYTES, LIST_TIMEOUT_MS, `The ${list} list`))
+  }
+
+  async #readLists(server: HubServer): Promise<{ blocked: string[]; allowed: string[] }> {
+    const [blocked, allowed] = await Promise.all([this.#readList(server, 'blocked'), this.#readList(server, 'allowed')])
+    return { blocked, allowed }
+  }
+
+  getListsBaseline(): ListsBaseline | null {
+    return readListsBaseline(this.#config.dataDir)
+  }
+
+  async listsCompare(): Promise<unknown> {
+    const baseline = this.getListsBaseline()
+    const reads = await Promise.all(
+      this.#config.servers.map(async (s) => {
+        try {
+          return { s, lists: await this.#readLists(s), error: undefined }
+        } catch (e) {
+          return { s, lists: null, error: reason(e) }
+        }
+      }),
+    )
+    const reference = baseline ?? reads.find((r) => r.lists !== null)?.lists ?? null
+    const refId = baseline === null ? reads.find((r) => r.lists !== null)?.s.id : undefined
+    const view = (live: string[], ref: string[], own: boolean) => {
+      const d = own ? { missing: [], extra: [] } : diffSets(ref, live)
+      return { count: live.length, missing: d.missing.length, extra: d.extra.length, missingSample: d.missing.slice(0, SAMPLE), extraSample: d.extra.slice(0, SAMPLE) }
+    }
+    const servers: ListsServerView[] = reads.map((r) => {
+      const writable = r.s.adminToken !== undefined
+      if (r.lists === null || reference === null) {
+        const none = { count: 0, missing: 0, extra: 0, missingSample: [], extraSample: [] }
+        return { id: r.s.id, ok: false, error: r.error, writable, blocked: none, allowed: none, status: 'unknown' }
+      }
+      const own = r.s.id === refId
+      const blocked = view(r.lists.blocked, reference.blocked, own)
+      const allowed = view(r.lists.allowed, reference.allowed, own)
+      const differs = blocked.missing + blocked.extra + allowed.missing + allowed.extra > 0
+      return { id: r.s.id, ok: true, writable, blocked, allowed, status: differs ? 'differs' : 'same' }
+    })
+    // Without a baseline the first server is the reference; it differs when any other does.
+    if (baseline === null) {
+      const first = servers.find((s) => s.id === refId)
+      if (first !== undefined && servers.some((s) => s.id !== first.id && s.status === 'differs')) first.status = 'differs'
+    }
+    return {
+      baseline: baseline === null ? null : { blocked: baseline.blocked.length, allowed: baseline.allowed.length, importedFrom: baseline.importedFrom, updatedAt: baseline.updatedAt, updatedBy: baseline.updatedBy },
+      withBaseline: baseline !== null,
+      servers,
+    }
+  }
+
+  async listsImport(serverId: string, user: string): Promise<{ detail: string }> {
+    const server = this.#server(serverId)
+    let lists: { blocked: string[]; allowed: string[] }
+    try {
+      lists = await this.#readLists(server)
+    } catch (e) {
+      throw new UserError(`${serverId}: ${reason(e)}`)
+    }
+    if (lists.blocked.length > MAX_LIST || lists.allowed.length > MAX_LIST) {
+      throw new UserError(`A list is longer than ${MAX_LIST} domains. The hub compares custom lists, not whole block lists.`)
+    }
+    writeListsBaseline(this.#config.dataDir, { ...lists, importedFrom: serverId, updatedAt: new Date(this.#now()).toISOString(), updatedBy: user })
+    return { detail: `from ${serverId}: ${lists.blocked.length} blocked, ${lists.allowed.length} allowed` }
+  }
+
+  async listsPreview(ids: string[], removeExtras: boolean, override?: ListsBaseline): Promise<{ targets: (ListsServerView & { tooMany?: string })[] }> {
+    const targets = this.#targets(ids)
+    const baseline = override ?? this.getListsBaseline()
+    if (baseline === null) throw new UserError('There is no lists baseline yet. Accept one from a server first.')
+    const out = await Promise.all(
+      targets.map(async (s) => {
+        const writable = s.adminToken !== undefined
+        const none = { count: 0, missing: 0, extra: 0, missingSample: [], extraSample: [] }
+        try {
+          const live = await this.#readLists(s)
+          const mk = (l: string[], b: string[]) => {
+            const d = diffSets(b, l)
+            return { count: l.length, missing: d.missing.length, extra: d.extra.length, missingSample: d.missing.slice(0, SAMPLE), extraSample: d.extra.slice(0, SAMPLE) }
+          }
+          const blocked = mk(live.blocked, baseline.blocked)
+          const allowed = mk(live.allowed, baseline.allowed)
+          const adds = blocked.missing + allowed.missing
+          const deletes = removeExtras ? blocked.extra + allowed.extra : 0
+          const tooMany = adds > MAX_ADDS ? `${adds} domains to add; at most ${MAX_ADDS} at a time.` : deletes > MAX_DELETES ? `${deletes} domains to remove; at most ${MAX_DELETES} at a time.` : undefined
+          return { id: s.id, ok: true, writable, blocked, allowed, status: adds + (removeExtras ? blocked.extra + allowed.extra : 0) > 0 ? ('differs' as const) : ('same' as const), ...(tooMany === undefined ? {} : { tooMany }) }
+        } catch (e) {
+          return { id: s.id, ok: false, error: reason(e), writable, blocked: none, allowed: none, status: 'unknown' as const }
+        }
+      }),
+    )
+    return { targets: out }
+  }
+
+  /* Adds the missing domains and, only when asked, deletes the extras; one call per domain, a few at a time. */
+  async #applyLists(s: HubServer, baseline: ListsBaseline, removeExtras: boolean): Promise<{ result: TargetResult; detail?: string }> {
+    const fail = (message: string): { result: TargetResult } => ({ result: { server: s.id, ok: false, message } })
+    if (s.adminToken === undefined) return fail('no admin token configured')
+    try {
+      const live = await this.#readLists(s)
+      const plan = (['blocked', 'allowed'] as const).map((list) => {
+        const d = diffSets(baseline[list], live[list])
+        return { list, add: d.missing, remove: removeExtras ? d.extra : [] }
+      })
+      const adds = plan.reduce((n, p) => n + p.add.length, 0)
+      const removals = plan.reduce((n, p) => n + p.remove.length, 0)
+      if (adds + removals === 0) return { result: { server: s.id, ok: true, message: 'Already in sync; nothing was written.' } }
+      if (adds > MAX_ADDS) return fail(`${adds} domains to add; at most ${MAX_ADDS} at a time, so nothing was changed`)
+      if (removals > MAX_DELETES) return fail(`${removals} domains to remove; at most ${MAX_DELETES} at a time, so nothing was changed`)
+      const calls: { path: string; domain: string }[] = []
+      let invalid = 0
+      for (const p of plan) {
+        for (const d of p.add) (cleanDomain(d) === null ? invalid++ : calls.push({ path: `${p.list}/add`, domain: d }))
+        for (const d of p.remove) (cleanDomain(d) === null ? invalid++ : calls.push({ path: `${p.list}/delete`, domain: d }))
+      }
+      if (removals > 0) {
+        try {
+          writeHistoryIn(settingsHistoryDir(this.#config.dataDir), s.id, JSON.stringify(live), 'before-lists', this.#now())
+        } catch (e) {
+          return fail(`the old lists could not be saved first (${reason(e)}); nothing was changed`)
+        }
+      }
+      let failed = 0
+      let firstError = ''
+      for (let i = 0; i < calls.length; i += 10) {
+        await Promise.all(
+          calls.slice(i, i + 10).map(async (c) => {
+            try {
+              await this.#upstream.write(s, c.path, { domain: c.domain })
+            } catch (e) {
+              failed++
+              if (firstError === '') firstError = reason(e)
+            }
+          }),
+        )
+      }
+      const detail = `${adds} to add, ${removals} to remove`
+      if (failed > 0 || invalid > 0) {
+        return { result: { server: s.id, ok: false, message: `${failed} change(s) failed${firstError === '' ? '' : ` (${firstError})`}${invalid > 0 ? `, ${invalid} domain(s) skipped as invalid` : ''}` }, detail }
+      }
+      const after = await this.#readLists(s)
+      const left = (['blocked', 'allowed'] as const).reduce((n, l) => n + diffSets(baseline[l], after[l]).missing.length + (removeExtras ? diffSets(baseline[l], after[l]).extra.length : 0), 0)
+      if (left > 0) return { result: { server: s.id, ok: false, message: `the changes were accepted but ${left} difference(s) remain when read back` }, detail }
+      return { result: { server: s.id, ok: true, message: `Added ${adds}, removed ${removals}.` }, detail }
+    } catch (e) {
+      return fail(reason(e))
+    }
+  }
+
+  async listsApply(ids: string[], removeExtras: boolean, override?: ListsBaseline): Promise<{ results: TargetResult[]; details: Record<string, string> }> {
+    const targets = this.#targets(ids)
+    const baseline = override ?? this.getListsBaseline()
+    if (baseline === null) throw new UserError('There is no lists baseline yet. Accept one from a server first.')
+    const done = await Promise.all(targets.map((s) => this.#applyLists(s, baseline, removeExtras)))
+    const details: Record<string, string> = {}
+    for (const d of done) if (d.detail !== undefined) details[d.result.server] = d.detail
+    return { results: done.map((d) => d.result), details }
+  }
+}
+

hub/settings-sync.test.ts

@@ -0,0 +1,630 @@
+// @vitest-environment node
+/*
+Stage 7, settings and blocking: the pure rules (local, ignored and secret fields, the encoding for
+settings/set) and the endpoints against fake servers that keep their settings and lists.
+*/
+import { existsSync, readdirSync, readFileSync, rmSync } from 'node:fs'
+import path from 'node:path'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { fake, startHub, ROLES, type Call, type Fake, type TestHub } from './fake-servers.ts'
+import {
+  SECRET_PLACEHOLDER,
+  changedFields,
+  compareSettings,
+  diffSets,
+  isBuiltInLocal,
+  maskSettings,
+  normalize,
+  parseDomains,
+  prepare,
+  restoreSecrets,
+  sameValue,
+  toParams,
+} from './settings-sync.ts'
+import { routeFor } from './routes.ts'
+
+const TSIG = 'TSIG-SECRET-9f3a'
+const PROXY = 'PROXY-PASS-77c1'
+const CERT = 'CERT-PASS-4b2d'
+
+function settingsOf(over: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    version: '15.5',
+    uptimestamp: '2026-01-01T00:00:00Z',
+    clusterInitialized: false,
+    dnsServerDomain: 'a.example.net',
+    dnsServerLocalEndPoints: ['0.0.0.0:53'],
+    dnsServerIPv4SourceAddresses: null,
+    webServiceHttpPort: 5380,
+    webServiceTlsCertificatePassword: CERT,
+    dnsTlsCertificatePath: '/etc/a.pfx',
+    dnsTlsCertificatePassword: CERT,
+    logFolder: 'logs-a',
+    defaultRecordTtl: 3600,
+    dnssecValidation: true,
+    recursion: 'AllowOnlyForPrivateNetworks',
+    forwarders: ['1.1.1.1', '9.9.9.9'],
+    qpmPrefixLimitsIPv4: [{ prefix: 32, udpLimit: 600, tcpLimit: 600 }],
+    tsigKeys: [{ keyName: 'transfer.key', sharedSecret: TSIG, algorithmName: 'hmac-sha256' }],
+    proxy: null,
+    enableBlocking: true,
+    temporaryDisableBlockingTill: null,
+    blockingType: 'NxDomain',
+    blockingAnswerTtl: 30,
+    blockListUrls: ['https://a.example/hosts', '# Privacy'],
+    blockListNextUpdatedOn: '2026-02-01T00:00:00Z',
+    ...over,
+  }
+}
+
+interface Box {
+  settings: Record<string, unknown>
+  blocked: string[]
+  allowed: string[]
+  /* The settings a read answers with after a write, to pretend the server ignored the change. */
+  ignoreWrites: boolean
+}
+
+const boxes = new Map<Fake, Box>()
+
+function decode(old: unknown, value: string): unknown {
+  if (typeof old === 'boolean') return value === 'true'
+  if (typeof old === 'number') return Number(value)
+  if (Array.isArray(old)) return value === 'false' ? [] : value.split(',')
+  return value
+}
+
+function install(f: Fake, box: Box): void {
+  boxes.set(f, box)
+  f.on('settings/get', () => ({ response: box.settings }))
+  f.on('settings/set', (c: Call) => {
+    if (box.ignoreWrites) return { response: box.settings }
+    for (const [k, v] of c.form) {
+      if (k === 'tsigKeys') {
+        box.settings.tsigKeys = v === 'false' ? [] : (() => {
+          const cells = v.split('|')
+          const keys = []
+          for (let i = 0; i < cells.length; i += 3) keys.push({ keyName: cells[i], sharedSecret: cells[i + 1], algorithmName: cells[i + 2] })
+          return keys
+        })()
+      } else if (k === 'qpmPrefixLimitsIPv4') {
+        const cells = v.split('|')
+        box.settings[k] = v === 'false' ? [] : [{ prefix: Number(cells[0]), udpLimit: Number(cells[1]), tcpLimit: Number(cells[2]) }]
+      } else if (k === 'proxyType') {
+        box.settings.proxy = v === 'none' ? null : { type: v, address: c.form.get('proxyAddress'), port: Number(c.form.get('proxyPort')), username: c.form.get('proxyUsername'), password: c.form.get('proxyPassword'), bypass: [] }
+      } else if (!k.startsWith('proxy') && k in box.settings) box.settings[k] = decode(box.settings[k], v)
+    }
+    return { response: box.settings }
+  })
+  for (const list of ['blocked', 'allowed'] as const) {
+    f.on(`${list}/export`, () => ({ text: box[list].join('\n') + '\n' }))
+    f.on(`${list}/add`, (c) => {
+      box[list].push(c.form.get('domain') ?? '')
+      return { response: {} }
+    })
+    f.on(`${list}/delete`, (c) => {
+      box[list] = box[list].filter((d) => d !== c.form.get('domain'))
+      return { response: {} }
+    })
+  }
+}
+
+let a: Fake
+let b: Fake
+let hub: TestHub
+
+const boxB = (): Box => boxes.get(b)!
+const writes = (f: Fake): Call[] => f.calls.filter((c) => c.method === 'POST')
+
+beforeAll(async () => {
+  a = await fake()
+  b = await fake()
+  hub = await startHub(a, b)
+})
+
+afterAll(() => {
+  hub.close()
+  a.server.close()
+  b.server.close()
+})
+
+beforeEach(() => {
+  a.calls = []
+  b.calls = []
+  a.down = false
+  b.down = false
+  install(a, { settings: settingsOf(), blocked: ['ads.example', 'both.example'], allowed: ['ok.example'], ignoreWrites: false })
+  install(
+    b,
+    {
+      settings: settingsOf({
+        dnsServerDomain: 'b.example.net',
+        logFolder: 'logs-b',
+        webServiceHttpPort: 5381,
+        uptimestamp: '2026-03-03T00:00:00Z',
+        defaultRecordTtl: 7200,
+        forwarders: ['8.8.8.8'],
+        blockingType: 'CustomAddress',
+      }),
+      blocked: ['both.example', 'only-b.example'],
+      allowed: ['ok.example', 'extra-allowed.example'],
+      ignoreWrites: false,
+    },
+  )
+  // A baseline and its history from an earlier test must not leak into the next one.
+  for (const dir of ['settings', 'blocking']) rmSync(path.join(hub.dataDir, dir), { recursive: true, force: true })
+})
+
+describe('which fields count', () => {
+  it('treats the host identity, certificates, paths and every webService field as local', () => {
+    for (const n of ['dnsServerDomain', 'dnsServerLocalEndPoints', 'dnsServerIPv4SourceAddresses', 'dnsServerIPv6SourceAddresses', 'webServiceHttpPort', 'webServiceTlsCertificatePath', 'dnsTlsCertificatePath', 'dnsTlsCertificatePassword', 'logFolder', 'ssoEnabled', 'ldapServer']) {
+      expect(isBuiltInLocal(n), n).toBe(true)
+    }
+    for (const n of ['forwarders', 'defaultRecordTtl', 'blockListUrls', 'tsigKeys']) expect(isBuiltInLocal(n), n).toBe(false)
+  })
+
+  it('drops the read-only and runtime fields', () => {
+    const n = normalize(settingsOf())
+    for (const gone of ['version', 'uptimestamp', 'clusterInitialized', 'temporaryDisableBlockingTill', 'blockListNextUpdatedOn']) expect(n).not.toHaveProperty(gone)
+    expect(n).toHaveProperty('forwarders')
+  })
+
+  it('leaves the blocking switch alone while blocking is paused', () => {
+    const p = prepare(settingsOf({ enableBlocking: false, temporaryDisableBlockingTill: '2026-12-01T00:00:00Z' }))
+    expect(p.paused).toBe(true)
+    expect(p.settings).not.toHaveProperty('enableBlocking')
+  })
+
+  it('compares by meaning: absent, null, empty text and an empty list are the same, key order does not count', () => {
+    expect(sameValue('x', null, '')).toBe(true)
+    expect(sameValue('x', [], null)).toBe(true)
+    expect(sameValue('x', { a: 1, b: 2 }, { b: 2, a: 1 })).toBe(true)
+    expect(sameValue('x', ['a', 'b'], ['b', 'a'])).toBe(false)
+    expect(sameValue('proxy', null, { type: 'None', address: 'x' })).toBe(true)
+    expect(sameValue('x', 5, '5')).toBe(false)
+  })
+})
+
+describe('comparing', () => {
+  const live = (x: Record<string, unknown>, y: Record<string, unknown>) => ({ a: { settings: settingsOf(x) }, b: { settings: settingsOf(y) } })
+
+  it('says same, differs and local-only against a baseline', () => {
+    const baseline = settingsOf()
+    const out = compareSettings({
+      baseline,
+      live: live({}, { dnsServerDomain: 'b.example.net', defaultRecordTtl: 1 }),
+      extraLocal: [],
+    })
+    const by = Object.fromEntries(out.servers.map((s) => [s.id, s]))
+    expect(by.a.status).toBe('same')
+    expect(by.b).toMatchObject({ status: 'differs', differing: 1, localDiffering: 1 })
+    expect(out.fields.map((f) => f.name)).toEqual(['defaultRecordTtl', 'dnsServerDomain'])
+
+    const onlyLocal = compareSettings({ baseline, live: live({}, { dnsServerDomain: 'b.example.net' }), extraLocal: [] })
+    expect(onlyLocal.servers.find((s) => s.id === 'b')!.status).toBe('local-only')
+  })
+
+  it('compares with the first server when there is no baseline', () => {
+    const out = compareSettings({ baseline: null, live: live({}, { forwarders: ['8.8.8.8'] }), extraLocal: [] })
+    expect(out.withBaseline).toBe(false)
+    expect(out.fields.map((f) => f.name)).toEqual(['forwarders'])
+    expect(out.servers.every((s) => s.status === 'differs')).toBe(true)
+  })
+
+  it('marks an unreachable server unknown', () => {
+    const out = compareSettings({ baseline: settingsOf(), live: { a: { settings: settingsOf() }, b: { error: 'timed out' } }, extraLocal: [] })
+    expect(out.servers.find((s) => s.id === 'b')).toMatchObject({ ok: false, status: 'unknown', error: 'timed out' })
+  })
+
+  it('counts an extra local field as local, not as a difference', () => {
+    const out = compareSettings({ baseline: settingsOf(), live: live({}, { defaultRecordTtl: 1 }), extraLocal: ['defaultRecordTtl'] })
+    expect(out.servers.find((s) => s.id === 'b')!.status).toBe('local-only')
+  })
+
+  it('never puts a secret value in a row, and says only that a secret differs', () => {
+    const other = [{ keyName: 'transfer.key', sharedSecret: 'DIFFERENT-SECRET', algorithmName: 'hmac-sha256' }]
+    const out = compareSettings({ baseline: settingsOf(), live: live({}, { tsigKeys: other, proxy: { type: 'Http', address: 'p', port: 1, username: 'u', password: PROXY, bypass: [] } }), extraLocal: [] })
+    const text = JSON.stringify(out)
+    for (const secret of [TSIG, PROXY, CERT, 'DIFFERENT-SECRET']) expect(text).not.toContain(secret)
+    expect(out.fields.find((f) => f.name === 'tsigKeys')).toMatchObject({ secret: true, cells: { b: { differs: true, secretChanged: true } } })
+  })
+
+  it('masks every secret of a settings object', () => {
+    const text = JSON.stringify(maskSettings(settingsOf({ proxy: { type: 'Http', address: 'p', port: 1, username: 'u', password: PROXY, bypass: [] } })))
+    for (const secret of [TSIG, PROXY, CERT]) expect(text).not.toContain(secret)
+    expect(text).toContain(SECRET_PLACEHOLDER)
+  })
+})
+
+describe('what an apply would send', () => {
+  it('lists only shared fields that differ and that both sides have', () => {
+    const baseline = settingsOf({ defaultRecordTtl: 1, webServiceHttpPort: 1, dnsServerDomain: 'x', onlyInBaseline: true })
+    expect(changedFields(baseline, settingsOf({ defaultRecordTtl: 2 }), [])).toEqual(['defaultRecordTtl'])
+    expect(changedFields(baseline, settingsOf({ defaultRecordTtl: 2 }), ['defaultRecordTtl'])).toEqual([])
+    expect(changedFields(baseline, settingsOf({ defaultRecordTtl: 2, blockingType: 'x' }), [], ['blockingType'])).toEqual(['blockingType'])
+  })
+})
+
+describe('encoding for settings/set', () => {
+  const one = (name: string, value: unknown) => {
+    const r = toParams(name, value)
+    if ('error' in r) throw new Error(r.error)
+    return r.params
+  }
+
+  it('encodes flags, numbers and text', () => {
+    expect(one('dnssecValidation', true)).toEqual({ dnssecValidation: 'true' })
+    expect(one('blockingAnswerTtl', 30)).toEqual({ blockingAnswerTtl: '30' })
+    expect(one('recursion', 'Allow')).toEqual({ recursion: 'Allow' })
+    expect(one('eDnsClientSubnetIpv4Override', null)).toEqual({ eDnsClientSubnetIpv4Override: '' })
+  })
+
+  it('joins lists with commas and sends an empty one as false', () => {
+    expect(one('forwarders', ['1.1.1.1', '9.9.9.9'])).toEqual({ forwarders: '1.1.1.1,9.9.9.9' })
+    expect(one('forwarders', [])).toEqual({ forwarders: 'false' })
+    expect(one('blockListUrls', ['https://a.example/x', '# Privacy'])).toEqual({ blockListUrls: 'https://a.example/x,# Privacy' })
+  })
+
+  it('refuses an entry the server would split', () => {
+    expect(toParams('blockListUrls', ['https://a.example/x,y'])).toHaveProperty('error')
+  })
+
+  it('sends tsigKeys and the QPM limits as pipe-joined rows, and false when empty', () => {
+    expect(one('tsigKeys', [{ keyName: 'k', sharedSecret: 's', algorithmName: 'hmac-sha256' }])).toEqual({ tsigKeys: 'k|s|hmac-sha256' })
+    expect(one('tsigKeys', [])).toEqual({ tsigKeys: 'false' })
+    expect(one('qpmPrefixLimitsIPv4', [{ prefix: 32, udpLimit: 600, tcpLimit: 500 }, { prefix: 24, udpLimit: 1, tcpLimit: 2 }])).toEqual({ qpmPrefixLimitsIPv4: '32|600|500|24|1|2' })
+    expect(one('qpmPrefixLimitsIPv6', [])).toEqual({ qpmPrefixLimitsIPv6: 'false' })
+    expect(toParams('tsigKeys', [{ keyName: 'k|x', sharedSecret: 's', algorithmName: 'a' }])).toHaveProperty('error')
+  })
+
+  it('sends the proxy flat, and only its type when there is none', () => {
+    expect(one('proxy', null)).toEqual({ proxyType: 'none' })
+    expect(one('proxy', { type: 'None', address: 'x' })).toEqual({ proxyType: 'none' })
+    expect(one('proxy', { type: 'Socks5', address: 'p.example', port: 1080, username: 'u', password: 'pw', bypass: ['127.0.0.1', 'lan'] })).toEqual({
+      proxyType: 'socks5',
+      proxyAddress: 'p.example',
+      proxyPort: '1080',
+      proxyUsername: 'u',
+      proxyPassword: 'pw',
+      proxyBypass: '127.0.0.1,lan',
+    })
+  })
+
+  it('refuses a name that is not a setting', () => {
+    expect(toParams('a=b&c', 1)).toHaveProperty('error')
+  })
+})
+
+describe('secrets coming back from the baseline editor', () => {
+  const old = normalize(settingsOf())
+
+  it('keeps the stored secret behind a placeholder, and takes a typed one', () => {
+    const kept = restoreSecrets({ tsigKeys: [{ keyName: 'transfer.key', sharedSecret: SECRET_PLACEHOLDER, algorithmName: 'x' }] }, old)
+    expect((kept.tsigKeys as { sharedSecret: string }[])[0].sharedSecret).toBe(TSIG)
+    const typed = restoreSecrets({ tsigKeys: [{ keyName: 'transfer.key', sharedSecret: 'NEW', algorithmName: 'x' }] }, old)
+    expect((typed.tsigKeys as { sharedSecret: string }[])[0].sharedSecret).toBe('NEW')
+  })
+
+  it('refuses a placeholder with nothing to stand for', () => {
+    expect(() => restoreSecrets({ tsigKeys: [{ keyName: 'new.key', sharedSecret: SECRET_PLACEHOLDER, algorithmName: 'x' }] }, old)).toThrow(/transfer|new\.key/)
+    expect(() => restoreSecrets({ proxy: { type: 'Http', password: SECRET_PLACEHOLDER } }, old)).toThrow(/proxy/)
+  })
+})
+
+describe('lists', () => {
+  it('parses an export and compares as sets', () => {
+    expect(parseDomains('B.example.\n# note\n\na.example\r\na.example\n')).toEqual(['a.example', 'b.example'])
+    expect(diffSets(['a', 'b'], ['b', 'c'])).toEqual({ missing: ['a'], extra: ['c'] })
+  })
+})
+
+describe('routes and roles', () => {
+  const PATHS = [
+    'hub/settings/compare',
+    'hub/settings/preview',
+    'hub/settings/history/list',
+    'hub/settings/history/get',
+    'hub/settings/baseline/import',
+    'hub/settings/baseline/save',
+    'hub/settings/overrides/save',
+    'hub/settings/apply',
+    'hub/blocking/lists/compare',
+    'hub/blocking/lists/preview',
+    'hub/blocking/lists/baseline/import',
+    'hub/blocking/lists/apply',
+  ]
+
+  it('needs admin on every endpoint, reads included', () => {
+    for (const p of PATHS) expect(routeFor(p)?.capability, p).toBe('admin')
+    for (const p of PATHS.filter((x) => /import|save|apply/.test(x))) expect(routeFor(p)?.kind, p).toBe('write-all')
+  })
+
+  it('refuses the other roles without touching a server', async () => {
+    for (const role of ROLES.filter((r) => r !== 'admin')) {
+      for (const p of PATHS) expect((await hub.call(role, p, { server: 'a', targets: 'a', text: '{}' })).code, `${role} ${p}`).toBe(403)
+    }
+    expect(a.calls).toHaveLength(0)
+    expect(b.calls).toHaveLength(0)
+  })
+})
+
+describe('hub/settings', () => {
+  const importFrom = (server: string, extra: Record<string, string> = {}) => hub.call('admin', 'hub/settings/baseline/import', { server, ...extra })
+  const historyFiles = (): string[] => {
+    const dir = path.join(hub.dataDir, 'settings', 'history')
+    return existsSync(dir) ? readdirSync(dir) : []
+  }
+
+  it('compares the servers with each other before there is a baseline', async () => {
+    const r = await hub.call('admin', 'hub/settings/compare', {}, 'GET')
+    expect(r.body.response.withBaseline).toBe(false)
+    expect(r.body.response.baseline).toBeNull()
+    const names = r.body.response.fields.map((f: { name: string }) => f.name)
+    expect(names).toContain('defaultRecordTtl')
+    expect(names).toContain('dnsServerDomain')
+    expect(names).not.toContain('uptimestamp')
+    expect(r.body.response.localFields.builtIn).toContain('dnsServerDomain')
+  })
+
+  it('accepts a server as the baseline, then reports b as differing and a as same', async () => {
+    expect((await importFrom('a')).body.status).toBe('ok')
+    const r = await hub.call('admin', 'hub/settings/compare', {}, 'GET')
+    const by = Object.fromEntries(r.body.response.servers.map((s: { id: string }) => [s.id, s]))
+    expect(by.a.status).toBe('same')
+    expect(by.b.status).toBe('differs')
+    expect(by.b.differing).toBe(3)
+    expect(r.body.response.baseline.importedFrom).toBe('a')
+    // The import read with the admin token.
+    expect(a.calls.find((c) => c.path === 'settings/get')!.token).toBe('admin-a')
+  })
+
+  it('imports only the blocking fields with group=blocking', async () => {
+    await importFrom('a')
+    boxB().settings.blockingType = 'AnyAddress'
+    boxB().settings.defaultRecordTtl = 5
+    await importFrom('b', { group: 'blocking' })
+    const r = await hub.call('admin', 'hub/settings/compare', {}, 'GET')
+    const rows = r.body.response.fields.map((f: { name: string }) => f.name)
+    // The baseline now has b's blocking type and a's TTL: a differs in blocking, b in TTL.
+    expect(rows).toContain('blockingType')
+    expect(r.body.response.baseline.settings.blockingType).toBe('AnyAddress')
+    expect(r.body.response.baseline.settings.defaultRecordTtl).toBe(3600)
+  })
+
+  it('previews the fields that would change per target, and writes nothing', async () => {
+    await importFrom('a')
+    a.calls = []
+    b.calls = []
+    const r = await hub.call('admin', 'hub/settings/preview', { targets: 'a,b' })
+    const [pa, pb] = r.body.response.targets
+    expect(pa).toMatchObject({ server: 'a', ok: true, unchanged: true })
+    expect(pb.unchanged).toBe(false)
+    expect(pb.changes.kind).toBe('json')
+    expect(pb.changes.changes.map((c: { path: string }) => c.path)).toEqual(['blockingType', 'defaultRecordTtl', 'forwarders'])
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('applies only the changed shared fields, never a local or webService field, and saves the old settings first', async () => {
+    await importFrom('a')
+    b.calls = []
+    const preview = await hub.call('admin', 'hub/settings/preview', { targets: 'b' })
+    const expectHash = { b: preview.body.response.targets[0].currentHash }
+    const r = await hub.call('admin', 'hub/settings/apply', { targets: 'b', expect: JSON.stringify(expectHash) })
+    expect(r.body.response.results[0]).toMatchObject({ server: 'b', ok: true })
+    expect(r.body.response.results[0].message).toMatch(/verified \(3 fields\)/)
+    const set = writes(b).filter((c) => c.path === 'settings/set')
+    expect(set).toHaveLength(1)
+    expect(set[0].token).toBe('admin-b')
+    expect(Object.fromEntries(set[0].form)).toEqual({ blockingType: 'NxDomain', defaultRecordTtl: '3600', forwarders: '1.1.1.1,9.9.9.9' })
+    expect(Object.keys(Object.fromEntries(set[0].form)).some((k) => k.startsWith('webService') || k === 'dnsServerDomain' || k === 'logFolder')).toBe(false)
+    expect(boxB().settings.dnsServerDomain).toBe('b.example.net')
+    // The history entry holds b's settings from before the change.
+    const files = historyFiles()
+    expect(files).toHaveLength(1)
+    expect(files[0]).toMatch(/__b__before-apply\.txt$/)
+    expect(JSON.parse(readFileSync(path.join(hub.dataDir, 'settings', 'history', files[0]), 'utf8')).defaultRecordTtl).toBe(7200)
+  })
+
+  it('limits an apply to the blocking fields with group=blocking', async () => {
+    await importFrom('a')
+    b.calls = []
+    const r = await hub.call('admin', 'hub/settings/apply', { targets: 'b', group: 'blocking' })
+    expect(r.body.response.results[0].ok).toBe(true)
+    expect(Object.keys(Object.fromEntries(writes(b)[0].form))).toEqual(['blockingType'])
+    expect(boxB().settings.defaultRecordTtl).toBe(7200)
+  })
+
+  it('refuses to apply when the server changed since the preview', async () => {
+    await importFrom('a')
+    const preview = await hub.call('admin', 'hub/settings/preview', { targets: 'b' })
+    boxB().settings.defaultRecordTtl = 99
+    b.calls = []
+    const r = await hub.call('admin', 'hub/settings/apply', { targets: 'b', expect: JSON.stringify({ b: preview.body.response.targets[0].currentHash }) })
+    expect(r.body.response.results[0]).toMatchObject({ ok: false })
+    expect(r.body.response.results[0].message).toMatch(/changed on the server since the preview/)
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('reports a server that accepts the write but reads back something else', async () => {
+    await importFrom('a')
+    boxB().ignoreWrites = true
+    const r = await hub.call('admin', 'hub/settings/apply', { targets: 'b' })
+    expect(r.body.response.results[0].ok).toBe(false)
+    expect(r.body.response.results[0].message).toMatch(/reads back different values for: blockingType, defaultRecordTtl, forwarders/)
+  })
+
+  it('reports an unreachable target and a rejected write per server', async () => {
+    await importFrom('a')
+    b.on('settings/set', () => ({ error: 'Access was denied.' }))
+    const r = await hub.call('admin', 'hub/settings/apply', { targets: 'b' })
+    expect(r.body.response.results[0]).toMatchObject({ server: 'b', ok: false, message: 'Access was denied.' })
+    b.down = true
+    const down = await hub.call('admin', 'hub/settings/apply', { targets: 'b' })
+    expect(down.body.response.results[0]).toMatchObject({ server: 'b', ok: false })
+  })
+
+  it('leaves the blocking switch alone while blocking is paused on the server', async () => {
+    await importFrom('a')
+    boxB().settings.enableBlocking = false
+    boxB().settings.temporaryDisableBlockingTill = '2026-12-01T00:00:00Z'
+    b.calls = []
+    await hub.call('admin', 'hub/settings/apply', { targets: 'b' })
+    expect(Object.fromEntries(writes(b)[0].form)).not.toHaveProperty('enableBlocking')
+  })
+
+  it('does not send a field the server does not report', async () => {
+    await importFrom('a')
+    delete boxB().settings.forwarders
+    b.calls = []
+    await hub.call('admin', 'hub/settings/apply', { targets: 'b' })
+    expect(Object.fromEntries(writes(b)[0].form)).not.toHaveProperty('forwarders')
+  })
+
+  it('treats a field added to the local list as not to be sent', async () => {
+    await importFrom('a')
+    expect((await hub.call('admin', 'hub/settings/overrides/save', { paths: 'defaultRecordTtl' })).body.status).toBe('ok')
+    b.calls = []
+    await hub.call('admin', 'hub/settings/apply', { targets: 'b' })
+    expect(Object.fromEntries(writes(b)[0].form)).not.toHaveProperty('defaultRecordTtl')
+    expect((await hub.call('admin', 'hub/settings/overrides/save', { paths: 'bad name' })).code).toBe(400)
+    await hub.call('admin', 'hub/settings/overrides/save', { paths: '' })
+  })
+
+  it('keeps a secret behind the placeholder when the baseline is saved from the editor', async () => {
+    await importFrom('a')
+    const shown = (await hub.call('admin', 'hub/settings/compare', {}, 'GET')).body.response.baseline.settings
+    expect(shown.tsigKeys[0].sharedSecret).toBe(SECRET_PLACEHOLDER)
+    const saved = await hub.call('admin', 'hub/settings/baseline/save', { text: JSON.stringify({ ...shown, defaultRecordTtl: 1800 }) })
+    expect(saved.body.status).toBe('ok')
+    const stored = JSON.parse(readFileSync(path.join(hub.dataDir, 'settings', 'baseline.json'), 'utf8'))
+    expect(stored.settings.tsigKeys[0].sharedSecret).toBe(TSIG)
+    expect(stored.settings.defaultRecordTtl).toBe(1800)
+    // The old baseline went to the history.
+    expect(historyFiles().some((f) => f.endsWith('__baseline__before-save.txt'))).toBe(true)
+    const bad = await hub.call('admin', 'hub/settings/baseline/save', { text: JSON.stringify({ tsigKeys: [{ keyName: 'other', sharedSecret: SECRET_PLACEHOLDER, algorithmName: 'x' }] }) })
+    expect(bad.code).toBe(400)
+    expect((await hub.call('admin', 'hub/settings/baseline/save', { text: '{nope' })).code).toBe(400)
+  })
+
+  it('lists and shows history without a secret', async () => {
+    await importFrom('a')
+    await hub.call('admin', 'hub/settings/apply', { targets: 'b' })
+    const list = await hub.call('admin', 'hub/settings/history/list', {}, 'GET')
+    expect(list.body.response.entries.length).toBeGreaterThan(0)
+    const entry = list.body.response.entries.find((e: { server: string }) => e.server === 'b')
+    const got = await hub.call('admin', 'hub/settings/history/get', { id: entry.id }, 'GET')
+    expect(got.body.response.settings.defaultRecordTtl).toBe(7200)
+    expect(JSON.stringify(got.body)).not.toContain(TSIG)
+    expect((await hub.call('admin', 'hub/settings/history/get', { id: '../../users.json' }, 'GET')).code).toBe(400)
+  })
+
+  it('never lets a secret into any response or into the audit log', async () => {
+    boxB().settings.tsigKeys = [{ keyName: 'transfer.key', sharedSecret: 'B-DIFFERENT-SECRET', algorithmName: 'hmac-sha256' }]
+    boxB().settings.proxy = { type: 'Http', address: 'p.example', port: 8080, username: 'u', password: PROXY, bypass: [] }
+    const bodies: string[] = []
+    const keep = async (p: Promise<{ body: unknown }>) => bodies.push(JSON.stringify((await p).body))
+    await keep(hub.call('admin', 'hub/settings/compare', {}, 'GET'))
+    await keep(hub.call('admin', 'hub/settings/baseline/import', { server: 'a' }))
+    await keep(hub.call('admin', 'hub/settings/compare', {}, 'GET'))
+    await keep(hub.call('admin', 'hub/settings/preview', { targets: 'a,b' }))
+    await keep(hub.call('admin', 'hub/settings/apply', { targets: 'b' }))
+    await keep(hub.call('admin', 'hub/settings/history/list', {}, 'GET'))
+    const list = await hub.call('admin', 'hub/settings/history/list', {}, 'GET')
+    for (const e of list.body.response.entries) await keep(hub.call('admin', 'hub/settings/history/get', { id: e.id }, 'GET'))
+    // And what the existing settings/get gives a viewer.
+    const viewer = await hub.call('viewer', 'settings/get', { node: 'b' }, 'GET')
+    bodies.push(JSON.stringify(viewer.body))
+    const audit = JSON.stringify(await hub.audit())
+    for (const secret of [TSIG, PROXY, CERT, 'B-DIFFERENT-SECRET']) {
+      for (const body of bodies) expect(body).not.toContain(secret)
+      expect(audit).not.toContain(secret)
+    }
+    // The apply did send the baseline's TSIG key to b; that travels to the server, not to the console.
+    expect(JSON.stringify(writes(b).map((c) => [...c.form]))).toContain(TSIG)
+  })
+
+  it('writes audit lines with field names and no values', async () => {
+    await importFrom('a')
+    await hub.call('admin', 'hub/settings/apply', { targets: 'b' })
+    const line = (await hub.audit()).find((e) => e.action === 'settings.apply' && e.server === 'b')!
+    expect(line.outcome).toBe('ok')
+    expect(line.detail).toMatch(/fields: blockingType, defaultRecordTtl, forwarders/)
+    expect(line.detail).not.toContain('8.8.8.8')
+  })
+})
+
+describe('hub/blocking/lists', () => {
+  const importFrom = (server: string) => hub.call('admin', 'hub/blocking/lists/baseline/import', { server })
+
+  it('compares as sets, per server: missing and extra', async () => {
+    await importFrom('a')
+    const r = await hub.call('admin', 'hub/blocking/lists/compare', {}, 'GET')
+    const by = Object.fromEntries(r.body.response.servers.map((s: { id: string }) => [s.id, s]))
+    expect(by.a.status).toBe('same')
+    expect(by.b.status).toBe('differs')
+    expect(by.b.blocked).toMatchObject({ count: 2, missing: 1, extra: 1, missingSample: ['ads.example'], extraSample: ['only-b.example'] })
+    expect(by.b.allowed).toMatchObject({ missing: 0, extra: 1, extraSample: ['extra-allowed.example'] })
+  })
+
+  it('compares with the first server before there is a baseline', async () => {
+    const r = await hub.call('admin', 'hub/blocking/lists/compare', {}, 'GET')
+    expect(r.body.response.withBaseline).toBe(false)
+    expect(r.body.response.servers.map((s: { status: string }) => s.status)).toEqual(['differs', 'differs'])
+  })
+
+  it('marks an unreachable server unknown', async () => {
+    await importFrom('a')
+    b.down = true
+    const r = await hub.call('admin', 'hub/blocking/lists/compare', {}, 'GET')
+    expect(r.body.response.servers.find((s: { id: string }) => s.id === 'b')).toMatchObject({ ok: false, status: 'unknown' })
+  })
+
+  it('adds the missing domains and leaves the extras unless asked', async () => {
+    await importFrom('a')
+    const r = await hub.call('admin', 'hub/blocking/lists/apply', { targets: 'b' })
+    expect(r.body.response.results[0]).toMatchObject({ server: 'b', ok: true })
+    // Extras remain: they are not what the user asked to change.
+    expect(boxB().blocked.sort()).toEqual(['ads.example', 'both.example', 'only-b.example'])
+    expect(writes(b).some((c) => c.path.endsWith('/delete'))).toBe(false)
+  })
+
+  it('removes the extras only when the user asked', async () => {
+    await importFrom('a')
+    b.calls = []
+    const r = await hub.call('admin', 'hub/blocking/lists/apply', { targets: 'b', removeExtras: 'true' })
+    expect(r.body.response.results[0]).toMatchObject({ server: 'b', ok: true })
+    expect(boxB().blocked.sort()).toEqual(['ads.example', 'both.example'])
+    expect(boxB().allowed).toEqual(['ok.example'])
+    expect(writes(b).every((c) => c.token === 'admin-b')).toBe(true)
+    // Before deleting, the server's lists went to the history.
+    const dir = path.join(hub.dataDir, 'settings', 'history')
+    expect(readdirSync(dir).some((f) => f.endsWith('__b__before-lists.txt'))).toBe(true)
+  })
+
+  it('previews the counts and warns when a change is too large', async () => {
+    await importFrom('a')
+    const r = await hub.call('admin', 'hub/blocking/lists/preview', { targets: 'b', removeExtras: 'true' })
+    expect(r.body.response.targets[0]).toMatchObject({ id: 'b', ok: true, status: 'differs' })
+    boxB().blocked = Array.from({ length: 1200 }, (_, i) => `d${i}.example`)
+    const big = await hub.call('admin', 'hub/blocking/lists/preview', { targets: 'b', removeExtras: 'true' })
+    expect(big.body.response.targets[0].tooMany).toMatch(/at most/)
+    const applied = await hub.call('admin', 'hub/blocking/lists/apply', { targets: 'b', removeExtras: 'true' })
+    expect(applied.body.response.results[0].ok).toBe(false)
+  })
+
+  it('is a no-op when the lists already match, and reports a rejected write', async () => {
+    await importFrom('a')
+    const same = await hub.call('admin', 'hub/blocking/lists/apply', { targets: 'a' })
+    expect(same.body.response.results[0].message).toMatch(/Already in sync/)
+    b.on('blocked/add', () => ({ error: 'Access was denied.' }))
+    const r = await hub.call('admin', 'hub/blocking/lists/apply', { targets: 'b' })
+    expect(r.body.response.results[0].ok).toBe(false)
+    expect(r.body.response.results[0].message).toMatch(/failed/)
+  })
+
+  it('refuses to apply without a baseline', async () => {
+    rmSync(path.join(hub.dataDir, 'blocking', 'baseline.json'), { force: true })
+    expect((await hub.call('admin', 'hub/blocking/lists/apply', { targets: 'b' })).code).toBe(400)
+  })
+})

hub/settings-sync.ts

@@ -0,0 +1,385 @@
+/*
+Pure functions behind the settings and blocking compare (stage 7): which fields are never
+compared, which stay local to a server, how two settings are compared, how a field is encoded for
+`settings/set`, and how secrets are kept out of every view. No I/O here.
+
+Three kinds of field:
+
+  * IGNORED: runtime or read-only (`version`, `uptimestamp`, cluster info, the temporary blocking
+    switch, the next list update). Never compared, never sent.
+  * LOCAL: the identity and the bind addresses of the host, certificates and paths, and every
+    `webService*` field. Compared across servers for information, never sent by an apply. The
+    built-in list cannot be shortened; more fields can be added (`overrides.json`).
+  * everything else is SHARED: compared with the baseline and sent by an apply when it differs.
+
+Secrets (TSIG shared secrets and the forward proxy password, plus the two certificate passwords,
+which are local anyway) are compared inside the hub by their real value, but no view ever carries
+a value or a hash of one: a view shows `SECRET_PLACEHOLDER`, and the row says only that the secret
+differs.
+*/
+import { isForbiddenSettingsField } from './guards.ts'
+
+export const SECRET_PLACEHOLDER = '<secret>'
+
+/* Read-only or runtime: never compared. */
+const IGNORED = new Set([
+  'version',
+  'uptimestamp',
+  'clusterInitialized',
+  'clusterNodes',
+  'temporaryDisableBlockingTill',
+  'blockListNextUpdatedOn',
+])
+
+export const isIgnoredField = (name: string): boolean => IGNORED.has(name) || /^cluster/i.test(name)
+
+/*
+Local by default, and why:
+  dnsServerDomain, dnsServerLocalEndPoints, dnsServerIPv4SourceAddresses,
+  dnsServerIPv6SourceAddresses  the identity of the host and the addresses it binds
+  webService*                   how the hub itself reaches the server (guards.ts refuses them anyway)
+  dnsTlsCertificatePath, dnsTlsCertificatePassword   the certificate file of this host
+  dnsOverHttpUnixSocket, dnsOverHttpsUnixSocket, enableDnsOverHttpUnixSocket,
+  enableDnsOverHttpsUnixSocket  sockets of this host
+  dnsReverseProxyNetworkACL     the address of the reverse proxy in front of this host
+  logFolder                     a path on this host
+*/
+export const LOCAL_DEFAULTS = [
+  'dnsServerDomain',
+  'dnsServerLocalEndPoints',
+  'dnsServerIPv4SourceAddresses',
+  'dnsServerIPv6SourceAddresses',
+  'dnsTlsCertificatePath',
+  'dnsTlsCertificatePassword',
+  'dnsOverHttpUnixSocket',
+  'dnsOverHttpsUnixSocket',
+  'enableDnsOverHttpUnixSocket',
+  'enableDnsOverHttpsUnixSocket',
+  'dnsReverseProxyNetworkACL',
+  'logFolder',
+] as const
+
+const LOCAL_SET = new Set<string>(LOCAL_DEFAULTS)
+
+/* True for a field that is local whatever the overrides say: the built-in list, `webService*`, and anything user, SSO or LDAP. */
+export const isBuiltInLocal = (name: string): boolean => LOCAL_SET.has(name) || name.startsWith('webService') || isForbiddenSettingsField(name)
+
+/* The fields of the Blocking group (the rest are General). */
+export const BLOCKING_FIELDS = [
+  'enableBlocking',
+  'allowTxtBlockingReport',
+  'blockingBypassList',
+  'blockingType',
+  'customBlockingAddresses',
+  'blockingAnswerTtl',
+  'blockListUrls',
+  'blockListUpdateIntervalHours',
+] as const
+
+const BLOCKING_SET = new Set<string>(BLOCKING_FIELDS)
+export const groupOf = (name: string): 'blocking' | 'general' => (BLOCKING_SET.has(name) ? 'blocking' : 'general')
+
+const PASSWORD_FIELDS = new Set(['webServiceTlsCertificatePassword', 'dnsTlsCertificatePassword'])
+const FIELD_NAME = /^[A-Za-z][A-Za-z0-9]{0,59}$/
+
+export const isFieldName = (name: string): boolean => FIELD_NAME.test(name)
+
+type Settings = Record<string, unknown>
+
+const isObject = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === 'object' && !Array.isArray(v)
+
+/* `settings/get` without the ignored fields. */
+export function normalize(raw: unknown): Settings {
+  const out: Settings = {}
+  if (!isObject(raw)) return out
+  for (const [k, v] of Object.entries(raw)) if (isFieldName(k) && !isIgnoredField(k)) out[k] = v
+  return out
+}
+
+/* Whether a field carries a secret anywhere inside it. */
+export const hasSecret = (name: string): boolean => name === 'tsigKeys' || name === 'proxy' || PASSWORD_FIELDS.has(name)
+
+/* A copy of a field's value that is safe to send out: every secret replaced by the placeholder. */
+export function maskValue(name: string, value: unknown): unknown {
+  if (PASSWORD_FIELDS.has(name)) return typeof value === 'string' && value !== '' ? SECRET_PLACEHOLDER : value
+  if (name === 'tsigKeys' && Array.isArray(value)) {
+    return value.map((k) => (isObject(k) ? { ...k, sharedSecret: typeof k.sharedSecret === 'string' && k.sharedSecret !== '' ? SECRET_PLACEHOLDER : k.sharedSecret } : k))
+  }
+  if (name === 'proxy' && isObject(value)) return { ...value, password: typeof value.password === 'string' && value.password !== '' ? SECRET_PLACEHOLDER : value.password }
+  return value
+}
+
+export function maskSettings(settings: Settings): Settings {
+  return Object.fromEntries(Object.entries(settings).map(([k, v]) => [k, maskValue(k, v)]))
+}
+
+/* ---- Comparing ---- */
+
+/* A value reduced to what it means: no key order, no difference between absent, null, empty text and an empty list. */
+function canon(name: string, value: unknown): unknown {
+  if (value === undefined || value === null || value === '') return null
+  if (typeof value === 'string') return value.trim()
+  if (Array.isArray(value)) return value.length === 0 ? null : value.map((v) => canon('', v))
+  if (isObject(value)) {
+    // A proxy of type None carries no other setting.
+    if (name === 'proxy' && String(value.type ?? '').toLowerCase() === 'none') return null
+    return Object.fromEntries(Object.keys(value).sort().map((k) => [k, canon('', value[k])]))
+  }
+  return value
+}
+
+export const sameValue = (name: string, a: unknown, b: unknown): boolean => JSON.stringify(canon(name, a)) === JSON.stringify(canon(name, b))
+
+/*
+Settings ready for comparing: normalised, and without `enableBlocking` while blocking is paused
+for a while (it is off for now, not by configuration).
+*/
+export function prepare(raw: unknown): { settings: Settings; paused: boolean } {
+  const settings = normalize(raw)
+  const paused = isObject(raw) && typeof raw.temporaryDisableBlockingTill === 'string' && raw.temporaryDisableBlockingTill !== ''
+  if (paused) delete settings.enableBlocking
+  return { settings, paused }
+}
+
+export interface Cell {
+  value: unknown
+  differs: boolean
+  /* The values look alike once masked: only a secret differs. */
+  secretChanged?: boolean
+}
+
+export interface FieldRow {
+  name: string
+  group: 'blocking' | 'general'
+  local: boolean
+  secret: boolean
+  /* The baseline's value (masked), when comparing with a baseline. */
+  expected?: unknown
+  cells: Record<string, Cell>
+}
+
+export interface ServerSummary {
+  id: string
+  ok: boolean
+  error?: string
+  /* same, differs, local-only (only local fields differ) or unknown (the server did not answer). */
+  status: 'same' | 'differs' | 'local-only' | 'unknown'
+  differing: number
+  localDiffering: number
+  paused: boolean
+}
+
+export interface Comparison {
+  /* True when compared with a baseline, false when the servers are compared with the first that answered. */
+  withBaseline: boolean
+  servers: ServerSummary[]
+  fields: FieldRow[]
+}
+
+export interface CompareInput {
+  baseline: Settings | null
+  /** Raw `settings/get` per server; null when it did not answer. */
+  live: Record<string, { settings: unknown } | { error: string }>
+  extraLocal: string[]
+}
+
+export function compareSettings({ baseline, live, extraLocal }: CompareInput): Comparison {
+  const isLocal = (name: string): boolean => isBuiltInLocal(name) || extraLocal.includes(name)
+  const base = baseline === null ? null : normalize(baseline)
+  const prepared: Record<string, { settings: Settings; paused: boolean }> = {}
+  const summaries: ServerSummary[] = []
+  for (const [id, answer] of Object.entries(live)) {
+    if ('error' in answer) summaries.push({ id, ok: false, error: answer.error, status: 'unknown', differing: 0, localDiffering: 0, paused: false })
+    else {
+      prepared[id] = prepare(answer.settings)
+      summaries.push({ id, ok: true, status: 'same', differing: 0, localDiffering: 0, paused: prepared[id].paused })
+    }
+  }
+  const ids = Object.keys(prepared)
+  const reference = base ?? (ids.length > 0 ? prepared[ids[0]].settings : null)
+  const names = new Set<string>()
+  if (base !== null) for (const k of Object.keys(base)) names.add(k)
+  for (const id of ids) for (const k of Object.keys(prepared[id].settings)) names.add(k)
+
+  const rows: FieldRow[] = []
+  for (const name of [...names].sort()) {
+    const local = isLocal(name)
+    const holders = ids.filter((id) => Object.hasOwn(prepared[id].settings, name))
+    const cells: Record<string, Cell> = {}
+    let any = false
+    if (local) {
+      // Compared with the baseline's value when it has one, else with the first server that has the field.
+      const useBase = base !== null && Object.hasOwn(base, name)
+      const ref = useBase ? base[name] : holders.length > 0 ? prepared[holders[0]].settings[name] : undefined
+      for (const id of holders) cells[id] = { value: maskValue(name, prepared[id].settings[name]), differs: !sameValue(name, ref, prepared[id].settings[name]) }
+      any = holders.some((id) => cells[id].differs)
+      if (any && !useBase) for (const id of holders) cells[id].differs = true
+    } else {
+      if (reference === null || !Object.hasOwn(reference, name)) continue
+      for (const id of holders) {
+        const differs = !sameValue(name, reference[name], prepared[id].settings[name])
+        const secretChanged = differs && hasSecret(name) && JSON.stringify(maskValue(name, canon(name, reference[name]))) === JSON.stringify(maskValue(name, canon(name, prepared[id].settings[name])))
+        cells[id] = { value: maskValue(name, prepared[id].settings[name]), differs, ...(secretChanged ? { secretChanged: true } : {}) }
+        if (differs) any = true
+      }
+      // Without a baseline the reference itself is one side of the difference.
+      if (any && base === null) cells[ids[0]].differs = true
+    }
+    if (!any) continue
+    rows.push({
+      name,
+      group: groupOf(name),
+      local,
+      secret: hasSecret(name),
+      ...(base !== null && !local ? { expected: maskValue(name, base[name]) } : {}),
+      cells,
+    })
+  }
+  for (const s of summaries) {
+    if (!s.ok) continue
+    s.differing = rows.filter((r) => !r.local && r.cells[s.id]?.differs === true && (base !== null || s.id !== ids[0])).length
+    s.localDiffering = rows.filter((r) => r.local && r.cells[s.id]?.differs === true).length
+    s.status = s.differing > 0 ? 'differs' : s.localDiffering > 0 ? 'local-only' : 'same'
+  }
+  // Without a baseline the first server is the reference: it differs only if another does.
+  if (base === null && ids.length > 0) {
+    const first = summaries.find((s) => s.id === ids[0])!
+    const others = summaries.some((s) => s.ok && s.id !== first.id && s.differing > 0)
+    first.differing = rows.filter((r) => !r.local).length
+    first.status = others ? 'differs' : first.localDiffering > 0 ? 'local-only' : 'same'
+  }
+  return { withBaseline: base !== null, servers: summaries, fields: rows }
+}
+
+/*
+The shared fields an apply would send to a server: in the baseline and reported by the server,
+different, not local, not forbidden. Sorted. A field only one side has is left alone.
+*/
+export function changedFields(baseline: unknown, current: unknown, extraLocal: string[], only?: readonly string[]): string[] {
+  const base = normalize(baseline)
+  const { settings } = prepare(current)
+  const out: string[] = []
+  for (const name of Object.keys(base)) {
+    if (isBuiltInLocal(name) || extraLocal.includes(name)) continue
+    if (only !== undefined && !only.includes(name)) continue
+    if (!Object.hasOwn(settings, name)) continue
+    if (!sameValue(name, base[name], settings[name])) out.push(name)
+  }
+  return out.sort()
+}
+
+/* ---- Encoding for `settings/set` (the same as the console's own save, model.ts `buildBody`) ---- */
+
+const text = (v: unknown): string => (v === null || v === undefined ? '' : String(v))
+
+function table(rows: unknown, keys: string[]): string | null {
+  if (!Array.isArray(rows)) return rows === null || rows === undefined ? 'false' : null
+  const cells: string[] = []
+  for (const row of rows) {
+    if (!isObject(row)) return null
+    for (const k of keys) {
+      const cell = text(row[k])
+      if (cell.includes('|')) return null
+      cells.push(cell)
+    }
+  }
+  return cells.length === 0 ? 'false' : cells.join('|')
+}
+
+/* The form parameters for one field, or an error that says why it cannot be sent. */
+export function toParams(name: string, value: unknown): { params: Record<string, string> } | { error: string } {
+  if (!isFieldName(name)) return { error: `"${name}" is not a setting name.` }
+  if (name === 'proxy') {
+    if (!isObject(value) || String(value.type ?? '').toLowerCase() === 'none' || String(value.type ?? '') === '') return { params: { proxyType: 'none' } }
+    const bypass = Array.isArray(value.bypass) ? value.bypass.map(String) : []
+    if (bypass.some((b) => b.includes(','))) return { error: 'A proxy bypass entry contains a comma.' }
+    return {
+      params: {
+        proxyType: String(value.type).toLowerCase(),
+        proxyAddress: text(value.address),
+        proxyPort: text(value.port),
+        proxyUsername: text(value.username),
+        proxyPassword: text(value.password),
+        proxyBypass: bypass.join(','),
+      },
+    }
+  }
+  if (name === 'tsigKeys') {
+    const t = table(value, ['keyName', 'sharedSecret', 'algorithmName'])
+    return t === null ? { error: 'A TSIG key cannot be sent (a "|" in it, or a malformed list).' } : { params: { tsigKeys: t } }
+  }
+  if (name === 'qpmPrefixLimitsIPv4' || name === 'qpmPrefixLimitsIPv6') {
+    const t = table(value, ['prefix', 'udpLimit', 'tcpLimit'])
+    return t === null ? { error: `${name} cannot be sent (a malformed list).` } : { params: { [name]: t } }
+  }
+  if (Array.isArray(value)) {
+    const items = value.map((v) => text(v).trim()).filter((v) => v !== '')
+    if (items.some((i) => i.includes(',') || i.includes('\n'))) return { error: `An entry of ${name} contains a comma or a line break, which the server takes as a separator.` }
+    return { params: { [name]: items.length === 0 ? 'false' : items.join(',') } }
+  }
+  if (isObject(value)) return { error: `${name} has a shape the hub does not send.` }
+  if (typeof value === 'boolean' || typeof value === 'number') return { params: { [name]: String(value) } }
+  return { params: { [name]: text(value) } }
+}
+
+/* ---- Baseline editing ---- */
+
+/*
+A baseline coming back from the editor shows secrets as the placeholder. Each placeholder stands for
+the secret the stored baseline already has (a TSIG key is found by its name); anything else is a new
+secret and replaces it. A placeholder with nothing to stand for is refused, so a save can never
+write the placeholder over a real key.
+*/
+export function restoreSecrets(next: Settings, old: Settings | null): Settings {
+  const out: Settings = structuredClone(next)
+  const fail = (what: string): never => {
+    throw new Error(`The secret of ${what} is shown as ${SECRET_PLACEHOLDER} but the stored baseline has none to keep. Type the value or remove the entry.`)
+  }
+  for (const name of PASSWORD_FIELDS) {
+    if (out[name] === SECRET_PLACEHOLDER) {
+      const was = old?.[name]
+      out[name] = typeof was === 'string' ? was : fail(name)
+    }
+  }
+  if (Array.isArray(out.tsigKeys)) {
+    const oldKeys = Array.isArray(old?.tsigKeys) ? (old.tsigKeys as unknown[]).filter(isObject) : []
+    out.tsigKeys = out.tsigKeys.map((k: unknown) => {
+      if (!isObject(k) || k.sharedSecret !== SECRET_PLACEHOLDER) return k
+      const was = oldKeys.find((o) => o.keyName === k.keyName)
+      return { ...k, sharedSecret: typeof was?.sharedSecret === 'string' ? was.sharedSecret : fail(`the TSIG key ${text(k.keyName)}`) }
+    })
+  }
+  if (isObject(out.proxy) && out.proxy.password === SECRET_PLACEHOLDER) {
+    const was = isObject(old?.proxy) ? old.proxy.password : undefined
+    out.proxy = { ...out.proxy, password: typeof was === 'string' ? was : fail('the proxy') }
+  }
+  return out
+}
+
+/* ---- Blocked and Allowed lists ---- */
+
+export const MAX_LIST = 5000
+export const MAX_DELETES = 500
+export const IMPORT_CHUNK = 500
+
+/* The domains of an export: lower case, no trailing dot, no blank or `#` line, once each, sorted. */
+export function parseDomains(textBody: string): string[] {
+  const set = new Set<string>()
+  for (const line of textBody.split(/\r?\n/)) {
+    const t = line.trim().toLowerCase().replace(/\.$/, '')
+    if (t !== '' && !t.startsWith('#')) set.add(t)
+  }
+  return [...set].sort()
+}
+
+export interface SetDiff {
+  /* In the baseline, not on the server. */
+  missing: string[]
+  /* On the server, not in the baseline. */
+  extra: string[]
+}
+
+export function diffSets(baseline: readonly string[], live: readonly string[]): SetDiff {
+  const [b, l] = [new Set(baseline), new Set(live)]
+  return { missing: baseline.filter((d) => !l.has(d)), extra: live.filter((d) => !b.has(d)) }
+}

hub/store.ts

@@ -0,0 +1,245 @@
+/*
+What the hub keeps per app under `${HUB_DATA}/apps/<app>/`:
+
+  baseline.json   the desired config, as TEXT (some configs are not JSON)
+  overrides.json  the fields that stay local to each server, and their values
+  history/        the config a server had before the hub changed it (50 files kept)
+
+Every write is a temp file and a rename, mode 0600: these files hold configs.
+*/
+import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+
+export const KEEP_HISTORY = 50
+
+/* A folder name for an app: letters, digits and . _ - only, with a short hash when anything had to change. */
+export function safeName(name: string): string {
+  const cleaned = name.replace(/[^A-Za-z0-9._-]/g, '_').replace(/\.{2,}/g, '_').replace(/^\.+/, '_').slice(0, 80)
+  if (cleaned === name && cleaned !== '') return cleaned
+  return `${cleaned === '' ? '_' : cleaned}-${createHash('sha256').update(name).digest('hex').slice(0, 6)}`
+}
+
+export const appDir = (dataDir: string, app: string): string => path.join(dataDir, 'apps', safeName(app))
+
+export function writeAtomic(file: string, text: string): void {
+  mkdirSync(path.dirname(file), { recursive: true })
+  const temp = `${file}.${process.pid}.tmp`
+  try {
+    writeFileSync(temp, text, { mode: 0o600 })
+    renameSync(temp, file)
+  } catch (e) {
+    rmSync(temp, { force: true })
+    throw e
+  }
+}
+
+export interface Baseline {
+  app: string
+  text: string
+  importedFrom?: string
+  updatedAt: string
+  updatedBy: string
+}
+
+export interface Overrides {
+  /** Dot paths (`a.b.0.c`) of the fields that are not synchronised. */
+  paths: string[]
+  /** server -> path -> value */
+  values: Record<string, Record<string, unknown>>
+}
+
+export const EMPTY_OVERRIDES: Overrides = { paths: [], values: {} }
+
+function readJson(file: string): unknown {
+  if (!existsSync(file)) return null
+  try {
+    return JSON.parse(readFileSync(file, 'utf8'))
+  } catch {
+    return null
+  }
+}
+
+export function readBaseline(dataDir: string, app: string): Baseline | null {
+  const data = readJson(path.join(appDir(dataDir, app), 'baseline.json')) as Partial<Baseline> | null
+  if (data === null || typeof data.text !== 'string') return null
+  return {
+    app,
+    text: data.text,
+    ...(typeof data.importedFrom === 'string' ? { importedFrom: data.importedFrom } : {}),
+    updatedAt: String(data.updatedAt ?? ''),
+    updatedBy: String(data.updatedBy ?? ''),
+  }
+}
+
+export function writeBaseline(dataDir: string, baseline: Baseline): void {
+  writeAtomic(path.join(appDir(dataDir, baseline.app), 'baseline.json'), JSON.stringify(baseline, null, 2) + '\n')
+}
+
+export function readOverrides(dataDir: string, app: string): Overrides {
+  const data = readJson(path.join(appDir(dataDir, app), 'overrides.json')) as Partial<Overrides> | null
+  if (data === null || !Array.isArray(data.paths)) return { paths: [], values: {} }
+  const values = data.values !== null && typeof data.values === 'object' ? data.values : {}
+  return { paths: data.paths.filter((p): p is string => typeof p === 'string'), values }
+}
+
+export function writeOverrides(dataDir: string, app: string, overrides: Overrides): void {
+  writeAtomic(path.join(appDir(dataDir, app), 'overrides.json'), JSON.stringify(overrides, null, 2) + '\n')
+}
+
+export type HistoryKind = 'before-uninstall' | 'before-apply' | 'before-set' | 'before-save' | 'before-lists'
+
+export interface HistoryEntry {
+  /** The file name; the only handle `readHistory` accepts. */
+  id: string
+  at: string
+  /** A server id, or `baseline` for an earlier text of the baseline. */
+  server: string
+  kind: string
+  size: number
+  hash: string
+}
+
+const HISTORY_FILE = /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2}\.\d{3}Z)__(.+)__(before-[a-z-]+)\.txt$/
+
+const historyDir = (dataDir: string, app: string): string => path.join(appDir(dataDir, app), 'history')
+
+export const listHistory = (dataDir: string, app: string): HistoryEntry[] => listHistoryIn(historyDir(dataDir, app))
+
+/* Newest first. Never the text. `dir` is a history folder. */
+export function listHistoryIn(dir: string): HistoryEntry[] {
+  if (!existsSync(dir)) return []
+  const entries: HistoryEntry[] = []
+  for (const name of readdirSync(dir)) {
+    const m = HISTORY_FILE.exec(name)
+    if (m === null) continue
+    try {
+      const text = readFileSync(path.join(dir, name), 'utf8')
+      entries.push({
+        id: name,
+        at: `${m[1]}T${m[2]}:${m[3]}:${m[4]}`,
+        server: m[5],
+        kind: m[6],
+        size: Buffer.byteLength(text),
+        hash: createHash('sha256').update(text).digest('hex').slice(0, 16),
+      })
+    } catch {
+      // A file that vanished while listing is skipped.
+    }
+  }
+  return entries.sort((a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0))
+}
+
+/* The text of one entry. `id` has to be a name the listing returns, so nothing outside the folder can be read. */
+export const readHistory = (dataDir: string, app: string, id: string): string | null => readHistoryIn(historyDir(dataDir, app), id)
+
+export function readHistoryIn(dir: string, id: string): string | null {
+  const known = listHistoryIn(dir).find((e) => e.id === id)
+  return known === undefined ? null : readFileSync(path.join(dir, known.id), 'utf8')
+}
+
+/* The apps that have a baseline, by the name stored in the file. */
+export function listBaselineApps(dataDir: string): string[] {
+  const root = path.join(dataDir, 'apps')
+  if (!existsSync(root)) return []
+  const apps: string[] = []
+  for (const dir of readdirSync(root)) {
+    try {
+      if (!statSync(path.join(root, dir)).isDirectory()) continue
+      const data = readJson(path.join(root, dir, 'baseline.json')) as Partial<Baseline> | null
+      if (data !== null && typeof data.app === 'string' && typeof data.text === 'string') apps.push(data.app)
+    } catch {
+      // Not an app folder.
+    }
+  }
+  return apps.sort((a, b) => a.localeCompare(b))
+}
+
+/*
+Saves a server's config as it was before the hub changed it. The name starts with the time, so
+sorting the names sorts the files by age; the oldest beyond KEEP_HISTORY are deleted.
+*/
+export function writeHistory(
+  dataDir: string,
+  app: string,
+  server: string,
+  text: string,
+  kind: HistoryKind,
+  now: number,
+): string {
+  return writeHistoryIn(historyDir(dataDir, app), server, text, kind, now)
+}
+
+export function writeHistoryIn(dir: string, server: string, text: string, kind: HistoryKind, now: number): string {
+  const stamp = new Date(now).toISOString().replace(/:/g, '-')
+  const file = path.join(dir, `${stamp}__${safeName(server)}__${kind}.txt`)
+  writeAtomic(file, text)
+  const files = readdirSync(dir).filter((f) => f.endsWith('.txt')).sort()
+  for (const old of files.slice(0, Math.max(0, files.length - KEEP_HISTORY))) rmSync(path.join(dir, old), { force: true })
+  return file
+}
+
+/* ---- Settings and blocking lists (stage 7): `${HUB_DATA}/settings/` and `${HUB_DATA}/blocking/` ---- */
+
+export const settingsDir = (dataDir: string): string => path.join(dataDir, 'settings')
+export const settingsHistoryDir = (dataDir: string): string => path.join(settingsDir(dataDir), 'history')
+export const blockingDir = (dataDir: string): string => path.join(dataDir, 'blocking')
+
+export interface SettingsBaseline {
+  /** The settings as `settings/get` gave them, secrets included (the file is 0600). */
+  settings: Record<string, unknown>
+  importedFrom?: string
+  updatedAt: string
+  updatedBy: string
+}
+
+/* Extra local fields, on top of the built-in ones. Values are read live from the servers, not stored. */
+export interface SettingsOverrides {
+  paths: string[]
+}
+
+export function readSettingsBaseline(dataDir: string): SettingsBaseline | null {
+  const data = readJson(path.join(settingsDir(dataDir), 'baseline.json')) as Partial<SettingsBaseline> | null
+  if (data === null || data.settings === null || typeof data.settings !== 'object' || Array.isArray(data.settings)) return null
+  return {
+    settings: data.settings,
+    ...(typeof data.importedFrom === 'string' ? { importedFrom: data.importedFrom } : {}),
+    updatedAt: String(data.updatedAt ?? ''),
+    updatedBy: String(data.updatedBy ?? ''),
+  }
+}
+
+export const writeSettingsBaseline = (dataDir: string, baseline: SettingsBaseline): void =>
+  writeAtomic(path.join(settingsDir(dataDir), 'baseline.json'), JSON.stringify(baseline, null, 2) + '\n')
+
+export function readSettingsOverrides(dataDir: string): SettingsOverrides {
+  const data = readJson(path.join(settingsDir(dataDir), 'overrides.json')) as Partial<SettingsOverrides> | null
+  return { paths: Array.isArray(data?.paths) ? data.paths.filter((p): p is string => typeof p === 'string') : [] }
+}
+
+export const writeSettingsOverrides = (dataDir: string, overrides: SettingsOverrides): void =>
+  writeAtomic(path.join(settingsDir(dataDir), 'overrides.json'), JSON.stringify(overrides, null, 2) + '\n')
+
+export interface ListsBaseline {
+  blocked: string[]
+  allowed: string[]
+  importedFrom?: string
+  updatedAt: string
+  updatedBy: string
+}
+
+export function readListsBaseline(dataDir: string): ListsBaseline | null {
+  const data = readJson(path.join(blockingDir(dataDir), 'baseline.json')) as Partial<ListsBaseline> | null
+  if (data === null || !Array.isArray(data.blocked) || !Array.isArray(data.allowed)) return null
+  const strings = (v: unknown[]): string[] => v.filter((x): x is string => typeof x === 'string')
+  return {
+    blocked: strings(data.blocked),
+    allowed: strings(data.allowed),
+    ...(typeof data.importedFrom === 'string' ? { importedFrom: data.importedFrom } : {}),
+    updatedAt: String(data.updatedAt ?? ''),
+    updatedBy: String(data.updatedBy ?? ''),
+  }
+}
+
+export const writeListsBaseline = (dataDir: string, baseline: ListsBaseline): void =>
+  writeAtomic(path.join(blockingDir(dataDir), 'baseline.json'), JSON.stringify(baseline, null, 2) + '\n')

hub/upstream.ts

@@ -21,6 +21,15 @@ export class UpstreamError extends Error {
 
 export type Params = Record<string, string>
 
+/* What `raw` hands back: the headers worth passing on and the body as a stream. */
+export interface RawAnswer {
+  contentType: string
+  disposition?: string
+  body: ReadableStream<Uint8Array> | null
+  /* Stops the download (the caller went away). */
+  abort: () => void
+}
+
 interface Cached {
   until: number
   response: unknown
@@ -28,6 +37,9 @@ interface Cached {
 
 const MAX_CACHED = 500
 
+/* An app download or upload can take a while. */
+const WRITE_TIMEOUT_MS = 60_000
+
 export class Upstream {
   #config: HubConfig
   #fetch: typeof fetch
@@ -41,45 +53,128 @@ export class Upstream {
   }
 
   /** The `response` of the server's envelope, or an `UpstreamError`. */
-  async call(server: HubServer, path: string, params: Params = {}): Promise<unknown> {
+  async call(server: HubServer, path: string, params: Params = {}, token: string = server.token): Promise<unknown> {
     const sorted = Object.entries(params).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
     const key = `${server.id}\n${path}\n${new URLSearchParams(sorted).toString()}`
     const hit = this.#cache.get(key)
     if (hit !== undefined && hit.until > this.#now()) return hit.response
 
-    const query = new URLSearchParams(sorted)
+    const response = await this.#exec(server, path, new URLSearchParams(sorted), token, { method: 'GET' }, this.#config.timeoutMs)
+    if (this.#config.cacheSeconds > 0) {
+      if (this.#cache.size >= MAX_CACHED) this.#prune()
+      this.#cache.set(key, { until: this.#now() + this.#config.cacheSeconds * 1000, response })
+    }
+    return response
+  }
+
+  /*
+  A write: POST with the server's ADMIN token and never the read one, never cached. Whatever
+  the outcome, what the hub remembers of this server is dropped afterwards, so the next read
+  shows the state the write left.
+
+  The parameters travel in a form body (`application/x-www-form-urlencoded`) and only `token`
+  stays in the query, so values (a record's data, a TSIG secret) do not end up in a proxy's
+  access log. The server reads query or form alike. A `body` that is already a form gets the
+  parameters added to it. Two bodies cannot be a form, so there the parameters stay in the
+  query: a `FormData` (a multipart upload, the app zip or a zone file) and a string (a zone
+  file as `text/plain`).
+  */
+  async write(server: HubServer, path: string, params: Params = {}, body?: FormData | URLSearchParams | string): Promise<unknown> {
+    if (server.adminToken === undefined) throw new UpstreamError('rejected', 'no admin token configured')
+    let query = new URLSearchParams()
+    const init: RequestInit = { method: 'POST' }
+    if (body === undefined || body instanceof URLSearchParams) {
+      const form = new URLSearchParams(body)
+      for (const [k, v] of Object.entries(params)) form.set(k, v)
+      init.body = form
+    } else {
+      query = new URLSearchParams(params)
+      init.body = body
+      if (typeof body === 'string') init.headers = { 'content-type': 'text/plain; charset=utf-8' }
+    }
+    try {
+      return await this.#exec(server, path, query, server.adminToken, init, Math.max(this.#config.timeoutMs, WRITE_TIMEOUT_MS))
+    } finally {
+      this.invalidate(server.id)
+    }
+  }
+
+  /*
+  A file the server answers with (a zone export, a log, a block list): read with the READ token,
+  never cached. A JSON answer is the usual envelope, so an error is raised as for any call; any
+  other answer is handed back as it is, to be streamed to the caller. The timeout covers the wait
+  for the headers only: a big log may take longer to arrive.
+  */
+  async raw(server: HubServer, path: string, params: Params = {}): Promise<RawAnswer> {
+    const query = new URLSearchParams(params)
     query.set('token', server.token)
     const controller = new AbortController()
     const timer = setTimeout(() => controller.abort(), this.#config.timeoutMs)
+    let res: Response
+    try {
+      res = await this.#fetch(`${server.url}/api/${path}?${query.toString()}`, { method: 'GET', signal: controller.signal })
+    } catch (e) {
+      throw new UpstreamError('unreachable', this.#describe(e, [server.token], this.#config.timeoutMs))
+    } finally {
+      clearTimeout(timer)
+    }
+    const type = res.headers.get('content-type') ?? 'application/octet-stream'
+    if (/json/i.test(type)) {
+      let envelope: { status?: string; response?: unknown; errorMessage?: string }
+      try {
+        envelope = (await res.json()) as typeof envelope
+      } catch (e) {
+        throw new UpstreamError('unreachable', this.#describe(e, [server.token], this.#config.timeoutMs))
+      }
+      throw envelope?.status === 'ok' ? new UpstreamError('rejected', 'it sent JSON where a file was expected') : this.#rejection(envelope)
+    }
+    if (!res.ok) throw new UpstreamError('rejected', `status ${res.status}`)
+    return {
+      contentType: type,
+      disposition: res.headers.get('content-disposition') ?? undefined,
+      body: res.body,
+      abort: () => controller.abort(),
+    }
+  }
+
+  /** Forget every cached answer of one server. */
+  invalidate(serverId: string): void {
+    const prefix = `${serverId}\n`
+    for (const key of this.#cache.keys()) if (key.startsWith(prefix)) this.#cache.delete(key)
+  }
+
+  async #exec(server: HubServer, path: string, query: URLSearchParams, token: string, init: RequestInit, timeoutMs: number): Promise<unknown> {
+    query.set('token', token)
+    const controller = new AbortController()
+    const timer = setTimeout(() => controller.abort(), timeoutMs)
 
     let envelope: { status?: string; response?: unknown; errorMessage?: string }
     try {
       const res = await this.#fetch(`${server.url}/api/${path}?${query.toString()}`, {
+        ...init,
         signal: controller.signal,
-        headers: { accept: 'application/json' },
+        headers: { accept: 'application/json', ...(init.headers as Record<string, string> | undefined) },
       })
       envelope = (await res.json()) as typeof envelope
     } catch (e) {
-      throw new UpstreamError('unreachable', this.#describe(e, server))
+      throw new UpstreamError('unreachable', this.#describe(e, [server.token, token], timeoutMs))
     } finally {
       clearTimeout(timer)
     }
 
-    if (envelope?.status !== 'ok') {
-      const why =
-        envelope?.status === 'invalid-token'
-          ? 'its API token was refused'
-          : (envelope?.errorMessage ?? `status ${String(envelope?.status)}`)
-      throw new UpstreamError('rejected', why)
-    }
-
-    if (this.#config.cacheSeconds > 0) {
-      if (this.#cache.size >= MAX_CACHED) this.#prune()
-      this.#cache.set(key, { until: this.#now() + this.#config.cacheSeconds * 1000, response: envelope.response })
-    }
+    if (envelope?.status !== 'ok') throw this.#rejection(envelope)
     return envelope.response
   }
 
+  /* The error for an envelope that is not `ok`. */
+  #rejection(envelope: { status?: string; errorMessage?: string } | undefined): UpstreamError {
+    const why =
+      envelope?.status === 'invalid-token'
+        ? 'its API token was refused'
+        : (envelope?.errorMessage ?? `status ${String(envelope?.status)}`)
+    return new UpstreamError('rejected', why)
+  }
+
   #prune(): void {
     const now = this.#now()
     for (const [key, entry] of this.#cache) if (entry.until <= now) this.#cache.delete(key)
@@ -88,10 +183,10 @@ export class Upstream {
   }
 
   /* A short reason without anything that could carry the token. */
-  #describe(e: unknown, server: HubServer): string {
-    if (e instanceof Error && e.name === 'AbortError') return `timed out after ${this.#config.timeoutMs} ms`
+  #describe(e: unknown, secrets: string[], timeoutMs: number): string {
+    if (e instanceof Error && e.name === 'AbortError') return `timed out after ${timeoutMs} ms`
     const cause = e instanceof Error ? (e.cause as { code?: unknown } | undefined) : undefined
     const text = typeof cause?.code === 'string' ? cause.code : e instanceof Error ? e.message : 'request failed'
-    return text.split(server.token).join('***')
+    return secrets.reduce((out, secret) => (secret === '' ? out : out.split(secret).join('***')), text)
   }
 }

hub/users-cli.ts

@@ -0,0 +1,134 @@
+/*
+Manage the hub's users file.
+
+    node --experimental-strip-types hub/users-cli.ts add <name> <role>
+    node --experimental-strip-types hub/users-cli.ts passwd <name>
+    node --experimental-strip-types hub/users-cli.ts role <name> <role>
+    node --experimental-strip-types hub/users-cli.ts disable <name>
+    node --experimental-strip-types hub/users-cli.ts enable <name>
+    node --experimental-strip-types hub/users-cli.ts list
+
+The file is `${HUB_DATA}/users.json` (default /data). On a terminal the password is asked
+twice without echo; in a script, put it in HUB_NEW_PASSWORD. In the container:
+
+    docker exec -it technitium-hub node --experimental-strip-types /app/hub/users-cli.ts add alice apps-admin
+*/
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import {
+  ROLES,
+  addUser,
+  describeUsers,
+  readUsersFile,
+  setDisabled,
+  setPassword,
+  setRole,
+  writeUsersFile,
+  type UserRecord,
+} from './users.ts'
+
+const USAGE = `Usage: users-cli <command>
+  add <name> <role>   roles: ${ROLES.join(', ')}
+  passwd <name>
+  role <name> <role>
+  disable <name>
+  enable <name>
+  list
+The password is asked twice, or taken from HUB_NEW_PASSWORD.`
+
+type Env = Record<string, string | undefined>
+
+/* Reads one line from the terminal without echoing it. */
+function askHidden(prompt: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const stdin = process.stdin
+    process.stderr.write(prompt)
+    let line = ''
+    stdin.setRawMode(true)
+    stdin.resume()
+    stdin.setEncoding('utf8')
+    const done = (): void => {
+      stdin.setRawMode(false)
+      stdin.pause()
+      stdin.off('data', onData)
+      process.stderr.write('\n')
+    }
+    const onData = (chunk: string): void => {
+      for (const ch of chunk) {
+        if (ch === '\r' || ch === '\n') {
+          done()
+          resolve(line)
+          return
+        }
+        if (ch === '\u0003') {
+          done()
+          reject(new Error('Cancelled.'))
+          return
+        }
+        if (ch === '\u007f' || ch === '\b') line = line.slice(0, -1)
+        else line += ch
+      }
+    }
+    stdin.on('data', onData)
+  })
+}
+
+async function newPassword(env: Env): Promise<string> {
+  if (env.HUB_NEW_PASSWORD !== undefined && env.HUB_NEW_PASSWORD !== '') return env.HUB_NEW_PASSWORD
+  if (!process.stdin.isTTY) throw new Error('No terminal: set HUB_NEW_PASSWORD.')
+  const first = await askHidden('New password: ')
+  const second = await askHidden('Repeat it: ')
+  if (first !== second) throw new Error('The passwords differ.')
+  return first
+}
+
+/*
+Runs one command against the users in `dir` and returns the lines to print. `password`
+supplies the password for add and passwd, so the tests need no terminal.
+*/
+export async function run(
+  args: string[],
+  dir: string,
+  password: () => Promise<string>,
+): Promise<string[]> {
+  const [command, name, extra] = args
+  const users: UserRecord[] = readUsersFile(dir) ?? []
+  const need = (value: string | undefined, what: string): string => {
+    if (value === undefined || value === '') throw new Error(`Missing ${what}.\n${USAGE}`)
+    return value
+  }
+  switch (command) {
+    case 'add':
+      writeUsersFile(dir, addUser(users, need(name, 'name'), need(extra, 'role'), await password()))
+      return [`Added ${name} (${extra}).`]
+    case 'passwd':
+      writeUsersFile(dir, setPassword(users, need(name, 'name'), await password()))
+      return [`Password of ${name} changed. Their sessions end on their next request.`]
+    case 'role':
+      writeUsersFile(dir, setRole(users, need(name, 'name'), need(extra, 'role')))
+      return [`${name} is now ${extra}.`]
+    case 'disable':
+      writeUsersFile(dir, setDisabled(users, need(name, 'name'), true))
+      return [`Disabled ${name}.`]
+    case 'enable':
+      writeUsersFile(dir, setDisabled(users, need(name, 'name'), false))
+      return [`Enabled ${name}.`]
+    case 'list':
+      return users.length === 0 ? ['No users.'] : describeUsers(users)
+    default:
+      throw new Error(USAGE)
+  }
+}
+
+if (process.argv[1] !== undefined && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const dir = process.env.HUB_DATA?.trim() || '/data'
+  run(process.argv.slice(2), dir, () => newPassword(process.env)).then(
+    (lines) => {
+      for (const line of lines) console.log(line)
+    },
+    (e: unknown) => {
+      console.error(`users: ${e instanceof Error ? e.message : String(e)}`)
+      process.exit(1)
+    },
+  )
+}

hub/users.ts

@@ -0,0 +1,299 @@
+/*
+The hub's own accounts: roles, password hashes and the users file.
+
+`${HUB_DATA}/users.json` holds `{ users: [{ name, role, hash, disabled? }] }`, where hash is
+`scrypt$N$r$p$saltB64$keyB64`. The pure functions below are what the CLI (users-cli.ts)
+and the tests call; `Users` is what the running hub asks.
+*/
+import { randomBytes, scrypt, scryptSync, timingSafeEqual } from 'node:crypto'
+import { existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync, chmodSync, rmSync } from 'node:fs'
+import path from 'node:path'
+import type { Bootstrap } from './config.ts'
+
+export type Role = 'viewer' | 'operator' | 'apps-admin' | 'admin'
+export type Capability = 'read' | 'apps.write' | 'operate' | 'audit.read' | 'admin'
+
+export const ROLES: readonly Role[] = ['viewer', 'operator', 'apps-admin', 'admin']
+
+/* A simple rank. operator and apps-admin are siblings: use `can` to ask about a capability. */
+const RANK: Record<Role, number> = { viewer: 1, operator: 2, 'apps-admin': 2, admin: 3 }
+
+const CAPABILITIES: Record<Role, readonly Capability[]> = {
+  viewer: ['read'],
+  operator: ['read', 'operate'],
+  'apps-admin': ['read', 'apps.write'],
+  admin: ['read', 'apps.write', 'operate', 'audit.read', 'admin'],
+}
+
+export const isRole = (value: unknown): value is Role => typeof value === 'string' && (ROLES as readonly string[]).includes(value)
+
+export const rank = (role: Role): number => RANK[role]
+
+/** True when `role` ranks at least as high as `min`. */
+export const requires = (role: Role, min: Role): boolean => RANK[role] >= RANK[min]
+
+export const can = (role: Role, capability: Capability): boolean => CAPABILITIES[role].includes(capability)
+
+export interface UserRecord {
+  name: string
+  role: Role
+  hash: string
+  disabled?: boolean
+}
+
+/* ---- Passwords ---- */
+
+const N = 16384
+const R = 8
+const P = 1
+const KEYLEN = 64
+
+export function hashPassword(password: string): string {
+  const salt = randomBytes(16)
+  const key = scryptSync(password, salt, KEYLEN, { N, r: R, p: P })
+  return `scrypt$${N}$${R}$${P}$${salt.toString('base64')}$${key.toString('base64')}`
+}
+
+interface Parsed {
+  n: number
+  r: number
+  p: number
+  salt: Buffer
+  key: Buffer
+}
+
+function parseHash(stored: string): Parsed | null {
+  const parts = stored.split('$')
+  if (parts.length !== 6 || parts[0] !== 'scrypt') return null
+  const [n, r, p] = [Number(parts[1]), Number(parts[2]), Number(parts[3])]
+  if (![n, r, p].every((v) => Number.isInteger(v) && v > 0) || n > 1 << 20 || r > 32 || p > 16) return null
+  const salt = Buffer.from(parts[4], 'base64')
+  const key = Buffer.from(parts[5], 'base64')
+  return salt.length === 0 || key.length === 0 ? null : { n, r, p, salt, key }
+}
+
+export const isHash = (stored: string): boolean => parseHash(stored) !== null
+
+function derive(password: string, h: Parsed): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    scrypt(password, h.salt, h.key.length, { N: h.n, r: h.r, p: h.p, maxmem: 256 * h.n * h.r }, (err, key) =>
+      err ? reject(err) : resolve(key),
+    )
+  })
+}
+
+/** Checks a password against a stored hash; false for a malformed hash. Does not block the loop. */
+export async function verifyPassword(password: string, stored: string): Promise<boolean> {
+  const h = parseHash(stored)
+  if (h === null) return false
+  try {
+    return timingSafeEqual(await derive(password, h), h.key)
+  } catch {
+    return false
+  }
+}
+
+/* ---- The file: parse and pure edits ---- */
+
+const NAME = /^[A-Za-z0-9._@-]{1,64}$/
+
+export function validateName(name: string): void {
+  if (!NAME.test(name)) throw new Error('A user name is 1 to 64 letters, digits, . _ @ -.')
+}
+
+const same = (a: string, b: string): boolean => a.toLowerCase() === b.toLowerCase()
+
+export function parseUsersFile(text: string): UserRecord[] {
+  let data: unknown
+  try {
+    data = JSON.parse(text)
+  } catch {
+    throw new Error('users.json is not valid JSON.')
+  }
+  const list = (data as { users?: unknown } | null)?.users
+  if (!Array.isArray(list)) throw new Error('users.json must be {"users": [...]}.')
+  const users: UserRecord[] = []
+  for (const item of list as Record<string, unknown>[]) {
+    const name = item?.name
+    if (typeof name !== 'string' || !NAME.test(name)) throw new Error(`users.json: invalid user name ${JSON.stringify(name)}.`)
+    if (!isRole(item.role)) throw new Error(`users.json: user "${name}" has an unknown role.`)
+    if (typeof item.hash !== 'string' || !isHash(item.hash)) throw new Error(`users.json: user "${name}" has no valid hash.`)
+    if (users.some((u) => same(u.name, name))) throw new Error(`users.json lists "${name}" twice.`)
+    const record: UserRecord = { name, role: item.role, hash: item.hash }
+    if (item.disabled === true) record.disabled = true
+    users.push(record)
+  }
+  return users
+}
+
+export const serializeUsers = (users: UserRecord[]): string => JSON.stringify({ users }, null, 2) + '\n'
+
+export const findUser = (users: UserRecord[], name: string): UserRecord | undefined => users.find((u) => same(u.name, name))
+
+function must(users: UserRecord[], name: string): UserRecord {
+  const found = findUser(users, name)
+  if (found === undefined) throw new Error(`No user "${name}".`)
+  return found
+}
+
+export function checkPassword(password: string): void {
+  if (password.length < 8) throw new Error('The password must be at least 8 characters.')
+}
+
+function checkRole(role: string): Role {
+  if (!isRole(role)) throw new Error(`The role must be one of: ${ROLES.join(', ')}.`)
+  return role
+}
+
+/* Each edit returns a new list; the caller writes it. */
+
+export function addUser(users: UserRecord[], name: string, role: string, password: string): UserRecord[] {
+  validateName(name)
+  const checked = checkRole(role)
+  checkPassword(password)
+  if (findUser(users, name) !== undefined) throw new Error(`User "${name}" already exists.`)
+  return [...users, { name, role: checked, hash: hashPassword(password) }]
+}
+
+export function setPassword(users: UserRecord[], name: string, password: string): UserRecord[] {
+  checkPassword(password)
+  const target = must(users, name)
+  return users.map((u) => (u === target ? { ...u, hash: hashPassword(password) } : u))
+}
+
+export function setRole(users: UserRecord[], name: string, role: string): UserRecord[] {
+  const checked = checkRole(role)
+  const target = must(users, name)
+  return users.map((u) => (u === target ? { ...u, role: checked } : u))
+}
+
+export function setDisabled(users: UserRecord[], name: string, disabled: boolean): UserRecord[] {
+  const target = must(users, name)
+  return users.map((u) => {
+    if (u !== target) return u
+    const { disabled: _old, ...rest } = u
+    return disabled ? { ...rest, disabled: true } : rest
+  })
+}
+
+/** One line per user, never the hash. */
+export const describeUsers = (users: UserRecord[]): string[] =>
+  users.map((u) => `${u.name}\t${u.role}${u.disabled === true ? '\tdisabled' : ''}`)
+
+/* ---- The file: disk ---- */
+
+export const usersPath = (dir: string): string => path.join(dir, 'users.json')
+
+/** The users, or null when the file does not exist. Throws when it exists but is invalid. */
+export function readUsersFile(dir: string): UserRecord[] | null {
+  const file = usersPath(dir)
+  if (!existsSync(file)) return null
+  return parseUsersFile(readFileSync(file, 'utf8'))
+}
+
+/** Writes a temp file and renames it over users.json; mode 0600 where the platform honours it. */
+export function writeUsersFile(dir: string, users: UserRecord[]): void {
+  mkdirSync(dir, { recursive: true })
+  const file = usersPath(dir)
+  const temp = `${file}.${process.pid}.tmp`
+  try {
+    writeFileSync(temp, serializeUsers(users), { mode: 0o600 })
+    try {
+      chmodSync(temp, 0o600)
+    } catch {
+      // Not every platform has modes.
+    }
+    renameSync(temp, file)
+  } catch (e) {
+    rmSync(temp, { force: true })
+    throw e
+  }
+}
+
+/* ---- The running hub's view ---- */
+
+/* Compared when the role, hash or disabled flag changes, so a session can tell its user was edited. */
+export const signature = (u: UserRecord): string => `${u.role}|${u.hash}|${u.disabled === true ? 'x' : ''}`
+
+export class Users {
+  #dir: string
+  #users: UserRecord[]
+  #fromFile: boolean
+  #mtime = 0
+  #warned = false
+  /** A one-line notice for the start-up log, or null. */
+  readonly notice: string | null
+  /* Checked when a name is unknown, so an unknown name costs the same as a wrong password. */
+  #dummy = hashPassword(randomBytes(8).toString('hex'))
+
+  private constructor(dir: string, users: UserRecord[], fromFile: boolean, notice: string | null) {
+    this.#dir = dir
+    this.#users = users
+    this.#fromFile = fromFile
+    this.notice = notice
+    if (fromFile) this.#mtime = this.#stat()
+  }
+
+  /*
+  users.json when it exists (HUB_USER and HUB_PASSWORD are then ignored); else the one admin
+  from the environment; else an error. The file is never written here.
+  */
+  static load(dir: string, bootstrap: Bootstrap | null): Users {
+    const fromFile = readUsersFile(dir)
+    if (fromFile !== null) {
+      const notice =
+        bootstrap === null ? null : `${usersPath(dir)} exists: HUB_USER and HUB_PASSWORD are ignored.`
+      return new Users(dir, fromFile, true, notice)
+    }
+    if (bootstrap === null) {
+      throw new Error(
+        `No users: ${usersPath(dir)} does not exist and HUB_USER / HUB_PASSWORD are not set. ` +
+          'Create a user with: node --experimental-strip-types hub/users-cli.ts add <name> admin',
+      )
+    }
+    validateName(bootstrap.user)
+    const record: UserRecord = { name: bootstrap.user, role: 'admin', hash: hashPassword(bootstrap.password) }
+    return new Users(dir, [record], false, `No ${usersPath(dir)}: the single admin from HUB_USER / HUB_PASSWORD is in use.`)
+  }
+
+  #stat(): number {
+    try {
+      return statSync(usersPath(this.#dir)).mtimeMs
+    } catch {
+      return 0
+    }
+  }
+
+  /* Cheap enough to run on every request: one stat. A broken file keeps the last good list. */
+  refresh(): void {
+    if (!this.#fromFile) return
+    const mtime = this.#stat()
+    if (mtime === this.#mtime) return
+    this.#mtime = mtime
+    try {
+      const next = readUsersFile(this.#dir)
+      this.#users = next ?? []
+      this.#warned = false
+    } catch (e) {
+      if (!this.#warned) console.error(`hub: ${e instanceof Error ? e.message : String(e)} Keeping the previous users.`)
+      this.#warned = true
+    }
+  }
+
+  find(name: string): UserRecord | undefined {
+    this.refresh()
+    return findUser(this.#users, name)
+  }
+
+  /** The enabled user when the password is right, else null. Costs the same either way. */
+  async authenticate(name: string, password: string): Promise<UserRecord | null> {
+    const user = this.find(name.trim())
+    const good = await verifyPassword(password, user?.hash ?? this.#dummy)
+    return user !== undefined && good && user.disabled !== true ? user : null
+  }
+
+  /** True when the user was found and is disabled (for the audit log only). */
+  isDisabled(name: string): boolean {
+    return this.find(name.trim())?.disabled === true
+  }
+}

hub/webhook.test.ts

@@ -0,0 +1,230 @@
+// @vitest-environment node
+/*
+Stage 8, notices to the outside: the signature, the timeout and the single retry, what is logged,
+and the two endpoints (status and test) on a hub that has a webhook.
+*/
+import { createHmac } from 'node:crypto'
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
+import { parseConfig } from './config.ts'
+import { fake, startHub, type Fake, type TestHub } from './fake-servers.ts'
+import { Webhook, sign } from './webhook.ts'
+
+const urlConfig = (webhookUrl: string | null, webhookSecret: string | null = null) => ({ webhookUrl, webhookSecret, domain: 'hub.example' })
+
+describe('Webhook', () => {
+  afterEach(() => vi.restoreAllMocks())
+
+  it('posts {event, at, hub, items} and signs the exact body when there is a secret', async () => {
+    const seen: { url: string; init: RequestInit }[] = []
+    const hook = new Webhook(urlConfig('https://hooks.example/in?token=abc', 's3cret'), {
+      now: () => Date.parse('2026-11-01T00:00:00Z'),
+      fetch: (async (url: string, init: RequestInit) => {
+        seen.push({ url, init })
+        return new Response('ok')
+      }) as typeof fetch,
+    })
+    hook.send('drift.found', [{ kind: 'settings', server: 'a', fields: ['blockingType'] }])
+    await hook.settled()
+    expect(seen).toHaveLength(1)
+    const body = String(seen[0].init.body)
+    expect(JSON.parse(body)).toEqual({ event: 'drift.found', at: '2026-11-01T00:00:00.000Z', hub: 'hub.example', items: [{ kind: 'settings', server: 'a', fields: ['blockingType'] }] })
+    const headers = seen[0].init.headers as Record<string, string>
+    expect(headers['x-hub-signature']).toBe(`sha256=${createHmac('sha256', 's3cret').update(body).digest('hex')}`)
+    expect(headers['x-hub-signature']).toBe(sign('s3cret', body))
+  })
+
+  it('sends no signature without a secret and nothing at all without a URL', async () => {
+    const seen: RequestInit[] = []
+    const fetcher = (async (_u: string, init: RequestInit) => {
+      seen.push(init)
+      return new Response('ok')
+    }) as typeof fetch
+    const hook = new Webhook(urlConfig('https://hooks.example/in'), { fetch: fetcher })
+    hook.send('drift.cleared', [{ kind: 'app' }])
+    await hook.settled()
+    expect((seen[0].headers as Record<string, string>)['x-hub-signature']).toBeUndefined()
+    const none = new Webhook(urlConfig(null), { fetch: fetcher })
+    none.send('drift.found', [{ kind: 'app' }])
+    await none.settled()
+    expect(seen).toHaveLength(1)
+    expect(none.status()).toEqual({ configured: false, host: null, signed: false, last: null })
+  })
+
+  it('retries once and logs the host only, never the query string or the error text', async () => {
+    const log = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+    const out = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+    let tries = 0
+    const hook = new Webhook(urlConfig('https://hooks.example:8443/in?token=TOPSECRET'), {
+      retryDelayMs: 0,
+      fetch: (async () => {
+        tries++
+        throw new TypeError('fetch failed https://hooks.example:8443/in?token=TOPSECRET', { cause: { code: 'ECONNREFUSED' } })
+      }) as typeof fetch,
+    })
+    hook.send('drift.found', [{ kind: 'app' }])
+    await hook.settled()
+    expect(tries).toBe(2)
+    const text = [...log.mock.calls, ...out.mock.calls].flat().join(' ')
+    expect(text).toContain('hooks.example:8443')
+    expect(text).toContain('ECONNREFUSED')
+    expect(text).not.toContain('TOPSECRET')
+    expect(text).not.toContain('token=')
+    expect(hook.status()).toMatchObject({ configured: true, host: 'hooks.example:8443', last: { event: 'drift.found', ok: false, error: 'ECONNREFUSED' } })
+    expect(JSON.stringify(hook.status())).not.toContain('TOPSECRET')
+  })
+
+  it('treats a non-2xx answer as a failure and stops after the one retry; a second success is ok', async () => {
+    vi.spyOn(console, 'error').mockImplementation(() => undefined)
+    const answers = [500, 204]
+    let tries = 0
+    const hook = new Webhook(urlConfig('https://hooks.example/in'), { retryDelayMs: 0, fetch: (async () => new Response(null, { status: answers[tries++] ?? 500 })) as typeof fetch })
+    hook.send('drift.found', [{ kind: 'app' }])
+    await hook.settled()
+    expect(tries).toBe(2)
+    expect(hook.status().last).toMatchObject({ ok: true })
+    const bad = new Webhook(urlConfig('https://hooks.example/in'), { retryDelayMs: 0, fetch: (async () => new Response(null, { status: 503 })) as typeof fetch })
+    expect(await bad.deliver('webhook.test', [{}])).toMatchObject({ ok: false, error: 'HTTP 503' })
+  })
+
+  it('does not wait for a hanging receiver when sending, and gives up after the timeout', async () => {
+    vi.spyOn(console, 'error').mockImplementation(() => undefined)
+    const hook = new Webhook(urlConfig('https://hooks.example/in'), {
+      timeoutMs: 40,
+      retryDelayMs: 0,
+      fetch: ((_u: string, init: RequestInit) =>
+        new Promise((_resolve, reject) => {
+          init.signal?.addEventListener('abort', () => reject((init.signal as AbortSignal).reason))
+        })) as typeof fetch,
+    })
+    const started = Date.now()
+    hook.send('drift.found', [{ kind: 'app' }])
+    expect(Date.now() - started).toBeLessThan(30)
+    await hook.settled()
+    expect(hook.status().last).toMatchObject({ ok: false, error: 'TimeoutError' })
+  })
+
+  it('collects failed writes of one moment into a single notice', async () => {
+    const bodies: { event: string; items: unknown[] }[] = []
+    const hook = new Webhook(urlConfig('https://hooks.example/in'), {
+      batchMs: 5,
+      fetch: (async (_u: string, init: RequestInit) => {
+        bodies.push(JSON.parse(String(init.body)))
+        return new Response('ok')
+      }) as typeof fetch,
+    })
+    hook.queueFailed({ action: 'a', server: 'x' })
+    hook.queueFailed({ action: 'a', server: 'y' })
+    await hook.settled()
+    expect(bodies).toHaveLength(1)
+    expect(bodies[0]).toMatchObject({ event: 'write.failed', items: [{ server: 'x' }, { server: 'y' }] })
+  })
+})
+
+describe('HUB_WEBHOOK_URL', () => {
+  const base = { HUB_SERVERS: 'a=http://a.example', A_TOKEN: 't', HUB_USER: 'u', HUB_PASSWORD: 'p' }
+  it('accepts http and https and refuses anything else', () => {
+    expect(parseConfig({ ...base, HUB_WEBHOOK_URL: 'https://h.example/x?y=1', HUB_WEBHOOK_SECRET: 's' })).toMatchObject({ webhookUrl: 'https://h.example/x?y=1', webhookSecret: 's' })
+    expect(parseConfig(base)).toMatchObject({ webhookUrl: null, webhookSecret: null })
+    expect(() => parseConfig({ ...base, HUB_WEBHOOK_URL: 'ftp://h.example' })).toThrow(/HUB_WEBHOOK_URL/)
+    expect(() => parseConfig({ ...base, HUB_WEBHOOK_URL: 'h.example' })).toThrow(/HUB_WEBHOOK_URL/)
+  })
+})
+
+describe('the webhook on a hub', () => {
+  let a: Fake
+  let b: Fake
+  let hub: TestHub
+  let receiver: http.Server
+  let received: { headers: http.IncomingHttpHeaders; body: string }[]
+  let status = 200
+
+  beforeAll(async () => {
+    a = await fake()
+    b = await fake()
+    receiver = http.createServer((req, res) => {
+      const chunks: Buffer[] = []
+      req.on('data', (c: Buffer) => chunks.push(c))
+      req.on('end', () => {
+        received.push({ headers: req.headers, body: Buffer.concat(chunks).toString('utf8') })
+        res.writeHead(status).end('ok')
+      })
+    })
+    await new Promise<void>((r) => receiver.listen(0, '127.0.0.1', r))
+    hub = await startHub(a, b, {
+      HUB_WEBHOOK_URL: `http://127.0.0.1:${(receiver.address() as AddressInfo).port}/in?token=QUERYSECRET`,
+      HUB_WEBHOOK_SECRET: 'shared-secret',
+    })
+  })
+
+  afterAll(() => {
+    hub.close()
+    a.server.close()
+    b.server.close()
+    receiver.close()
+  })
+
+  beforeEach(() => {
+    received = []
+    status = 200
+  })
+
+  it('shows its status to an admin only, with the host and never the query', async () => {
+    for (const role of ['viewer', 'operator', 'apps-admin']) expect((await hub.call(role, 'hub/webhook/status', {}, 'GET')).code).toBe(403)
+    const got = await hub.call('admin', 'hub/webhook/status', {}, 'GET')
+    expect(got.code).toBe(200)
+    expect(got.body.response).toMatchObject({ configured: true, signed: true, last: null })
+    expect(got.body.response.host).toMatch(/^127\.0\.0\.1:\d+$/)
+    expect(JSON.stringify(got.body)).not.toContain('QUERYSECRET')
+  })
+
+  it('refuses the test to every role but admin, and refuses GET', async () => {
+    for (const role of ['viewer', 'operator', 'apps-admin']) expect((await hub.call(role, 'hub/webhook/test')).code).toBe(403)
+    expect((await hub.call('admin', 'hub/webhook/test', {}, 'GET')).code).toBe(405)
+    expect(received).toEqual([])
+  })
+
+  it('sends a signed test notice, records the result and audits it', async () => {
+    const done = await hub.call('admin', 'hub/webhook/test')
+    expect(done.code).toBe(200)
+    expect(done.body.response.last).toMatchObject({ event: 'webhook.test', ok: true })
+    expect(received).toHaveLength(1)
+    const { headers, body } = received[0]
+    expect(headers['x-hub-signature']).toBe(`sha256=${createHmac('sha256', 'shared-secret').update(body).digest('hex')}`)
+    expect(JSON.parse(body)).toMatchObject({ event: 'webhook.test', hub: 'hub' })
+    const audit = await hub.audit()
+    expect(audit.find((e) => e.action === 'webhook.test')).toMatchObject({ user: 'admin', outcome: 'ok' })
+    expect(JSON.stringify(audit)).not.toContain('QUERYSECRET')
+    expect(JSON.stringify(audit)).not.toContain('shared-secret')
+  })
+
+  it('reports a failing receiver in the answer without failing the request', async () => {
+    vi.spyOn(console, 'error').mockImplementation(() => undefined)
+    status = 500
+    const done = await hub.call('admin', 'hub/webhook/test')
+    expect(done.code).toBe(200)
+    expect(done.body.status).toBe('ok')
+    expect(done.body.response.last).toMatchObject({ ok: false, error: 'HTTP 500' })
+    expect(received).toHaveLength(2)
+    expect((await hub.audit()).find((e) => e.action === 'webhook.test' && e.outcome === 'failed')).toBeTruthy()
+    vi.restoreAllMocks()
+  })
+
+  it('sends write.failed for a failed audited write, without the values', async () => {
+    a.on('zones/records/add', () => ({ error: 'The record already exists.' }))
+    const done = await hub.call('admin', 'zones/records/add', { node: 'a', zone: 'example.com', domain: 'www.example.com', type: 'TXT', text: 'TOPVALUE' })
+    expect(done.body.status).toBe('error')
+    for (let i = 0; i < 40 && received.length === 0; i++) await new Promise((r) => setTimeout(r, 50))
+    expect(received).toHaveLength(1)
+    const sent = JSON.parse(received[0].body)
+    expect(sent.event).toBe('write.failed')
+    expect(sent.items[0]).toMatchObject({ action: 'zones.records.add', server: 'a', user: 'admin' })
+    expect(received[0].body).not.toContain('TOPVALUE')
+    // A failed login is not a write.
+    received = []
+    await fetch(`${hub.base}/api/user/login`, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded' }, body: 'user=admin&pass=wrong' })
+    await new Promise((r) => setTimeout(r, 500))
+    expect(received).toEqual([])
+  })
+})

hub/webhook.ts

@@ -0,0 +1,141 @@
+/*
+Notices to the outside (stage 8): when HUB_WEBHOOK_URL is set, the hub POSTs a small JSON body
+`{event, at, hub, items}` on a change of drift (`drift.found`, `drift.cleared`) and when an audited
+write failed (`write.failed`). With HUB_WEBHOOK_SECRET the exact bytes of the body are signed:
+`X-Hub-Signature: sha256=<hex hmac>`.
+
+A delivery never blocks or fails the request that caused it: it runs on its own with a timeout and
+at most one retry. Errors go to the server log with the HOST only (the URL may carry a token in its
+query string) and never the error text. Items carry names and field names, never values.
+*/
+import { createHmac } from 'node:crypto'
+import type { HubConfig } from './config.ts'
+
+export type WebhookEvent = 'drift.found' | 'drift.cleared' | 'write.failed' | 'webhook.test'
+
+export interface Delivery {
+  event: WebhookEvent
+  at: string
+  ok: boolean
+  /** `HTTP 500`, or an error code or name. Never a URL. */
+  error?: string
+}
+
+export interface WebhookStatus {
+  configured: boolean
+  /** Host and port only. */
+  host: string | null
+  signed: boolean
+  last: Delivery | null
+}
+
+export interface WebhookOptions {
+  fetch?: typeof fetch
+  timeoutMs?: number
+  retryDelayMs?: number
+  /** How long write.failed items are collected into one notice. */
+  batchMs?: number
+  now?: () => number
+}
+
+export const sign = (secret: string, body: string): string => `sha256=${createHmac('sha256', secret).update(body).digest('hex')}`
+
+export class Webhook {
+  #url: string | null
+  #secret: string | null
+  #domain: string
+  #host: string | null
+  #fetch: typeof fetch
+  #timeoutMs: number
+  #retryDelayMs: number
+  #batchMs: number
+  #now: () => number
+  #last: Delivery | null = null
+  #pending = new Set<Promise<unknown>>()
+  #queued: Record<string, unknown>[] = []
+  #timer: NodeJS.Timeout | null = null
+
+  constructor(config: Pick<HubConfig, 'webhookUrl' | 'webhookSecret' | 'domain'>, options: WebhookOptions = {}) {
+    this.#url = config.webhookUrl
+    this.#secret = config.webhookSecret
+    this.#domain = config.domain
+    this.#host = config.webhookUrl === null ? null : new URL(config.webhookUrl).host
+    this.#fetch = options.fetch ?? fetch
+    this.#timeoutMs = options.timeoutMs ?? 5000
+    this.#retryDelayMs = options.retryDelayMs ?? 1000
+    this.#batchMs = options.batchMs ?? 250
+    this.#now = options.now ?? Date.now
+  }
+
+  get configured(): boolean {
+    return this.#url !== null
+  }
+
+  status(): WebhookStatus {
+    return { configured: this.#url !== null, host: this.#host, signed: this.#secret !== null, last: this.#last }
+  }
+
+  /** Resolves when every delivery started so far has ended (for the tests). */
+  async settled(): Promise<void> {
+    if (this.#timer !== null) this.#flush()
+    while (this.#pending.size > 0) await Promise.allSettled([...this.#pending])
+  }
+
+  #body(event: WebhookEvent, items: Record<string, unknown>[]): string {
+    return JSON.stringify({ event, at: new Date(this.#now()).toISOString(), hub: this.#domain, items })
+  }
+
+  async #attempt(body: string): Promise<string | null> {
+    const headers: Record<string, string> = { 'content-type': 'application/json' }
+    if (this.#secret !== null) headers['x-hub-signature'] = sign(this.#secret, body)
+    try {
+      const res = await this.#fetch(this.#url as string, { method: 'POST', headers, body, redirect: 'manual', signal: AbortSignal.timeout(this.#timeoutMs) })
+      await res.body?.cancel().catch(() => undefined)
+      return res.status >= 200 && res.status < 300 ? null : `HTTP ${res.status}`
+    } catch (e) {
+      const code = (e as { cause?: { code?: unknown } })?.cause?.code ?? (e as { code?: unknown })?.code
+      return typeof code === 'string' ? code : e instanceof Error ? e.name : 'failed'
+    }
+  }
+
+  /* Delivers one notice and records the result. Returns it. */
+  async deliver(event: WebhookEvent, items: Record<string, unknown>[]): Promise<Delivery> {
+    if (this.#url === null) return { event, at: new Date(this.#now()).toISOString(), ok: false, error: 'not configured' }
+    const body = this.#body(event, items)
+    let error = await this.#attempt(body)
+    if (error !== null) {
+      if (this.#retryDelayMs > 0) await new Promise((r) => setTimeout(r, this.#retryDelayMs))
+      error = await this.#attempt(body)
+    }
+    const result: Delivery = { event, at: new Date(this.#now()).toISOString(), ok: error === null, ...(error === null ? {} : { error }) }
+    this.#last = result
+    if (error !== null) console.error(`hub: webhook delivery to ${this.#host} failed (${error}).`)
+    return result
+  }
+
+  /* Fire and forget: never throws, never waits. */
+  send(event: WebhookEvent, items: Record<string, unknown>[]): void {
+    if (this.#url === null || items.length === 0) return
+    const run = this.deliver(event, items).catch(() => undefined)
+    this.#pending.add(run)
+    void run.finally(() => this.#pending.delete(run))
+  }
+
+  /* Collects `write.failed` items for a moment, so ten failed servers make one notice. */
+  queueFailed(item: Record<string, unknown>): void {
+    if (this.#url === null) return
+    this.#queued.push(item)
+    if (this.#timer === null) {
+      this.#timer = setTimeout(() => this.#flush(), this.#batchMs)
+      this.#timer.unref()
+    }
+  }
+
+  #flush(): void {
+    if (this.#timer !== null) clearTimeout(this.#timer)
+    this.#timer = null
+    const items = this.#queued.splice(0, 50)
+    this.#queued = []
+    this.send('write.failed', items)
+  }
+}

hub/zones-sync.test.ts

@@ -0,0 +1,530 @@
+// @vitest-environment node
+/*
+Stage 7, zones: the merged list, copying a zone and making a Secondary. The servers are fakes that
+keep a zone table and record every call.
+*/
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { fake, startHub, ROLES, type Call, type Fake, type TestHub } from './fake-servers.ts'
+import { classifyRow, countRecordLines, secondaryOptions, validNameServer, cleanZone, type ZoneCell } from './zones-sync.ts'
+import { routeFor } from './routes.ts'
+
+interface Z {
+  name: string
+  type: string
+  soaSerial?: number | null
+  disabled?: boolean
+  dnssecStatus?: string
+  internal?: boolean
+  catalog?: string | null
+}
+
+interface State {
+  zones: Z[]
+  files: Record<string, string>
+  options: Record<string, Record<string, unknown>>
+  tsig: string[]
+}
+
+const state = new Map<Fake, State>()
+
+function install(f: Fake, init: Partial<State>): void {
+  const s: State = { zones: [], files: {}, options: {}, tsig: [], ...init }
+  state.set(f, s)
+  f.on('zones/list', (c) => {
+    const per = Number(c.query.get('zonesPerPage') ?? 10)
+    const page = Number(c.query.get('pageNumber') ?? 1)
+    return { response: { zones: s.zones.slice((page - 1) * per, page * per), pageNumber: page, totalPages: Math.max(1, Math.ceil(s.zones.length / per)), totalZones: s.zones.length } }
+  })
+  f.on('zones/export', (c) => {
+    const text = s.files[c.query.get('zone') ?? '']
+    return text === undefined ? { error: 'No such zone was found.' } : { text }
+  })
+  f.on('zones/create', (c) => {
+    s.zones.push({ name: c.query.get('zone') ?? c.form.get('zone') ?? '', type: c.query.get('type') ?? c.form.get('type') ?? 'Primary', soaSerial: 1 })
+    return { response: {} }
+  })
+  f.on('zones/import', (c) => {
+    s.files[c.query.get('zone') ?? ''] = c.text
+    return { response: {} }
+  })
+  f.on('zones/options/get', (c) => ({
+    response: { zoneTransfer: 'Deny', zoneTransferNetworkACL: [], zoneTransferTsigKeyNames: [], notify: 'None', notifyNameServers: [], ...(s.options[c.query.get('zone') ?? ''] ?? {}) },
+  }))
+  f.on('zones/options/set', (c) => {
+    const zone = c.form.get('zone') ?? ''
+    const next: Record<string, unknown> = { ...(s.options[zone] ?? {}) }
+    for (const [k, v] of c.form) {
+      if (k === 'zone') continue
+      next[k] = k.endsWith('ACL') || k.endsWith('Names') || k.endsWith('Servers') ? (v === 'false' ? [] : v.split(',')) : v
+    }
+    s.options[zone] = next
+    return { response: {} }
+  })
+  f.on('settings/getTsigKeyNames', () => ({ response: { tsigKeyNames: s.tsig } }))
+}
+
+const SOA = '$ORIGIN example.net.\n@ IN SOA ns1 hostmaster 5 900 300 604800 900\n@ IN NS ns1\nwww IN A 203.0.113.5\n; a comment\n'
+
+let a: Fake
+let b: Fake
+let hub: TestHub
+
+beforeAll(async () => {
+  a = await fake()
+  b = await fake()
+  hub = await startHub(a, b)
+})
+
+afterAll(() => {
+  hub.close()
+  a.server.close()
+  b.server.close()
+})
+
+beforeEach(() => {
+  a.calls = []
+  b.calls = []
+  a.down = false
+  b.down = false
+  install(a, {
+    zones: [
+      { name: 'example.net', type: 'Primary', soaSerial: 5, dnssecStatus: 'Unsigned' },
+      { name: 'lan', type: 'Primary', soaSerial: 3 },
+      { name: 'only-a.test', type: 'Primary', soaSerial: 1 },
+      { name: 'fwd.test', type: 'Forwarder', disabled: false },
+      { name: 'signed.test', type: 'Primary', soaSerial: 2, dnssecStatus: 'SignedWithNSEC' },
+      { name: 'sec.test', type: 'Secondary', soaSerial: 9 },
+      { name: 'primary.test', type: 'Primary', soaSerial: 10 },
+      { name: 'localhost', type: 'Primary', soaSerial: 1, internal: true },
+    ],
+    files: { 'example.net': SOA, 'fwd.test': 'fwd.test. 3600 IN FWD this-server\n', 'signed.test': SOA, 'lan': SOA, 'primary.test': SOA },
+    tsig: ['transfer.key'],
+  })
+  install(b, {
+    zones: [
+      { name: 'example.net', type: 'Primary', soaSerial: 5, dnssecStatus: 'Unsigned' },
+      { name: 'lan', type: 'Primary', soaSerial: 4 },
+      { name: 'only-b.test', type: 'Primary', soaSerial: 1 },
+      { name: 'primary.test', type: 'Secondary', soaSerial: 8 },
+      { name: 'localhost', type: 'Primary', soaSerial: 1, internal: true },
+    ],
+    files: {},
+    tsig: ['transfer.key'],
+  })
+})
+
+afterEach(() => {
+  a.down = false
+  b.down = false
+})
+
+const writes = (f: Fake): Call[] => f.calls.filter((c) => c.method === 'POST')
+
+const present = (type: string, serial: number | null = null): ZoneCell => ({ state: 'present', type, serial })
+const absent: ZoneCell = { state: 'absent' }
+const unknown: ZoneCell = { state: 'unknown' }
+
+describe('the status of a row', () => {
+  it('is the same when the zone is alike everywhere', () => {
+    expect(classifyRow('z', { a: present('Primary', 5), b: present('Primary', 5) }).status).toBe('same')
+  })
+
+  it('says serial-differs for two Primary zones with different serials', () => {
+    expect(classifyRow('z', { a: present('Primary', 5), b: present('Primary', 6) }).status).toBe('serial-differs')
+  })
+
+  it('says type-differs for types that do not follow one another', () => {
+    expect(classifyRow('z', { a: present('Primary', 5), b: present('Forwarder') }).status).toBe('type-differs')
+    expect(classifyRow('z', { a: present('Stub', 5), b: present('Secondary', 5) }).status).toBe('type-differs')
+  })
+
+  it('treats a Primary and its Secondary as the intended setup, and a lagging Secondary as behind, not as a divergence', () => {
+    const same = classifyRow('z', { a: present('Primary', 10), b: present('Secondary', 10) })
+    expect(same).toMatchObject({ status: 'same', note: 'primary-secondary' })
+    const behind = classifyRow('z', { a: present('Primary', 10), b: present('Secondary', 8) })
+    expect(behind).toMatchObject({ status: 'same', note: 'secondary-behind' })
+    expect(classifyRow('z', { a: present('Secondary', 8), b: present('Primary', 10) })).toMatchObject({ status: 'same', note: 'secondary-behind' })
+    // A Secondary that is AHEAD of its primary is not normal.
+    expect(classifyRow('z', { a: present('Primary', 8), b: present('Secondary', 10) }).status).toBe('serial-differs')
+  })
+
+  it('says only-on for a zone one server lacks', () => {
+    expect(classifyRow('z', { a: present('Primary', 1), b: absent })).toMatchObject({ status: 'only-on', onlyOn: 'a' })
+  })
+
+  it('does not call a zone only-on while the other server did not answer', () => {
+    expect(classifyRow('z', { a: present('Primary', 1), b: unknown }).status).toBe('unknown')
+    expect(classifyRow('z', { a: present('Primary', 1), b: present('Primary', 2), c: unknown }).status).toBe('serial-differs')
+  })
+
+  it('names the reachable servers that lack a zone the others have', () => {
+    expect(classifyRow('z', { a: present('Primary', 1), b: present('Primary', 1), c: absent })).toMatchObject({ status: 'missing-on', missingOn: ['c'] })
+  })
+
+  it('ignores serials of types that have none', () => {
+    expect(classifyRow('z', { a: present('Forwarder'), b: present('Forwarder') }).status).toBe('same')
+  })
+})
+
+describe('small helpers', () => {
+  it('counts the record lines of a zone file', () => {
+    expect(countRecordLines(SOA)).toBe(3)
+    expect(countRecordLines('')).toBe(0)
+  })
+
+  it('checks the zone name', () => {
+    expect(cleanZone('Example.NET.')).toBe('Example.NET')
+    for (const bad of [undefined, '', '*.example.net', 'a b', 'a..b']) expect(cleanZone(bad)).toBeNull()
+  })
+
+  it('checks a name server address', () => {
+    for (const ok of ['10.0.0.1', '10.0.0.1:53', 'ns.example.net', 'ns.example.net:5353', '2001:db8::1', '[2001:db8::1]:53']) expect(validNameServer(ok), ok).toBe(true)
+    for (const bad of ['', 'a b', '10.0.0.1,10.0.0.2', 'ns.example.net:abc', '*.example.net', 'x:1:2']) expect(validNameServer(bad), bad).toBe(false)
+  })
+})
+
+describe('secondaryOptions', () => {
+  it('widens the modes that cannot hold a list and keeps what is listed', () => {
+    const out = secondaryOptions('z', { zoneTransfer: 'Deny', zoneTransferNetworkACL: ['10.0.0.9'], notify: 'ZoneNameServers', notifyNameServers: ['10.0.0.8'] }, '10.0.0.2', '')
+    expect(out).toEqual({
+      zone: 'z',
+      zoneTransfer: 'UseSpecifiedNetworkACL',
+      zoneTransferNetworkACL: '10.0.0.9,10.0.0.2',
+      zoneTransferTsigKeyNames: 'false',
+      notify: 'BothZoneAndSpecifiedNameServers',
+      notifyNameServers: '10.0.0.8,10.0.0.2',
+    })
+  })
+
+  it('maps the other modes and keeps those that already take a list', () => {
+    expect(secondaryOptions('z', { zoneTransfer: 'AllowOnlyZoneNameServers', notify: 'None' }, '10.0.0.2', 'k')).toMatchObject({
+      zoneTransfer: 'AllowZoneNameServersAndUseSpecifiedNetworkACL',
+      notify: 'SpecifiedNameServers',
+      zoneTransferTsigKeyNames: 'k',
+    })
+    expect(secondaryOptions('z', { zoneTransfer: 'UseSpecifiedNetworkACL', notify: 'SpecifiedNameServers' }, '10.0.0.2', '')).toMatchObject({
+      zoneTransfer: 'UseSpecifiedNetworkACL',
+      notify: 'SpecifiedNameServers',
+    })
+  })
+
+  it('is null when the address and key are already there', () => {
+    expect(
+      secondaryOptions('z', { zoneTransfer: 'UseSpecifiedNetworkACL', zoneTransferNetworkACL: ['10.0.0.2'], zoneTransferTsigKeyNames: ['k'], notify: 'SpecifiedNameServers', notifyNameServers: ['10.0.0.2'] }, '10.0.0.2', 'k'),
+    ).toBeNull()
+  })
+})
+
+describe('routes', () => {
+  it('lists the zone endpoints with their kinds and rights', () => {
+    expect(routeFor('hub/zones/matrix')).toEqual({ kind: 'read', capability: 'read' })
+    expect(routeFor('hub/zones/copy/preview')).toEqual({ kind: 'read', capability: 'admin' })
+    expect(routeFor('hub/zones/copy')).toEqual({ kind: 'write-all', capability: 'admin' })
+    expect(routeFor('hub/zones/secondary/preview')).toEqual({ kind: 'read', capability: 'admin' })
+    expect(routeFor('hub/zones/secondary')).toEqual({ kind: 'write-all', capability: 'admin' })
+  })
+
+  it('refuses the admin endpoints to every other role, without touching a server', async () => {
+    const params = { source: 'a', zone: 'example.net', targets: 'b', target: 'b', primaryAddress: '10.0.0.1', targetAddress: '10.0.0.2' }
+    for (const role of ROLES.filter((r) => r !== 'admin')) {
+      for (const p of ['hub/zones/copy/preview', 'hub/zones/copy', 'hub/zones/secondary/preview', 'hub/zones/secondary']) {
+        const r = await hub.call(role, p, params)
+        expect(r.code, `${role} ${p}`).toBe(403)
+      }
+    }
+    expect(a.calls).toHaveLength(0)
+    expect(b.calls).toHaveLength(0)
+  })
+
+  it('lets a viewer read the merged list', async () => {
+    expect((await hub.call('viewer', 'hub/zones/matrix', {}, 'GET')).body.status).toBe('ok')
+  })
+})
+
+describe('hub/zones/matrix', () => {
+  it('merges the zones of both servers, one row per name, skipping the internal ones', async () => {
+    const r = await hub.call('admin', 'hub/zones/matrix', {}, 'GET')
+    const rows = r.body.response.zones as { name: string; status: string; onlyOn?: string; note?: string; cells: Record<string, any> }[]
+    const by = Object.fromEntries(rows.map((z) => [z.name, z]))
+    expect(Object.keys(by).sort()).toEqual(['example.net', 'fwd.test', 'lan', 'only-a.test', 'only-b.test', 'primary.test', 'sec.test', 'signed.test'])
+    expect(by['example.net'].status).toBe('same')
+    expect(by.lan.status).toBe('serial-differs')
+    expect(by['only-a.test']).toMatchObject({ status: 'only-on', onlyOn: 'a' })
+    expect(by['only-b.test']).toMatchObject({ status: 'only-on', onlyOn: 'b' })
+    expect(by['primary.test']).toMatchObject({ status: 'same', note: 'secondary-behind' })
+    expect(by['example.net'].cells.a).toMatchObject({ state: 'present', type: 'Primary', serial: 5, dnssec: 'Unsigned' })
+    expect(by.lan.cells.b.serial).toBe(4)
+    expect(r.body.response.servers).toEqual([
+      { id: 'a', ok: true, writable: true, host: '127.0.0.1' },
+      { id: 'b', ok: true, writable: true, host: '127.0.0.1' },
+    ])
+    expect(r.body.hubWarnings).toEqual([])
+  })
+
+  it('asks for every page', async () => {
+    const s = state.get(a)!
+    s.zones = Array.from({ length: 1200 }, (_, i) => ({ name: `z${String(i).padStart(4, '0')}.test`, type: 'Primary', soaSerial: 1 }))
+    const r = await hub.call('admin', 'hub/zones/matrix', {}, 'GET')
+    expect(r.body.response.zones.filter((z: { name: string }) => z.name.startsWith('z'))).toHaveLength(1200)
+    expect(a.calls.filter((c) => c.path === 'zones/list').length).toBeGreaterThanOrEqual(3)
+  })
+
+  it('marks the cells of a server that did not answer as unknown, warns, and does not call the zone only-on', async () => {
+    b.down = true
+    const r = await hub.call('admin', 'hub/zones/matrix', {}, 'GET')
+    expect(r.body.status).toBe('ok')
+    expect(r.body.hubWarnings).toHaveLength(1)
+    expect(r.body.hubWarnings[0]).toMatch(/^b:/)
+    const rows = r.body.response.zones as { name: string; status: string; cells: Record<string, any> }[]
+    const lan = rows.find((z) => z.name === 'lan')!
+    expect(lan.cells.b).toEqual({ state: 'unknown' })
+    expect(lan.status).toBe('unknown')
+    expect(rows.some((z) => z.status === 'only-on')).toBe(false)
+    expect(r.body.response.servers.find((s: { id: string }) => s.id === 'b').ok).toBe(false)
+  })
+
+  it('fails when no server answers', async () => {
+    a.down = true
+    b.down = true
+    const r = await hub.call('admin', 'hub/zones/matrix', {}, 'GET')
+    expect(r.body.status).toBe('error')
+  })
+})
+
+describe('hub/zones/copy/preview', () => {
+  const preview = (params: Record<string, string>) => hub.call('admin', 'hub/zones/copy/preview', params)
+
+  it('says what the source has and what each target holds', async () => {
+    const r = await preview({ source: 'a', zone: 'example.net', targets: 'b' })
+    expect(r.body.response).toMatchObject({
+      zone: 'example.net',
+      source: { server: 'a', type: 'Primary', serial: 5, records: 3 },
+      targets: [{ server: 'b', ok: true, exists: true, type: 'Primary', serial: 5, needsOverwrite: true }],
+    })
+    // only-a.test has no file on the fake, so the export fails and the preview says so.
+    expect((await preview({ source: 'a', zone: 'only-a.test', targets: 'b' })).body.status).toBe('error')
+  })
+
+  it('shows a target that lacks the zone as free to create', async () => {
+    state.get(a)!.files['only-a.test'] = SOA
+    const r = await preview({ source: 'a', zone: 'only-a.test', targets: 'b' })
+    expect(r.body.response.targets[0]).toMatchObject({ server: 'b', ok: true, exists: false, needsOverwrite: false })
+  })
+
+  it('refuses a Secondary, a signed zone, a missing zone and the source itself, each with a reason', async () => {
+    expect((await preview({ source: 'a', zone: 'sec.test', targets: 'b' })).body.errorMessage).toMatch(/Only Primary and Forwarder/)
+    expect((await preview({ source: 'a', zone: 'signed.test', targets: 'b' })).body.errorMessage).toMatch(/DNSSEC-signed/)
+    expect((await preview({ source: 'a', zone: 'nope.test', targets: 'b' })).body.errorMessage).toMatch(/no zone nope\.test/)
+    expect((await preview({ source: 'a', zone: 'example.net', targets: 'a' })).body.errorMessage).toMatch(/server it comes from/)
+    expect((await preview({ source: 'x', zone: 'example.net', targets: 'b' })).body.errorMessage).toMatch(/Unknown source server/)
+    expect((await preview({ source: 'a', zone: 'example.net', targets: '' })).body.errorMessage).toMatch(/at least one server/)
+    expect((await preview({ source: 'a', zone: '*.net', targets: 'b' })).body.errorMessage).toMatch(/valid zone name/)
+  })
+
+  it('refuses a target that holds the zone as another type', async () => {
+    state.get(b)!.zones.push({ name: 'fwd.test', type: 'Primary', soaSerial: 1 })
+    const r = await preview({ source: 'a', zone: 'fwd.test', targets: 'b' })
+    expect(r.body.response.targets[0]).toMatchObject({ ok: false, exists: true, type: 'Primary' })
+    expect(r.body.response.targets[0].error).toMatch(/cannot replace it/)
+  })
+
+  it('does not write anything and reads the export with the read token', async () => {
+    await preview({ source: 'a', zone: 'example.net', targets: 'b' })
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b)).toHaveLength(0)
+    expect(a.calls.find((c) => c.path === 'zones/export')!.token).toBe('read-a')
+  })
+})
+
+describe('hub/zones/copy', () => {
+  const copy = (params: Record<string, string>) => hub.call('admin', 'hub/zones/copy', params)
+
+  it('creates the zone on a target that lacks it and imports the text, with the admin token', async () => {
+    state.get(a)!.files['only-a.test'] = SOA
+    const r = await copy({ source: 'a', zone: 'only-a.test', targets: 'b' })
+    expect(r.body.response.results).toEqual([{ server: 'b', ok: true, message: 'Zone created and filled.' }])
+    const [create, imp] = writes(b)
+    expect(create.path).toBe('zones/create')
+    expect(create.token).toBe('admin-b')
+    expect(Object.fromEntries(new URLSearchParams([...create.form, ...create.query]))).toMatchObject({ zone: 'only-a.test', type: 'Primary' })
+    expect(imp.path).toBe('zones/import')
+    expect(imp.token).toBe('admin-b')
+    expect(Object.fromEntries(imp.query)).toMatchObject({ zone: 'only-a.test', overwrite: 'true', overwriteZone: 'true', overwriteSoaSerial: 'true' })
+    expect(imp.text).toBe(SOA)
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('creates a Forwarder empty and fills it by the import', async () => {
+    const r = await copy({ source: 'a', zone: 'fwd.test', targets: 'b' })
+    expect(r.body.response.results[0].ok).toBe(true)
+    const create = writes(b)[0]
+    expect(Object.fromEntries(new URLSearchParams([...create.form, ...create.query]))).toMatchObject({ zone: 'fwd.test', type: 'Forwarder', initializeForwarder: 'false' })
+  })
+
+  it('refuses to replace a zone that exists unless Overwrite was ticked for that server', async () => {
+    const refused = await copy({ source: 'a', zone: 'example.net', targets: 'b' })
+    expect(refused.body.response.results[0]).toMatchObject({ server: 'b', ok: false })
+    expect(refused.body.response.results[0].message).toMatch(/Tick Overwrite/)
+    expect(writes(b)).toHaveLength(0)
+
+    const other = await copy({ source: 'a', zone: 'example.net', targets: 'b', overwrite: 'a' })
+    expect(other.body.response.results[0].ok).toBe(false)
+    expect(writes(b)).toHaveLength(0)
+
+    const done = await copy({ source: 'a', zone: 'example.net', targets: 'b', overwrite: 'b' })
+    expect(done.body.response.results[0]).toEqual({ server: 'b', ok: true, message: 'Records replaced.' })
+    expect(writes(b).map((c) => c.path)).toEqual(['zones/import'])
+  })
+
+  it('refuses a Secondary source and a copy to the source, and writes nothing', async () => {
+    expect((await copy({ source: 'a', zone: 'sec.test', targets: 'b' })).code).toBe(400)
+    expect((await copy({ source: 'a', zone: 'example.net', targets: 'a' })).code).toBe(400)
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('reports a target that fails and still finishes the others', async () => {
+    state.get(a)!.files['only-a.test'] = SOA
+    b.on('zones/create', () => ({ error: 'Access was denied.' }))
+    const r = await copy({ source: 'a', zone: 'only-a.test', targets: 'b' })
+    expect(r.body.response.results[0]).toMatchObject({ server: 'b', ok: false })
+    expect(r.body.response.results[0].message).toMatch(/could not be created: Access was denied/)
+  })
+
+  it('says an empty zone was left when the import fails after the create', async () => {
+    state.get(a)!.files['only-a.test'] = SOA
+    b.on('zones/import', () => ({ error: 'Bad record.' }))
+    const r = await copy({ source: 'a', zone: 'only-a.test', targets: 'b' })
+    expect(r.body.response.results[0].ok).toBe(false)
+    expect(r.body.response.results[0].message).toMatch(/created but the import failed \(Bad record\.\); an empty zone is left on b/)
+  })
+
+  it('reports an unreachable target without failing the request', async () => {
+    state.get(a)!.files['only-a.test'] = SOA
+    b.down = true
+    const r = await copy({ source: 'a', zone: 'only-a.test', targets: 'b' })
+    expect(r.body.status).toBe('ok')
+    expect(r.body.response.results[0]).toMatchObject({ server: 'b', ok: false })
+  })
+
+  it('writes one audit line per target with the zone, the source, the target and the outcome, and no zone content', async () => {
+    state.get(a)!.files['only-a.test'] = SOA
+    await copy({ source: 'a', zone: 'only-a.test', targets: 'b' })
+    const line = (await hub.audit()).find((e) => e.action === 'zones.copy')!
+    expect(line).toMatchObject({ target: 'only-a.test', server: 'b', outcome: 'ok' })
+    expect(line.detail).toMatch(/from a/)
+    expect(JSON.stringify(line)).not.toContain('203.0.113.5')
+  })
+})
+
+describe('hub/zones/secondary', () => {
+  const input = { source: 'a', target: 'b', zone: 'example.net', primaryAddress: '10.0.0.1', targetAddress: '10.0.0.2' }
+  const secondary = (params: Record<string, string>) => hub.call('admin', 'hub/zones/secondary', params)
+  const preview = (params: Record<string, string>) => hub.call('admin', 'hub/zones/secondary/preview', params)
+
+  beforeEach(() => {
+    // The target does not have the zone yet.
+    state.get(b)!.zones = state.get(b)!.zones.filter((z) => z.name !== 'example.net')
+  })
+
+  it('previews the two steps and writes nothing', async () => {
+    const r = await preview(input)
+    expect(r.body.response.steps).toHaveLength(2)
+    expect(r.body.response.steps[0]).toMatchObject({ server: 'a', role: 'source', changes: true })
+    expect(r.body.response.steps[1]).toMatchObject({ server: 'b', role: 'target', changes: true })
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('adds the target to transfer and notify on the source, keeping what is there, then creates the Secondary', async () => {
+    state.get(a)!.options['example.net'] = { zoneTransfer: 'UseSpecifiedNetworkACL', zoneTransferNetworkACL: ['10.9.9.9'], notify: 'SpecifiedNameServers', notifyNameServers: ['10.9.9.8'] }
+    const r = await secondary({ ...input, tsigKeyName: 'transfer.key' })
+    expect(r.body.response.results.map((x: { server: string; ok: boolean }) => [x.server, x.ok])).toEqual([['a', true], ['b', true]])
+    const set = writes(a)[0]
+    expect(set.path).toBe('zones/options/set')
+    expect(set.token).toBe('admin-a')
+    expect(Object.fromEntries(set.form)).toEqual({
+      zone: 'example.net',
+      zoneTransfer: 'UseSpecifiedNetworkACL',
+      zoneTransferNetworkACL: '10.9.9.9,10.0.0.2',
+      zoneTransferTsigKeyNames: 'transfer.key',
+      notify: 'SpecifiedNameServers',
+      notifyNameServers: '10.9.9.8,10.0.0.2',
+    })
+    const create = writes(b)[0]
+    expect(create.path).toBe('zones/create')
+    expect(create.token).toBe('admin-b')
+    expect(Object.fromEntries(new URLSearchParams([...create.form, ...create.query]))).toMatchObject({
+      zone: 'example.net',
+      type: 'Secondary',
+      primaryNameServerAddresses: '10.0.0.1',
+      zoneTransferProtocol: 'Tcp',
+      tsigKeyName: 'transfer.key',
+    })
+    // The source went first.
+    expect(a.calls.findIndex((c) => c.path === 'zones/options/set')).toBeGreaterThan(-1)
+  })
+
+  it('can run again: an address already allowed is not written twice and the source is left alone', async () => {
+    await secondary(input)
+    state.get(b)!.zones = state.get(b)!.zones.filter((z) => z.name !== 'example.net')
+    a.calls = []
+    const again = await secondary(input)
+    expect(again.body.response.results[0]).toMatchObject({ server: 'a', ok: true })
+    expect(again.body.response.results[0].message).toMatch(/already allow/)
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b).filter((c) => c.path === 'zones/create')).toHaveLength(2)
+  })
+
+  it('reports a failing target and says the source change was left in place', async () => {
+    b.on('zones/create', () => ({ error: 'The primary is not reachable.' }))
+    const r = await secondary(input)
+    const [source, target] = r.body.response.results
+    expect(source).toMatchObject({ server: 'a', ok: true })
+    expect(target).toMatchObject({ server: 'b', ok: false })
+    expect(target.message).toMatch(/could not be created: The primary is not reachable\./)
+    expect(target.message).toMatch(/left in place/)
+  })
+
+  it('does not touch the target when the source step fails', async () => {
+    a.on('zones/options/set', () => ({ error: 'Access was denied.' }))
+    const r = await secondary(input)
+    expect(r.body.response.results[0]).toMatchObject({ server: 'a', ok: false })
+    expect(r.body.response.results[1]).toMatchObject({ server: 'b', ok: false })
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('refuses what cannot work, with a reason and no write', async () => {
+    const refused = async (patch: Record<string, string>, text: RegExp) => {
+      const r = await secondary({ ...input, ...patch })
+      expect(r.code).toBe(400)
+      expect(r.body.errorMessage).toMatch(text)
+    }
+    await refused({ zone: 'sec.test' }, /Only a Primary zone/)
+    await refused({ zone: 'nope.test' }, /no zone nope\.test/)
+    await refused({ target: 'a' }, /two different servers/)
+    await refused({ primaryAddress: '' }, /primary's address/)
+    await refused({ targetAddress: 'ns.example.net' }, /IP address/)
+    await refused({ tsigKeyName: 'missing.key' }, /does not exist on a/)
+    state.get(b)!.zones.push({ name: 'example.net', type: 'Primary' })
+    await refused({}, /already has example\.net/)
+    expect(writes(a)).toHaveLength(0)
+    expect(writes(b)).toHaveLength(0)
+  })
+
+  it('needs the TSIG key on both servers', async () => {
+    state.get(b)!.tsig = []
+    const r = await secondary({ ...input, tsigKeyName: 'transfer.key' })
+    expect(r.body.errorMessage).toMatch(/does not exist on b/)
+    expect(writes(a)).toHaveLength(0)
+  })
+
+  it('audits both steps', async () => {
+    await secondary(input)
+    // Newest first: the two lines of this call are the first two of their action.
+    const lines = (await hub.audit()).filter((e) => e.action === 'zones.secondary').slice(0, 2)
+    expect(lines.map((l) => [l.server, l.outcome]).sort()).toEqual([['a', 'ok'], ['b', 'ok']])
+    expect(lines[0].target).toBe('example.net')
+    expect(lines[0].detail).toMatch(/from a to b/)
+  })
+})

hub/zones-sync.ts

@@ -0,0 +1,656 @@
+/*
+Zones across the servers (stage 7): the merged list, copying a zone to another server, and
+making a zone a Secondary of another server's Primary (zone transfer, AXFR, done by Technitium
+itself rather than by copying records over and over).
+
+  * `buildZoneMatrix` reads `zones/list` of every server (all pages, read token) and puts one row
+    per zone name. `classifyRow` is the pure part that decides the status of a row.
+  * `ZonesSync.copyPreview` / `copy` move a zone as a zone file: `zones/export` from the source
+    with the READ token, then `zones/create` (when the zone is missing) and `zones/import` on each
+    target with that server's ADMIN token. The parameter names are the ones the console's own
+    Import zone dialog sends (`importZone` and `saveZoneFile` in src/api/zones.ts) and its Add
+    zone dialog sends (add-zone.ts).
+  * `ZonesSync.secondaryPreview` / `makeSecondary` do the two steps of a transfer: on the source
+    `zones/options/set` adds the target to the transfer ACL and to the notify list (what is there
+    stays), then on the target `zones/create` type Secondary. Each step can run again with no harm.
+
+Only Primary and Forwarder zones are copied; a Secondary, Stub or Catalog zone is refused with a
+reason. A DNSSEC-signed Primary is refused too: its keys do not travel in a zone file, so the copy
+would carry signatures nobody can renew. No zone content is written to the audit log.
+*/
+import { isIP } from 'node:net'
+import type { HubConfig, HubServer } from './config.ts'
+import { cleanDomain } from './servers-run.ts'
+import { Upstream, UpstreamError, type RawAnswer } from './upstream.ts'
+
+export class ZoneUserError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'ZoneUserError'
+  }
+}
+
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+
+export const PAGE_SIZE = 500
+export const MAX_PAGES = 40
+/* The largest zone file the hub carries between servers. */
+export const MAX_ZONE_BYTES = 5 * 1024 * 1024
+const EXPORT_TIMEOUT_MS = 30_000
+
+/* ---- The merged list ---- */
+
+export interface ListedZone {
+  name: string
+  type: string
+  lastModified?: string
+  disabled?: boolean
+  soaSerial?: number | null
+  dnssecStatus?: string
+  catalog?: string | null
+  internal?: boolean
+}
+
+export interface ZoneCell {
+  state: 'present' | 'absent' | 'unknown'
+  type?: string
+  serial?: number | null
+  disabled?: boolean
+  dnssec?: string
+  lastModified?: string
+}
+
+export type RowStatus = 'same' | 'serial-differs' | 'type-differs' | 'only-on' | 'missing-on' | 'unknown'
+
+export interface MatrixRow {
+  name: string
+  cells: Record<string, ZoneCell>
+  status: RowStatus
+  /* The server that has the zone, for `only-on`. */
+  onlyOn?: string
+  /* The reachable servers that lack it, for `missing-on`. */
+  missingOn?: string[]
+  /* A Primary with a Secondary of it is the intended setup: said here, not counted as a divergence. */
+  note?: 'primary-secondary' | 'secondary-behind'
+}
+
+export interface ZoneMatrix {
+  /* `host` is where the server's address points, to prefill the primary's address of a Secondary. */
+  servers: { id: string; ok: boolean; writable: boolean; host: string; error?: string }[]
+  zones: MatrixRow[]
+}
+
+/* The pairs where one side follows the other: not a type difference. */
+const FOLLOWS: Record<string, string> = { Secondary: 'Primary', SecondaryForwarder: 'Forwarder', SecondaryCatalog: 'Catalog' }
+
+function pairedTypes(types: Set<string>): boolean {
+  if (types.size !== 2) return false
+  const [a, b] = [...types]
+  return FOLLOWS[a] === b || FOLLOWS[b] === a
+}
+
+/* The status of one row, from its cells. Pure. */
+export function classifyRow(name: string, cells: Record<string, ZoneCell>): MatrixRow {
+  const entries = Object.entries(cells)
+  const present = entries.filter(([, c]) => c.state === 'present')
+  const absent = entries.filter(([, c]) => c.state === 'absent').map(([id]) => id)
+  const unknown = entries.some(([, c]) => c.state === 'unknown')
+  const row: MatrixRow = { name, cells, status: 'same' }
+
+  if (present.length < 2) {
+    // Whether it is "only on" one server cannot be said while a server that might have it did not answer.
+    if (unknown || present.length === 0) return { ...row, status: 'unknown' }
+    return { ...row, status: 'only-on', onlyOn: present[0][0] }
+  }
+
+  const types = new Set(present.map(([, c]) => c.type ?? ''))
+  if (types.size > 1 && !pairedTypes(types)) return { ...row, status: 'type-differs', ...(absent.length > 0 ? { missingOn: absent } : {}) }
+
+  if (types.size > 1) {
+    // Primary and its Secondary: lagging is expected, ahead is not.
+    const master = present.find(([, c]) => FOLLOWS[c.type ?? ''] === undefined)![1]
+    const followers = present.filter(([, c]) => FOLLOWS[c.type ?? ''] !== undefined)
+    const ahead = followers.some(([, c]) => typeof c.serial === 'number' && typeof master.serial === 'number' && c.serial > master.serial)
+    const behind = followers.some(([, c]) => typeof c.serial === 'number' && typeof master.serial === 'number' && c.serial < master.serial)
+    if (ahead) return { ...row, status: 'serial-differs', ...(absent.length > 0 ? { missingOn: absent } : {}) }
+    return finish({ ...row, note: behind ? 'secondary-behind' : 'primary-secondary' }, absent, unknown)
+  }
+
+  const [type] = [...types]
+  const serials = new Set(present.map(([, c]) => c.serial))
+  if (type === 'Primary' && serials.size > 1) return { ...row, status: 'serial-differs', ...(absent.length > 0 ? { missingOn: absent } : {}) }
+  return finish(row, absent, unknown)
+}
+
+function finish(row: MatrixRow, absent: string[], unknown: boolean): MatrixRow {
+  if (absent.length > 0) return { ...row, status: 'missing-on', missingOn: absent }
+  return unknown ? { ...row, status: 'unknown' } : row
+}
+
+/* Every page of one server's zones, without the server's own internal zones. */
+export async function listAllZones(upstream: Upstream, server: HubServer, fresh = false): Promise<{ zones: ListedZone[]; truncated: boolean }> {
+  if (fresh) upstream.invalidate(server.id)
+  const zones: ListedZone[] = []
+  let truncated = false
+  for (let page = 1; ; page++) {
+    const r = (await upstream.call(server, 'zones/list', { pageNumber: String(page), zonesPerPage: String(PAGE_SIZE) })) as {
+      zones?: ListedZone[]
+      totalPages?: number
+    } | null
+    for (const z of r?.zones ?? []) if (z.internal !== true) zones.push(z)
+    const total = typeof r?.totalPages === 'number' ? r.totalPages : 1
+    if (page >= total) break
+    if (page >= MAX_PAGES) {
+      truncated = true
+      break
+    }
+  }
+  return { zones, truncated }
+}
+
+export async function buildZoneMatrix(config: HubConfig, upstream: Upstream): Promise<{ matrix: ZoneMatrix; warnings: string[] }> {
+  const perServer = await Promise.all(
+    config.servers.map(async (server) => {
+      try {
+        return { server, error: undefined, ...(await listAllZones(upstream, server)) }
+      } catch (e) {
+        return { server, error: reason(e), zones: [] as ListedZone[], truncated: false }
+      }
+    }),
+  )
+  const warnings = perServer.filter((p) => p.error !== undefined).map((p) => `${p.server.id}: ${p.error}`)
+  if (warnings.length === perServer.length) throw new Error(`No server answered. ${warnings.join('; ')}`.trim())
+  for (const p of perServer) if (p.truncated) warnings.push(`${p.server.id}: more than ${MAX_PAGES * PAGE_SIZE} zones, the list is cut short.`)
+
+  const names = [...new Set(perServer.flatMap((p) => p.zones.map((z) => z.name)))].sort((a, b) => a.localeCompare(b))
+  const zones = names.map((name) => {
+    const cells: Record<string, ZoneCell> = {}
+    for (const p of perServer) {
+      if (p.error !== undefined) {
+        cells[p.server.id] = { state: 'unknown' }
+        continue
+      }
+      const z = p.zones.find((x) => x.name === name)
+      cells[p.server.id] =
+        z === undefined
+          ? { state: 'absent' }
+          : {
+              state: 'present',
+              type: z.type,
+              serial: typeof z.soaSerial === 'number' ? z.soaSerial : null,
+              disabled: z.disabled === true,
+              ...(z.dnssecStatus === undefined ? {} : { dnssec: z.dnssecStatus }),
+              ...(z.lastModified === undefined ? {} : { lastModified: z.lastModified }),
+            }
+    }
+    return classifyRow(name, cells)
+  })
+  const servers = perServer.map((p) => ({
+    id: p.server.id,
+    ok: p.error === undefined,
+    writable: p.server.adminToken !== undefined,
+    host: new URL(p.server.url).hostname,
+    ...(p.error === undefined ? {} : { error: p.error }),
+  }))
+  return { matrix: { servers, zones }, warnings }
+}
+
+/* ---- Copy ---- */
+
+const COPYABLE = new Set(['Primary', 'Forwarder'])
+
+/* A zone name as the hub accepts it: a plain domain, not a wildcard and not the root. */
+export function cleanZone(value: string | undefined): string | null {
+  const zone = cleanDomain(value)
+  return zone === null || zone.startsWith('*') ? null : zone
+}
+
+/* Records in a zone file, roughly: the lines that are not blank, a comment or a `$` directive. */
+export function countRecordLines(text: string): number {
+  let n = 0
+  for (const line of text.split(/\r?\n/)) {
+    const t = line.trim()
+    if (t !== '' && !t.startsWith(';') && !t.startsWith('$')) n++
+  }
+  return n
+}
+
+/* The body of an answer as text, stopping at `cap` bytes and `ms` milliseconds. */
+export async function readCapped(file: RawAnswer, cap: number, ms: number, what = 'The zone file'): Promise<string> {
+  if (file.body === null) return ''
+  const timer = setTimeout(() => file.abort(), ms)
+  const reader = file.body.getReader()
+  const chunks: Uint8Array[] = []
+  let size = 0
+  try {
+    for (;;) {
+      const { done, value } = await reader.read()
+      if (done) break
+      size += value.length
+      if (size > cap) {
+        file.abort()
+        throw new ZoneUserError(`${what} is larger than ${Math.round(cap / 1024 / 1024)} MB.`)
+      }
+      chunks.push(value)
+    }
+  } catch (e) {
+    if (e instanceof ZoneUserError) throw e
+    throw new UpstreamError('unreachable', 'the zone file could not be read in time')
+  } finally {
+    clearTimeout(timer)
+  }
+  return Buffer.concat(chunks).toString('utf8')
+}
+
+export interface TargetResult {
+  server: string
+  ok: boolean
+  message?: string
+}
+
+interface SourceZone {
+  zone: ListedZone
+  warnings: string[]
+}
+
+export interface CopyTargetPreview {
+  server: string
+  ok: boolean
+  error?: string
+  writable: boolean
+  exists: boolean
+  type?: string
+  serial?: number | null
+  /* The user has to tick Overwrite for this one. */
+  needsOverwrite: boolean
+}
+
+export interface CopyPreview {
+  zone: string
+  source: { server: string; type: string; serial: number | null; records: number; bytes: number }
+  warnings: string[]
+  targets: CopyTargetPreview[]
+}
+
+export class ZonesSync {
+  #config: HubConfig
+  #upstream: Upstream
+
+  constructor(config: HubConfig, upstream: Upstream) {
+    this.#config = config
+    this.#upstream = upstream
+  }
+
+  #server(id: string, what = 'server'): HubServer {
+    const found = this.#config.servers.find((s) => s.id === id)
+    if (found === undefined) throw new ZoneUserError(`Unknown ${what}: ${id}.`)
+    return found
+  }
+
+  #targets(ids: string[], source: string): HubServer[] {
+    const unique = [...new Set(ids)]
+    if (unique.length === 0) throw new ZoneUserError('Choose at least one server.')
+    if (unique.includes(source)) throw new ZoneUserError('A zone cannot be copied to the server it comes from.')
+    return unique.map((id) => this.#server(id))
+  }
+
+  /* The zone as the source lists it, and why it may not be copied. */
+  async #source(server: HubServer, zone: string): Promise<SourceZone> {
+    let listed: ListedZone[]
+    try {
+      listed = (await listAllZones(this.#upstream, server, true)).zones
+    } catch (e) {
+      throw new ZoneUserError(`${server.id}: ${reason(e)}`)
+    }
+    const found = listed.find((z) => z.name.toLowerCase() === zone.toLowerCase())
+    if (found === undefined) throw new ZoneUserError(`${server.id} has no zone ${zone}.`)
+    if (!COPYABLE.has(found.type)) {
+      throw new ZoneUserError(`${zone} is a ${found.type} zone on ${server.id}. Only Primary and Forwarder zones are copied; a ${found.type} zone gets its content from elsewhere.`)
+    }
+    if (found.dnssecStatus !== undefined && found.dnssecStatus !== 'Unsigned') {
+      throw new ZoneUserError(`${zone} is DNSSEC-signed. The signing keys do not travel in a zone file, so a copy would carry signatures that cannot be renewed. Sign the copy on the target instead.`)
+    }
+    const warnings: string[] = []
+    if (found.disabled === true) warnings.push(`${zone} is disabled on ${server.id}; the copy is created enabled.`)
+    if (typeof found.catalog === 'string' && found.catalog !== '') warnings.push(`${zone} is a member of the catalog ${found.catalog} on ${server.id}; the copy is not added to a catalog.`)
+    return { zone: found, warnings }
+  }
+
+  async #readZone(server: HubServer, zone: string): Promise<string> {
+    let file: RawAnswer
+    try {
+      file = await this.#upstream.raw(server, 'zones/export', { zone })
+    } catch (e) {
+      throw new ZoneUserError(`${server.id}: ${reason(e)}`)
+    }
+    try {
+      return await readCapped(file, MAX_ZONE_BYTES, EXPORT_TIMEOUT_MS)
+    } catch (e) {
+      throw new ZoneUserError(`${server.id}: ${reason(e)}`)
+    }
+  }
+
+  /* What is on a target for this zone, or why the target cannot take it. */
+  async #look(target: HubServer, zone: string, type: string): Promise<CopyTargetPreview> {
+    const base = { server: target.id, writable: target.adminToken !== undefined }
+    if (target.adminToken === undefined) return { ...base, ok: false, error: 'The hub has no admin token for this server, so it cannot write to it.', exists: false, needsOverwrite: false }
+    let listed: ListedZone[]
+    try {
+      listed = (await listAllZones(this.#upstream, target, true)).zones
+    } catch (e) {
+      return { ...base, ok: false, error: reason(e), exists: false, needsOverwrite: false }
+    }
+    const found = listed.find((z) => z.name.toLowerCase() === zone.toLowerCase())
+    if (found === undefined) return { ...base, ok: true, exists: false, needsOverwrite: false }
+    const here = { ...base, exists: true, type: found.type, serial: typeof found.soaSerial === 'number' ? found.soaSerial : null }
+    if (found.type !== type) {
+      return { ...here, ok: false, needsOverwrite: false, error: `${target.id} has ${zone} as a ${found.type} zone; a ${type} zone cannot replace it. Delete or convert it there first.` }
+    }
+    return { ...here, ok: true, needsOverwrite: true }
+  }
+
+  async copyPreview(sourceId: string, zoneRaw: string, ids: string[]): Promise<CopyPreview> {
+    const zone = cleanZone(zoneRaw)
+    if (zone === null) throw new ZoneUserError('A valid zone name is required.')
+    const source = this.#server(sourceId, 'source server')
+    const targets = this.#targets(ids, source.id)
+    const { zone: listed, warnings } = await this.#source(source, zone)
+    const text = await this.#readZone(source, zone)
+    return {
+      zone,
+      source: {
+        server: source.id,
+        type: listed.type,
+        serial: typeof listed.soaSerial === 'number' ? listed.soaSerial : null,
+        records: countRecordLines(text),
+        bytes: Buffer.byteLength(text),
+      },
+      warnings,
+      targets: await Promise.all(targets.map((t) => this.#look(t, zone, listed.type))),
+    }
+  }
+
+  /*
+  Copies the zone to each target. A target that has the zone is touched only when its id is in
+  `overwrite` (checked here, not only by the console). Create and import are two calls: when the
+  import fails after the create, the answer says an empty zone was left behind.
+  */
+  async copy(sourceId: string, zoneRaw: string, ids: string[], overwrite: string[]): Promise<{ results: TargetResult[] }> {
+    const zone = cleanZone(zoneRaw)
+    if (zone === null) throw new ZoneUserError('A valid zone name is required.')
+    const source = this.#server(sourceId, 'source server')
+    const targets = this.#targets(ids, source.id)
+    const { zone: listed } = await this.#source(source, zone)
+    const text = await this.#readZone(source, zone)
+    if (text.trim() === '') throw new ZoneUserError(`${source.id} exported an empty zone file for ${zone}.`)
+
+    const results = await Promise.all(
+      targets.map(async (target): Promise<TargetResult> => {
+        const look = await this.#look(target, zone, listed.type)
+        if (!look.ok) return { server: target.id, ok: false, message: look.error }
+        if (look.exists && !overwrite.includes(target.id)) {
+          return { server: target.id, ok: false, message: `${zone} already exists here (${look.type}, serial ${look.serial ?? 'none'}). Tick Overwrite to replace its records.` }
+        }
+        try {
+          if (!look.exists) {
+            const params: Record<string, string> = { zone, type: listed.type }
+            // A Forwarder is created empty and filled by the import (add-zone.ts: `initializeForwarder=false`).
+            if (listed.type === 'Forwarder') params.initializeForwarder = 'false'
+            await this.#upstream.write(target, 'zones/create', params)
+          }
+        } catch (e) {
+          return { server: target.id, ok: false, message: `the zone could not be created: ${reason(e)}` }
+        }
+        try {
+          // The three switches and a text/plain body, as `importZone` sends them.
+          await this.#upstream.write(target, 'zones/import', { zone, overwrite: 'true', overwriteZone: 'true', overwriteSoaSerial: 'true' }, text)
+        } catch (e) {
+          return {
+            server: target.id,
+            ok: false,
+            message: look.exists
+              ? `the import failed: ${reason(e)}`
+              : `the zone was created but the import failed (${reason(e)}); an empty zone is left on ${target.id}`,
+          }
+        }
+        return { server: target.id, ok: true, message: look.exists ? 'Records replaced.' : 'Zone created and filled.' }
+      }),
+    )
+    return { results }
+  }
+
+  /* ---- Make secondary ---- */
+
+  async #tsigNames(server: HubServer): Promise<string[]> {
+    this.#upstream.invalidate(server.id)
+    const r = (await this.#upstream.call(server, 'settings/getTsigKeyNames', {})) as { tsigKeyNames?: string[] } | null
+    return r?.tsigKeyNames ?? []
+  }
+
+  async secondaryPreview(input: SecondaryInput): Promise<SecondaryPreview> {
+    const plan = await this.#plan(input)
+    return { zone: plan.zone, source: plan.source.id, target: plan.target.id, warnings: plan.warnings, steps: plan.steps }
+  }
+
+  async makeSecondary(input: SecondaryInput): Promise<{ results: TargetResult[] }> {
+    const plan = await this.#plan(input)
+    const results: TargetResult[] = []
+    const { source, target } = plan
+
+    if (plan.sourceParams === null) {
+      results.push({ server: source.id, ok: true, message: 'Transfer and notify already allow the target; nothing was changed.' })
+    } else {
+      try {
+        await this.#upstream.write(source, 'zones/options/set', plan.sourceParams)
+        results.push({ server: source.id, ok: true, message: 'Zone transfer and notify now include the target.' })
+      } catch (e) {
+        results.push({ server: source.id, ok: false, message: `${reason(e)}; nothing was changed on either server` })
+        results.push({ server: target.id, ok: false, message: 'Not attempted: the source step failed.' })
+        return { results }
+      }
+    }
+    try {
+      await this.#upstream.write(target, 'zones/create', plan.targetParams)
+      results.push({ server: target.id, ok: true, message: 'Secondary zone created; it will fetch the zone from the primary.' })
+    } catch (e) {
+      results.push({
+        server: target.id,
+        ok: false,
+        message: `the Secondary zone could not be created: ${reason(e)}. The change on ${source.id} (zone transfer and notify) was left in place; run this again after fixing the cause.`,
+      })
+    }
+    return { results }
+  }
+
+  async #plan(input: SecondaryInput): Promise<SecondaryPlan> {
+    const zone = cleanZone(input.zone)
+    if (zone === null) throw new ZoneUserError('A valid zone name is required.')
+    const source = this.#server(input.source, 'source server')
+    const target = this.#server(input.target, 'target server')
+    if (source.id === target.id) throw new ZoneUserError('The source and the target must be two different servers.')
+    if (source.adminToken === undefined || target.adminToken === undefined) {
+      throw new ZoneUserError(`The hub has no admin token for ${source.adminToken === undefined ? source.id : target.id}, so it cannot change it.`)
+    }
+    const primaryAddress = input.primaryAddress.trim()
+    const targetAddress = input.targetAddress.trim()
+    if (!validNameServer(primaryAddress)) throw new ZoneUserError("Give the primary's address as the target reaches it: an IP address or a host name.")
+    if (isIP(targetAddress) === 0) throw new ZoneUserError("Give the target's IP address as the source sees it (the zone transfer list takes addresses).")
+    const key = (input.tsigKeyName ?? '').trim()
+
+    const warnings: string[] = []
+    const { zone: listed } = await this.#sourceForSecondary(source, zone)
+    let existsOnTarget: ListedZone | undefined
+    try {
+      existsOnTarget = (await listAllZones(this.#upstream, target, true)).zones.find((z) => z.name.toLowerCase() === zone.toLowerCase())
+    } catch (e) {
+      throw new ZoneUserError(`${target.id}: ${reason(e)}`)
+    }
+    if (existsOnTarget !== undefined) {
+      throw new ZoneUserError(`${target.id} already has ${zone} as a ${existsOnTarget.type} zone. Delete it there first if it should become a Secondary.`)
+    }
+    if (key !== '') {
+      for (const s of [source, target]) {
+        let names: string[]
+        try {
+          names = await this.#tsigNames(s)
+        } catch (e) {
+          throw new ZoneUserError(`${s.id}: ${reason(e)}`)
+        }
+        if (!names.includes(key)) throw new ZoneUserError(`The TSIG key "${key}" does not exist on ${s.id}. Add it in that server's Settings first.`)
+      }
+    }
+
+    let options: ZoneOptionsLike
+    try {
+      this.#upstream.invalidate(source.id)
+      options = (await this.#upstream.call(source, 'zones/options/get', { zone })) as ZoneOptionsLike
+    } catch (e) {
+      throw new ZoneUserError(`${source.id}: ${reason(e)}`)
+    }
+    if (typeof options.catalog === 'string' && options.catalog !== '' && options.overrideCatalogZoneTransfer !== true) {
+      warnings.push(`${zone} follows the zone transfer settings of the catalog ${options.catalog} on ${source.id}; the change may have no effect until the zone overrides them.`)
+    }
+
+    const sourceParams = secondaryOptions(zone, options, targetAddress, key)
+    const targetParams: Record<string, string> = {
+      zone,
+      type: 'Secondary',
+      catalog: '',
+      primaryNameServerAddresses: primaryAddress,
+      zoneTransferProtocol: 'Tcp',
+      tsigKeyName: key,
+      validateZone: 'false',
+    }
+    const steps: SecondaryStep[] = [
+      {
+        server: source.id,
+        role: 'source',
+        summary:
+          sourceParams === null
+            ? `${zone} on ${source.id} already allows zone transfer and notify to ${targetAddress}.`
+            : `On ${source.id}: allow zone transfer of ${zone} to ${targetAddress}${key === '' ? '' : ` with TSIG key ${key}`} and notify ${targetAddress}. What is already listed stays.`,
+        changes: sourceParams !== null,
+      },
+      {
+        server: target.id,
+        role: 'target',
+        summary: `On ${target.id}: create ${zone} as a Secondary zone that fetches it from ${primaryAddress} over TCP${key === '' ? '' : ` signed with ${key}`}.`,
+        changes: true,
+      },
+    ]
+    return { zone, source, target, warnings, steps, sourceParams, targetParams, type: listed.type }
+  }
+
+  async #sourceForSecondary(source: HubServer, zone: string): Promise<SourceZone> {
+    let listed: ListedZone[]
+    try {
+      listed = (await listAllZones(this.#upstream, source, true)).zones
+    } catch (e) {
+      throw new ZoneUserError(`${source.id}: ${reason(e)}`)
+    }
+    const found = listed.find((z) => z.name.toLowerCase() === zone.toLowerCase())
+    if (found === undefined) throw new ZoneUserError(`${source.id} has no zone ${zone}.`)
+    if (found.type !== 'Primary') throw new ZoneUserError(`${zone} is a ${found.type} zone on ${source.id}. Only a Primary zone can be transferred to a Secondary.`)
+    return { zone: found, warnings: [] }
+  }
+}
+
+export interface SecondaryInput {
+  source: string
+  target: string
+  zone: string
+  primaryAddress: string
+  targetAddress: string
+  tsigKeyName?: string
+}
+
+export interface SecondaryStep {
+  server: string
+  role: 'source' | 'target'
+  summary: string
+  /* False when nothing would be written there. */
+  changes: boolean
+}
+
+export interface SecondaryPreview {
+  zone: string
+  source: string
+  target: string
+  warnings: string[]
+  steps: SecondaryStep[]
+}
+
+interface SecondaryPlan {
+  zone: string
+  source: HubServer
+  target: HubServer
+  warnings: string[]
+  steps: SecondaryStep[]
+  /* Null when the source already allows it. */
+  sourceParams: Record<string, string> | null
+  targetParams: Record<string, string>
+  type: string
+}
+
+interface ZoneOptionsLike {
+  catalog?: string | null
+  overrideCatalogZoneTransfer?: boolean
+  zoneTransfer?: string
+  zoneTransferNetworkACL?: string[]
+  zoneTransferTsigKeyNames?: string[]
+  notify?: string
+  notifyNameServers?: string[]
+}
+
+/* An address for a name server: an IP, a host name, either with a port. */
+export function validNameServer(value: string): boolean {
+  if (value === '' || value.length > 255 || /[\s,]/.test(value)) return false
+  if (isIP(value) !== 0) return true
+  const bracketed = /^\[([^\]]+)\](?::(\d{1,5}))?$/.exec(value)
+  if (bracketed !== null) return isIP(bracketed[1]) === 6
+  const [host, port, extra] = value.split(':')
+  if (extra !== undefined || (port !== undefined && !/^\d{1,5}$/.test(port))) return false
+  return isIP(host) === 4 || cleanZone(host) !== null
+}
+
+/* A transfer mode that also lets the listed networks in. */
+const TRANSFER_WITH_ACL: Record<string, string> = { Deny: 'UseSpecifiedNetworkACL', AllowOnlyZoneNameServers: 'AllowZoneNameServersAndUseSpecifiedNetworkACL' }
+const NOTIFY_WITH_LIST: Record<string, string> = { None: 'SpecifiedNameServers', ZoneNameServers: 'BothZoneAndSpecifiedNameServers' }
+
+const joined = (list: string[]): string => (list.length === 0 ? 'false' : list.join(','))
+
+/*
+The `zones/options/set` that lets `address` transfer and be notified, keeping what is there: modes
+that cannot hold a list are widened to the matching one that can, an address already listed is not
+added twice. Null when nothing needs to change. Only the transfer and notify fields are sent, so
+the rest of the zone's options is not touched.
+*/
+export function secondaryOptions(zone: string, current: ZoneOptionsLike, address: string, tsigKey: string): Record<string, string> | null {
+  const mode = current.zoneTransfer ?? 'Deny'
+  const acl = [...(current.zoneTransferNetworkACL ?? [])]
+  const keys = [...(current.zoneTransferTsigKeyNames ?? [])]
+  const notify = current.notify ?? 'None'
+  const notifyList = [...(current.notifyNameServers ?? [])]
+
+  const nextMode = TRANSFER_WITH_ACL[mode] ?? mode
+  if (nextMode !== 'Allow' && !acl.includes(address)) acl.push(address)
+  if (tsigKey !== '' && !keys.includes(tsigKey)) keys.push(tsigKey)
+  const nextNotify = NOTIFY_WITH_LIST[notify] ?? notify
+  if (!notifyList.includes(address)) notifyList.push(address)
+
+  const unchanged =
+    nextMode === mode &&
+    nextNotify === notify &&
+    acl.length === (current.zoneTransferNetworkACL ?? []).length &&
+    keys.length === (current.zoneTransferTsigKeyNames ?? []).length &&
+    notifyList.length === (current.notifyNameServers ?? []).length
+  if (unchanged) return null
+  return {
+    zone,
+    zoneTransfer: nextMode,
+    zoneTransferNetworkACL: joined(acl),
+    zoneTransferTsigKeyNames: joined(keys),
+    notify: nextNotify,
+    notifyNameServers: joined(notifyList),
+  }
+}

package.json

@@ -10,6 +10,7 @@
     "dev": "vite",
     "mock": "node dev/mock-api.mjs",
     "hub": "node --experimental-strip-types hub/server.ts",
+    "hub:user": "node --experimental-strip-types hub/users-cli.ts",
     "build": "tsc -b && vite build",
     "lint": "oxlint",
     "lint:language": "node dev/check-language.mjs",

src/api/apps.ts

@@ -1,4 +1,5 @@
-import { apiRequest, type ApiOutcome } from './client'
+import { urlApi } from '../app/base'
+import { apiRequest, envelopeOutcome, type ApiOutcome } from './client'
 
 /*
 The nine endpoints of the `apps` family (upstream/master:www/js/apps.js).
@@ -61,6 +62,8 @@ export interface StoreApp {
   installedVersion?: string
   /** Only when `installed` is true. */
   updateAvailable?: boolean
+  /** Only behind a hub, in the All servers listing: the servers that have it installed. */
+  installedOn?: string[]
 }
 
 /*
@@ -88,28 +91,32 @@ export function listApps(
 
 export function listStoreApps(
   token: string | null,
+  node = '',
 ): Promise<ApiOutcome<{ response: { storeApps: StoreApp[] } }>> {
-  return apiRequest('apps/listStoreApps', { token })
+  return apiRequest('apps/listStoreApps', { token, node })
 }
 
+/* Behind a hub, `node` is the one server the call is for; elsewhere it stays empty and is not sent. */
 export function downloadAndInstall(
   token: string | null,
   name: string,
   url: string,
+  node = '',
 ): Promise<ApiOutcome<{ response: { installedApp: InstalledApp } }>> {
-  return apiRequest('apps/downloadAndInstall', { token, body: { name, url } })
+  return apiRequest('apps/downloadAndInstall', { token, body: { name, url }, node })
 }
 
 export function downloadAndUpdate(
   token: string | null,
   name: string,
   url: string,
+  node = '',
 ): Promise<ApiOutcome<{ response: { updatedApp: InstalledApp } }>> {
-  return apiRequest('apps/downloadAndUpdate', { token, body: { name, url } })
+  return apiRequest('apps/downloadAndUpdate', { token, body: { name, url }, node })
 }
 
-export function uninstallApp(token: string | null, name: string): Promise<ApiOutcome> {
-  return apiRequest('apps/uninstall', { token, body: { name } })
+export function uninstallApp(token: string | null, name: string, node = ''): Promise<ApiOutcome> {
+  return apiRequest('apps/uninstall', { token, body: { name }, node })
 }
 
 /*
@@ -136,8 +143,9 @@ export function setAppConfig(
   token: string | null,
   name: string,
   config: string,
+  node = '',
 ): Promise<ApiOutcome> {
-  return apiRequest('apps/config/set', { token, method: 'POST', body: { name, config } })
+  return apiRequest('apps/config/set', { token, method: 'POST', body: { name, config }, node })
 }
 
 /*
@@ -174,16 +182,286 @@ export function installApp(
   token: string | null,
   name: string,
   file: File,
+  node = '',
 ): Promise<ApiOutcome<{ response: { installedApp: InstalledApp } }>> {
   const { path, form } = buildUpload('apps/install', name, file)
-  return apiRequest(path, { token, form })
+  return apiRequest(path, { token, form, node })
 }
 
 export function updateApp(
   token: string | null,
   name: string,
   file: File,
+  node = '',
 ): Promise<ApiOutcome<{ response: { updatedApp: InstalledApp } }>> {
   const { path, form } = buildUpload('apps/update', name, file)
-  return apiRequest(path, { token, form })
+  return apiRequest(path, { token, form, node })
+}
+
+/*
+The hub's matrix of apps over servers (`hub/apps/matrix`, hub/apps-matrix.ts). Only on a
+hub. A config is never in it: just a hash per server and the comparison.
+*/
+export interface MatrixCell {
+  installed: boolean
+  version?: string
+  updateAvailable?: boolean
+  updateVersion?: string
+  configStatus: 'none' | 'read' | 'unreadable'
+  configHash?: string
+  configError?: string
+}
+
+export interface MatrixApp {
+  name: string
+  description: string
+  servers: Record<string, MatrixCell>
+  configs: 'same' | 'different' | 'single' | 'unknown'
+  diffCount?: number
+  versions: 'same' | 'different' | 'single'
+}
+
+export interface AppsMatrixData {
+  servers: { id: string; ok: boolean; writable: boolean; error?: string }[]
+  apps: MatrixApp[]
+}
+
+export function getAppsMatrix(token: string | null): Promise<ApiOutcome<{ response: AppsMatrixData }>> {
+  return apiRequest('hub/apps/matrix', { token })
+}
+
+/*
+App writes across servers (hub/apps-write.ts). The hub resolves a store URL from each
+server's own store; the client never sends one.
+*/
+export type HubRunAction = 'install-store' | 'update-store' | 'uninstall'
+
+export interface StepResult {
+  step: 'apps' | 'configs' | 'settings' | 'lists'
+  status: 'ok' | 'failed' | 'skipped' | 'not-run'
+  message: string
+}
+
+export interface TargetResult {
+  server: string
+  ok: boolean
+  message?: string
+  /* Only a profile apply: what each step did on this server. */
+  steps?: StepResult[]
+}
+
+export type RunOutcome = ApiOutcome<{ response: { results: TargetResult[] } }>
+
+export function runHubApps(token: string | null, action: HubRunAction, name: string, targets: string[]): Promise<RunOutcome> {
+  return apiRequest('hub/apps/run', { token, method: 'POST', body: { action, name, targets: targets.join(',') } })
+}
+
+/* The same zip to several servers: multipart, as the hub's `hub/apps/upload` expects. */
+export function uploadHubApp(
+  token: string | null,
+  action: 'install' | 'update',
+  name: string,
+  file: File,
+  targets: string[],
+): Promise<RunOutcome> {
+  const form = new FormData()
+  form.append('action', action)
+  form.append('name', name)
+  form.append('targets', targets.join(','))
+  form.append('fileAppZip', file)
+  return apiRequest('hub/apps/upload', { token, form })
+}
+
+/*
+Config sync behind a hub (hub/apps-config.ts): the comparison of one app's config across
+servers, the baseline kept in the hub, the fields that stay local to each server, and the
+preview and apply that bring servers to baseline plus their own local values.
+*/
+export interface FieldChange {
+  path: string
+  type: 'added' | 'removed' | 'changed'
+  a?: unknown
+  b?: unknown
+}
+
+export interface LineOp {
+  op: 'same' | 'add' | 'remove'
+  text: string
+}
+
+export type ConfigDiff =
+  | { kind: 'json'; changes: FieldChange[] }
+  | { kind: 'text'; lines: LineOp[]; truncated: boolean; changed: number }
+
+export interface Baseline {
+  app: string
+  text: string
+  importedFrom?: string
+  updatedAt: string
+  updatedBy: string
+}
+
+export interface Overrides {
+  paths: string[]
+  values: Record<string, Record<string, unknown>>
+}
+
+export interface ServerConfigView {
+  id: string
+  ok: boolean
+  error?: string
+  installed: boolean
+  writable: boolean
+  version?: string
+  text: string | null
+  hash: string
+  effective?: string
+  warnings: string[]
+  /** Null without a baseline. */
+  inSync: boolean | null
+  vsBaseline?: ConfigDiff
+  vsOthers: Record<string, ConfigDiff>
+}
+
+export interface ConfigCompare {
+  name: string
+  baseline: Baseline | null
+  overrides: Overrides
+  servers: ServerConfigView[]
+}
+
+export interface TargetPreview {
+  server: string
+  ok: boolean
+  error?: string
+  writable: boolean
+  current: string | null
+  currentHash: string
+  effective: string
+  changes: ConfigDiff
+  unchanged: boolean
+  warnings: string[]
+}
+
+export const getConfigCompare = (token: string | null, name: string): Promise<ApiOutcome<{ response: ConfigCompare }>> =>
+  apiRequest('hub/apps/config/compare', { token, body: { name } })
+
+export const importBaseline = (token: string | null, name: string, server: string): Promise<ApiOutcome<{ response: { baseline: Baseline } }>> =>
+  apiRequest('hub/apps/baseline/import', { token, method: 'POST', body: { name, server } })
+
+export const saveBaseline = (token: string | null, name: string, text: string): Promise<ApiOutcome<{ response: { baseline: Baseline } }>> =>
+  apiRequest('hub/apps/baseline/save', { token, method: 'POST', body: { name, text } })
+
+export const saveOverrides = (token: string | null, name: string, overrides: Overrides): Promise<ApiOutcome<{ response: { overrides: Overrides } }>> =>
+  apiRequest('hub/apps/overrides/save', {
+    token,
+    method: 'POST',
+    body: { name, paths: overrides.paths.join(','), values: JSON.stringify(overrides.values) },
+  })
+
+export const getCurrentLocalValues = (token: string | null, name: string, paths: string[]): Promise<ApiOutcome<{ response: { values: Overrides['values'] } }>> =>
+  apiRequest('hub/apps/overrides/current', { token, method: 'POST', body: { name, paths: paths.join(',') } })
+
+export const previewConfig = (token: string | null, name: string, targets: string[]): Promise<ApiOutcome<{ response: { targets: TargetPreview[] } }>> =>
+  apiRequest('hub/apps/config/preview', { token, method: 'POST', body: { name, targets: targets.join(',') } })
+
+export const applyConfig = (token: string | null, name: string, targets: string[], expect: Record<string, string>): Promise<RunOutcome> =>
+  apiRequest('hub/apps/config/apply', { token, method: 'POST', body: { name, targets: targets.join(','), expect: JSON.stringify(expect) } })
+
+/*
+History, rollback, drift, and moving baselines between hubs (hub/apps-config.ts, hub/drift.ts).
+*/
+export interface HistoryEntry {
+  id: string
+  at: string
+  /** A server id, or `baseline` for an earlier text of the baseline. */
+  server: string
+  kind: string
+  size: number
+  hash: string
+}
+
+export interface HistoryDiff {
+  entry: HistoryEntry
+  server: string
+  writable: boolean
+  current: string
+  currentHash: string
+  version: string
+  changes: ConfigDiff
+  unchanged: boolean
+}
+
+export type DriftStatus = 'differs' | 'unreadable' | 'not-installed'
+/* What drifted: an app's config, the settings, or the Blocked and Allowed lists. */
+export type DriftKind = 'app' | 'settings' | 'lists'
+
+export interface DriftEntry {
+  kind: DriftKind
+  /* The app's name; absent for settings and lists. */
+  name?: string
+  app?: string
+  server: string
+  since: string
+  status: DriftStatus
+  /* Setting names, or blocked and allowed. Never values. */
+  fields?: string[]
+}
+
+export interface DriftState {
+  checkedAt: string | null
+  apps: DriftEntry[]
+  checking?: boolean
+}
+
+export interface ImportPlan {
+  apps: { app: string; status: 'new' | 'changed' | 'same'; ignoredServers: string[] }[]
+  applied: boolean
+}
+
+export const getHistory = (token: string | null, name: string): Promise<ApiOutcome<{ response: { entries: HistoryEntry[] } }>> =>
+  apiRequest('hub/apps/history/list', { token, body: { name } })
+
+export const getHistoryEntry = (token: string | null, name: string, id: string): Promise<ApiOutcome<{ response: { entry: HistoryEntry; text: string } }>> =>
+  apiRequest('hub/apps/history/get', { token, body: { name, id } })
+
+export const getHistoryDiff = (token: string | null, name: string, id: string, server: string): Promise<ApiOutcome<{ response: HistoryDiff }>> =>
+  apiRequest('hub/apps/history/diff', { token, body: { name, id, server } })
+
+export const rollbackHistory = (token: string | null, name: string, id: string, targets: string[], expect: Record<string, string>): Promise<RunOutcome> =>
+  apiRequest('hub/apps/history/rollback', { token, method: 'POST', body: { name, id, targets: targets.join(','), expect: JSON.stringify(expect) } })
+
+export const restoreBaseline = (token: string | null, name: string, id: string): Promise<ApiOutcome<{ response: { baseline: Baseline } }>> =>
+  apiRequest('hub/apps/baseline/restore', { token, method: 'POST', body: { name, id } })
+
+export const getDrift = (token: string | null): Promise<ApiOutcome<{ response: DriftState }>> => apiRequest('hub/drift/get', { token })
+
+export const checkDrift = (token: string | null): Promise<ApiOutcome<{ response: DriftState }>> => apiRequest('hub/drift/check', { token, method: 'POST', body: {} })
+
+export const acceptDrift = (token: string | null, name: string, server: string): Promise<ApiOutcome<{ response: { baseline: Baseline } }>> =>
+  apiRequest('hub/apps/drift/accept', { token, method: 'POST', body: { name, server } })
+
+export const importBaselines = (token: string | null, data: string, apply: boolean): Promise<ApiOutcome<{ response: ImportPlan }>> =>
+  apiRequest('hub/apps/import', { token, method: 'POST', body: { data, apply: apply ? 'true' : 'false' } })
+
+/* The file of all baselines. A download is not the usual envelope, so it is read here (as the rule lists are). */
+export async function exportBaselines(token: string | null): Promise<ApiOutcome<{ filename: string; text: string }>> {
+  const headers: Record<string, string> = {}
+  if (token) headers.Authorization = `Bearer ${token}`
+  let res: Response
+  let text: string
+  try {
+    res = await fetch(urlApi('api/hub/apps/export'), { headers })
+    text = await res.text()
+  } catch {
+    return { kind: 'error', message: 'Unable to connect to the server. Please try again.' }
+  }
+  try {
+    const env = JSON.parse(text) as { status?: string; hub?: string }
+    if (env.status !== undefined && env.status !== 'ok') return envelopeOutcome<{ filename: string; text: string }>(env, token)
+  } catch {
+    return { kind: 'error', message: 'The hub did not send a file.' }
+  }
+  const filename = /filename="([^"]+)"/.exec(res.headers.get('content-disposition') ?? '')?.[1] ?? 'hub-baselines.json'
+  return { kind: 'ok', data: { filename, text } }
 }

src/api/blocking.ts

@@ -2,6 +2,7 @@ import { urlApi } from '../app/base'
 import { envelopeOutcome, type ApiOutcome } from './client'
 import { queryLogs, type QueryLogEntry } from './logs'
 import { addDomain, deleteDomain, type DomainList } from './zonelists'
+import { nodeArg } from '../lib/hub-write'
 
 /*
 The Blocking section's own reads. Every endpoint here already exists in the server;
@@ -144,15 +145,15 @@ export async function recentBlocked(
 }
 
 /* `allowDomain` (other-zones.js:637): out of Blocked first, then into Allowed. */
-export async function allowDomain(token: string | null, domain: string): Promise<ApiOutcome> {
-  const removed = await deleteDomain('blocked', token, domain)
+export async function allowDomain(token: string | null, domain: string, node = ''): Promise<ApiOutcome> {
+  const removed = await deleteDomain('blocked', token, domain, ...nodeArg(node))
   if (removed.kind !== 'ok') return removed
-  return addDomain('allowed', token, domain)
+  return addDomain('allowed', token, domain, ...nodeArg(node))
 }
 
 /* `blockDomain` (other-zones.js:686): out of Allowed first, then into Blocked. */
-export async function blockDomain(token: string | null, domain: string): Promise<ApiOutcome> {
-  const removed = await deleteDomain('allowed', token, domain)
+export async function blockDomain(token: string | null, domain: string, node = ''): Promise<ApiOutcome> {
+  const removed = await deleteDomain('allowed', token, domain, ...nodeArg(node))
   if (removed.kind !== 'ok') return removed
-  return addDomain('blocked', token, domain)
+  return addDomain('blocked', token, domain, ...nodeArg(node))
 }

src/api/hub-drift.ts

@@ -0,0 +1,47 @@
+import { useSyncExternalStore } from 'react'
+import { getDrift, type DriftEntry } from './apps'
+
+/*
+Which servers no longer have the config the hub says they should. The session carries only a count
+(`info.hubDrift`), cheap enough to read at sign-in; the list comes from `hub/drift/get`. The header
+subscribes to this store, and the screens that fix a drift call `refreshHubDrift` afterwards.
+*/
+interface State {
+  count: number
+  entries: readonly DriftEntry[]
+}
+
+let current: State = { count: 0, entries: [] }
+const listeners = new Set<() => void>()
+
+function set(next: State): void {
+  if (next.count === current.count && next.entries.length === current.entries.length && next.entries.every((e, i) => e === current.entries[i])) return
+  current = next
+  for (const fn of listeners) fn()
+}
+
+/** The count from the session, before the list is read. */
+export function setHubDriftCount(count: number): void {
+  if (current.entries.length === 0) set({ count, entries: [] })
+}
+
+export async function refreshHubDrift(token: string | null): Promise<void> {
+  const outcome = await getDrift(token)
+  if (outcome.kind !== 'ok') return
+  const entries = Array.isArray(outcome.data.response?.apps) ? outcome.data.response.apps : []
+  set({ count: entries.filter((e) => e.status === 'differs').length, entries })
+}
+
+export function resetHubDrift(): void {
+  set({ count: 0, entries: [] })
+}
+
+const subscribe = (fn: () => void): (() => void) => {
+  listeners.add(fn)
+  return () => listeners.delete(fn)
+}
+const snapshot = (): State => current
+
+export function useHubDrift(): State {
+  return useSyncExternalStore(subscribe, snapshot, snapshot)
+}

src/api/hub-profiles.ts

@@ -0,0 +1,126 @@
+import { urlApi } from '../app/base'
+import { apiRequest, envelopeOutcome, type ApiOutcome } from './client'
+import type { ConfigDiff, RunOutcome } from './apps'
+
+/*
+Profiles and the webhook, behind a hub (hub/profiles.ts, hub/webhook.ts). Every endpoint needs the
+hub's admin role. A profile holds settings secrets, so no answer shows one: they appear as
+`<secret>`, and an app's config text is never sent, only whether there is one.
+*/
+
+export interface ProfileSummary {
+  name: string
+  apps: number
+  configs: number
+  settings: number | null
+  lists: { blocked: number; allowed: number } | null
+  updatedAt: string
+  updatedBy: string
+}
+
+export interface ProfileView {
+  name: string
+  createdAt: string
+  updatedAt: string
+  updatedBy: string
+  apps: { name: string; version: string; hasConfig: boolean; bytes: number }[]
+  settings: Record<string, unknown> | null
+  lists: { blocked: number; allowed: number } | null
+}
+
+export const listProfiles = (token: string | null): Promise<ApiOutcome<{ response: { profiles: ProfileSummary[] } }>> => apiRequest('hub/profiles/list', { token })
+
+export const getProfile = (token: string | null, name: string): Promise<ApiOutcome<{ response: ProfileView }>> => apiRequest('hub/profiles/get', { token, body: { name } })
+
+export interface SnapshotParts {
+  /* Undefined takes every app that has a baseline. */
+  apps?: string[]
+  settings: boolean
+  lists: boolean
+}
+
+export const snapshotProfile = (token: string | null, name: string, parts: SnapshotParts): Promise<ApiOutcome<{ response: { profile: ProfileView; replaced: boolean } }>> =>
+  apiRequest('hub/profiles/snapshot', {
+    token,
+    method: 'POST',
+    body: { name, settings: String(parts.settings), lists: String(parts.lists), ...(parts.apps === undefined ? {} : { apps: parts.apps.join(',') }) },
+  })
+
+export const deleteProfile = (token: string | null, name: string): Promise<ApiOutcome<{ response: { deleted: boolean } }>> =>
+  apiRequest('hub/profiles/delete', { token, method: 'POST', body: { name } })
+
+export interface ProfileImportPlan {
+  profiles: { name: string; status: 'new' | 'changed' | 'same'; apps: number; settings: boolean; lists: boolean }[]
+  applied: boolean
+}
+
+export const importProfiles = (token: string | null, data: string, apply: boolean): Promise<ApiOutcome<{ response: ProfileImportPlan }>> =>
+  apiRequest('hub/profiles/import', { token, method: 'POST', body: { data, apply: apply ? 'true' : 'false' } })
+
+/* The file of profiles (it holds settings secrets). A download is not the usual envelope. */
+export async function exportProfiles(token: string | null, name?: string): Promise<ApiOutcome<{ filename: string; text: string }>> {
+  const headers: Record<string, string> = {}
+  if (token) headers.Authorization = `Bearer ${token}`
+  let res: Response
+  let text: string
+  try {
+    res = await fetch(urlApi(`api/hub/profiles/export${name === undefined ? '' : `?name=${encodeURIComponent(name)}`}`), { headers })
+    text = await res.text()
+  } catch {
+    return { kind: 'error', message: 'Unable to connect to the server. Please try again.' }
+  }
+  try {
+    const env = JSON.parse(text) as { status?: string }
+    if (env.status !== undefined && env.status !== 'ok') return envelopeOutcome<{ filename: string; text: string }>(env, token)
+  } catch {
+    return { kind: 'error', message: 'The hub did not send a file.' }
+  }
+  const filename = /filename="([^"]+)"/.exec(res.headers.get('content-disposition') ?? '')?.[1] ?? 'hub-profiles.json'
+  return { kind: 'ok', data: { filename, text } }
+}
+
+export interface ProfileAppState {
+  name: string
+  state: 'install' | 'present' | 'version-differs' | 'not-in-store'
+  profileVersion: string
+  storeVersion?: string
+  installedVersion?: string
+}
+
+export interface ProfileConfigState {
+  name: string
+  state: 'same' | 'differs' | 'after-install' | 'unreadable'
+  error?: string
+  changes?: ConfigDiff
+}
+
+export interface ProfileTargetPreview {
+  server: string
+  writable: boolean
+  ok: boolean
+  error?: string
+  apps: ProfileAppState[]
+  configs: ProfileConfigState[]
+  settings: { fields: string[]; secretFields: string[]; unchanged: boolean; changes: ConfigDiff; warnings: string[]; error?: string } | null
+  lists: { blocked: { missing: number; extra: number }; allowed: { missing: number; extra: number }; status: string; tooMany?: string; error?: string } | null
+  warnings: string[]
+}
+
+export const previewProfile = (token: string | null, name: string, targets: string[], removeExtras: boolean): Promise<ApiOutcome<{ response: { profile: string; targets: ProfileTargetPreview[] } }>> =>
+  apiRequest('hub/profiles/preview', { token, method: 'POST', body: { name, targets: targets.join(','), removeExtras: String(removeExtras) } })
+
+/* `from` says, per server, the step to start at: the retry of a failed run. */
+export const applyProfile = (token: string | null, name: string, targets: string[], removeExtras: boolean, from: Record<string, string>): Promise<RunOutcome> =>
+  apiRequest('hub/profiles/apply', { token, method: 'POST', body: { name, targets: targets.join(','), removeExtras: String(removeExtras), from: JSON.stringify(from) } })
+
+export interface WebhookStatus {
+  configured: boolean
+  /* Host and port only, never the address. */
+  host: string | null
+  signed: boolean
+  last: { event: string; at: string; ok: boolean; error?: string } | null
+}
+
+export const getWebhookStatus = (token: string | null): Promise<ApiOutcome<{ response: WebhookStatus }>> => apiRequest('hub/webhook/status', { token })
+
+export const testWebhook = (token: string | null): Promise<ApiOutcome<{ response: WebhookStatus }>> => apiRequest('hub/webhook/test', { token, method: 'POST', body: {} })

src/api/hub-run.ts

@@ -0,0 +1,26 @@
+import { apiRequest } from './client'
+import type { RunOutcome } from './apps'
+
+/*
+The same change on several servers, outside Apps (hub/servers-run.ts). The hub checks the
+parameters and sends the one matching call to each target with that server's admin token; the
+answer is a result per server.
+*/
+export type ServerAction =
+  | 'blocked.add'
+  | 'blocked.delete'
+  | 'allowed.add'
+  | 'allowed.delete'
+  | 'blocking.disable'
+  | 'blocking.enable'
+  | 'blocking.updateLists'
+  | 'blocking.setListUrls'
+  | 'cache.flush'
+  | 'cache.delete'
+
+/** What an action needs besides the servers: a `domain`, `minutes`, or `urls` (one per line). */
+export type ServerParams = Partial<Record<'domain' | 'minutes' | 'urls', string>>
+
+export function runOnServers(token: string | null, action: ServerAction, params: ServerParams, targets: string[]): Promise<RunOutcome> {
+  return apiRequest('hub/servers/run', { token, method: 'POST', body: { action, ...params, targets: targets.join(',') } })
+}

src/api/hub-settings.ts

@@ -0,0 +1,124 @@
+import { apiRequest, type ApiOutcome } from './client'
+import type { ConfigDiff, HistoryEntry, RunOutcome } from './apps'
+
+/*
+Settings and blocking across the servers, behind a hub (hub/settings-config.ts). Every endpoint
+needs the hub's admin role: settings carry secrets, so even the comparison is for admins. No answer
+ever carries a secret; a secret shows as `<secret>` and a row says only that it differs.
+*/
+export const SECRET_PLACEHOLDER = '<secret>'
+
+export interface SettingsCell {
+  value: unknown
+  differs: boolean
+  secretChanged?: boolean
+}
+
+export interface SettingsRow {
+  name: string
+  group: 'blocking' | 'general'
+  local: boolean
+  secret: boolean
+  expected?: unknown
+  cells: Record<string, SettingsCell>
+}
+
+export interface SettingsServer {
+  id: string
+  ok: boolean
+  error?: string
+  writable: boolean
+  status: 'same' | 'differs' | 'local-only' | 'unknown'
+  differing: number
+  localDiffering: number
+  paused: boolean
+}
+
+export interface SettingsBaselineView {
+  settings: Record<string, unknown>
+  importedFrom?: string
+  updatedAt: string
+  updatedBy: string
+}
+
+export interface SettingsCompare {
+  baseline: SettingsBaselineView | null
+  withBaseline: boolean
+  localFields: { builtIn: string[]; builtInPatterns: string[]; extra: string[] }
+  blockingFields: string[]
+  servers: SettingsServer[]
+  fields: SettingsRow[]
+}
+
+export const getSettingsCompare = (token: string | null): Promise<ApiOutcome<{ response: SettingsCompare }>> => apiRequest('hub/settings/compare', { token })
+
+export const importSettingsBaseline = (token: string | null, server: string, group?: 'blocking'): Promise<ApiOutcome<{ response: { baseline: SettingsBaselineView } }>> =>
+  apiRequest('hub/settings/baseline/import', { token, method: 'POST', body: { server, ...(group === undefined ? {} : { group }) } })
+
+export const saveSettingsBaseline = (token: string | null, text: string): Promise<ApiOutcome<{ response: { baseline: SettingsBaselineView } }>> =>
+  apiRequest('hub/settings/baseline/save', { token, method: 'POST', body: { text } })
+
+export const saveSettingsOverrides = (token: string | null, paths: string[]): Promise<ApiOutcome<{ response: { overrides: { paths: string[] } } }>> =>
+  apiRequest('hub/settings/overrides/save', { token, method: 'POST', body: { paths: paths.join(',') } })
+
+export interface SettingsPreview {
+  server: string
+  ok: boolean
+  error?: string
+  writable: boolean
+  currentHash: string
+  changes: ConfigDiff
+  secretFields: string[]
+  unchanged: boolean
+  warnings: string[]
+}
+
+export const previewSettings = (token: string | null, targets: string[], group?: 'blocking'): Promise<ApiOutcome<{ response: { targets: SettingsPreview[] } }>> =>
+  apiRequest('hub/settings/preview', { token, method: 'POST', body: { targets: targets.join(','), ...(group === undefined ? {} : { group }) } })
+
+export const applySettings = (token: string | null, targets: string[], expect: Record<string, string>, group?: 'blocking'): Promise<RunOutcome> =>
+  apiRequest('hub/settings/apply', { token, method: 'POST', body: { targets: targets.join(','), expect: JSON.stringify(expect), ...(group === undefined ? {} : { group }) } })
+
+export const getSettingsHistory = (token: string | null): Promise<ApiOutcome<{ response: { entries: HistoryEntry[] } }>> => apiRequest('hub/settings/history/list', { token })
+
+export const getSettingsHistoryEntry = (
+  token: string | null,
+  id: string,
+): Promise<ApiOutcome<{ response: { entry: HistoryEntry; settings?: Record<string, unknown>; lists?: { blocked: number; allowed: number } } }>> =>
+  apiRequest('hub/settings/history/get', { token, body: { id } })
+
+export interface ListCounts {
+  count: number
+  missing: number
+  extra: number
+  missingSample: string[]
+  extraSample: string[]
+}
+
+export interface ListsServer {
+  id: string
+  ok: boolean
+  error?: string
+  writable: boolean
+  blocked: ListCounts
+  allowed: ListCounts
+  status: 'same' | 'differs' | 'unknown'
+  tooMany?: string
+}
+
+export interface ListsCompare {
+  baseline: { blocked: number; allowed: number; importedFrom?: string; updatedAt: string; updatedBy: string } | null
+  withBaseline: boolean
+  servers: ListsServer[]
+}
+
+export const getListsCompare = (token: string | null): Promise<ApiOutcome<{ response: ListsCompare }>> => apiRequest('hub/blocking/lists/compare', { token })
+
+export const importListsBaseline = (token: string | null, server: string): Promise<ApiOutcome<{ response: { imported: boolean } }>> =>
+  apiRequest('hub/blocking/lists/baseline/import', { token, method: 'POST', body: { server } })
+
+export const previewLists = (token: string | null, targets: string[], removeExtras: boolean): Promise<ApiOutcome<{ response: { targets: ListsServer[] } }>> =>
+  apiRequest('hub/blocking/lists/preview', { token, method: 'POST', body: { targets: targets.join(','), removeExtras: String(removeExtras) } })
+
+export const applyLists = (token: string | null, targets: string[], removeExtras: boolean): Promise<RunOutcome> =>
+  apiRequest('hub/blocking/lists/apply', { token, method: 'POST', body: { targets: targets.join(','), removeExtras: String(removeExtras) } })

src/api/hub-zones.ts

@@ -0,0 +1,80 @@
+import { apiRequest, type ApiOutcome } from './client'
+import type { RunOutcome } from './apps'
+
+/*
+Zones across the servers, behind a hub (hub/zones-sync.ts): the merged list, copying a zone to
+other servers, and making a zone a Secondary of another server's Primary.
+*/
+export interface ZoneCell {
+  state: 'present' | 'absent' | 'unknown'
+  type?: string
+  serial?: number | null
+  disabled?: boolean
+  dnssec?: string
+  lastModified?: string
+}
+
+export type RowStatus = 'same' | 'serial-differs' | 'type-differs' | 'only-on' | 'missing-on' | 'unknown'
+
+export interface MatrixZone {
+  name: string
+  cells: Record<string, ZoneCell>
+  status: RowStatus
+  onlyOn?: string
+  missingOn?: string[]
+  note?: 'primary-secondary' | 'secondary-behind'
+}
+
+export interface ZonesMatrixData {
+  servers: { id: string; ok: boolean; writable: boolean; host: string; error?: string }[]
+  zones: MatrixZone[]
+}
+
+export const getZonesMatrix = (token: string | null): Promise<ApiOutcome<{ response: ZonesMatrixData }>> => apiRequest('hub/zones/matrix', { token })
+
+export interface CopyTargetPreview {
+  server: string
+  ok: boolean
+  error?: string
+  writable: boolean
+  exists: boolean
+  type?: string
+  serial?: number | null
+  needsOverwrite: boolean
+}
+
+export interface CopyPreview {
+  zone: string
+  source: { server: string; type: string; serial: number | null; records: number; bytes: number }
+  warnings: string[]
+  targets: CopyTargetPreview[]
+}
+
+export const previewZoneCopy = (token: string | null, source: string, zone: string, targets: string[]): Promise<ApiOutcome<{ response: CopyPreview }>> =>
+  apiRequest('hub/zones/copy/preview', { token, method: 'POST', body: { source, zone, targets: targets.join(',') } })
+
+export const copyZone = (token: string | null, source: string, zone: string, targets: string[], overwrite: string[]): Promise<RunOutcome> =>
+  apiRequest('hub/zones/copy', { token, method: 'POST', body: { source, zone, targets: targets.join(','), overwrite: overwrite.join(',') } })
+
+export interface SecondaryInput {
+  source: string
+  target: string
+  zone: string
+  primaryAddress: string
+  targetAddress: string
+  tsigKeyName: string
+}
+
+export interface SecondaryPreview {
+  zone: string
+  source: string
+  target: string
+  warnings: string[]
+  steps: { server: string; role: 'source' | 'target'; summary: string; changes: boolean }[]
+}
+
+export const previewSecondary = (token: string | null, input: SecondaryInput): Promise<ApiOutcome<{ response: SecondaryPreview }>> =>
+  apiRequest('hub/zones/secondary/preview', { token, method: 'POST', body: { ...input } })
+
+export const makeSecondary = (token: string | null, input: SecondaryInput): Promise<RunOutcome> =>
+  apiRequest('hub/zones/secondary', { token, method: 'POST', body: { ...input } })

src/api/settings.ts

@@ -277,8 +277,8 @@ export async function setSettings(
 }
 
 /** `settings/forceUpdateBlockLists`. No parameters: forces the download now. */
-export async function forceUpdateBlockLists(token: string | null): Promise<boolean> {
-  const outcome = await apiRequest('settings/forceUpdateBlockLists', { token })
+export async function forceUpdateBlockLists(token: string | null, node = ''): Promise<boolean> {
+  const outcome = await apiRequest('settings/forceUpdateBlockLists', { token, ...(node ? { node } : {}) })
   return outcome.kind === 'ok'
 }
 
@@ -286,10 +286,11 @@ export async function forceUpdateBlockLists(token: string | null): Promise<boole
 export async function temporaryDisableBlocking(
   token: string | null,
   minutes: string,
+  node = '',
 ): Promise<string | null> {
   const outcome = await apiRequest<{ response: { temporaryDisableBlockingTill: string } }>(
     'settings/temporaryDisableBlocking',
-    { token, body: { minutes } },
+    { token, body: { minutes }, ...(node ? { node } : {}) },
   )
   if (outcome.kind === 'ok') settingsChanged()
   return outcome.kind === 'ok' ? (outcome.data.response?.temporaryDisableBlockingTill ?? null) : null

src/api/zonelists.ts

@@ -172,8 +172,9 @@ export function addDomain(
   list: DomainList,
   token: string | null,
   domain: string,
+  node = '',
 ): Promise<ApiOutcome> {
-  return apiRequest(`${list}/add`, { token, body: { domain } })
+  return apiRequest(`${list}/add`, { token, body: { domain }, ...(node ? { node } : {}) })
 }
 
 /** `deleteAllowedZone` / `deleteBlockedZone`. */
@@ -181,13 +182,14 @@ export function deleteDomain(
   list: DomainList,
   token: string | null,
   domain: string,
+  node = '',
 ): Promise<ApiOutcome> {
-  return apiRequest(`${list}/delete`, { token, body: { domain } })
+  return apiRequest(`${list}/delete`, { token, body: { domain }, ...(node ? { node } : {}) })
 }
 
-  /** `flushAllowedZone` / `flushBlockedZone`. No `node`, unlike the rest. */
-export function flushList(list: DomainList, token: string | null): Promise<ApiOutcome> {
-  return apiRequest(`${list}/flush`, { token })
+  /** `flushAllowedZone` / `flushBlockedZone`. No `node`, unlike the rest, except behind a hub (a change is made on one server). */
+export function flushList(list: DomainList, token: string | null, node = ''): Promise<ApiOutcome> {
+  return apiRequest(`${list}/flush`, { token, ...(node ? { node } : {}) })
 }
 
 /*
@@ -200,9 +202,10 @@ export function importDomains(
   list: DomainList,
   token: string | null,
   zones: string,
+  node = '',
 ): Promise<ApiOutcome> {
   const field = list === 'allowed' ? 'allowedZones' : 'blockedZones'
-  return apiRequest(`${list}/import`, { token, method: 'POST', body: { [field]: zones } })
+  return apiRequest(`${list}/import`, { token, method: 'POST', body: { [field]: zones }, ...(node ? { node } : {}) })
 }
 
 /*
@@ -214,6 +217,7 @@ token.
 export function exportDomains(
   list: DomainList,
   token: string | null,
+  node = '',
 ): Promise<{ ok: boolean; url?: string }> {
-  return openDownload(token, `${list}/export`)
+  return openDownload(token, `${list}/export`, node ? { node } : {})
 }

src/app/DriftStatus.kinds.test.tsx

@@ -0,0 +1,167 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { act, render, screen, waitFor, within } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { DriftStatus } from './DriftStatus'
+import { refreshHubDrift, resetHubDrift } from '../api/hub-drift'
+import * as apps from '../api/apps'
+import * as settings from '../api/hub-settings'
+import * as profiles from '../api/hub-profiles'
+import type { DriftEntry } from '../api/apps'
+
+/*
+The drift list with the kinds added in stage 8: settings and the Blocked and Allowed lists. Their
+actions are for an admin and reuse the compare views and the apply flows of Settings and Blocking;
+an entry names fields, never values. The webhook line is the admin's.
+*/
+
+beforeEach(() => resetHubDrift())
+afterEach(() => vi.restoreAllMocks())
+
+const ENTRIES: DriftEntry[] = [
+  { kind: 'app', name: 'Blocker', app: 'Blocker', server: 'ndns', since: '2026-10-31T09:00:00Z', status: 'differs' },
+  { kind: 'settings', server: 'ndns', since: '2026-10-31T09:00:00Z', status: 'differs', fields: ['blockingType', 'defaultRecordTtl'] },
+  { kind: 'lists', server: 'ndns', since: '2026-10-31T09:00:00Z', status: 'differs', fields: ['blocked'] },
+  { kind: 'settings', server: 'mdns', since: '2026-10-31T09:00:00Z', status: 'unreadable' },
+]
+
+async function load(entries: DriftEntry[] = ENTRIES) {
+  vi.spyOn(apps, 'getDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: 'x', apps: entries } } } as never)
+  await act(async () => refreshHubDrift('t'))
+}
+
+const NOT_SET = { configured: false, host: null, signed: false, last: null }
+
+describe('DriftStatus kinds', () => {
+  it('counts every kind that differs and lists settings and lists with their field names', async () => {
+    vi.spyOn(profiles, 'getWebhookStatus').mockResolvedValue({ kind: 'ok', data: { response: NOT_SET } } as never)
+    render(<DriftStatus token="t" canWrite writable={['ndns']} isAdmin />)
+    await load()
+    expect(await screen.findByRole('button', { name: 'Config drift' })).toHaveTextContent('3 config drifts')
+    await userEvent.click(screen.getByRole('button', { name: 'Config drift' }))
+    const settingsRow = await screen.findByRole('listitem', { name: 'Settings on ndns' })
+    expect(within(settingsRow).getByText('blockingType, defaultRecordTtl')).toBeInTheDocument()
+    expect(within(screen.getByRole('listitem', { name: 'Blocked and Allowed lists on ndns' })).getByText('blocked')).toBeInTheDocument()
+    expect(within(screen.getByRole('listitem', { name: 'Settings on mdns' })).getByText('Cannot read')).toBeInTheDocument()
+  })
+
+  it('gives a non-admin no action on settings or lists, but keeps the app actions of a writer', async () => {
+    render(<DriftStatus token="t" canWrite writable={['ndns']} />)
+    await load()
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    const settingsRow = await screen.findByRole('listitem', { name: 'Settings on ndns' })
+    expect(within(settingsRow).queryByRole('button')).not.toBeInTheDocument()
+    expect(within(screen.getByRole('listitem', { name: 'Blocked and Allowed lists on ndns' })).queryByRole('button')).not.toBeInTheDocument()
+    expect(within(screen.getByRole('listitem', { name: 'Blocker on ndns' })).getByRole('button', { name: 'Accept as baseline' })).toBeInTheDocument()
+    expect(screen.queryByText('Webhook')).not.toBeInTheDocument()
+  })
+
+  it('opens the settings compare, and the blocking compare for lists', async () => {
+    vi.spyOn(profiles, 'getWebhookStatus').mockResolvedValue({ kind: 'ok', data: { response: NOT_SET } } as never)
+    const compare = vi.spyOn(settings, 'getSettingsCompare').mockResolvedValue({ kind: 'error', message: 'stop here' } as never)
+    const lists = vi.spyOn(settings, 'getListsCompare').mockResolvedValue({ kind: 'error', message: 'stop here' } as never)
+    vi.spyOn(apps, 'checkDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: 'y', apps: ENTRIES } } } as never)
+    render(<DriftStatus token="t" canWrite writable={['ndns']} isAdmin />)
+    await load()
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Settings on ndns' })).getByRole('button', { name: 'Open compare' }))
+    await waitFor(() => expect(compare).toHaveBeenCalledTimes(1))
+    expect(lists).not.toHaveBeenCalled()
+    expect(await screen.findByRole('dialog', { name: 'Compare settings across servers' })).toBeInTheDocument()
+  })
+
+  it('accepts the server settings as the baseline after a confirmation, then checks again', async () => {
+    vi.spyOn(profiles, 'getWebhookStatus').mockResolvedValue({ kind: 'ok', data: { response: NOT_SET } } as never)
+    const accept = vi.spyOn(settings, 'importSettingsBaseline').mockResolvedValue({ kind: 'ok', data: { response: { baseline: { settings: {}, updatedAt: '', updatedBy: '' } } } } as never)
+    const check = vi.spyOn(apps, 'checkDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: 'y', apps: [] } } } as never)
+    render(<DriftStatus token="t" canWrite writable={['ndns']} isAdmin />)
+    await load()
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Settings on ndns' })).getByRole('button', { name: 'Accept as baseline' }))
+    await userEvent.click(await screen.findByRole('button', { name: 'Accept' }))
+    expect(accept).toHaveBeenCalledWith('t', 'ndns')
+    await waitFor(() => expect(check).toHaveBeenCalled())
+  })
+
+  it('accepts the lists of a server as the baseline', async () => {
+    vi.spyOn(profiles, 'getWebhookStatus').mockResolvedValue({ kind: 'ok', data: { response: NOT_SET } } as never)
+    const accept = vi.spyOn(settings, 'importListsBaseline').mockResolvedValue({ kind: 'ok', data: { response: { imported: true } } } as never)
+    vi.spyOn(apps, 'checkDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: 'y', apps: [] } } } as never)
+    render(<DriftStatus token="t" canWrite writable={['ndns']} isAdmin />)
+    await load()
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Blocked and Allowed lists on ndns' })).getByRole('button', { name: 'Accept as baseline' }))
+    await userEvent.click(await screen.findByRole('button', { name: 'Accept' }))
+    expect(accept).toHaveBeenCalledWith('t', 'ndns')
+  })
+
+  it('reverts settings to the baseline through the preview', async () => {
+    vi.spyOn(profiles, 'getWebhookStatus').mockResolvedValue({ kind: 'ok', data: { response: NOT_SET } } as never)
+    const preview = vi.spyOn(settings, 'previewSettings').mockResolvedValue({
+      kind: 'ok',
+      data: { response: { targets: [{ server: 'ndns', ok: true, writable: true, currentHash: 'h1', changes: { kind: 'json', changes: [{ path: 'blockingType', type: 'changed', a: 'AnyAddress', b: 'NxDomain' }] }, secretFields: [], unchanged: false, warnings: [] }] } },
+    } as never)
+    const apply = vi.spyOn(settings, 'applySettings').mockResolvedValue({ kind: 'ok', data: { response: { results: [{ server: 'ndns', ok: true, message: 'Applied and verified (1 field).' }] } } } as never)
+    vi.spyOn(apps, 'checkDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: 'y', apps: [] } } } as never)
+    render(<DriftStatus token="t" canWrite writable={['ndns']} isAdmin />)
+    await load()
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Settings on ndns' })).getByRole('button', { name: 'Revert to baseline' }))
+    const picker = await screen.findByRole('dialog', { name: 'Apply the settings baseline' })
+    await userEvent.click(within(picker).getByRole('button', { name: 'Preview' }))
+    expect(preview).toHaveBeenCalledWith('t', ['ndns'], undefined)
+    await userEvent.click(within(await screen.findByRole('dialog', { name: 'Apply the settings baseline: review' })).getByRole('button', { name: 'Apply' }))
+    expect(apply).toHaveBeenCalledWith('t', ['ndns'], { ndns: 'h1' }, undefined)
+    expect(await screen.findByText('Applied and verified (1 field).')).toBeInTheDocument()
+  })
+
+  it('reverts the lists through the lists preview', async () => {
+    vi.spyOn(profiles, 'getWebhookStatus').mockResolvedValue({ kind: 'ok', data: { response: NOT_SET } } as never)
+    const none = { count: 0, missing: 0, extra: 0, missingSample: [], extraSample: [] }
+    vi.spyOn(settings, 'previewLists').mockResolvedValue({ kind: 'ok', data: { response: { targets: [{ id: 'ndns', ok: true, writable: true, status: 'differs', blocked: { ...none, extra: 1 }, allowed: none }] } } } as never)
+    const apply = vi.spyOn(settings, 'applyLists').mockResolvedValue({ kind: 'ok', data: { response: { results: [{ server: 'ndns', ok: true, message: 'Added 0, removed 0.' }] } } } as never)
+    vi.spyOn(apps, 'checkDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: 'y', apps: [] } } } as never)
+    render(<DriftStatus token="t" canWrite writable={['ndns']} isAdmin />)
+    await load()
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Blocked and Allowed lists on ndns' })).getByRole('button', { name: 'Revert to baseline' }))
+    await userEvent.click(within(await screen.findByRole('dialog', { name: 'Apply the Blocked and Allowed lists' })).getByRole('button', { name: 'Preview' }))
+    await userEvent.click(within(await screen.findByRole('dialog', { name: 'Apply the Blocked and Allowed lists: review' })).getByRole('button', { name: 'Apply' }))
+    expect(apply).toHaveBeenCalledWith('t', ['ndns'], false)
+  })
+})
+
+describe('webhook line in the drift dialog', () => {
+  it('shows the host and the last delivery, and sends a test', async () => {
+    vi.spyOn(profiles, 'getWebhookStatus').mockResolvedValue({
+      kind: 'ok',
+      data: { response: { configured: true, host: 'hooks.example:8443', signed: true, last: { event: 'drift.found', at: '2026-11-01T00:00:00Z', ok: false, error: 'HTTP 500' } } },
+    } as never)
+    const test = vi.spyOn(profiles, 'testWebhook').mockResolvedValue({
+      kind: 'ok',
+      data: { response: { configured: true, host: 'hooks.example:8443', signed: true, last: { event: 'webhook.test', at: '2026-11-01T00:01:00Z', ok: true } } },
+    } as never)
+    render(<DriftStatus token="t" canWrite writable={['ndns']} isAdmin />)
+    await load()
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    expect(await screen.findByText('hooks.example:8443')).toBeInTheDocument()
+    expect(screen.getByText('Last delivery failed (HTTP 500)')).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Send test' }))
+    expect(test).toHaveBeenCalledWith('t')
+    expect(await screen.findByText('Last delivery ok')).toBeInTheDocument()
+  })
+
+  it('says it is not configured, with no test button, and shows an error with Retry', async () => {
+    const spy = vi
+      .spyOn(profiles, 'getWebhookStatus')
+      .mockResolvedValueOnce({ kind: 'error', message: 'boom' } as never)
+      .mockResolvedValue({ kind: 'ok', data: { response: NOT_SET } } as never)
+    render(<DriftStatus token="t" canWrite writable={['ndns']} isAdmin />)
+    await load()
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    expect(await screen.findByText(/boom/)).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Retry' }))
+    expect(await screen.findByText('Not configured')).toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Send test' })).not.toBeInTheDocument()
+    expect(spy).toHaveBeenCalledTimes(2)
+  })
+})

src/app/DriftStatus.tsx

@@ -0,0 +1,196 @@
+import { useState } from 'react'
+import { acceptDrift, checkDrift, type DriftEntry } from '../api/apps'
+import { refreshHubDrift, useHubDrift } from '../api/hub-drift'
+import { importListsBaseline, importSettingsBaseline } from '../api/hub-settings'
+import { AppConfigSync } from '../screens/apps/AppConfigSync'
+import { applyRequest } from '../screens/apps/applyRequest'
+import { RunFlow, pickerServers, type RunRequest } from '../screens/apps/RunFlow'
+import { SettingsCompare, listsApplyRequest, settingsApplyRequest, type Scope } from '../screens/settings/SettingsCompare'
+import { Alert } from '../ui/Alert'
+import { Button } from '../ui/Button'
+import { Confirm } from '../ui/Confirm'
+import { Dialog } from '../ui/Dialog'
+import { Empty } from '../ui/Empty'
+import { Tag } from '../ui/Tag'
+import styles from './Shell.module.css'
+import { WebhookStatus } from './WebhookStatus'
+
+/*
+Behind a hub: a mark in the header when a server no longer has what the hub says it should (someone
+changed it in that server's own console), and the list of them. Three kinds: an app's config, the
+settings, and the Blocked and Allowed lists. Each can be looked at (the app's config view, or the
+compare view of Settings or Blocking); a writer can accept the server's state as the new baseline,
+or revert the server to the baseline after a preview. Settings and lists are for an admin, as their
+compare views are. An entry names fields, never values.
+*/
+
+const STATUS = { differs: 'Differs', unreadable: 'Cannot read', 'not-installed': 'Not installed' } as const
+
+const label = (e: DriftEntry): string => (e.kind === 'app' ? (e.name ?? e.app ?? '') : e.kind === 'settings' ? 'Settings' : 'Blocked and Allowed lists')
+const keyOf = (e: DriftEntry): string => `${e.kind}/${e.name ?? e.app ?? ''}/${e.server}`
+
+export function DriftStatus({ token, canWrite, writable, isAdmin = false }: { token: string | null; canWrite: boolean; writable: string[]; isAdmin?: boolean }) {
+  const { count, entries } = useHubDrift()
+  const [open, setOpen] = useState(false)
+  const [configApp, setConfigApp] = useState<string | null>(null)
+  const [compare, setCompare] = useState<Scope | null>(null)
+  const [request, setRequest] = useState<RunRequest | null>(null)
+  const [accepting, setAccepting] = useState<DriftEntry | null>(null)
+  const [problem, setProblem] = useState<string | null>(null)
+  const [busy, setBusy] = useState(false)
+
+  if (count === 0 && !open) return null
+
+  async function recheck() {
+    await checkDrift(token)
+    await refreshHubDrift(token)
+  }
+
+  async function accept(entry: DriftEntry) {
+    setBusy(true)
+    setProblem(null)
+    const outcome =
+      entry.kind === 'app'
+        ? await acceptDrift(token, entry.name ?? entry.app ?? '', entry.server)
+        : entry.kind === 'settings'
+          ? await importSettingsBaseline(token, entry.server)
+          : await importListsBaseline(token, entry.server)
+    if (outcome.kind !== 'ok') setProblem(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+    // The hub checks again after an app is accepted; for settings and lists it is asked to.
+    if (entry.kind === 'app') await refreshHubDrift(token)
+    else await recheck()
+    setBusy(false)
+  }
+
+  async function checkNow() {
+    setBusy(true)
+    setProblem(null)
+    const outcome = await checkDrift(token)
+    setBusy(false)
+    if (outcome.kind !== 'ok') setProblem(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+    await refreshHubDrift(token)
+  }
+
+  function revert(entry: DriftEntry) {
+    const servers = pickerServers([{ id: entry.server, writable: writable.includes(entry.server) }])
+    setRequest(
+      entry.kind === 'app'
+        ? applyRequest(token, entry.name ?? entry.app ?? '', servers, [entry.server])
+        : entry.kind === 'settings'
+          ? settingsApplyRequest(token, servers, [entry.server], 'settings')
+          : listsApplyRequest(token, servers, [entry.server]),
+    )
+  }
+
+  /* Settings and lists are admin-only; an app's config follows the Apps right. */
+  const mayAct = (e: DriftEntry): boolean => (e.kind === 'app' ? canWrite : isAdmin)
+
+  return (
+    <>
+      <button type="button" className={styles.status} data-attention="true" aria-label="Config drift" onClick={() => setOpen(true)}>
+        <i className={styles.pdot} data-tone="warn" />
+        <span className={styles.lbl}>{`${count} config ${count === 1 ? 'drift' : 'drifts'}`}</span>
+      </button>
+
+      <Dialog
+        open={open}
+        onOpenChange={setOpen}
+        size="medium"
+        title="Config drift"
+        actions={
+          canWrite ? (
+            <Button disabled={busy} onClick={() => void checkNow()}>
+              Check now
+            </Button>
+          ) : undefined
+        }
+      >
+        <p className="m-0 text-sm text-muted-foreground">
+          These servers no longer have what the hub expects: an app&apos;s baseline plus the server&apos;s own local fields, or the shared settings and the Blocked and Allowed lists of their baselines.
+        </p>
+        {isAdmin && <WebhookStatus token={token} />}
+        {problem !== null && <Alert type="danger" title="Error">{problem}</Alert>}
+        {entries.length === 0 ? (
+          <Empty title="No drift">Every server has what the hub expects.</Empty>
+        ) : (
+          <ul className="m-0 flex list-none flex-col gap-3 p-0" aria-label="Drifts">
+            {entries.map((e) => (
+              <li key={keyOf(e)} aria-label={`${label(e)} on ${e.server}`} className="flex flex-wrap items-center gap-3 border-b border-border pb-3 text-sm last:border-b-0">
+                <span className="font-medium">{label(e)}</span>
+                <span>{e.server}</span>
+                <Tag tone={e.status === 'differs' ? 'warn' : 'neutral'}>{STATUS[e.status]}</Tag>
+                <span className="text-xs text-muted-foreground">since {new Date(e.since).toLocaleString()}</span>
+                {e.fields !== undefined && e.fields.length > 0 && (
+                  <span className="basis-full font-mono text-xs text-muted-foreground">{e.fields.slice(0, 8).join(', ')}{e.fields.length > 8 ? ` and ${e.fields.length - 8} more` : ''}</span>
+                )}
+                <span className="ml-auto flex flex-wrap gap-2">
+                  {e.kind === 'app' ? (
+                    <Button size="sm" onClick={() => setConfigApp(e.name ?? e.app ?? '')}>
+                      Open config
+                    </Button>
+                  ) : (
+                    isAdmin && (
+                      <Button size="sm" onClick={() => setCompare(e.kind === 'settings' ? 'settings' : 'blocking')}>
+                        Open compare
+                      </Button>
+                    )
+                  )}
+                  {mayAct(e) && e.status === 'differs' && (
+                    <>
+                      <Button size="sm" disabled={busy} onClick={() => setAccepting(e)}>
+                        Accept as baseline
+                      </Button>
+                      <Button size="sm" variant="primary" disabled={busy} onClick={() => revert(e)}>
+                        Revert to baseline
+                      </Button>
+                    </>
+                  )}
+                </span>
+              </li>
+            ))}
+          </ul>
+        )}
+      </Dialog>
+
+      <Confirm
+        open={accepting !== null}
+        title="Accept as baseline"
+        text={
+          accepting === null
+            ? ''
+            : accepting.kind === 'app'
+              ? `Make the config of ${label(accepting)} on ${accepting.server} the baseline? The other servers will then differ from it. The old baseline is kept in the history.`
+              : accepting.kind === 'settings'
+                ? `Make the settings of ${accepting.server} the baseline? The other servers will then differ from it. The old baseline is kept in the settings history.`
+                : `Make the Blocked and Allowed lists of ${accepting.server} the baseline? The other servers will then differ from it.`
+        }
+        label="Accept"
+        onClose={() => setAccepting(null)}
+        onConfirm={() => accepting && accept(accepting)}
+      />
+
+      {configApp !== null && (
+        <AppConfigSync token={token} name={configApp} canWrite={canWrite} onClose={() => setConfigApp(null)} onChanged={() => void refreshHubDrift(token)} />
+      )}
+      {compare !== null && (
+        <SettingsCompare
+          token={token}
+          scope={compare}
+          onClose={() => {
+            setCompare(null)
+            void recheck()
+          }}
+        />
+      )}
+      {request !== null && (
+        <RunFlow
+          request={request}
+          onClose={(changed) => {
+            setRequest(null)
+            if (changed) void recheck()
+          }}
+        />
+      )}
+    </>
+  )
+}

src/app/Shell.hub.test.tsx

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
-import { act, render, screen } from '@testing-library/react'
+import { act, cleanup, render, screen } from '@testing-library/react'
 import userEvent from '@testing-library/user-event'
 import { Shell, type ShellSession } from './Shell'
 import { ThemeProvider } from '../theme/ThemeProvider'
@@ -103,6 +103,25 @@ describe('Shell on a hub', () => {
     expect(status).not.toHaveTextContent('not answering')
   })
 
+  it('marks the header with the config drift the session and the list report, on a hub only', async () => {
+    vi.restoreAllMocks()
+    vi.spyOn(client, 'apiRequest').mockImplementation((async (path: string) =>
+      path === 'hub/drift/get'
+        ? { kind: 'ok', data: { status: 'ok', response: { checkedAt: 'x', apps: [{ kind: 'app', name: 'Blocker', app: 'Blocker', server: 'ndns', since: '2026-10-31T09:00:00Z', status: 'differs' }] } } }
+        : { kind: 'ok', data: { status: 'ok', response: {} } }) as never)
+    const hubSession = session(true)
+    hubSession.info = { ...hubSession.info!, hubDrift: 1 }
+    render(
+      <ThemeProvider>
+        <Shell session={hubSession} onLogout={() => {}} />
+      </ThemeProvider>,
+    )
+    expect(await screen.findByRole('button', { name: 'Config drift' })).toHaveTextContent('1 config drift')
+    cleanup()
+    mount(false)
+    expect(screen.queryByRole('button', { name: 'Config drift' })).not.toBeInTheDocument()
+  })
+
   /* Per-node calls (live metrics, settings) arrive right after an aggregate and say nothing. */
   it('an ok response without the field leaves the warning, and an empty list clears it', async () => {
     mount(true)

src/app/Shell.tsx

@@ -32,11 +32,16 @@ import { Menu } from '../ui/Menu'
 import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover'
 import { Command, CommandDialog, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from '@/components/ui/command'
 import { HubMode, ThisServerName } from '../ui/ClusterNodeSelect'
+import { HubWritable } from '../lib/hub-write'
+import { HubServerIds } from '../lib/hub-all'
 import { FooterLinks } from '../ui/FooterLinks'
 import { Versions } from './Versions'
 import { BlockingMark } from './BlockingMark'
 import { useBlockingMark } from '../screens/blocking/blocking-state'
 import { useHubWarnings } from '../api/hub-warnings'
+import { refreshHubDrift, resetHubDrift, setHubDriftCount } from '../api/hub-drift'
+import { DriftStatus } from './DriftStatus'
+import { WebhookStatus } from './WebhookStatus'
 import { Confirm } from '../ui/Confirm'
 import { Notifier } from '../ui/Notifier'
 import type { Notice } from '../lib/notice'
@@ -80,6 +85,9 @@ function keep(key: string, value: unknown) {
   }
 }
 
+/* A stable empty list for a session without the field (outside a hub). */
+const NO_SERVERS: string[] = []
+
 const ICONS: Record<string, IconName> = {
   dashboard: 'dashboard', zones: 'zones', cache: 'cache', blocking: 'blocked',
   apps: 'apps', dnsclient: 'dnsclient', settings: 'settings',
@@ -105,6 +113,10 @@ export interface ShellSession {
     clusterInitialized?: boolean
     /** Only on the hub (`hub/`): a read-only console over several servers. */
     hub?: boolean
+    /** Only on the hub: the servers it holds an admin token for. */
+    hubWritable?: string[]
+    /** Only on the hub: how many servers differ from their baseline when the session was read. */
+    hubDrift?: number
     /** Only on an initialised cluster (WebServiceAuthApi.cs:104-106). */
     clusterDomain?: string
     clusterNodes?: { name: string; type: string }[]
@@ -129,7 +141,23 @@ export function Shell({
 }) {
   const permissions = session.info?.permissions
   const hub = session.info?.hub === true
+  /* Behind a hub: the servers a change can be made on (the screens read it through `HubWritable`). */
+  const hubWritable = session.info?.hubWritable ?? NO_SERVERS
+  /* And every server it fronts, for the picker of a change made on several at once. */
+  const hubServers = useMemo(() => (session.info?.clusterNodes ?? []).map((n) => n.name), [session.info?.clusterNodes])
   const hubWarnings = useHubWarnings()
+  /* Behind a hub: the count comes with the session, the list is read now and then. */
+  const hubDriftAtStart = session.info?.hubDrift
+  useEffect(() => {
+    if (!hub) return
+    setHubDriftCount(hubDriftAtStart ?? 0)
+    void refreshHubDrift(session.token)
+    const timer = setInterval(() => void refreshHubDrift(session.token), 60_000)
+    return () => {
+      clearInterval(timer)
+      resetHubDrift()
+    }
+  }, [hub, session.token, hubDriftAtStart])
   // Memoised: if it is recreated on every render, the `hashchange` resubscribes on each.
   const sections = useMemo(() => visibleSections(permissions), [permissions])
   /* The starting section comes from the address bar if it carries one, and only
@@ -252,7 +280,7 @@ export function Shell({
 
   const current = sections.find((s) => s.id === active) ?? sections[0]
   /* Whether blocking is off or paused, marked on its entry (`BlockingMark.tsx`). */
-  const blockingNow = useBlockingMark(session.token, permissions, sections.some((s) => s.id === 'blocking'))
+  const blockingNow = useBlockingMark(session.token, permissions, sections.some((s) => s.id === 'blocking'), hub)
 
   /*
   The jumps other screens' row menus make (`app/handoff.ts`). The section switch
@@ -404,6 +432,8 @@ export function Shell({
     <HandoffContext.Provider value={handoff}>
     <ThisServerName.Provider value={info?.dnsServerDomain ?? session.info?.dnsServerDomain ?? ''}>
     <HubMode.Provider value={hub}>
+    <HubWritable.Provider value={hubWritable}>
+    <HubServerIds.Provider value={hubServers}>
     <SlotProvider>
       {(slot) => (
     <div className={styles.shell} data-rail={rail}>
@@ -584,6 +614,7 @@ export function Shell({
               <span key={i} data-last={i === crumbs.length - 1}>{c}</span>
             ))}
           </div>
+          {hub && <DriftStatus token={session.token} canWrite={permissions?.Apps?.canModify === true} writable={hubWritable} isAdmin={permissions?.Settings?.canModify === true} />}
           <Popover>
             <PopoverTrigger className={styles.status} data-attention={blockingNow != null || hubWarnings.length > 0} aria-label="Server status">
               <i className={styles.pdot} data-tone={hubWarnings.length > 0 ? 'warn' : (blockingNow?.tone ?? 'ok')} />
@@ -638,20 +669,22 @@ export function Shell({
             token={session.token}
             nodes={session.info?.clusterNodes ?? []}
             clusterInitialised={session.info?.clusterInitialized === true}
+            canBlock={hub ? permissions?.Blocked?.canModify !== false && permissions?.Allowed?.canModify !== false : true}
           />
         ) : current?.id === 'dnsclient' ? (
           <DnsClient
             token={session.token}
             nodes={session.info?.clusterNodes ?? []}
             clusterInitialised={session.info?.clusterInitialized === true}
+            canImport={permissions?.DnsClient?.canModify !== false}
             request={pending != null && 'dnsclient' in pending ? pending.dnsclient : undefined}
           />
         ) : current?.id === 'about' ? (
-          <About info={info} />
+          <About info={info} hubPanel={hub && permissions?.Settings?.canModify === true ? <WebhookStatus token={session.token} /> : undefined} />
         ) : current?.id === 'apps' ? (
-          <Apps token={session.token} />
+          <Apps token={session.token} nodes={session.info?.clusterNodes ?? []} clusterInitialised={session.info?.clusterInitialized === true} canWrite={permissions?.Apps?.canModify === true} writable={hubWritable} isAdmin={permissions?.Settings?.canModify === true} />
         ) : current?.id === 'cache' ? (
-          <Cache token={session.token} nodes={session.info?.clusterNodes ?? []} clusterInitialised={session.info?.clusterInitialized === true} />
+          <Cache token={session.token} nodes={session.info?.clusterNodes ?? []} clusterInitialised={session.info?.clusterInitialized === true} canDelete={hub ? permissions?.Cache?.canDelete !== false : undefined} />
         ) : current?.id === 'blocking' ? (
           <Blocking
             token={session.token}
@@ -697,6 +730,7 @@ export function Shell({
             clusterInitialised={session.info?.clusterInitialized === true}
             sub={currentSub ?? 'View Logs'}
             onSubChange={setSub}
+            canBlock={hub ? permissions?.Blocked?.canModify !== false && permissions?.Allowed?.canModify !== false : true}
             canDeleteLogs={permissions?.Logs?.canDelete !== false}
             canDeleteStats={permissions?.Dashboard?.canDelete !== false}
             request={pending != null && 'logs' in pending ? pending.logs : undefined}
@@ -818,6 +852,8 @@ export function Shell({
     </div>
       )}
     </SlotProvider>
+    </HubServerIds.Provider>
+    </HubWritable.Provider>
     </HubMode.Provider>
     </ThisServerName.Provider>
     </HandoffContext.Provider>

src/app/WebhookStatus.tsx

@@ -0,0 +1,78 @@
+import { useCallback, useEffect, useState } from 'react'
+import { getWebhookStatus, testWebhook, type WebhookStatus as Status } from '../api/hub-profiles'
+import { Button } from '../ui/Button'
+import { Tag } from '../ui/Tag'
+import { Skeleton } from '@/components/ui/skeleton'
+
+/*
+For the hub's admin: whether the hub sends notices to a webhook (HUB_WEBHOOK_URL), the host it
+sends them to (never the address, which may carry a token), how the last delivery went, and a
+button that sends a test notice. Read-only: the address and the secret are set on the hub.
+*/
+export function WebhookStatus({ token }: { token: string | null }) {
+  const [status, setStatus] = useState<Status | null>(null)
+  const [failure, setFailure] = useState<string | null>(null)
+  const [busy, setBusy] = useState(false)
+
+  const load = useCallback(async () => {
+    const outcome = await getWebhookStatus(token)
+    if (outcome.kind === 'ok') {
+      setStatus(outcome.data.response)
+      setFailure(null)
+    } else setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+  }, [token])
+
+  useEffect(() => {
+    void load()
+  }, [load])
+
+  async function test() {
+    setBusy(true)
+    const outcome = await testWebhook(token)
+    setBusy(false)
+    if (outcome.kind === 'ok') {
+      setStatus(outcome.data.response)
+      setFailure(null)
+    } else setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+  }
+
+  if (failure !== null && status === null) {
+    return (
+      <div className="flex flex-wrap items-center gap-2 text-sm" role="alert">
+        <span>Could not read the webhook status. {failure}</span>
+        <Button size="sm" onClick={() => void load()}>
+          Retry
+        </Button>
+      </div>
+    )
+  }
+  if (status === null) return <Skeleton className="h-6 w-2/3" role="status" aria-label="Loading the webhook status" />
+
+  return (
+    <div className="flex flex-col gap-2" aria-label="Webhook">
+      <div className="flex flex-wrap items-center gap-2 text-sm">
+        <span className="font-medium">Webhook</span>
+        {status.configured ? <Tag tone="ok">Configured</Tag> : <Tag tone="neutral">Not configured</Tag>}
+        {status.configured && <span className="text-muted-foreground">{status.host}</span>}
+        {status.configured && status.last === null && <span className="text-xs text-muted-foreground">Nothing sent yet.</span>}
+        {status.last !== null && (
+          <>
+            <Tag tone={status.last.ok ? 'ok' : 'dan'}>{status.last.ok ? 'Last delivery ok' : `Last delivery failed${status.last.error === undefined ? '' : ` (${status.last.error})`}`}</Tag>
+            <span className="text-xs text-muted-foreground">{new Date(status.last.at).toLocaleString()}</span>
+          </>
+        )}
+        {status.configured && (
+          <Button size="sm" className="ml-auto" disabled={busy} onClick={() => void test()}>
+            Send test
+          </Button>
+        )}
+      </div>
+      {!status.configured && <p className="m-0 text-xs text-muted-foreground">Set HUB_WEBHOOK_URL on the hub to get a notice when a server drifts or a write fails.</p>}
+      {failure !== null && (
+        <p className="m-0 text-xs text-muted-foreground" role="alert">
+          {failure}
+        </p>
+      )}
+    </div>
+  )
+}

src/lib/allow-block.ts

@@ -1,4 +1,5 @@
-import { useRef, useState } from 'react'
+import { useContext, useRef, useState } from 'react'
+import { HubNode, nodeArg } from './hub-write'
 import { allowDomain, blockDomain, BLOCKED_TYPES } from '../api/blocking'
 import { noticeFromFailure, type Notice } from './notice'
 
@@ -32,7 +33,11 @@ export function verbFor(responseType: string): DomainVerb {
   return BLOCKED.includes(responseType.toLowerCase()) ? 'Allow Domain' : 'Block Domain'
 }
 
-export function useDomainAction(token: string | null, onNotice: (n: Notice) => void) {
+/* `node` is the server the change is made on, for the hub; by default the one the section around it has chosen
+   (`HubNode`, empty outside a hub: the server the console talks to). */
+export function useDomainAction(token: string | null, onNotice: (n: Notice) => void, node?: string) {
+  const section = useContext(HubNode)
+  const target = node ?? section
   const [busy, setBusy] = useState<ReadonlySet<string>>(new Set())
   /* Read by `run` so two clicks inside one render cannot both pass the guard. */
   const inFlight = useRef(new Set<string>())
@@ -41,7 +46,7 @@ export function useDomainAction(token: string | null, onNotice: (n: Notice) => v
     if (inFlight.current.has(key)) return false
     inFlight.current.add(key)
     setBusy(new Set(inFlight.current))
-    const outcome = await (verb === 'Allow Domain' ? allowDomain(token, domain) : blockDomain(token, domain))
+    const outcome = await (verb === 'Allow Domain' ? allowDomain(token, domain, ...nodeArg(target)) : blockDomain(token, domain, ...nodeArg(target)))
     inFlight.current.delete(key)
     setBusy(new Set(inFlight.current))
     if (outcome.kind !== 'ok') {

src/lib/hub-all.tsx

@@ -0,0 +1,64 @@
+import { createContext, useContext, useState, type ReactNode } from 'react'
+import { runOnServers, type ServerAction, type ServerParams } from '../api/hub-run'
+import { RunFlow, pickerServers, type RunRequest } from '../screens/apps/RunFlow'
+import { HubWritable } from './hub-write'
+
+/*
+Behind a hub, a change that belongs on several servers at once (Blocked and Allowed, the blocking
+switch, the block lists, the cache) goes through the same three steps as the apps: pick the
+servers, wait, read what happened on each, with "Retry failed". This is the glue:
+
+  · `HubServerIds`: every server the hub fronts, given by the shell. The picker lists all of them
+    and disables the ones the hub holds no admin token for, with the reason.
+  · `useAllServersFlow`: `start(spec)` opens the flow, `flow` is the element to render.
+  · `HubAll`: what a section provides to the controls inside it when "All servers" is chosen and
+    the role may make the change. A control that finds `null` there behaves as it always did.
+*/
+export const HubServerIds = createContext<readonly string[]>([])
+
+export interface AllSpec {
+  action: ServerAction
+  params?: ServerParams
+  title: string
+  intro?: ReactNode
+  confirmLabel?: string
+  /** The servers ticked at the start; every writable one when absent. */
+  preselected?: string[]
+  /** After the flow closes, told whether any server changed. */
+  onDone?: (changed: boolean) => void
+}
+
+export type StartAll = (spec: AllSpec) => void
+
+export const HubAll = createContext<StartAll | null>(null)
+
+export function useAllServersFlow(token: string | null): { start: StartAll; flow: ReactNode } {
+  const ids = useContext(HubServerIds)
+  const writable = useContext(HubWritable)
+  const [open, setOpen] = useState<{ spec: AllSpec; request: RunRequest } | null>(null)
+
+  const start: StartAll = (spec) => {
+    const request: RunRequest = {
+      title: spec.title,
+      intro: spec.intro,
+      confirmLabel: spec.confirmLabel ?? 'Run',
+      servers: pickerServers(ids.map((id) => ({ id, writable: writable.includes(id) }))),
+      preselected: spec.preselected ?? ids.filter((id) => writable.includes(id)),
+      run: (targets) => runOnServers(token, spec.action, spec.params ?? {}, targets),
+    }
+    setOpen({ spec, request })
+  }
+
+  const flow =
+    open === null ? null : (
+      <RunFlow
+        request={open.request}
+        onClose={(changed) => {
+          const done = open.spec.onDone
+          setOpen(null)
+          done?.(changed)
+        }}
+      />
+    )
+  return { start, flow }
+}

src/lib/hub-write.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'vitest'
+import { CHOOSE_A_SERVER, NO_ADMIN_TOKEN, hubDefaultNode, hubWriteState, nodeArg } from './hub-write'
+
+const NODES = [
+  { name: 'mdns', type: 'Primary' },
+  { name: 'ndns', type: 'Secondary' },
+]
+
+describe('hubWriteState', () => {
+  const base = { hub: true, writable: ['mdns'], allowed: true }
+
+  it('allows a change on a chosen server the hub can write to', () => {
+    expect(hubWriteState({ ...base, node: 'mdns' })).toEqual({ allowed: true, reason: null })
+  })
+
+  it('says "Choose a server" for the aggregate and for no node', () => {
+    for (const node of ['', 'cluster']) {
+      expect(hubWriteState({ ...base, node })).toEqual({ allowed: false, reason: CHOOSE_A_SERVER })
+    }
+  })
+
+  it('says there is no admin token for a server the hub cannot write to', () => {
+    expect(hubWriteState({ ...base, node: 'ndns' })).toEqual({ allowed: false, reason: NO_ADMIN_TOKEN })
+    expect(NO_ADMIN_TOKEN).toBe('No admin token configured on the hub')
+  })
+
+  it('gives no reason when the role is what says no (the control carries its own tooltip)', () => {
+    expect(hubWriteState({ ...base, node: 'mdns', allowed: false })).toEqual({ allowed: false, reason: null })
+    expect(hubWriteState({ ...base, node: '', allowed: false })).toEqual({ allowed: false, reason: null })
+  })
+
+  it('is the permission and nothing more outside a hub', () => {
+    expect(hubWriteState({ hub: false, node: '', writable: [], allowed: true })).toEqual({ allowed: true, reason: null })
+    expect(hubWriteState({ hub: false, node: 'x', writable: [], allowed: false })).toEqual({ allowed: false, reason: null })
+  })
+})
+
+describe('hubDefaultNode', () => {
+  it('names the first server behind a hub and nothing elsewhere', () => {
+    expect(hubDefaultNode(true, NODES)).toBe('mdns')
+    expect(hubDefaultNode(false, NODES)).toBe('')
+    expect(hubDefaultNode(true, [])).toBe('')
+  })
+})
+
+describe('nodeArg', () => {
+  it('adds the argument only when a server is chosen', () => {
+    expect(nodeArg('')).toEqual([])
+    expect(nodeArg('mdns')).toEqual(['mdns'])
+  })
+})

src/lib/hub-write.ts

@@ -0,0 +1,75 @@
+import { createContext, useContext } from 'react'
+import { HubMode, type ClusterNode } from '../ui/ClusterNodeSelect'
+
+/*
+Behind a hub a change is made on ONE server at a time (`node=<id>`), and only on a server the
+hub holds an admin token for. These are the pieces every screen that changes something shares,
+so the rule is written once.
+
+  · `HubWritable`: the servers the hub can write to (`hubWritable` of the session), given by the shell.
+  · `hubWriteState`: whether a change is allowed on `node`, and if not, the sentence that says why.
+  · `useHubWrite`: that, read from the contexts, for a screen's own node.
+  · `useHubDefaultNode`: the server a screen without an "All servers" choice starts on.
+  · `HubNode`: the server a section's controls act on, for the controls that are not handed it.
+
+Outside a hub none of this applies: `allowed` is the permission and there is no reason.
+*/
+export const HubWritable = createContext<readonly string[]>([])
+
+/* The server the controls of a section act on (`''` outside a hub, or when none is chosen). */
+export const HubNode = createContext('')
+
+export const CHOOSE_A_SERVER = 'Choose a server'
+export const NO_ADMIN_TOKEN = 'No admin token configured on the hub'
+
+export interface HubWrite {
+  /** The role may change it, a server is chosen and the hub can write to it. */
+  allowed: boolean
+  /** Why not, when it is the node and not the role; `null` when allowed or when the role says no. */
+  reason: string | null
+}
+
+export function hubWriteState({
+  hub,
+  node,
+  writable,
+  allowed,
+}: {
+  hub: boolean
+  node: string
+  writable: readonly string[]
+  /** What the role allows (`canModify`). */
+  allowed: boolean
+}): HubWrite {
+  if (!hub) return { allowed, reason: null }
+  if (!allowed) return { allowed: false, reason: null }
+  if (node === '' || node === 'cluster') return { allowed: false, reason: CHOOSE_A_SERVER }
+  if (!writable.includes(node)) return { allowed: false, reason: NO_ADMIN_TOKEN }
+  return { allowed: true, reason: null }
+}
+
+/* The write state of a screen on its own `node`. `allowed` is the role's permission, true when it has none. */
+export function useHubWrite(node: string, allowed = true): HubWrite {
+  const hub = useContext(HubMode)
+  const writable = useContext(HubWritable)
+  return hubWriteState({ hub, node, writable, allowed })
+}
+
+/*
+Where a screen without an "All servers" choice starts. Outside a hub that is the empty string (this
+server, which the real console takes as its own node). Behind a hub the empty string is the hub, not
+a server, so the first server is named: the selector shows it and the requests carry it.
+*/
+export function hubDefaultNode(hub: boolean, nodes: readonly ClusterNode[]): string {
+  return hub && nodes.length > 0 ? nodes[0].name : ''
+}
+
+export function useHubDefaultNode(nodes: readonly ClusterNode[]): string {
+  return hubDefaultNode(useContext(HubMode), nodes)
+}
+
+/*
+The optional `node` argument of an API function, spread into its call: nothing when no server is
+chosen, so a console used without a hub makes exactly the calls it always made.
+*/
+export const nodeArg = (node: string): [] | [string] => (node === '' ? [] : [node])

src/screens/about/About.tsx

@@ -61,7 +61,8 @@ four literals the product does not say.
 What stays is the `Server` panel: version and uptime are facts upstream's own
 About shows, right under the title.
 */
-export function About({ info }: { info?: Info }) {
+/* `hubPanel`: behind a hub, the hub's own status for its admin (the webhook), under Server. */
+export function About({ info, hubPanel }: { info?: Info; hubPanel?: ReactNode }) {
   return (
     <>
       <SectionHeader title="About" subtitle="Technitium DNS Server and this console" />
@@ -166,6 +167,12 @@ export function About({ info }: { info?: Info }) {
           </Body>
         </Panel>
 
+        {hubPanel != null && (
+          <Panel title="Hub notifications" description="Where the hub sends drift and failure notices">
+            <Body>{hubPanel}</Body>
+          </Panel>
+        )}
+
         {/*
         This console's own panels (deviation 9 of CONVENTIONS.md, approved on
         2026-10-16), under Server. Technitium's card on the left stays literal.

src/screens/apps/AppCard.tsx

@@ -21,6 +21,8 @@ export function AppCard({
   onUpdate,
   onStoreUpdate,
   onUninstall,
+  readOnly = false,
+  hideConfig = false,
 }: {
   app: InstalledApp
   busy: boolean
@@ -28,6 +30,10 @@ export function AppCard({
   onUpdate: () => void
   onStoreUpdate: () => void
   onUninstall: () => void
+  /** Behind a hub nothing can be changed yet: no action buttons. */
+  readOnly?: boolean
+  /** Behind a hub the config is not edited from here yet. */
+  hideConfig?: boolean
 }) {
   const hasUpdate = app.updateAvailable === true && Boolean(app.updateUrl)
 
@@ -54,10 +60,15 @@ export function AppCard({
         <Tag tone={hasUpdate ? 'warn' : 'ok'}>{hasUpdate ? 'Update available' : 'Up to date'}</Tag>
       </div>
 
+      {readOnly ? (
+        <div />
+      ) : (
       <div className={styles.rowActs}>
-        <Button disabled={busy} onClick={onConfig}>
-          Config
-        </Button>
+        {!hideConfig && (
+          <Button disabled={busy} onClick={onConfig}>
+            Config
+          </Button>
+        )}
         <Button disabled={busy} onClick={onUpdate}>
           Update
         </Button>
@@ -94,6 +105,7 @@ export function AppCard({
           )}
         </Menu>
       </div>
+      )}
 
       {app.dnsApps.length > 0 && (
         <Details className={styles.rowDetails} summary="More Details">

src/screens/apps/AppConfig.tsx

@@ -25,6 +25,7 @@ export function AppConfig({
   name,
   initialConfig,
   onSaved,
+  node = '',
 }: {
   open: boolean
   onOpenChange: (o: boolean) => void
@@ -32,6 +33,8 @@ export function AppConfig({
   name: string
   initialConfig: string
   onSaved: (name: string) => void
+  /** Behind a hub: the one server it goes to. */
+  node?: string
 }) {
   const [config, setConfig] = useState(initialConfig)
   const [alert, setAlert] = useState<AlertState | null>(null)
@@ -47,7 +50,7 @@ export function AppConfig({
 
   async function save() {
     setBusy(true)
-    const outcome = await setAppConfig(token, name, config)
+    const outcome = await setAppConfig(token, name, config, node)
     setBusy(false)
 
     if (outcome.kind !== 'ok') {

src/screens/apps/AppConfigSync.test.tsx

@@ -0,0 +1,281 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { render, screen, waitFor, within } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { AppConfigSync } from './AppConfigSync'
+import { AppsMatrix } from './AppsMatrix'
+import * as api from '../../api/apps'
+import type { ConfigCompare, ServerConfigView } from '../../api/apps'
+
+afterEach(() => vi.restoreAllMocks())
+
+const A = JSON.stringify({ ttl: 30, db: '/a.db' }, null, 2)
+const B = JSON.stringify({ ttl: 60, db: '/b.db' }, null, 2)
+
+function view(id: string, over: Partial<ServerConfigView> = {}): ServerConfigView {
+  return { id, ok: true, installed: true, writable: true, version: '1.0', text: A, hash: 'h-' + id, warnings: [], inSync: null, vsOthers: {}, ...over }
+}
+
+/* mdns is the baseline; ndns differs in ttl and in the local db path. */
+const WITH_BASELINE: ConfigCompare = {
+  name: 'Blocker',
+  baseline: { app: 'Blocker', text: A, importedFrom: 'mdns', updatedAt: '2026-10-31T08:00:00Z', updatedBy: 'alice' },
+  overrides: { paths: ['db'], values: { mdns: { db: '/a.db' }, ndns: { db: '/b.db' } } },
+  servers: [
+    view('mdns', { inSync: true, vsBaseline: { kind: 'json', changes: [] } }),
+    view('ndns', {
+      text: B,
+      inSync: false,
+      vsBaseline: { kind: 'json', changes: [{ path: 'ttl', type: 'changed', a: 60, b: 30 }, { path: 'db', type: 'changed', a: '/x.db', b: '/b.db' }] },
+    }),
+  ],
+}
+
+const NO_BASELINE: ConfigCompare = {
+  name: 'Blocker',
+  baseline: null,
+  overrides: { paths: [], values: {} },
+  servers: [
+    view('mdns', { vsOthers: { ndns: { kind: 'json', changes: [{ path: 'ttl', type: 'changed', a: 60, b: 30 }] } } }),
+    view('ndns', { text: B }),
+  ],
+}
+
+function mockCompare(data: ConfigCompare) {
+  return vi.spyOn(api, 'getConfigCompare').mockResolvedValue({ kind: 'ok', data: { response: data } } as never)
+}
+
+const ok = (results: api.TargetResult[]) => ({ kind: 'ok', data: { response: { results } } }) as never
+
+function mount(canWrite: boolean) {
+  const onChanged = vi.fn()
+  render(<AppConfigSync token="t" name="Blocker" canWrite={canWrite} onClose={() => {}} onChanged={onChanged} />)
+  return { onChanged }
+}
+
+describe('Compare', () => {
+  it('shows a skeleton while loading, then the badges and the fields that differ', async () => {
+    mockCompare(WITH_BASELINE)
+    mount(false)
+    expect(screen.getByRole('status', { name: 'Loading the configs' })).toBeInTheDocument()
+    expect(await screen.findByText('In sync')).toBeInTheDocument()
+    expect(screen.getByText('Differs')).toBeInTheDocument()
+    const dialog = screen.getByRole('dialog', { name: 'App config: Blocker' })
+    expect(within(dialog).getByRole('columnheader', { name: 'ndns' })).toBeInTheDocument()
+    expect(within(dialog).getByText('ttl')).toBeInTheDocument()
+    expect(within(dialog).getByText('60')).toBeInTheDocument()
+    expect(within(dialog).getByText('expected 30')).toBeInTheDocument()
+    // The local field is marked.
+    const dbRow = within(dialog).getByText('db').closest('tr') as HTMLElement
+    expect(within(dbRow).getByText('Local')).toBeInTheDocument()
+  })
+
+  it('compares the servers to each other when there is no baseline', async () => {
+    mockCompare(NO_BASELINE)
+    mount(false)
+    expect(await screen.findByText('No baseline yet')).toBeInTheDocument()
+    expect(screen.getAllByText('No baseline').length).toBeGreaterThan(0)
+    expect(screen.getByText('ttl')).toBeInTheDocument()
+    expect(screen.getByText('30')).toBeInTheDocument()
+    expect(screen.getByText('60')).toBeInTheDocument()
+  })
+
+  it('shows lines for a text config', async () => {
+    mockCompare({
+      ...WITH_BASELINE,
+      servers: [
+        view('mdns', { inSync: true, vsBaseline: { kind: 'text', lines: [], truncated: false, changed: 0 } }),
+        view('ndns', { inSync: false, vsBaseline: { kind: 'text', lines: [{ op: 'same', text: 'a = 1' }, { op: 'remove', text: 'b = 2' }, { op: 'add', text: 'b = 3' }], truncated: false, changed: 2 } }),
+      ],
+    })
+    mount(false)
+    const lines = await screen.findByLabelText('Line differences')
+    expect(within(lines).getByText(/b = 3/)).toHaveAttribute('data-op', 'add')
+    expect(within(lines).getByText(/b = 2/)).toHaveAttribute('data-op', 'remove')
+  })
+
+  it('shows an error with Retry', async () => {
+    const spy = vi.spyOn(api, 'getConfigCompare').mockResolvedValueOnce({ kind: 'error', message: 'No server answered.' } as never)
+    mount(false)
+    expect(await screen.findByText(/Could not read the configs/)).toHaveTextContent('No server answered.')
+    spy.mockResolvedValue({ kind: 'ok', data: { response: WITH_BASELINE } } as never)
+    await userEvent.click(screen.getByRole('button', { name: 'Retry' }))
+    expect(await screen.findByText('In sync')).toBeInTheDocument()
+  })
+})
+
+describe('role gating', () => {
+  it('gives a reader no editing controls', async () => {
+    mockCompare(WITH_BASELINE)
+    mount(false)
+    await screen.findByText('In sync')
+    expect(screen.queryByRole('button', { name: 'Apply…' })).not.toBeInTheDocument()
+    await userEvent.click(screen.getByRole('tab', { name: 'Baseline' }))
+    expect(screen.getByLabelText('Baseline text')).toHaveAttribute('readonly')
+    expect(screen.queryByRole('button', { name: 'Save baseline' })).not.toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: /^Import from/ })).not.toBeInTheDocument()
+    await userEvent.click(screen.getByRole('tab', { name: 'Local fields' }))
+    expect(screen.getByLabelText('db on mdns')).toHaveAttribute('readonly')
+    expect(screen.queryByRole('button', { name: 'Save local fields' })).not.toBeInTheDocument()
+  })
+
+  it('tells a reader who sees no baseline to ask, without import buttons', async () => {
+    mockCompare(NO_BASELINE)
+    mount(false)
+    await screen.findByText('No baseline yet')
+    expect(screen.queryByRole('button', { name: /^Import from/ })).not.toBeInTheDocument()
+    expect(screen.getByText(/apps administrator/)).toBeInTheDocument()
+  })
+
+  it('offers a writer the import from each server when there is no baseline', async () => {
+    const spy = mockCompare(NO_BASELINE)
+    const imp = vi.spyOn(api, 'importBaseline').mockResolvedValue({ kind: 'ok', data: { response: { baseline: WITH_BASELINE.baseline } } } as never)
+    const { onChanged } = mount(true)
+    await userEvent.click(await screen.findByRole('button', { name: 'Import from mdns' }))
+    expect(imp).toHaveBeenCalledWith('t', 'Blocker', 'mdns')
+    await waitFor(() => expect(spy.mock.calls.length).toBeGreaterThan(1))
+    expect(onChanged).toHaveBeenCalled()
+    expect(await screen.findByText('Baseline imported from mdns.')).toBeInTheDocument()
+  })
+})
+
+describe('Baseline and Local fields', () => {
+  it('shows who updated the baseline and a validation error from the hub inline', async () => {
+    mockCompare(WITH_BASELINE)
+    const save = vi.spyOn(api, 'saveBaseline').mockResolvedValue({ kind: 'error', message: 'The baseline is not valid JSON: Unexpected token at position 4' } as never)
+    mount(true)
+    await userEvent.click(await screen.findByRole('tab', { name: 'Baseline' }))
+    expect(screen.getByText(/Imported from mdns\. Updated by alice/)).toBeInTheDocument()
+    const box = screen.getByLabelText('Baseline text')
+    await userEvent.clear(box)
+    await userEvent.type(box, 'x')
+    await userEvent.click(screen.getByRole('button', { name: 'Save baseline' }))
+    expect(save).toHaveBeenCalledWith('t', 'Blocker', 'x')
+    expect(await screen.findByText(/not valid JSON: Unexpected token at position 4/)).toBeInTheDocument()
+  })
+
+  it('adds a local field, fills the values from the servers and saves them', async () => {
+    mockCompare({ ...WITH_BASELINE, overrides: { paths: [], values: {} } })
+    const current = vi.spyOn(api, 'getCurrentLocalValues').mockResolvedValue({ kind: 'ok', data: { response: { values: { mdns: { db: '/a.db' }, ndns: { db: '/b.db' } } } } } as never)
+    const save = vi.spyOn(api, 'saveOverrides').mockResolvedValue({ kind: 'ok', data: { response: { overrides: WITH_BASELINE.overrides } } } as never)
+    mount(true)
+    await userEvent.click(await screen.findByRole('tab', { name: 'Local fields' }))
+    expect(screen.getByText('No local fields yet.')).toBeInTheDocument()
+    await userEvent.type(screen.getByLabelText('New local field'), 'db')
+    await userEvent.click(screen.getByRole('button', { name: 'Add field' }))
+    await userEvent.click(screen.getByRole('button', { name: 'Fill from the current server configs' }))
+    expect(current).toHaveBeenCalledWith('t', 'Blocker', ['db'])
+    expect(await screen.findByDisplayValue('"/b.db"')).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Save local fields' }))
+    expect(save).toHaveBeenCalledWith('t', 'Blocker', { paths: ['db'], values: { mdns: { db: '/a.db' }, ndns: { db: '/b.db' } } })
+  })
+
+  it('refuses a value that is not JSON, without calling the hub', async () => {
+    mockCompare(WITH_BASELINE)
+    const save = vi.spyOn(api, 'saveOverrides')
+    mount(true)
+    await userEvent.click(await screen.findByRole('tab', { name: 'Local fields' }))
+    const box = screen.getByLabelText('db on mdns')
+    await userEvent.clear(box)
+    await userEvent.type(box, 'no quotes')
+    await userEvent.click(screen.getByRole('button', { name: 'Save local fields' }))
+    expect(await screen.findByText(/db on mdns is not valid JSON/)).toBeInTheDocument()
+    expect(save).not.toHaveBeenCalled()
+  })
+})
+
+describe('Apply', () => {
+  it('picks servers, previews, applies with the preview hashes, and retries only what failed', async () => {
+    mockCompare({ ...WITH_BASELINE, servers: [WITH_BASELINE.servers[0], { ...WITH_BASELINE.servers[1], writable: true }] })
+    const preview = vi.spyOn(api, 'previewConfig').mockResolvedValue({
+      kind: 'ok',
+      data: {
+        response: {
+          targets: [
+            { server: 'ndns', ok: true, writable: true, current: B, currentHash: 'hash-ndns', effective: A, unchanged: false, warnings: ['Version 0.9 here, 1.0 on mdns: versions differ, align versions first.'], changes: WITH_BASELINE.servers[1].vsBaseline },
+          ],
+        },
+      },
+    } as never)
+    const apply = vi
+      .spyOn(api, 'applyConfig')
+      .mockResolvedValueOnce(ok([{ server: 'ndns', ok: false, message: 'The config is invalid.' }]))
+      .mockResolvedValueOnce(ok([{ server: 'ndns', ok: true, message: 'Applied and verified.' }]))
+    const { onChanged } = mount(true)
+    await userEvent.click(await screen.findByRole('button', { name: 'Apply…' }))
+
+    const picker = await screen.findByRole('dialog', { name: 'Apply the baseline of Blocker' })
+    // Only the server that differs is ticked.
+    expect(within(picker).getByRole('checkbox', { name: 'ndns' })).toBeChecked()
+    expect(within(picker).getByRole('checkbox', { name: 'mdns' })).not.toBeChecked()
+    await userEvent.click(within(picker).getByRole('button', { name: 'Preview' }))
+    expect(preview).toHaveBeenCalledWith('t', 'Blocker', ['ndns'])
+
+    const review = await screen.findByRole('dialog', { name: 'Apply the baseline of Blocker: review' })
+    expect(within(review).getByText('Will change')).toBeInTheDocument()
+    expect(within(review).getByText(/align versions first/)).toBeInTheDocument()
+    expect(within(review).getByText('ttl')).toBeInTheDocument()
+    expect(apply).not.toHaveBeenCalled()
+    await userEvent.click(within(review).getByRole('button', { name: 'Apply' }))
+    expect(apply).toHaveBeenLastCalledWith('t', 'Blocker', ['ndns'], { ndns: 'hash-ndns' })
+
+    const results = await screen.findByRole('dialog', { name: 'Apply the baseline of Blocker: results' })
+    expect(within(results).getByText('The config is invalid.')).toBeInTheDocument()
+    await userEvent.click(within(results).getByRole('button', { name: 'Retry failed' }))
+    expect(apply).toHaveBeenCalledTimes(2)
+    await waitFor(() => expect(within(screen.getByRole('dialog')).getByText('Applied and verified.')).toBeInTheDocument())
+    await userEvent.click(within(screen.getByRole('dialog')).getAllByRole('button', { name: 'Close' }).at(-1) as HTMLElement)
+    await waitFor(() => expect(onChanged).toHaveBeenCalled())
+  })
+
+  it('cannot apply without a baseline, and lists a server without a token as disabled', async () => {
+    mockCompare(NO_BASELINE)
+    mount(true)
+    await screen.findByText('No baseline yet')
+    expect(screen.getByRole('button', { name: 'Apply…' })).toBeDisabled()
+  })
+
+  it('disables a server the hub cannot write to in the picker', async () => {
+    mockCompare({ ...WITH_BASELINE, servers: [WITH_BASELINE.servers[0], { ...WITH_BASELINE.servers[1], writable: false }] })
+    mount(true)
+    await userEvent.click(await screen.findByRole('button', { name: 'Apply…' }))
+    const picker = await screen.findByRole('dialog', { name: 'Apply the baseline of Blocker' })
+    expect(within(picker).getByRole('checkbox', { name: 'ndns' })).toBeDisabled()
+    expect(within(picker).getByText('No admin token configured on the hub')).toBeInTheDocument()
+  })
+
+  it('shows a review failure from the hub', async () => {
+    mockCompare(WITH_BASELINE)
+    vi.spyOn(api, 'previewConfig').mockResolvedValue({ kind: 'error', message: 'This app has no baseline yet.' } as never)
+    mount(true)
+    await userEvent.click(await screen.findByRole('button', { name: 'Apply…' }))
+    await userEvent.click(within(await screen.findByRole('dialog', { name: 'Apply the baseline of Blocker' })).getByRole('button', { name: 'Preview' }))
+    expect(await screen.findByText('This app has no baseline yet.')).toBeInTheDocument()
+  })
+})
+
+describe('from the matrix', () => {
+  const DATA: api.AppsMatrixData = {
+    servers: [{ id: 'mdns', ok: true, writable: true }, { id: 'ndns', ok: true, writable: true }],
+    apps: [
+      {
+        name: 'Blocker', description: '', configs: 'different', diffCount: 2, versions: 'same',
+        servers: { mdns: { installed: true, version: '1.0', configStatus: 'read' }, ndns: { installed: true, version: '1.0', configStatus: 'read' } },
+      },
+    ],
+  }
+
+  it('opens the config view from the row menu for a reader and from the Different badge', async () => {
+    vi.spyOn(api, 'getAppsMatrix').mockResolvedValue({ kind: 'ok', data: { response: DATA } } as never)
+    const compare = mockCompare(WITH_BASELINE)
+    render(<AppsMatrix token="t" />)
+    await userEvent.click(await screen.findByRole('button', { name: 'Actions: Blocker' }))
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Config…' }))
+    expect(await screen.findByRole('dialog', { name: 'App config: Blocker' })).toBeInTheDocument()
+    expect(compare).toHaveBeenCalledWith('t', 'Blocker')
+    await userEvent.click(within(screen.getByRole('dialog')).getAllByRole('button', { name: 'Close' }).at(-1) as HTMLElement)
+    await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument())
+
+    await userEvent.click(screen.getByRole('button', { name: 'Compare the configs of Blocker' }))
+    expect(await screen.findByRole('dialog', { name: 'App config: Blocker' })).toBeInTheDocument()
+  })
+})

src/screens/apps/AppConfigSync.tsx

@@ -0,0 +1,545 @@
+import { useCallback, useEffect, useState } from 'react'
+import {
+  getConfigCompare,
+  getCurrentLocalValues,
+  getHistory,
+  getHistoryDiff,
+  getHistoryEntry,
+  importBaseline,
+  restoreBaseline,
+  saveBaseline,
+  saveOverrides,
+  type ConfigCompare,
+  type HistoryDiff,
+  type HistoryEntry,
+  type ServerConfigView,
+} from '../../api/apps'
+import { Alert } from '../../ui/Alert'
+import { Button } from '../../ui/Button'
+import { Dialog } from '../../ui/Dialog'
+import { Empty, Failure } from '../../ui/Empty'
+import { Input, LabeledTextarea } from '../../ui/Field'
+import { Tag } from '../../ui/Tag'
+import { Skeleton } from '@/components/ui/skeleton'
+import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'
+import { DiffView, FieldTable, fieldRows } from './ConfigDiffView'
+import { RunFlow, pickerServers, type RunRequest } from './RunFlow'
+import { applyRequest, rollbackRequest } from './applyRequest'
+import { Select } from '../../ui/Field'
+
+/*
+One app's config across the servers, behind a hub: compare them, keep a baseline in the hub,
+mark the fields that stay local to each server, then preview and apply. Everyone who can read
+sees the comparison; the editors and Apply are for roles that may change apps.
+*/
+
+type Tab = 'compare' | 'baseline' | 'local' | 'history'
+
+const when = (iso: string): string => (iso === '' ? '' : new Date(iso).toLocaleString())
+
+function ServerBadge({ s, hasBaseline }: { s: ServerConfigView; hasBaseline: boolean }) {
+  if (!s.ok) return <Tag tone="dan">Did not answer</Tag>
+  if (!s.installed) return <Tag>Not installed</Tag>
+  if (!hasBaseline || s.inSync === null) return <Tag>No baseline</Tag>
+  return s.inSync ? <Tag tone="ok">In sync</Tag> : <Tag tone="warn">Differs</Tag>
+}
+
+export function AppConfigSync({
+  token,
+  name,
+  canWrite,
+  onClose,
+  onChanged,
+}: {
+  token: string | null
+  name: string
+  canWrite: boolean
+  onClose: () => void
+  onChanged: () => void
+}) {
+  const [data, setData] = useState<ConfigCompare | null>(null)
+  const [failure, setFailure] = useState<string | null>(null)
+  const [tab, setTab] = useState<Tab>('compare')
+  const [request, setRequest] = useState<RunRequest | null>(null)
+  const [busy, setBusy] = useState(false)
+  const [problem, setProblem] = useState<string | null>(null)
+  const [saved, setSaved] = useState<string | null>(null)
+  // The editors.
+  const [baselineText, setBaselineText] = useState('')
+  const [paths, setPaths] = useState<string[]>([])
+  const [values, setValues] = useState<Record<string, Record<string, string>>>({})
+  const [newPath, setNewPath] = useState('')
+  const [loads, setLoads] = useState(0)
+
+  const load = useCallback(async () => {
+    const outcome = await getConfigCompare(token, name)
+    if (outcome.kind !== 'ok') {
+      setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+      return
+    }
+    const d = outcome.data.response
+    setData(d)
+    setLoads((n) => n + 1)
+    setFailure(null)
+    setBaselineText(d.baseline?.text ?? '')
+    setPaths(d.overrides.paths)
+    setValues(
+      Object.fromEntries(d.servers.map((s) => [s.id, Object.fromEntries(d.overrides.paths.flatMap((p) => (Object.hasOwn(d.overrides.values[s.id] ?? {}, p) ? [[p, JSON.stringify(d.overrides.values[s.id][p])]] : [])))])),
+    )
+  }, [token, name])
+
+  useEffect(() => {
+    void load()
+  }, [load])
+
+  const hasBaseline = data?.baseline != null
+
+  async function run(call: () => Promise<{ kind: string; message?: string }>, done: string) {
+    setBusy(true)
+    setProblem(null)
+    setSaved(null)
+    const outcome = await call()
+    setBusy(false)
+    if (outcome.kind !== 'ok') {
+      setProblem(outcome.kind === 'error' ? (outcome.message ?? 'Failed.') : 'The session ended.')
+      return
+    }
+    setSaved(done)
+    await load()
+    onChanged()
+  }
+
+  function saveLocal() {
+    const parsed: Record<string, Record<string, unknown>> = {}
+    for (const [server, mine] of Object.entries(values)) {
+      parsed[server] = {}
+      for (const [path, text] of Object.entries(mine)) {
+        if (!paths.includes(path) || text.trim() === '') continue
+        try {
+          parsed[server][path] = JSON.parse(text)
+        } catch {
+          setProblem(`The value of ${path} on ${server} is not valid JSON (a string needs quotes).`)
+          return
+        }
+      }
+    }
+    void run(() => saveOverrides(token, name, { paths, values: parsed }), 'Local fields saved.')
+  }
+
+  async function fill() {
+    setBusy(true)
+    setProblem(null)
+    const outcome = await getCurrentLocalValues(token, name, paths)
+    setBusy(false)
+    if (outcome.kind !== 'ok') {
+      setProblem(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+      return
+    }
+    setValues(Object.fromEntries(Object.entries(outcome.data.response.values).map(([s, mine]) => [s, Object.fromEntries(Object.entries(mine).map(([p, v]) => [p, JSON.stringify(v)]))])))
+  }
+
+  function pickable(extra: Record<string, string> = {}) {
+    return pickerServers(
+      (data?.servers ?? []).map((s) => ({ id: s.id, writable: s.writable, ok: s.ok })),
+      { ...Object.fromEntries((data?.servers ?? []).filter((s) => s.ok && !s.installed).map((s) => [s.id, 'Not installed'])), ...extra },
+    )
+  }
+
+  function startApply() {
+    if (data === null) return
+    setRequest(applyRequest(token, name, pickable(), data.servers.filter((s) => s.installed && s.inSync === false).map((s) => s.id)))
+  }
+
+  function startRollback(entry: HistoryEntry) {
+    // Preselect the server the version came from: putting it on another server would also carry its local fields.
+    setRequest(rollbackRequest(token, name, entry, pickable(), entry.server === 'baseline' ? [] : [entry.server]))
+  }
+
+  const importButtons = (data?.servers ?? []).filter((s) => s.installed && s.ok)
+
+  return (
+    <>
+      <Dialog
+        open
+        onOpenChange={(o) => {
+          if (!o) onClose()
+        }}
+        size="wide"
+        title={`App config: ${name}`}
+        actions={
+          canWrite ? (
+            <Button variant="primary" disabled={!hasBaseline || busy} onClick={startApply}>
+              Apply…
+            </Button>
+          ) : undefined
+        }
+      >
+        {failure !== null && data === null ? (
+          <Failure>
+            <span className="flex flex-col items-center gap-3">
+              <span>Could not read the configs. {failure}</span>
+              <Button variant="primary" onClick={() => void load()}>
+                Retry
+              </Button>
+            </span>
+          </Failure>
+        ) : data === null ? (
+          <div className="flex flex-col gap-3" role="status" aria-label="Loading the configs">
+            <Skeleton className="h-6 w-1/3" />
+            <Skeleton className="h-24 w-full" />
+            <Skeleton className="h-24 w-full" />
+          </div>
+        ) : (
+          <Tabs
+            value={tab}
+            onValueChange={(v) => setTab(v as Tab)}
+            // The project has no `data-horizontal` variant, so the component's own flex-col never applies.
+            className="flex-col"
+          >
+            <TabsList variant="line">
+              <TabsTrigger value="compare">Compare</TabsTrigger>
+              <TabsTrigger value="baseline">Baseline</TabsTrigger>
+              <TabsTrigger value="local">Local fields</TabsTrigger>
+              <TabsTrigger value="history">History</TabsTrigger>
+            </TabsList>
+            {problem !== null && <Alert type="danger" title="Error">{problem}</Alert>}
+            {saved !== null && <Alert type="success" title="Saved." onDismiss={() => setSaved(null)}>{saved}</Alert>}
+
+            <TabsContent value="compare" className="flex flex-col gap-4 pt-3">
+              <div className="flex flex-wrap items-center gap-4">
+                {data.servers.map((s) => (
+                  <span key={s.id} className="flex items-center gap-2 text-sm">
+                    <span className="font-medium">{s.id}</span>
+                    {s.version !== undefined && <span className="text-xs text-muted-foreground">v{s.version}</span>}
+                    <ServerBadge s={s} hasBaseline={hasBaseline} />
+                  </span>
+                ))}
+              </div>
+              {data.servers.filter((s) => s.error !== undefined).map((s) => (
+                <Alert key={s.id} type="warning" title={s.id}>
+                  {s.error}
+                </Alert>
+              ))}
+              {!hasBaseline && (
+                <Empty
+                  title="No baseline yet"
+                  actions={
+                    canWrite ? (
+                      <span className="flex flex-wrap justify-center gap-2">
+                        {importButtons.map((s) => (
+                          <Button key={s.id} variant="primary" disabled={busy} onClick={() => void run(() => importBaseline(token, name, s.id), `Baseline imported from ${s.id}.`)}>
+                            Import from {s.id}
+                          </Button>
+                        ))}
+                      </span>
+                    ) : undefined
+                  }
+                >
+                  The baseline is the config every server should have. Import it from a server that is right, then mark the fields that must differ per server as local.
+                  {!canWrite && ' An apps administrator can set it up.'}
+                </Empty>
+              )}
+              <CompareBody data={data} />
+            </TabsContent>
+
+            <TabsContent value="baseline" className="flex flex-col gap-3 pt-3">
+              <p className="m-0 text-xs text-muted-foreground">
+                {data.baseline === null
+                  ? 'No baseline saved.'
+                  : `${data.baseline.importedFrom !== undefined ? `Imported from ${data.baseline.importedFrom}. ` : ''}Updated by ${data.baseline.updatedBy} at ${when(data.baseline.updatedAt)}.`}
+              </p>
+              <LabeledTextarea
+                label="Baseline text"
+                mono
+                rows={16}
+                spellCheck={false}
+                readOnly={!canWrite}
+                value={baselineText}
+                onChange={(e) => setBaselineText(e.target.value)}
+              />
+              {canWrite && (
+                <div className="flex flex-wrap items-center gap-2">
+                  <Button variant="primary" disabled={busy} onClick={() => void run(() => saveBaseline(token, name, baselineText), 'Baseline saved.')}>
+                    Save baseline
+                  </Button>
+                  {importButtons.map((s) => (
+                    <Button key={s.id} disabled={busy} onClick={() => void run(() => importBaseline(token, name, s.id), `Baseline imported from ${s.id}.`)}>
+                      Import from {s.id}
+                    </Button>
+                  ))}
+                </div>
+              )}
+            </TabsContent>
+
+            <TabsContent value="local" className="flex flex-col gap-4 pt-3">
+              <p className="m-0 text-xs text-muted-foreground">
+                Local fields are not synchronised: each server keeps its own value. Name a field by its path, for example <code>db.path</code> or <code>networks.0.cidr</code>. A value is JSON, so a string needs quotes.
+              </p>
+              {paths.length === 0 && <p className="m-0 text-sm text-muted-foreground">No local fields yet.</p>}
+              {paths.map((p) => (
+                <fieldset key={p} className="m-0 flex flex-col gap-2 border-0 p-0">
+                  <legend className="flex items-center gap-2 p-0 font-mono text-xs">
+                    {p}
+                    {canWrite && (
+                      <Button size="sm" onClick={() => setPaths((old) => old.filter((x) => x !== p))}>
+                        Remove {p}
+                      </Button>
+                    )}
+                  </legend>
+                  <div className="flex flex-wrap gap-3">
+                    {data.servers.map((s) => (
+                      <label key={s.id} className="flex min-w-48 flex-1 flex-col gap-1 text-xs">
+                        <span>
+                          {p} on {s.id}
+                        </span>
+                        <Input
+                          mono
+                          readOnly={!canWrite}
+                          aria-label={`${p} on ${s.id}`}
+                          placeholder="(no local value)"
+                          value={values[s.id]?.[p] ?? ''}
+                          onChange={(e) => setValues((old) => ({ ...old, [s.id]: { ...(old[s.id] ?? {}), [p]: e.target.value } }))}
+                        />
+                      </label>
+                    ))}
+                  </div>
+                </fieldset>
+              ))}
+              {canWrite && (
+                <>
+                  <div className="flex flex-wrap items-end gap-2">
+                    <label className="flex flex-col gap-1 text-xs">
+                      <span>New local field</span>
+                      <Input mono aria-label="New local field" placeholder="db.path" value={newPath} onChange={(e) => setNewPath(e.target.value)} />
+                    </label>
+                    <Button
+                      disabled={newPath.trim() === '' || paths.includes(newPath.trim())}
+                      onClick={() => {
+                        setPaths((old) => [...old, newPath.trim()])
+                        setNewPath('')
+                      }}
+                    >
+                      Add field
+                    </Button>
+                  </div>
+                  <div className="flex flex-wrap gap-2">
+                    <Button variant="primary" disabled={busy} onClick={saveLocal}>
+                      Save local fields
+                    </Button>
+                    <Button disabled={busy || paths.length === 0} onClick={() => void fill()}>
+                      Fill from the current server configs
+                    </Button>
+                  </div>
+                </>
+              )}
+            </TabsContent>
+
+            <TabsContent value="history" className="pt-3">
+              <HistoryTab
+                token={token}
+                name={name}
+                canWrite={canWrite}
+                servers={data.servers.filter((s) => s.installed && s.ok).map((s) => s.id)}
+                active={tab === 'history'}
+                onRollback={startRollback}
+                onRestore={(entry) => run(() => restoreBaseline(token, name, entry.id), 'Baseline restored.')}
+                version={String(loads)}
+              />
+            </TabsContent>
+          </Tabs>
+        )}
+      </Dialog>
+      {request !== null && (
+        <RunFlow
+          request={request}
+          onClose={(changed) => {
+            setRequest(null)
+            if (changed) {
+              void load()
+              onChanged()
+            }
+          }}
+        />
+      )}
+    </>
+  )
+}
+
+const KIND: Record<string, string> = {
+  'before-apply': 'Before an apply',
+  'before-set': 'Before a save',
+  'before-uninstall': 'Before an uninstall',
+  'before-save': 'Old baseline',
+}
+
+/*
+Every earlier config the hub kept: what a server had before the hub changed it, and what the
+baseline was before it was replaced. Pick one to read it and to see how it differs from a server now.
+*/
+function HistoryTab({
+  token,
+  name,
+  canWrite,
+  servers,
+  active,
+  onRollback,
+  onRestore,
+  version,
+}: {
+  token: string | null
+  name: string
+  canWrite: boolean
+  servers: string[]
+  active: boolean
+  onRollback: (entry: HistoryEntry) => void
+  onRestore: (entry: HistoryEntry) => void
+  /** Changes when the baseline does, so the list is read again. */
+  version: string
+}) {
+  const [entries, setEntries] = useState<HistoryEntry[] | null>(null)
+  const [failure, setFailure] = useState<string | null>(null)
+  const [picked, setPicked] = useState<string | null>(null)
+  const [text, setText] = useState<string | null>(null)
+  const [server, setServer] = useState(servers[0] ?? '')
+  const [diff, setDiff] = useState<HistoryDiff | null>(null)
+  const [diffFailure, setDiffFailure] = useState<string | null>(null)
+
+  const load = useCallback(async () => {
+    const outcome = await getHistory(token, name)
+    if (outcome.kind === 'ok') {
+      setEntries(outcome.data.response.entries)
+      setFailure(null)
+    } else setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+  }, [token, name])
+
+  useEffect(() => {
+    if (active) void load()
+  }, [active, load, version])
+
+  const entry = entries?.find((e) => e.id === picked) ?? null
+
+  useEffect(() => {
+    if (picked === null) return
+    void getHistoryEntry(token, name, picked).then((o) => setText(o.kind === 'ok' ? o.data.response.text : null))
+  }, [token, name, picked])
+
+  useEffect(() => {
+    if (entry === null || entry.kind === 'before-save' || server === '') return
+    void getHistoryDiff(token, name, entry.id, server).then((o) => {
+      setDiff(o.kind === 'ok' ? o.data.response : null)
+      setDiffFailure(o.kind === 'ok' ? null : o.kind === 'error' ? o.message : 'The session ended.')
+    })
+  }, [token, name, entry, server])
+
+  if (failure !== null && entries === null) {
+    return (
+      <Failure>
+        <span className="flex flex-col items-center gap-3">
+          <span>Could not read the history. {failure}</span>
+          <Button variant="primary" onClick={() => void load()}>
+            Retry
+          </Button>
+        </span>
+      </Failure>
+    )
+  }
+  if (entries === null) return <Skeleton className="h-24 w-full" />
+  if (entries.length === 0) {
+    return (
+      <Empty title="No history yet">
+        The hub keeps a copy of a server's config before it changes it, and of the baseline before it is replaced. They appear here.
+      </Empty>
+    )
+  }
+
+  return (
+    <div className="flex flex-col gap-4">
+      <ul className="m-0 flex max-h-56 list-none flex-col gap-1 overflow-auto p-0" aria-label="History">
+        {entries.map((e) => (
+          <li key={e.id}>
+            <button
+              type="button"
+              aria-pressed={e.id === picked}
+              aria-label={`${new Date(e.at).toLocaleString()} ${e.server === 'baseline' ? 'baseline' : e.server} ${KIND[e.kind] ?? e.kind}`}
+              className="flex w-full cursor-pointer flex-wrap items-center gap-3 rounded-md border border-border bg-transparent px-3 py-2 text-left text-sm transition-colors hover:bg-muted aria-pressed:bg-muted"
+              onClick={() => {
+                setPicked(e.id)
+                // A server's own version is compared with that server by default.
+                if (servers.includes(e.server)) setServer(e.server)
+                setText(null)
+                setDiff(null)
+              }}
+            >
+              <span className="tabular-nums">{new Date(e.at).toLocaleString()}</span>
+              <Tag tone={e.server === 'baseline' ? 'info' : 'neutral'}>{e.server === 'baseline' ? 'Baseline' : e.server}</Tag>
+              <span className="text-muted-foreground">{KIND[e.kind] ?? e.kind}</span>
+              <span className="text-xs text-muted-foreground">{e.size} bytes</span>
+            </button>
+          </li>
+        ))}
+      </ul>
+
+      {entry !== null && (
+        <div className="flex flex-col gap-3">
+          <LabeledTextarea label="Version text" mono rows={10} readOnly spellCheck={false} value={text ?? ''} onChange={() => {}} />
+          {entry.kind !== 'before-save' && servers.length > 0 && (
+            <div className="flex flex-col gap-3">
+              <label className="flex max-w-60 flex-col gap-1 text-xs">
+                <span>Compare with the config now on</span>
+                <Select value={server} onChange={(e) => setServer(e.target.value)}>
+                  {servers.map((id) => (
+                    <option key={id} value={id}>
+                      {id}
+                    </option>
+                  ))}
+                </Select>
+              </label>
+              {diffFailure !== null ? (
+                <Alert type="danger" title="Error">{diffFailure}</Alert>
+              ) : diff === null ? (
+                <Skeleton className="h-16 w-full" />
+              ) : diff.unchanged ? (
+                <p className="m-0 text-sm text-muted-foreground">{server} already has this version.</p>
+              ) : (
+                <DiffView diff={diff.changes} from={`${server} now`} to="This version" />
+              )}
+            </div>
+          )}
+          {canWrite && (
+            <div className="flex gap-2">
+              {entry.kind === 'before-save' ? (
+                <Button variant="primary" onClick={() => onRestore(entry)}>
+                  Restore as baseline
+                </Button>
+              ) : (
+                <Button variant="primary" onClick={() => onRollback(entry)}>
+                  Roll back…
+                </Button>
+              )}
+            </div>
+          )}
+        </div>
+      )}
+    </div>
+  )
+}
+
+/* The differences, as a field table for JSON and as line diffs for text. */
+function CompareBody({ data }: { data: ConfigCompare }) {
+  if (fieldRows(data) !== null) return <FieldTable data={data} />
+  const live = data.servers.filter((s) => s.installed && s.ok)
+  return (
+    <div className="flex flex-col gap-4">
+      {live.map((s) => {
+        const diff = data.baseline !== null ? s.vsBaseline : Object.values(s.vsOthers)[0]
+        return diff === undefined ? null : (
+          <section key={s.id} aria-label={`Differences ${s.id}`} className="flex flex-col gap-2">
+            <h3 className="m-0 text-sm font-semibold">{s.id}</h3>
+            <DiffView diff={diff} from={data.baseline !== null ? 'Now' : 'Other'} to={data.baseline !== null ? 'Expected' : s.id} />
+          </section>
+        )
+      })}
+      {live.length === 0 && <p className="m-0 text-sm text-muted-foreground">Nothing to compare: the app is not installed on any server that answered.</p>}
+    </div>
+  )
+}

src/screens/apps/Apps.tsx

@@ -1,7 +1,8 @@
 import { KpiStrip } from '../../ui/KpiStrip'
-import { useCallback, useEffect, useState } from 'react'
+import { useCallback, useContext, useEffect, useState } from 'react'
 import {
   downloadAndUpdate,
+  exportBaselines,
   getAppConfig,
   listApps,
   uninstallApp,
@@ -25,6 +26,14 @@ import styles from './Apps.module.css'
 import { noticeFromFailure } from '../../lib/notice'
 import { Notifier } from '../../ui/Notifier'
 import { Confirm } from '../../ui/Confirm'
+import { InChrome } from '../../app/ChromeSlot'
+import { AGGREGATE, ClusterNodeSelect, HubMode, type ClusterNode } from '../../ui/ClusterNodeSelect'
+import { AppsMatrix } from './AppsMatrix'
+import { HubStore } from './HubStore'
+import { HubInstallFile } from './HubInstallFile'
+import { ImportBaselines } from './ImportBaselines'
+import { Profiles } from './Profiles'
+import { RunFlow, type RunRequest } from './RunFlow'
 
 /*
 A replica of the Apps tab (apps.js + index.html:807-835).
@@ -56,7 +65,59 @@ export function error(outcome: { kind: string; message?: string }): AlertState {
   return noticeFromFailure(outcome)
 }
 
-export function Apps({ token }: { token: string | null }) {
+export function Apps({
+  token,
+  nodes = [],
+  clusterInitialised = false,
+  canWrite = false,
+  writable = [],
+  isAdmin = false,
+}: {
+  token: string | null
+  /** Only used behind a hub: its servers. */
+  nodes?: ClusterNode[]
+  clusterInitialised?: boolean
+  /** Behind a hub: the role may change apps (`Apps.canModify`). */
+  canWrite?: boolean
+  /** Behind a hub: the servers the hub holds an admin token for. */
+  writable?: string[]
+  /** Behind a hub: the role is the hub's admin (profiles are for admins). */
+  isAdmin?: boolean
+}) {
+  /*
+  Behind a hub (`info.hub`) this screen is read-only and has a server selector: "All
+  servers" shows the matrix of apps over servers, one server shows that server's list.
+  Anywhere else none of this exists.
+  */
+  const hub = useContext(HubMode)
+  const [node, setNode] = useState<string>(() => {
+    try {
+      return localStorage.getItem('hubAppsNode') || AGGREGATE
+    } catch {
+      return AGGREGATE
+    }
+  })
+  const chosen = node === AGGREGATE || nodes.some((n) => n.name === node) ? node : AGGREGATE
+  const matrixMode = hub && chosen === AGGREGATE
+  const requestNode = hub && !matrixMode ? chosen : ''
+  /* Behind a hub, changes are for writers and only on a server the hub can write to. Elsewhere always. */
+  const mayChange = !hub || (!matrixMode && canWrite && writable.includes(chosen))
+  const hubServers = nodes.map((n) => ({ id: n.name, writable: writable.includes(n.name) }))
+  const [hubView, setHubView] = useState<'matrix' | 'store'>('matrix')
+  const [fileDialog, setFileDialog] = useState(false)
+  const [importDialog, setImportDialog] = useState(false)
+  const [profilesDialog, setProfilesDialog] = useState(false)
+  const [fileRequest, setFileRequest] = useState<RunRequest | null>(null)
+  const [refresh, setRefresh] = useState(0)
+  useEffect(() => {
+    if (!hub) return
+    try {
+      localStorage.setItem('hubAppsNode', node)
+    } catch {
+      // Not remembered; the choice still works.
+    }
+  }, [hub, node])
+
   const [apps, setApps] = useState<InstalledApp[] | null>(null)
   const [alert, setAlert] = useState<AlertState | null>(null)
   const [modal, setModal] = useState<Modal | null>(null)
@@ -67,7 +128,7 @@ export function Apps({ token }: { token: string | null }) {
   const [pendingUninstall, setPendingUninstall] = useState<string | null>(null)
 
   const reload = useCallback(async () => {
-    const outcome = await listApps(token)
+    const outcome = await listApps(token, requestNode)
     if (outcome.kind === 'ok') {
       setApps(outcome.data.response.apps)
       setFailure(false)
@@ -87,11 +148,11 @@ export function Apps({ token }: { token: string | null }) {
     setApps([])
     setFailure(true)
     setAlert(error(outcome))
-  }, [token])
+  }, [token, requestNode])
 
   useEffect(() => {
-    void reload()
-  }, [reload])
+    if (!matrixMode) void reload()
+  }, [reload, matrixMode])
 
   /*
   apps.js:425-449. The confirmation and the alert are upstream literals.
@@ -103,7 +164,7 @@ export function Apps({ token }: { token: string | null }) {
   */
   async function uninstall(name: string) {
     setBusy(name)
-    const outcome = await uninstallApp(token, name)
+    const outcome = await uninstallApp(token, name, requestNode)
     setBusy(null)
 
     if (outcome.kind !== 'ok') {
@@ -123,7 +184,7 @@ export function Apps({ token }: { token: string | null }) {
     if (!app.updateUrl) return
 
     setBusy(app.name)
-    const outcome = await downloadAndUpdate(token, app.name, app.updateUrl)
+    const outcome = await downloadAndUpdate(token, app.name, app.updateUrl, requestNode)
     setBusy(null)
 
     if (outcome.kind !== 'ok') {
@@ -142,7 +203,7 @@ export function Apps({ token }: { token: string | null }) {
   // cluster's primary node. `config` can come null; the editor is left empty.
   async function openConfig(name: string) {
     setBusy(name)
-    const outcome = await getAppConfig(token, name)
+    const outcome = await getAppConfig(token, name, requestNode)
     setBusy(null)
 
     if (outcome.kind !== 'ok') {
@@ -169,13 +230,78 @@ export function Apps({ token }: { token: string | null }) {
   */
   const { rows: sortedApps, toggle } = useSort({ name: (a: InstalledApp) => a.name }, apps ?? [])
 
+  /* The file of every baseline, saved by the browser. */
+  async function exportFile() {
+    const outcome = await exportBaselines(token)
+    if (outcome.kind !== 'ok') {
+      setAlert(error(outcome))
+      return
+    }
+    const url = URL.createObjectURL(new Blob([outcome.data.text], { type: 'application/json' }))
+    const link = document.createElement('a')
+    link.href = url
+    link.download = outcome.data.filename
+    link.click()
+    URL.revokeObjectURL(url)
+  }
+
+  const selector = hub ? (
+    <InChrome>
+      <ClusterNodeSelect nodes={nodes} initialised={clusterInitialised} aggregate value={chosen} onChange={setNode} />
+    </InChrome>
+  ) : null
+
+  if (matrixMode) {
+    return (
+      <>
+        {selector}
+        <Notifier notice={alert} onClose={() => setAlert(null)} />
+        <SectionHeader
+          title="Apps"
+          subtitle="DNS apps on every server"
+          actions={
+            canWrite ? (
+              <>
+                <Button variant="primary" onClick={() => setHubView((v) => (v === 'store' ? 'matrix' : 'store'))}>
+                  {hubView === 'store' ? 'Back to apps' : 'App Store'}
+                </Button>
+                <Button onClick={() => setFileDialog(true)}>Install from file</Button>
+                <Button onClick={() => void exportFile()}>Export baselines</Button>
+                <Button onClick={() => setImportDialog(true)}>Import baselines</Button>
+                {isAdmin && <Button onClick={() => setProfilesDialog(true)}>Profiles</Button>}
+              </>
+            ) : undefined
+          }
+        />
+        {hubView === 'store' && canWrite ? (
+          <HubStore token={token} servers={hubServers} canWrite={canWrite} onChanged={() => setRefresh((n) => n + 1)} />
+        ) : (
+          <AppsMatrix token={token} canWrite={canWrite} refresh={refresh} />
+        )}
+        {isAdmin && <Profiles open={profilesDialog} onOpenChange={setProfilesDialog} token={token} servers={hubServers} />}
+        <ImportBaselines open={importDialog} onOpenChange={setImportDialog} token={token} onImported={() => setRefresh((n) => n + 1)} />
+        <HubInstallFile open={fileDialog} onOpenChange={setFileDialog} token={token} servers={hubServers} onRequest={setFileRequest} />
+        {fileRequest !== null && (
+          <RunFlow
+            request={fileRequest}
+            onClose={(changed) => {
+              setFileRequest(null)
+              if (changed) setRefresh((n) => n + 1)
+            }}
+          />
+        )}
+      </>
+    )
+  }
+
   return (
     <>
+      {selector}
       <Notifier notice={alert} onClose={() => setAlert(null)} />
 
       <SectionHeader
         title="Apps"
-        subtitle="DNS apps installed on this server"
+        subtitle={hub ? `DNS apps installed on ${chosen}` : 'DNS apps installed on this server'}
         labels={
           withUpdate > 0 ? (
             <Tag tone="warn">
@@ -184,12 +310,14 @@ export function Apps({ token }: { token: string | null }) {
           ) : undefined
         }
         actions={
-          <>
-            <Button variant="primary" onClick={() => setTab('store')}>
-              App Store
-            </Button>
-            <Button onClick={() => setModal({ kind: 'install' })}>Install from file</Button>
-          </>
+          !mayChange ? undefined : (
+            <>
+              <Button variant="primary" onClick={() => setTab('store')}>
+                App Store
+              </Button>
+              <Button onClick={() => setModal({ kind: 'install' })}>Install from file</Button>
+            </>
+          )
         }
       />
 
@@ -204,18 +332,20 @@ export function Apps({ token }: { token: string | null }) {
         />
       )}
 
-      <PageTabs
-        label="Apps"
-        tabs={[
-          { id: 'installed', label: 'Installed', count: apps?.length },
-          { id: 'store', label: 'App Store' },
-        ]}
-        active={tab}
-        onChoose={setTab}
-      />
+      {mayChange && (
+        <PageTabs
+          label="Apps"
+          tabs={[
+            { id: 'installed', label: 'Installed', count: apps?.length },
+            { id: 'store', label: 'App Store' },
+          ]}
+          active={tab}
+          onChoose={setTab}
+        />
+      )}
 
-      {tab === 'store' ? (
-        <StoreApps token={token} onChanged={() => void reload()} />
+      {tab === 'store' && mayChange ? (
+        <StoreApps token={token} node={requestNode} onChanged={() => void reload()} />
       ) : apps === null ? (
         <Loading />
       ) : failure ? (
@@ -227,13 +357,16 @@ export function Apps({ token }: { token: string | null }) {
         <Empty
           title="No apps installed"
           actions={
-            <Button variant="primary" onClick={() => setTab('store')}>
-              Open App Store
-            </Button>
+            !mayChange ? undefined : (
+              <Button variant="primary" onClick={() => setTab('store')}>
+                Open App Store
+              </Button>
+            )
           }
         >
           DNS Apps add behaviour to the server (query logging, advanced blocking, split horizon)
-          without touching the base configuration. Open the store to see what is available.
+          without touching the base configuration.
+          {!mayChange && hub ? ' Install apps in the console of this server.' : ' Open the store to see what is available.'}
         </Empty>
       ) : (
         <Panel title="Installed" description="Version, DNS apps and status">
@@ -258,6 +391,7 @@ export function Apps({ token }: { token: string | null }) {
                 key={app.name}
                 app={app}
                 busy={busy === app.name}
+                readOnly={!mayChange}
                 onConfig={() => void openConfig(app.name)}
                 onUpdate={() => setModal({ kind: 'update', name: app.name })}
                 onStoreUpdate={() => void updateFromStore(app)}
@@ -281,6 +415,7 @@ export function Apps({ token }: { token: string | null }) {
         open={modal?.kind === 'install'}
         onOpenChange={(o) => setModal(o ? { kind: 'install' } : null)}
         token={token}
+        node={requestNode}
         onInstalled={(name) => {
           setModal(null)
           setAlert({
@@ -296,6 +431,7 @@ export function Apps({ token }: { token: string | null }) {
         onOpenChange={(o) => setModal(o && modal?.kind === 'update' ? modal : null)}
         token={token}
         name={modal?.kind === 'update' ? modal.name : ''}
+        node={requestNode}
         onUpdated={(name) => {
           setModal(null)
           setAlert({
@@ -311,6 +447,7 @@ export function Apps({ token }: { token: string | null }) {
         onOpenChange={(o) => setModal(o && modal?.kind === 'config' ? modal : null)}
         token={token}
         name={modal?.kind === 'config' ? modal.name : ''}
+        node={requestNode}
         initialConfig={modal?.kind === 'config' ? modal.config : ''}
         onSaved={(name) => {
           setModal(null)

src/screens/apps/AppsHistory.test.tsx

@@ -0,0 +1,288 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { act, render, screen, waitFor, within } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { AppConfigSync } from './AppConfigSync'
+import { AppsMatrix } from './AppsMatrix'
+import { Apps } from './Apps'
+import { ImportBaselines } from './ImportBaselines'
+import { DriftStatus } from '../../app/DriftStatus'
+import { SlotProvider } from '../../app/ChromeSlot'
+import { HubMode } from '../../ui/ClusterNodeSelect'
+import { refreshHubDrift, resetHubDrift } from '../../api/hub-drift'
+import * as api from '../../api/apps'
+import type { ConfigCompare, DriftEntry, HistoryEntry, ServerConfigView } from '../../api/apps'
+
+beforeEach(() => resetHubDrift())
+afterEach(() => vi.restoreAllMocks())
+
+const A = JSON.stringify({ ttl: 30 }, null, 2)
+const B = JSON.stringify({ ttl: 60 }, null, 2)
+
+const view = (id: string, over: Partial<ServerConfigView> = {}): ServerConfigView => ({
+  id, ok: true, installed: true, writable: true, version: '1.0', text: A, hash: 'h-' + id, warnings: [], inSync: true, vsOthers: {}, vsBaseline: { kind: 'json', changes: [] }, ...over,
+})
+
+const COMPARE: ConfigCompare = {
+  name: 'Blocker',
+  baseline: { app: 'Blocker', text: A, updatedAt: '2026-10-31T08:00:00Z', updatedBy: 'alice' },
+  overrides: { paths: [], values: {} },
+  servers: [view('mdns'), view('ndns', { text: B, inSync: false })],
+}
+
+const SERVER_ENTRY: HistoryEntry = { id: '2026-10-31T09-00-00.000Z__ndns__before-apply.txt', at: '2026-10-31T09:00:00.000Z', server: 'ndns', kind: 'before-apply', size: 20, hash: 'aaaa' }
+const BASELINE_ENTRY: HistoryEntry = { id: '2026-10-31T08-00-00.000Z__baseline__before-save.txt', at: '2026-10-31T08:00:00.000Z', server: 'baseline', kind: 'before-save', size: 18, hash: 'bbbb' }
+
+const ok = (results: api.TargetResult[]) => ({ kind: 'ok', data: { response: { results } } }) as never
+
+function mockHistory(entries: HistoryEntry[]) {
+  vi.spyOn(api, 'getConfigCompare').mockResolvedValue({ kind: 'ok', data: { response: COMPARE } } as never)
+  vi.spyOn(api, 'getHistory').mockResolvedValue({ kind: 'ok', data: { response: { entries } } } as never)
+  vi.spyOn(api, 'getHistoryEntry').mockImplementation((async (_t: string | null, _n: string, id: string) => ({
+    kind: 'ok',
+    data: { response: { entry: entries.find((e) => e.id === id), text: id.includes('baseline') ? '{"old":true}' : '{"ttl":10}' } },
+  })) as never)
+  vi.spyOn(api, 'getHistoryDiff').mockImplementation((async (_t: string | null, _n: string, id: string, server: string) => ({
+    kind: 'ok',
+    data: {
+      response: {
+        entry: entries.find((e) => e.id === id), server, writable: true, current: B, currentHash: `hash-${server}`, version: '{"ttl":10}',
+        changes: { kind: 'json', changes: [{ path: 'ttl', type: 'changed', a: 60, b: 10 }] }, unchanged: false,
+      },
+    },
+  })) as never)
+}
+
+async function openHistory(canWrite: boolean) {
+  render(<AppConfigSync token="t" name="Blocker" canWrite={canWrite} onClose={() => {}} onChanged={() => {}} />)
+  await userEvent.click(await screen.findByRole('tab', { name: 'History' }))
+}
+
+describe('History tab', () => {
+  it('has an empty state', async () => {
+    mockHistory([])
+    await openHistory(false)
+    expect(await screen.findByText('No history yet')).toBeInTheDocument()
+  })
+
+  it('lists the saved versions and shows one with its difference from a server', async () => {
+    mockHistory([SERVER_ENTRY, BASELINE_ENTRY])
+    await openHistory(false)
+    const list = await screen.findByRole('list', { name: 'History' })
+    expect(within(list).getAllByRole('button')).toHaveLength(2)
+    expect(within(list).getByText('Before an apply')).toBeInTheDocument()
+    expect(within(list).getByText('Old baseline')).toBeInTheDocument()
+    await userEvent.click(within(list).getByRole('button', { name: /ndns Before an apply/ }))
+    expect(await screen.findByLabelText('Version text')).toHaveValue('{"ttl":10}')
+    expect(await screen.findByText('ttl')).toBeInTheDocument()
+    // Compared with the server the version came from.
+    expect(api.getHistoryDiff).toHaveBeenCalledWith('t', 'Blocker', SERVER_ENTRY.id, 'ndns')
+    // A reader may look, not act.
+    expect(screen.queryByRole('button', { name: 'Roll back…' })).not.toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Restore as baseline' })).not.toBeInTheDocument()
+  })
+
+  it('shows an error with Retry', async () => {
+    mockHistory([])
+    const spy = vi.spyOn(api, 'getHistory').mockResolvedValueOnce({ kind: 'error', message: 'No such app.' } as never)
+    await openHistory(false)
+    expect(await screen.findByText(/Could not read the history/)).toHaveTextContent('No such app.')
+    spy.mockResolvedValue({ kind: 'ok', data: { response: { entries: [] } } } as never)
+    await userEvent.click(screen.getByRole('button', { name: 'Retry' }))
+    expect(await screen.findByText('No history yet')).toBeInTheDocument()
+  })
+
+  it('rolls back after a preview, with the preview hash, and shows the results', async () => {
+    mockHistory([SERVER_ENTRY])
+    const rollback = vi.spyOn(api, 'rollbackHistory').mockResolvedValue(ok([{ server: 'ndns', ok: true, message: 'Applied and verified.' }]))
+    await openHistory(true)
+    await userEvent.click(await screen.findByRole('button', { name: /ndns Before an apply/ }))
+    await userEvent.click(await screen.findByRole('button', { name: 'Roll back…' }))
+    const picker = await screen.findByRole('dialog', { name: 'Roll back Blocker' })
+    // The version's own server is ticked already, and only it.
+    expect(within(picker).getByRole('checkbox', { name: 'ndns' })).toBeChecked()
+    expect(within(picker).getByRole('checkbox', { name: 'mdns' })).not.toBeChecked()
+    await userEvent.click(within(picker).getByRole('button', { name: 'Preview' }))
+    const review = await screen.findByRole('dialog', { name: 'Roll back Blocker: review' })
+    expect(within(review).getByText('Will change')).toBeInTheDocument()
+    expect(rollback).not.toHaveBeenCalled()
+    await userEvent.click(within(review).getByRole('button', { name: 'Roll back' }))
+    expect(rollback).toHaveBeenCalledWith('t', 'Blocker', SERVER_ENTRY.id, ['ndns'], { ndns: 'hash-ndns' })
+    const results = await screen.findByRole('dialog', { name: 'Roll back Blocker: results' })
+    expect(within(results).getByText('Applied and verified.')).toBeInTheDocument()
+  })
+
+  it('restores an old baseline as the baseline, for writers only', async () => {
+    mockHistory([BASELINE_ENTRY])
+    const restore = vi.spyOn(api, 'restoreBaseline').mockResolvedValue({ kind: 'ok', data: { response: { baseline: COMPARE.baseline } } } as never)
+    await openHistory(true)
+    await userEvent.click(await screen.findByRole('button', { name: /baseline Old baseline/ }))
+    expect(screen.queryByRole('button', { name: 'Roll back…' })).not.toBeInTheDocument()
+    await userEvent.click(await screen.findByRole('button', { name: 'Restore as baseline' }))
+    expect(restore).toHaveBeenCalledWith('t', 'Blocker', BASELINE_ENTRY.id)
+    expect(await screen.findByText('Baseline restored.')).toBeInTheDocument()
+  })
+})
+
+describe('DriftStatus in the header', () => {
+  const DRIFTS: DriftEntry[] = [
+    { kind: 'app', name: 'Blocker', app: 'Blocker', server: 'ndns', since: '2026-10-31T09:00:00Z', status: 'differs' },
+    { kind: 'app', name: 'Blocker', app: 'Blocker', server: 'mdns', since: '2026-10-31T09:00:00Z', status: 'unreadable' },
+  ]
+
+  async function load(entries: DriftEntry[]) {
+    vi.spyOn(api, 'getDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: 'x', apps: entries } } } as never)
+    await act(async () => refreshHubDrift('t'))
+  }
+
+  it('shows nothing without a drift and a count with one', async () => {
+    const { container } = render(<DriftStatus token="t" canWrite writable={['ndns']} />)
+    expect(container).toBeEmptyDOMElement()
+    await load(DRIFTS)
+    // Only a server that differs is counted; the unreadable one is listed in the dialog.
+    expect(await screen.findByRole('button', { name: 'Config drift' })).toHaveTextContent('1 config drift')
+  })
+
+  it('lists app, server, kind and since, and opens the config', async () => {
+    vi.spyOn(api, 'getConfigCompare').mockResolvedValue({ kind: 'ok', data: { response: COMPARE } } as never)
+    render(<DriftStatus token="t" canWrite={false} writable={[]} />)
+    await load(DRIFTS)
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    const row = await screen.findByRole('listitem', { name: 'Blocker on ndns' })
+    expect(within(row).getByText('Differs')).toBeInTheDocument()
+    expect(within(screen.getByRole('listitem', { name: 'Blocker on mdns' })).getByText('Cannot read')).toBeInTheDocument()
+    // A reader gets no actions.
+    expect(screen.queryByRole('button', { name: 'Accept as baseline' })).not.toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Revert to baseline' })).not.toBeInTheDocument()
+    await userEvent.click(within(row).getByRole('button', { name: 'Open config' }))
+    expect(await screen.findByRole('dialog', { name: 'App config: Blocker' })).toBeInTheDocument()
+  })
+
+  it('accepts a server config as the baseline after a confirmation and refreshes', async () => {
+    render(<DriftStatus token="t" canWrite writable={['ndns']} />)
+    await load(DRIFTS)
+    const accept = vi.spyOn(api, 'acceptDrift').mockResolvedValue({ kind: 'ok', data: { response: { baseline: COMPARE.baseline } } } as never)
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    await userEvent.click(await screen.findByRole('button', { name: 'Accept as baseline' }))
+    await userEvent.click(await screen.findByRole('button', { name: 'Accept' }))
+    expect(accept).toHaveBeenCalledWith('t', 'Blocker', 'ndns')
+    await waitFor(() => expect(api.getDrift).toHaveBeenCalledTimes(2))
+  })
+
+  it('reverts to the baseline through the preview', async () => {
+    render(<DriftStatus token="t" canWrite writable={['ndns']} />)
+    await load(DRIFTS)
+    const preview = vi.spyOn(api, 'previewConfig').mockResolvedValue({
+      kind: 'ok',
+      data: { response: { targets: [{ server: 'ndns', ok: true, writable: true, current: B, currentHash: 'hash-ndns', effective: A, unchanged: false, warnings: [], changes: { kind: 'json', changes: [{ path: 'ttl', type: 'changed', a: 60, b: 30 }] } }] } },
+    } as never)
+    const apply = vi.spyOn(api, 'applyConfig').mockResolvedValue(ok([{ server: 'ndns', ok: true, message: 'Applied and verified.' }]))
+    vi.spyOn(api, 'checkDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: 'y', apps: [] } } } as never)
+    await userEvent.click(await screen.findByRole('button', { name: 'Config drift' }))
+    await userEvent.click(await screen.findByRole('button', { name: 'Revert to baseline' }))
+    const picker = await screen.findByRole('dialog', { name: 'Apply the baseline of Blocker' })
+    expect(within(picker).getByRole('checkbox', { name: 'ndns' })).toBeChecked()
+    await userEvent.click(within(picker).getByRole('button', { name: 'Preview' }))
+    expect(preview).toHaveBeenCalledWith('t', 'Blocker', ['ndns'])
+    await userEvent.click(within(await screen.findByRole('dialog', { name: 'Apply the baseline of Blocker: review' })).getByRole('button', { name: 'Apply' }))
+    expect(apply).toHaveBeenCalledWith('t', 'Blocker', ['ndns'], { ndns: 'hash-ndns' })
+    expect(await screen.findByText('Applied and verified.')).toBeInTheDocument()
+  })
+})
+
+describe('Drift badge in the matrix', () => {
+  it('marks the server cell whose config differs', async () => {
+    vi.spyOn(api, 'getAppsMatrix').mockResolvedValue({
+      kind: 'ok',
+      data: {
+        response: {
+          servers: [{ id: 'mdns', ok: true, writable: true }, { id: 'ndns', ok: true, writable: true }],
+          apps: [{ name: 'Blocker', description: '', configs: 'same', versions: 'same', servers: { mdns: { installed: true, version: '1.0', configStatus: 'read' }, ndns: { installed: true, version: '1.0', configStatus: 'read' } } }],
+        },
+      },
+    } as never)
+    vi.spyOn(api, 'getDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: 'x', apps: [{ kind: 'app', name: 'Blocker', app: 'Blocker', server: 'ndns', since: 'z', status: 'differs' }, { kind: 'app', name: 'Blocker', app: 'Blocker', server: 'mdns', since: 'z', status: 'unreadable' }] } } } as never)
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Blocker')
+    await waitFor(() => expect(screen.getAllByText('Drift')).toHaveLength(1))
+    const row = screen.getByText('Blocker').closest('tr') as HTMLElement
+    expect(within(row).getByText('Drift')).toBeInTheDocument()
+  })
+})
+
+describe('Export and import of baselines', () => {
+  const NODES = [{ name: 'mdns', type: 'Primary' }]
+
+  function mountApps(canWrite: boolean) {
+    vi.spyOn(api, 'getAppsMatrix').mockResolvedValue({ kind: 'ok', data: { response: { servers: [], apps: [] } } } as never)
+    vi.spyOn(api, 'getDrift').mockResolvedValue({ kind: 'ok', data: { response: { checkedAt: null, apps: [] } } } as never)
+    return render(
+      <HubMode.Provider value>
+        <SlotProvider>
+          {(ref) => (
+            <>
+              <div ref={ref} />
+              <Apps token="t" nodes={NODES} clusterInitialised canWrite={canWrite} writable={['mdns']} />
+            </>
+          )}
+        </SlotProvider>
+      </HubMode.Provider>,
+    )
+  }
+
+  it('offers both to writers only', async () => {
+    const { unmount } = mountApps(false)
+    await screen.findByText('No apps installed')
+    expect(screen.queryByRole('button', { name: 'Export baselines' })).not.toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Import baselines' })).not.toBeInTheDocument()
+    unmount()
+    mountApps(true)
+    expect(await screen.findByRole('button', { name: 'Export baselines' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Import baselines' })).toBeInTheDocument()
+  })
+
+  it('downloads the file the hub sends', async () => {
+    mountApps(true)
+    const exported = vi.spyOn(api, 'exportBaselines').mockResolvedValue({ kind: 'ok', data: { filename: 'hub-baselines-2026-10-31.json', text: '{"hub":"technitium-hub"}' } } as never)
+    const create = vi.fn(() => 'blob:x')
+    Object.assign(URL, { createObjectURL: create, revokeObjectURL: vi.fn() })
+    const click = vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => {})
+    await userEvent.click(await screen.findByRole('button', { name: 'Export baselines' }))
+    await waitFor(() => expect(click).toHaveBeenCalled())
+    expect(exported).toHaveBeenCalledWith('t')
+    expect(create).toHaveBeenCalled()
+  })
+
+  it('checks the file first, summarises it, and imports only on confirmation', async () => {
+    const spy = vi
+      .spyOn(api, 'importBaselines')
+      .mockResolvedValueOnce({ kind: 'ok', data: { response: { applied: false, apps: [{ app: 'Blocker', status: 'changed', ignoredServers: [] }, { app: 'Wild IP', status: 'new', ignoredServers: ['zzz'] }, { app: 'Split', status: 'same', ignoredServers: [] }] } } } as never)
+      .mockResolvedValueOnce({ kind: 'ok', data: { response: { applied: true, apps: [{ app: 'Blocker', status: 'changed', ignoredServers: [] }] } } } as never)
+    const onImported = vi.fn()
+    render(<ImportBaselines open onOpenChange={() => {}} token="t" onImported={onImported} />)
+    const file = new File(['{"hub":"technitium-hub"}'], 'b.json', { type: 'application/json' })
+    await userEvent.upload(screen.getByLabelText('Baselines file'), file)
+    await userEvent.click(screen.getByRole('button', { name: 'Check file' }))
+    const list = await screen.findByRole('list', { name: 'Apps in the file' })
+    expect(within(list).getByText('Changed')).toBeInTheDocument()
+    expect(within(list).getByText('New')).toBeInTheDocument()
+    expect(within(list).getByText('Same')).toBeInTheDocument()
+    expect(within(list).getByText(/zzz are ignored/)).toBeInTheDocument()
+    expect(spy).toHaveBeenCalledTimes(1)
+    expect(spy.mock.calls[0][2]).toBe(false)
+    await userEvent.click(screen.getByRole('button', { name: 'Import' }))
+    await waitFor(() => expect(spy).toHaveBeenCalledTimes(2))
+    expect(spy.mock.calls[1][2]).toBe(true)
+    expect(await screen.findByText('Imported.')).toBeInTheDocument()
+    expect(onImported).toHaveBeenCalled()
+  })
+
+  it('shows why a file was refused and imports nothing', async () => {
+    const spy = vi.spyOn(api, 'importBaselines').mockResolvedValue({ kind: 'error', message: 'This is not a baselines file of the hub.' } as never)
+    render(<ImportBaselines open onOpenChange={() => {}} token="t" onImported={() => {}} />)
+    await userEvent.upload(screen.getByLabelText('Baselines file'), new File(['{}'], 'x.json'))
+    await userEvent.click(screen.getByRole('button', { name: 'Check file' }))
+    expect(await screen.findByText('This is not a baselines file of the hub.')).toBeInTheDocument()
+    expect(spy).toHaveBeenCalledTimes(1)
+    expect(screen.queryByRole('button', { name: 'Import' })).not.toBeInTheDocument()
+  })
+})

src/screens/apps/AppsMatrix.test.tsx

@@ -0,0 +1,243 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { render, screen, within } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { AppsMatrix } from './AppsMatrix'
+import { Apps } from './Apps'
+import { SlotProvider } from '../../app/ChromeSlot'
+import { HubMode } from '../../ui/ClusterNodeSelect'
+import * as api from '../../api/apps'
+import type { AppsMatrixData } from '../../api/apps'
+
+afterEach(() => {
+  vi.restoreAllMocks()
+  localStorage.clear()
+})
+
+const DATA: AppsMatrixData = {
+  servers: [
+    { id: 'mdns', ok: true, writable: false },
+    { id: 'ndns', ok: true, writable: false },
+  ],
+  apps: [
+    {
+      name: 'Advanced Blocking',
+      description: 'Group based blocking',
+      configs: 'different',
+      diffCount: 2,
+      versions: 'same',
+      servers: {
+        mdns: { installed: true, version: '10.1', configStatus: 'read', configHash: 'aaaa' },
+        ndns: { installed: true, version: '10.1', configStatus: 'read', configHash: 'bbbb' },
+      },
+    },
+    {
+      name: 'Query Logs (Sqlite)',
+      description: 'Logs queries',
+      configs: 'same',
+      versions: 'different',
+      servers: {
+        mdns: { installed: true, version: '7.0', configStatus: 'read', configHash: 'cccc' },
+        ndns: { installed: true, version: '7.1', updateAvailable: true, updateVersion: '7.2', configStatus: 'read', configHash: 'cccc' },
+      },
+    },
+    {
+      name: 'Split Horizon',
+      description: '',
+      configs: 'single',
+      versions: 'single',
+      servers: {
+        mdns: { installed: true, version: '9.0', configStatus: 'read', configHash: 'dddd' },
+        ndns: { installed: false, configStatus: 'none' },
+      },
+    },
+    {
+      name: 'Wild IP',
+      description: '',
+      configs: 'unknown',
+      versions: 'same',
+      servers: {
+        mdns: { installed: true, version: '5.1', configStatus: 'unreadable', configError: 'Access was denied' },
+        ndns: { installed: true, version: '5.1', configStatus: 'read', configHash: 'eeee' },
+      },
+    },
+  ],
+}
+
+function mockMatrix(data: AppsMatrixData = DATA) {
+  return vi.spyOn(api, 'getAppsMatrix').mockResolvedValue({ kind: 'ok', data: { response: data } } as never)
+}
+
+const row = (name: string) => screen.getByText(name).closest('tr') as HTMLElement
+
+describe('AppsMatrix', () => {
+  it('draws a row per app and a column per server', async () => {
+    mockMatrix()
+    render(<AppsMatrix token="t" />)
+    expect(await screen.findByText('Advanced Blocking')).toBeInTheDocument()
+    expect(screen.getByRole('columnheader', { name: 'mdns' })).toBeInTheDocument()
+    expect(screen.getByRole('columnheader', { name: 'ndns' })).toBeInTheDocument()
+    expect(within(row('Split Horizon')).getByText('v9.0')).toBeInTheDocument()
+    expect(within(row('Split Horizon')).getByText('Not installed')).toBeInTheDocument()
+    expect(within(row('Query Logs (Sqlite)')).getByText('v7.1')).toBeInTheDocument()
+    expect(within(row('Query Logs (Sqlite)')).getByText('Update v7.2')).toBeInTheDocument()
+  })
+
+  it('shows the config state of each app', async () => {
+    mockMatrix()
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Advanced Blocking')
+    expect(within(row('Advanced Blocking')).getByText('Different · 2')).toBeInTheDocument()
+    expect(within(row('Query Logs (Sqlite)')).getByText('Same')).toBeInTheDocument()
+    expect(within(row('Split Horizon')).getByText('Single server')).toBeInTheDocument()
+    expect(within(row('Wild IP')).getByText('Unknown')).toBeInTheDocument()
+  })
+
+  it('explains an unknown config in a tooltip', async () => {
+    mockMatrix()
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Advanced Blocking')
+    await userEvent.hover(within(row('Wild IP')).getByText('Unknown'))
+    expect(await screen.findByRole('tooltip')).toHaveTextContent('mdns: Access was denied')
+  })
+
+  it('filters with counts', async () => {
+    mockMatrix()
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Advanced Blocking')
+    const filters = screen.getByRole('group', { name: 'Filter apps' })
+    expect(within(filters).getByRole('button', { name: 'All · 4' })).toBeInTheDocument()
+    expect(within(filters).getByRole('button', { name: 'Version differs · 1' })).toBeInTheDocument()
+    expect(within(filters).getByRole('button', { name: 'Config differs · 1' })).toBeInTheDocument()
+    expect(within(filters).getByRole('button', { name: 'Update available · 1' })).toBeInTheDocument()
+
+    await userEvent.click(within(filters).getByRole('button', { name: 'Config differs · 1' }))
+    expect(screen.getByText('Advanced Blocking')).toBeInTheDocument()
+    expect(screen.queryByText('Split Horizon')).not.toBeInTheDocument()
+
+    await userEvent.click(within(filters).getByRole('button', { name: 'Update available · 1' }))
+    expect(screen.getByText('Query Logs (Sqlite)')).toBeInTheDocument()
+    expect(screen.queryByText('Advanced Blocking')).not.toBeInTheDocument()
+  })
+
+  it('marks a server that did not answer as unknown', async () => {
+    mockMatrix({
+      servers: [{ id: 'mdns', ok: true, writable: false }, { id: 'ndns', ok: false, writable: false, error: 'timed out' }],
+      apps: [
+        {
+          name: 'Split Horizon',
+          description: '',
+          configs: 'unknown',
+          versions: 'single',
+          servers: {
+            mdns: { installed: true, version: '9.0', configStatus: 'read' },
+            ndns: { installed: false, configStatus: 'unreadable', configError: 'ndns did not answer' },
+          },
+        },
+      ],
+    })
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Split Horizon')
+    expect(within(row('Split Horizon')).getByText('v9.0')).toBeInTheDocument()
+    expect(within(row('Split Horizon')).getAllByText('Unknown')).toHaveLength(2)
+  })
+
+  it('says so when no filter matches, with a way back', async () => {
+    mockMatrix({ ...DATA, apps: [DATA.apps[2]] })
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Split Horizon')
+    await userEvent.click(screen.getByRole('button', { name: 'Config differs · 0' }))
+    expect(screen.getByText('No apps match')).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Show all' }))
+    expect(screen.getByText('Split Horizon')).toBeInTheDocument()
+  })
+
+  it('has an empty state with an action', async () => {
+    const spy = mockMatrix({ servers: DATA.servers, apps: [] })
+    render(<AppsMatrix token="t" />)
+    expect(await screen.findByText('No apps installed')).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Refresh' }))
+    expect(spy).toHaveBeenCalledTimes(2)
+  })
+
+  it('shows loading and then an error with Retry', async () => {
+    const spy = vi.spyOn(api, 'getAppsMatrix').mockResolvedValueOnce({ kind: 'error', message: 'No server answered.' } as never)
+    render(<AppsMatrix token="t" />)
+    expect(screen.getByRole('status')).toHaveTextContent('Loading')
+    expect(await screen.findByText(/Could not load the apps/)).toHaveTextContent('No server answered.')
+    spy.mockResolvedValue({ kind: 'ok', data: { response: DATA } } as never)
+    await userEvent.click(screen.getByRole('button', { name: 'Retry' }))
+    expect(await screen.findByText('Advanced Blocking')).toBeInTheDocument()
+  })
+})
+
+describe('Apps behind a hub', () => {
+  const INSTALLED = {
+    name: 'Split Horizon',
+    description: 'Different answers',
+    version: '9.0',
+    dnsApps: [],
+  }
+  const NODES = [{ name: 'mdns', type: 'Primary' }, { name: 'ndns', type: 'Secondary' }]
+
+  function mount(hub: boolean) {
+    return render(
+      <HubMode.Provider value={hub}>
+        <SlotProvider>
+          {(ref) => (
+            <>
+              <div ref={ref} />
+              <Apps token="t" nodes={NODES} clusterInitialised />
+            </>
+          )}
+        </SlotProvider>
+      </HubMode.Provider>,
+    )
+  }
+
+  it('shows the selector and the matrix for All servers', async () => {
+    const matrix = mockMatrix()
+    const list = vi.spyOn(api, 'listApps')
+    mount(true)
+    expect(await screen.findByText('Advanced Blocking')).toBeInTheDocument()
+    expect(await screen.findByRole('combobox', { name: 'Server' })).toHaveTextContent('All servers')
+    expect(matrix).toHaveBeenCalled()
+    expect(list).not.toHaveBeenCalled()
+    expect(screen.queryByRole('button', { name: 'App Store' })).not.toBeInTheDocument()
+  })
+
+  it('shows one server read-only and remembers the choice', async () => {
+    mockMatrix()
+    const list = vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    mount(true)
+    await userEvent.click(await screen.findByRole('combobox', { name: 'Server' }))
+    await userEvent.click(screen.getByRole('option', { name: 'ndns' }))
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    expect(list).toHaveBeenCalledWith('t', 'ndns')
+    expect(localStorage.getItem('hubAppsNode')).toBe('ndns')
+    for (const name of ['Config', 'Update', 'Store Update', 'App Store', 'Install from file']) {
+      expect(screen.queryByRole('button', { name })).not.toBeInTheDocument()
+    }
+    expect(screen.queryByRole('button', { name: /Actions: Split Horizon/ })).not.toBeInTheDocument()
+    expect(screen.queryByRole('tab', { name: /App Store/ })).not.toBeInTheDocument()
+  })
+
+  it('starts on the remembered server', async () => {
+    localStorage.setItem('hubAppsNode', 'mdns')
+    const list = vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    mount(true)
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    expect(list).toHaveBeenCalledWith('t', 'mdns')
+  })
+
+  it('is unchanged outside a hub', async () => {
+    const matrix = mockMatrix()
+    const list = vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    mount(false)
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    expect(list).toHaveBeenCalledWith('t', '')
+    expect(matrix).not.toHaveBeenCalled()
+    expect(screen.queryByRole('combobox', { name: 'Server' })).not.toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'App Store' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Config' })).toBeInTheDocument()
+  })
+})

src/screens/apps/AppsMatrix.tsx

@@ -0,0 +1,310 @@
+import { useCallback, useEffect, useMemo, useState } from 'react'
+import { getAppsMatrix, getDrift, listStoreApps, runHubApps, type AppsMatrixData, type HubRunAction, type MatrixApp } from '../../api/apps'
+import { Badge } from '@/components/reui/badge'
+import { Button } from '../../ui/Button'
+import { DataTable, type Col } from '../../ui/DataTable'
+import { Empty, Failure, Loading } from '../../ui/Empty'
+import { Panel } from '../../ui/Panel'
+import { Segmented } from '../../ui/Segmented'
+import { Menu } from '../../ui/Menu'
+import { Tag } from '../../ui/Tag'
+import { RunFlow, pickerServers, type RunRequest } from './RunFlow'
+import { AppConfigSync } from './AppConfigSync'
+import { Tooltip } from '../../ui/Tooltip'
+import { head } from '../../ui/Table'
+
+/*
+Behind a hub: one row per app, one column per server, and whether the configs agree.
+Read-only. The numbers come from `hub/apps/matrix`, which never sends a config, only its
+hash and the comparison, so nothing here can show a secret.
+*/
+
+type Filter = 'all' | 'version' | 'config' | 'update'
+
+/* 9.10 is newer than 9.9: compared part by part, numerically. */
+export function compareVersions(a: string, b: string): number {
+  const [x, y] = [a, b].map((v) => v.split('.').map((p) => Number.parseInt(p, 10) || 0))
+  for (let i = 0; i < Math.max(x.length, y.length); i++) {
+    const d = (x[i] ?? 0) - (y[i] ?? 0)
+    if (d !== 0) return d
+  }
+  return 0
+}
+
+const hasUpdate = (app: MatrixApp): boolean => Object.values(app.servers).some((c) => c.installed && c.updateAvailable === true)
+
+const FILTERS: Record<Filter, (app: MatrixApp) => boolean> = {
+  all: () => true,
+  version: (app) => app.versions === 'different',
+  config: (app) => app.configs === 'different',
+  update: hasUpdate,
+}
+
+/* Why a config's state is not known, from the cells and the servers that did not answer. */
+function unknownReason(app: MatrixApp, data: AppsMatrixData): string {
+  const reasons = new Set<string>()
+  for (const [id, cell] of Object.entries(app.servers)) {
+    if (cell.installed && cell.configStatus === 'unreadable') reasons.add(`${id}: ${cell.configError ?? 'config not readable'}`)
+  }
+  for (const s of data.servers) if (!s.ok) reasons.add(`${s.id}: ${s.error ?? 'did not answer'}`)
+  return [...reasons].join('; ') || 'The configs could not be compared.'
+}
+
+function ConfigState({ app, data, onOpen }: { app: MatrixApp; data: AppsMatrixData; onOpen: () => void }) {
+  switch (app.configs) {
+    case 'same':
+      return <Tag tone="ok">Same</Tag>
+    case 'different':
+      return (
+        <button type="button" className="cursor-pointer border-0 bg-transparent p-0" aria-label={`Compare the configs of ${app.name}`} onClick={onOpen}>
+          <Tag tone="warn">{app.diffCount === undefined ? 'Different' : `Different · ${app.diffCount}`}</Tag>
+        </button>
+      )
+    case 'single':
+      return <Badge variant="outline">Single server</Badge>
+    default:
+      return (
+        <Tooltip text={unknownReason(app, data)} placement="top">
+          <Badge variant="outline" tabIndex={0}>
+            Unknown
+          </Badge>
+        </Tooltip>
+      )
+  }
+}
+
+function ServerCell({ app, id, data, drifted }: { app: MatrixApp; id: string; data: AppsMatrixData; drifted: boolean }) {
+  const cell = app.servers[id]
+  if (data.servers.find((s) => s.id === id)?.ok === false) return <span className="text-sm text-muted-foreground">Unknown</span>
+  if (cell === undefined || !cell.installed) return <span className="text-sm text-muted-foreground">Not installed</span>
+  return (
+    <span className="flex flex-wrap items-center gap-2 text-sm">
+      <span>{cell.version === undefined ? 'Installed' : `v${cell.version}`}</span>
+      {drifted && <Tag tone="warn">Drift</Tag>}
+      {cell.updateAvailable === true && (
+        <Tag tone="warn">{cell.updateVersion === undefined ? 'Update' : `Update v${cell.updateVersion}`}</Tag>
+      )}
+    </span>
+  )
+}
+
+export function AppsMatrix({ token, canWrite = false, refresh = 0 }: { token: string | null; canWrite?: boolean; refresh?: number }) {
+  const [data, setData] = useState<AppsMatrixData | null>(null)
+  const [request, setRequest] = useState<RunRequest | null>(null)
+  const [configApp, setConfigApp] = useState<string | null>(null)
+  /** `app/server` pairs whose config differs from what the hub expects. */
+  const [drifted, setDrifted] = useState<Set<string>>(new Set())
+  /** What the store offers, by app: needed to know whether "Align versions" can be done. */
+  const [store, setStore] = useState<Record<string, string> | null>(null)
+  const [failure, setFailure] = useState<string | null>(null)
+  const [filter, setFilter] = useState<Filter>('all')
+
+  const load = useCallback(async () => {
+    const outcome = await getAppsMatrix(token)
+    if (outcome.kind === 'ok') {
+      setData(outcome.data.response)
+      setFailure(null)
+      return
+    }
+    setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+  }, [token])
+
+  useEffect(() => {
+    void load()
+    void getDrift(token)
+      .then((o) => {
+        if (o.kind === 'ok' && Array.isArray(o.data.response?.apps)) {
+          setDrifted(new Set(o.data.response.apps.filter((e) => e.kind === 'app' && e.status === 'differs').map((e) => `${e.name ?? e.app}\n${e.server}`)))
+        }
+      })
+      .catch(() => {})
+  }, [load, refresh, token])
+
+  useEffect(() => {
+    if (!canWrite) return
+    void listStoreApps(token).then((outcome) => {
+      if (outcome.kind === 'ok') setStore(Object.fromEntries(outcome.data.response.storeApps.map((s) => [s.name, s.version])))
+    })
+  }, [canWrite, token])
+
+  const counts = useMemo(() => {
+    const apps = data?.apps ?? []
+    return (Object.keys(FILTERS) as Filter[]).reduce(
+      (acc, f) => ({ ...acc, [f]: apps.filter(FILTERS[f]).length }),
+      {} as Record<Filter, number>,
+    )
+  }, [data])
+
+  const retry = (
+    <Button
+      variant="primary"
+      onClick={() => {
+        setFailure(null)
+        setData(null)
+        void load()
+      }}
+    >
+      Retry
+    </Button>
+  )
+
+  if (failure !== null && data === null) {
+    return (
+      <Failure>
+        <span className="flex flex-col items-center gap-3">
+          <span>Could not load the apps of the servers. {failure}</span>
+          {retry}
+        </span>
+      </Failure>
+    )
+  }
+  if (data === null) return <Loading>Loading the apps of every server…</Loading>
+
+  if (data.apps.length === 0) {
+    return (
+      <Empty
+        title="No apps installed"
+        actions={
+          <Button variant="primary" onClick={() => void load()}>
+            Refresh
+          </Button>
+        }
+      >
+        None of the servers that answered has a DNS app installed. Install apps in a server's own console.
+      </Empty>
+    )
+  }
+
+  const rows = data.apps.filter(FILTERS[filter])
+
+  /* The change menu of one app. Each item opens the server picker with the servers it makes sense on. */
+  function actions(app: MatrixApp, d: AppsMatrixData) {
+    const has = (id: string) => app.servers[id]?.installed === true
+    const installed = d.servers.filter((s) => has(s.id)).map((s) => s.id)
+    const outdated = installed.filter((id) => app.servers[id].updateAvailable === true)
+    const missing = d.servers.filter((s) => s.ok && !has(s.id)).map((s) => s.id)
+    const versions = installed.map((id) => app.servers[id].version ?? '0')
+    const newest = versions.reduce((best, v) => (compareVersions(v, best) > 0 ? v : best), versions[0] ?? '0')
+    const behind = installed.filter((id) => compareVersions(app.servers[id].version ?? '0', newest) < 0)
+    const canAlign = behind.length > 0 && store?.[app.name] === newest
+
+    const open = (
+      action: HubRunAction,
+      title: string,
+      confirmLabel: string,
+      preselected: string[],
+      allowed: string[],
+      why: string,
+      intro?: string,
+    ) =>
+      setRequest({
+        title,
+        intro,
+        confirmLabel,
+        servers: pickerServers(d.servers, Object.fromEntries(d.servers.filter((s) => !allowed.includes(s.id)).map((s) => [s.id, why]))),
+        preselected,
+        run: (targets) => runHubApps(token, action, app.name, targets),
+      })
+
+    return (
+      <Menu label={`Actions: ${app.name}`}>
+        {(close) => (
+          <>
+            <Button size="sm" onClick={() => { close(); setConfigApp(app.name) }}>
+              Config…
+            </Button>
+            {canWrite && (
+              <>
+            <Button size="sm" disabled={outdated.length === 0} onClick={() => { close(); open('update-store', `Update ${app.name}`, 'Update', outdated, installed, 'Not installed') }}>
+              Update on all
+            </Button>
+            <Button size="sm" disabled={missing.length === 0} onClick={() => { close(); open('install-store', `Install ${app.name}`, 'Install', missing, missing, 'Already installed') }}>
+              Install on missing servers
+            </Button>
+            <Button size="sm" disabled={!canAlign} onClick={() => { close(); open('update-store', `Align ${app.name} to v${newest}`, 'Update', behind, behind, 'Already at the newest version') }}>
+              Align versions
+            </Button>
+            <Button
+              size="sm"
+              variant="danger"
+              disabled={installed.length === 0}
+              onClick={() => {
+                close()
+                open('uninstall', `Uninstall ${app.name}`, 'Uninstall', [], installed, 'Not installed', "Each server's config of this app is saved to the hub's history before it is uninstalled.")
+              }}
+            >
+              Uninstall from…
+            </Button>
+              </>
+            )}
+          </>
+        )}
+      </Menu>
+    )
+  }
+  const columns: Col<MatrixApp>[] = [
+    {
+      id: 'name',
+      ...head({ children: 'App' }),
+      cell: (a) => (
+        <span className="flex flex-col gap-1">
+          <span className="text-sm font-medium">{a.name}</span>
+          {a.description !== '' && <span className="text-xs text-muted-foreground">{a.description}</span>}
+        </span>
+      ),
+    },
+    ...data.servers.map(
+      (s): Col<MatrixApp> => ({
+        id: `server-${s.id}`,
+        ...head({ children: s.id }),
+        cell: (a) => <ServerCell app={a} id={s.id} data={data} drifted={drifted.has(`${a.name}\n${s.id}`)} />,
+      }),
+    ),
+    { id: 'config', ...head({ children: 'Config' }), cell: (a) => <ConfigState app={a} data={data} onOpen={() => setConfigApp(a.name)} /> },
+    { id: 'actions', ...head({ children: <span className="sr-only">Actions</span> }), cell: (a) => actions(a, data) },
+  ]
+
+  return (
+    <div className="flex flex-col gap-4">
+      <Segmented
+        label="Filter apps"
+        active={filter}
+        onChoose={setFilter}
+        options={[
+          { id: 'all', label: `All · ${counts.all}` },
+          { id: 'version', label: `Version differs · ${counts.version}` },
+          { id: 'config', label: `Config differs · ${counts.config}` },
+          { id: 'update', label: `Update available · ${counts.update}` },
+        ]}
+      />
+      <Panel title="Apps by server" description="Installed version and whether the configs match">
+        {rows.length === 0 ? (
+          <Empty
+            title="No apps match"
+            actions={
+              <Button variant="primary" onClick={() => setFilter('all')}>
+                Show all
+              </Button>
+            }
+          >
+            No app fits this filter.
+          </Empty>
+        ) : (
+          <DataTable data={rows} columns={columns} getRowId={(a) => a.name} />
+        )}
+      </Panel>
+      {configApp !== null && (
+        <AppConfigSync token={token} name={configApp} canWrite={canWrite} onClose={() => setConfigApp(null)} onChanged={() => void load()} />
+      )}
+      {request !== null && (
+        <RunFlow
+          request={request}
+          onClose={(changed) => {
+            setRequest(null)
+            if (changed) void load()
+          }}
+        />
+      )}
+    </div>
+  )
+}

src/screens/apps/AppsWrite.test.tsx

@@ -0,0 +1,292 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { render, screen, waitFor, within } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { AppsMatrix } from './AppsMatrix'
+import { Apps } from './Apps'
+import { SlotProvider } from '../../app/ChromeSlot'
+import { HubMode } from '../../ui/ClusterNodeSelect'
+import * as api from '../../api/apps'
+import type { AppsMatrixData, InstalledApp } from '../../api/apps'
+
+afterEach(() => {
+  vi.restoreAllMocks()
+  localStorage.clear()
+})
+
+/* mdns has Blocker v1.0 with an update; ndns has v2.0; Wild IP is only on mdns. ndns has no admin token. */
+const DATA: AppsMatrixData = {
+  servers: [
+    { id: 'mdns', ok: true, writable: true },
+    { id: 'ndns', ok: true, writable: false },
+  ],
+  apps: [
+    {
+      name: 'Blocker',
+      description: '',
+      configs: 'same',
+      versions: 'different',
+      servers: {
+        mdns: { installed: true, version: '1.0', updateAvailable: true, updateVersion: '2.0', configStatus: 'read' },
+        ndns: { installed: true, version: '2.0', configStatus: 'read' },
+      },
+    },
+    {
+      name: 'Wild IP',
+      description: '',
+      configs: 'single',
+      versions: 'single',
+      servers: {
+        mdns: { installed: true, version: '5.1', configStatus: 'read' },
+        ndns: { installed: false, configStatus: 'none' },
+      },
+    },
+  ],
+}
+
+const ok = (results: api.TargetResult[]) => ({ kind: 'ok', data: { response: { results } } }) as never
+
+function mockReads(data: AppsMatrixData = DATA, store = [{ name: 'Blocker', version: '2.0' }]) {
+  const matrix = vi.spyOn(api, 'getAppsMatrix').mockResolvedValue({ kind: 'ok', data: { response: data } } as never)
+  vi.spyOn(api, 'listStoreApps').mockResolvedValue({ kind: 'ok', data: { response: { storeApps: store } } } as never)
+  return matrix
+}
+
+async function openMenu(app: string) {
+  await userEvent.click(await screen.findByRole('button', { name: `Actions: ${app}` }))
+}
+
+describe('matrix actions', () => {
+  it('gives a reader a menu with Config only and a writer all the actions', async () => {
+    mockReads()
+    const { unmount } = render(<AppsMatrix token="t" />)
+    await openMenu('Blocker')
+    expect(screen.getByRole('menuitem', { name: 'Config…' })).toBeInTheDocument()
+    for (const name of ['Update on all', 'Install on missing servers', 'Align versions', 'Uninstall from…']) {
+      expect(screen.queryByRole('menuitem', { name })).not.toBeInTheDocument()
+      expect(screen.queryByRole('button', { name })).not.toBeInTheDocument()
+    }
+    unmount()
+
+    render(<AppsMatrix token="t" canWrite />)
+    expect(await screen.findByRole('button', { name: 'Actions: Blocker' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Actions: Wild IP' })).toBeInTheDocument()
+  })
+
+  it('enables only the actions that make sense for the row', async () => {
+    mockReads()
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    expect(screen.getByRole('menuitem', { name: 'Update on all' })).toBeEnabled()
+    expect(screen.getByRole('button', { name: 'Install on missing servers' })).toBeDisabled()
+    expect(screen.getByRole('menuitem', { name: 'Align versions' })).toBeEnabled()
+    expect(screen.getByRole('menuitem', { name: 'Uninstall from…' })).toBeEnabled()
+  })
+
+  it('offers Align versions only when the store has the newest installed version', async () => {
+    mockReads(DATA, [{ name: 'Blocker', version: '1.5' }])
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    await waitFor(() => expect(api.listStoreApps).toHaveBeenCalled())
+    expect(screen.getByRole('button', { name: 'Align versions' })).toBeDisabled()
+  })
+
+  it('offers Install on missing servers for an app that is not everywhere', async () => {
+    mockReads()
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Wild IP')
+    expect(screen.getByRole('menuitem', { name: 'Install on missing servers' })).toBeEnabled()
+    expect(screen.getByRole('button', { name: 'Update on all' })).toBeDisabled()
+  })
+
+  it('shows the picker with a server the hub cannot write to disabled, with the reason', async () => {
+    mockReads()
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Wild IP')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Install on missing servers' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Install Wild IP' })
+    // mdns already has it, ndns has no admin token: neither can be chosen.
+    expect(within(dialog).getByRole('checkbox', { name: 'mdns' })).toBeDisabled()
+    expect(within(dialog).getByRole('checkbox', { name: 'ndns' })).toBeDisabled()
+    expect(within(dialog).getByText('Already installed')).toBeInTheDocument()
+    expect(within(dialog).getByText('No admin token configured on the hub')).toBeInTheDocument()
+    expect(within(dialog).getByRole('button', { name: 'Install' })).toBeDisabled()
+  })
+
+  it('runs the change on the ticked servers, shows each result and refreshes', async () => {
+    const matrix = mockReads()
+    const run = vi.spyOn(api, 'runHubApps').mockResolvedValue(ok([{ server: 'mdns', ok: true, message: 'Updated from the store.' }]))
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Update on all' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Update Blocker' })
+    expect(within(dialog).getByRole('checkbox', { name: 'mdns' })).toBeChecked()
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Update' }))
+    expect(run).toHaveBeenCalledWith('t', 'update-store', 'Blocker', ['mdns'])
+
+    const results = await screen.findByRole('dialog', { name: 'Update Blocker: results' })
+    expect(within(results).getByText('Updated from the store.')).toBeInTheDocument()
+    expect(within(results).getByText('Done')).toBeInTheDocument()
+    expect(within(results).queryByRole('button', { name: 'Retry failed' })).not.toBeInTheDocument()
+    const before = matrix.mock.calls.length
+    await userEvent.click(within(results).getAllByRole('button', { name: 'Close' }).at(-1) as HTMLElement)
+    await waitFor(() => expect(matrix.mock.calls.length).toBeGreaterThan(before))
+  })
+
+  it('retries only the servers that failed', async () => {
+    const wide: AppsMatrixData = {
+      servers: [
+        { id: 'mdns', ok: true, writable: true },
+        { id: 'ndns', ok: true, writable: true },
+      ],
+      apps: [{ ...DATA.apps[0], servers: { ...DATA.apps[0].servers, ndns: { installed: true, version: '1.0', updateAvailable: true, configStatus: 'read' } } }],
+    }
+    mockReads(wide)
+    const run = vi
+      .spyOn(api, 'runHubApps')
+      .mockResolvedValueOnce(ok([{ server: 'mdns', ok: true, message: 'Updated from the store.' }, { server: 'ndns', ok: false, message: 'Disk full.' }]))
+      .mockResolvedValueOnce(ok([{ server: 'ndns', ok: true, message: 'Updated from the store.' }]))
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Update on all' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Update Blocker' })
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Update' }))
+    expect(run).toHaveBeenLastCalledWith('t', 'update-store', 'Blocker', ['mdns', 'ndns'])
+
+    const results = await screen.findByRole('dialog', { name: 'Update Blocker: results' })
+    expect(within(results).getByText('Disk full.')).toBeInTheDocument()
+    await userEvent.click(within(results).getByRole('button', { name: 'Retry failed' }))
+    expect(run).toHaveBeenLastCalledWith('t', 'update-store', 'Blocker', ['ndns'])
+    await waitFor(() => expect(within(screen.getByRole('dialog')).queryByText('Disk full.')).not.toBeInTheDocument())
+    expect(within(screen.getByRole('dialog')).getAllByText('Done')).toHaveLength(2)
+  })
+
+  it('says the config is saved before an uninstall and starts with nothing ticked', async () => {
+    mockReads()
+    const run = vi.spyOn(api, 'runHubApps').mockResolvedValue(ok([{ server: 'mdns', ok: true }]))
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Wild IP')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Uninstall from…' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Uninstall Wild IP' })
+    expect(within(dialog).getByText(/saved to the hub's history/)).toBeInTheDocument()
+    expect(within(dialog).getByRole('button', { name: 'Uninstall' })).toBeDisabled()
+    await userEvent.click(within(dialog).getByRole('checkbox', { name: 'mdns' }))
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Uninstall' }))
+    expect(run).toHaveBeenCalledWith('t', 'uninstall', 'Wild IP', ['mdns'])
+  })
+
+  it('shows a failed call as a failure with Retry', async () => {
+    mockReads()
+    const run = vi
+      .spyOn(api, 'runHubApps')
+      .mockResolvedValueOnce({ kind: 'error', message: 'Your role does not allow this.' } as never)
+      .mockResolvedValueOnce(ok([{ server: 'mdns', ok: true }]))
+    render(<AppsMatrix token="t" canWrite />)
+    await openMenu('Blocker')
+    await userEvent.click(screen.getByRole('menuitem', { name: 'Update on all' }))
+    await userEvent.click(within(await screen.findByRole('dialog', { name: 'Update Blocker' })).getByRole('button', { name: 'Update' }))
+    const results = await screen.findByRole('dialog', { name: 'Update Blocker: results' })
+    expect(within(results).getByText('Your role does not allow this.')).toBeInTheDocument()
+    await userEvent.click(within(results).getByRole('button', { name: 'Retry' }))
+    await waitFor(() => expect(run).toHaveBeenCalledTimes(2))
+    expect(await within(screen.getByRole('dialog')).findByText('Done')).toBeInTheDocument()
+  })
+})
+
+describe('Apps behind a hub, for writers', () => {
+  const INSTALLED: InstalledApp = { name: 'Split Horizon', description: 'd', version: '9.0', updateAvailable: true, updateVersion: '9.1', updateUrl: 'https://x/s.zip', dnsApps: [] }
+  const NODES = [{ name: 'mdns', type: 'Primary' }, { name: 'ndns', type: 'Secondary' }]
+
+  function mount(props: { canWrite: boolean; writable: string[] }) {
+    return render(
+      <HubMode.Provider value>
+        <SlotProvider>
+          {(ref) => (
+            <>
+              <div ref={ref} />
+              <Apps token="t" nodes={NODES} clusterInitialised {...props} />
+            </>
+          )}
+        </SlotProvider>
+      </HubMode.Provider>,
+    )
+  }
+
+  it('shows App Store and Install from file on All servers to writers only', async () => {
+    mockReads()
+    const { unmount } = mount({ canWrite: false, writable: ['mdns'] })
+    await screen.findByText('Blocker')
+    expect(screen.queryByRole('button', { name: 'App Store' })).not.toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Install from file' })).not.toBeInTheDocument()
+    unmount()
+    mount({ canWrite: true, writable: ['mdns'] })
+    expect(await screen.findByRole('button', { name: 'App Store' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Install from file' })).toBeInTheDocument()
+  })
+
+  it('opens the hub store and sends an install through the picker', async () => {
+    mockReads()
+    vi.spyOn(api, 'listStoreApps').mockResolvedValue({
+      kind: 'ok',
+      data: { response: { storeApps: [{ name: 'Failover', description: 'Backup', version: '8.0', url: 'u', size: '1', installed: false, installedOn: [] }] } },
+    } as never)
+    const run = vi.spyOn(api, 'runHubApps').mockResolvedValue(ok([{ server: 'mdns', ok: true }]))
+    mount({ canWrite: true, writable: ['mdns'] })
+    await userEvent.click(await screen.findByRole('button', { name: 'App Store' }))
+    expect(await screen.findByText('Failover')).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Install…' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Install Failover' })
+    expect(within(dialog).getByRole('checkbox', { name: 'ndns' })).toBeDisabled()
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Install' }))
+    expect(run).toHaveBeenCalledWith('t', 'install-store', 'Failover', ['mdns'])
+  })
+
+  it('uploads a zip to the chosen servers', async () => {
+    mockReads()
+    const upload = vi.spyOn(api, 'uploadHubApp').mockResolvedValue(ok([{ server: 'mdns', ok: true }]))
+    mount({ canWrite: true, writable: ['mdns'] })
+    await userEvent.click(await screen.findByRole('button', { name: 'Install from file' }))
+    const form = await screen.findByRole('dialog', { name: 'Install App' })
+    await userEvent.type(within(form).getByLabelText('App Name'), 'Mine')
+    const file = new File([new Uint8Array([1, 2, 3])], 'Mine.zip', { type: 'application/zip' })
+    await userEvent.upload(within(form).getByLabelText('App Zip File'), file)
+    await userEvent.click(within(form).getByRole('button', { name: 'Choose servers' }))
+    const picker = await screen.findByRole('dialog', { name: 'Install Mine from file' })
+    await userEvent.click(within(picker).getByRole('button', { name: 'Install' }))
+    expect(upload).toHaveBeenCalledWith('t', 'install', 'Mine', file, ['mdns'])
+  })
+
+  it('brings the single-server actions back for a writer, Config included, and sends them to that node', async () => {
+    mockReads()
+    localStorage.setItem('hubAppsNode', 'mdns')
+    vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    const store = vi.spyOn(api, 'downloadAndUpdate').mockResolvedValue({ kind: 'ok', data: { response: {} } } as never)
+    mount({ canWrite: true, writable: ['mdns'] })
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Update' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'App Store' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Install from file' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: /Actions: Split Horizon/ })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Config' })).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Store Update' }))
+    expect(store).toHaveBeenCalledWith('t', 'Split Horizon', 'https://x/s.zip', 'mdns')
+  })
+
+  it('keeps a viewer, and a server without an admin token, read-only', async () => {
+    mockReads()
+    vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    localStorage.setItem('hubAppsNode', 'ndns')
+    const { unmount } = mount({ canWrite: true, writable: ['mdns'] })
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    for (const name of ['Update', 'Store Update', 'App Store', 'Install from file']) {
+      expect(screen.queryByRole('button', { name })).not.toBeInTheDocument()
+    }
+    unmount()
+
+    localStorage.setItem('hubAppsNode', 'mdns')
+    mount({ canWrite: false, writable: ['mdns'] })
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    for (const name of ['Update', 'Store Update', 'App Store', 'Install from file']) {
+      expect(screen.queryByRole('button', { name })).not.toBeInTheDocument()
+    }
+  })
+})

src/screens/apps/ConfigDiffView.tsx

@@ -0,0 +1,131 @@
+import type { ConfigDiff, ConfigCompare, FieldChange } from '../../api/apps'
+import { DataTable, type Col } from '../../ui/DataTable'
+import { head } from '../../ui/Table'
+import { Tag } from '../../ui/Tag'
+
+/* A value as one short line: JSON for anything that is not a string. */
+export function show(value: unknown): string {
+  if (value === undefined) return '(absent)'
+  const text = typeof value === 'string' ? value : JSON.stringify(value)
+  return text.length > 90 ? `${text.slice(0, 87)}...` : text
+}
+
+/* Is this field (or a parent of it) one of the local ones? */
+export const isLocal = (path: string, localPaths: string[]): boolean => localPaths.some((p) => path === p || path.startsWith(`${p}.`))
+
+const LINE_CLASS = {
+  same: 'text-muted-foreground',
+  add: 'bg-success/10 text-foreground',
+  remove: 'bg-destructive/10 text-foreground',
+} as const
+const MARK = { same: ' ', add: '+', remove: '-' } as const
+
+/* The changes of one config to another: fields for JSON, lines for text. */
+export function DiffView({ diff, localPaths = [], from = 'Now', to = 'Expected' }: { diff: ConfigDiff; localPaths?: string[]; from?: string; to?: string }) {
+  if (diff.kind === 'text') {
+    if (diff.truncated) return <p className="m-0 text-sm text-muted-foreground">Too long to show line by line: about {diff.changed} lines differ.</p>
+    if (diff.changed === 0) return <p className="m-0 text-sm text-muted-foreground">No differences.</p>
+    return (
+      <pre className="m-0 max-h-80 overflow-auto rounded-md border border-border font-mono text-xs leading-5" aria-label="Line differences">
+        {diff.lines.map((l, i) => (
+          <div key={i} className={`px-3 whitespace-pre-wrap ${LINE_CLASS[l.op]}`} data-op={l.op}>
+            {MARK[l.op]} {l.text}
+          </div>
+        ))}
+      </pre>
+    )
+  }
+  if (diff.changes.length === 0) return <p className="m-0 text-sm text-muted-foreground">No differences.</p>
+  const columns: Col<FieldChange>[] = [
+    {
+      id: 'path',
+      ...head({ children: 'Field' }),
+      cell: (c) => (
+        <span className="flex flex-wrap items-center gap-2 font-mono text-xs">
+          {c.path}
+          {isLocal(c.path, localPaths) && <Tag tone="info">Local</Tag>}
+        </span>
+      ),
+    },
+    { id: 'a', ...head({ children: from }), cell: (c) => <span className="font-mono text-xs">{show(c.a)}</span> },
+    { id: 'b', ...head({ children: to }), cell: (c) => <span className="font-mono text-xs">{show(c.b)}</span> },
+  ]
+  return <DataTable data={diff.changes} columns={columns} getRowId={(c) => c.path} />
+}
+
+interface Row {
+  path: string
+  cells: Record<string, { now?: unknown; expected?: unknown; differs: boolean }>
+}
+
+/*
+The fields that differ, one row each and one column per server. With a baseline each server is
+compared to its own effective config (its local values in place); without one, every server is
+compared to the first.
+*/
+export function fieldRows(data: ConfigCompare): Row[] | null {
+  const live = data.servers.filter((s) => s.installed && s.ok)
+  const rows = new Map<string, Row>()
+  const cell = (path: string, server: string) => {
+    const row = rows.get(path) ?? { path, cells: {} }
+    rows.set(path, row)
+    return (row.cells[server] ??= { differs: false })
+  }
+  if (data.baseline !== null) {
+    if (live.some((s) => s.vsBaseline?.kind === 'text')) return null
+    for (const s of live) {
+      if (s.vsBaseline?.kind !== 'json') continue
+      for (const c of s.vsBaseline.changes) Object.assign(cell(c.path, s.id), { now: c.a, expected: c.b, differs: true })
+    }
+    return [...rows.values()]
+  }
+  const reference = live[0]
+  if (reference === undefined) return []
+  for (const o of live.slice(1)) {
+    const diff = reference.vsOthers[o.id]
+    if (diff === undefined) continue
+    if (diff.kind === 'text') return null
+    for (const c of diff.changes) {
+      Object.assign(cell(c.path, reference.id), { now: c.b, differs: true })
+      Object.assign(cell(c.path, o.id), { now: c.a, differs: true })
+    }
+  }
+  return [...rows.values()]
+}
+
+export function FieldTable({ data }: { data: ConfigCompare }) {
+  const rows = fieldRows(data)
+  if (rows === null) return null
+  if (rows.length === 0) return <p className="m-0 text-sm text-muted-foreground">No differences between the servers.</p>
+  const servers = data.servers.filter((s) => s.installed && s.ok)
+  const withBaseline = data.baseline !== null
+  const columns: Col<Row>[] = [
+    {
+      id: 'path',
+      ...head({ children: 'Field' }),
+      cell: (r) => (
+        <span className="flex flex-wrap items-center gap-2 font-mono text-xs">
+          {r.path}
+          {isLocal(r.path, data.overrides.paths) && <Tag tone="info">Local</Tag>}
+        </span>
+      ),
+    },
+    ...servers.map(
+      (s): Col<Row> => ({
+        id: `s-${s.id}`,
+        ...head({ children: s.id }),
+        cell: (r) => {
+          const c = r.cells[s.id]
+          if (c === undefined || !c.differs) return <span className="text-xs text-muted-foreground">{withBaseline ? 'Matches' : 'Same'}</span>
+          return (
+            <span className="flex flex-col gap-0.5 font-mono text-xs">
+              <span>{show(c.now)}</span>
+              {withBaseline && <span className="text-muted-foreground">expected {show(c.expected)}</span>}
+            </span>
+          )
+        },
+      }),
+    ),
+  ]
+  return <DataTable data={rows} columns={columns} getRowId={(r) => r.path} />
+}

src/screens/apps/HubInstallFile.tsx

@@ -0,0 +1,76 @@
+import { useEffect, useRef, useState } from 'react'
+import { uploadHubApp } from '../../api/apps'
+import { Button } from '../../ui/Button'
+import { Dialog } from '../../ui/Dialog'
+import { LabeledInput } from '../../ui/Field'
+import { Notifier } from '../../ui/Notifier'
+import { pickerServers, type RunRequest } from './RunFlow'
+import type { AlertState } from './Apps'
+
+/*
+"Install from file" in the All servers view: the same two fields as the single-server dialog
+(the validation order and wording are the same), then the server picker. The zip goes to the
+hub once and the hub sends it to every chosen server.
+*/
+export function HubInstallFile({
+  open,
+  onOpenChange,
+  token,
+  servers,
+  onRequest,
+}: {
+  open: boolean
+  onOpenChange: (o: boolean) => void
+  token: string | null
+  servers: { id: string; writable: boolean }[]
+  onRequest: (request: RunRequest) => void
+}) {
+  const [name, setName] = useState('')
+  const [alert, setAlert] = useState<AlertState | null>(null)
+  const fileRef = useRef<HTMLInputElement>(null)
+
+  useEffect(() => {
+    if (open) {
+      setName('')
+      setAlert(null)
+    }
+  }, [open])
+
+  function next() {
+    if (name === '') {
+      setAlert({ type: 'warning', title: 'Missing!', text: 'Please enter an application name.' })
+      return
+    }
+    const file = fileRef.current?.files?.[0]
+    if (!file) {
+      setAlert({ type: 'warning', title: 'Missing!', text: 'Please select an application zip file to install.' })
+      return
+    }
+    onRequest({
+      title: `Install ${name} from file`,
+      intro: 'The zip is sent to every server you choose.',
+      confirmLabel: 'Install',
+      servers: pickerServers(servers),
+      preselected: servers.filter((s) => s.writable).map((s) => s.id),
+      run: (targets) => uploadHubApp(token, 'install', name, file, targets),
+    })
+    onOpenChange(false)
+  }
+
+  return (
+    <Dialog
+      open={open}
+      onOpenChange={onOpenChange}
+      title="Install App"
+      actions={
+        <Button variant="primary" onClick={next}>
+          Choose servers
+        </Button>
+      }
+    >
+      <Notifier notice={alert} onClose={() => setAlert(null)} />
+      <LabeledInput label="App Name" value={name} onChange={(e) => setName(e.target.value)} />
+      <LabeledInput label="App Zip File" type="file" ref={fileRef} />
+    </Dialog>
+  )
+}

src/screens/apps/HubStore.tsx

@@ -0,0 +1,133 @@
+import { useCallback, useEffect, useState } from 'react'
+import { listStoreApps, runHubApps, type HubRunAction, type StoreApp } from '../../api/apps'
+import { Button } from '../../ui/Button'
+import { Empty, Failure, Loading } from '../../ui/Empty'
+import { Panel } from '../../ui/Panel'
+import { Tag } from '../../ui/Tag'
+import { RunFlow, pickerServers, type RunRequest } from './RunFlow'
+
+/*
+The App Store behind a hub, in the All servers view. It lists what the first server's store
+offers and where each app is installed. Install and Update go to the servers the user picks;
+the hub takes each download URL from that server's own store, so none is sent from here.
+*/
+export function HubStore({
+  token,
+  servers,
+  canWrite,
+  onChanged,
+}: {
+  token: string | null
+  servers: { id: string; writable: boolean }[]
+  canWrite: boolean
+  onChanged: () => void
+}) {
+  const [apps, setApps] = useState<StoreApp[] | null>(null)
+  const [failure, setFailure] = useState<string | null>(null)
+  const [request, setRequest] = useState<RunRequest | null>(null)
+
+  const load = useCallback(async () => {
+    const outcome = await listStoreApps(token)
+    if (outcome.kind === 'ok') {
+      setApps(outcome.data.response.storeApps)
+      setFailure(null)
+    } else setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+  }, [token])
+
+  useEffect(() => {
+    void load()
+  }, [load])
+
+  function open(app: StoreApp, action: HubRunAction, verb: string) {
+    const on = app.installedOn ?? []
+    const allowed = action === 'install-store' ? servers.map((s) => s.id).filter((id) => !on.includes(id)) : on
+    setRequest({
+      title: `${verb} ${app.name}`,
+      confirmLabel: verb,
+      servers: pickerServers(
+        servers,
+        Object.fromEntries(servers.filter((s) => !allowed.includes(s.id)).map((s) => [s.id, action === 'install-store' ? 'Already installed' : 'Not installed'])),
+      ),
+      preselected: allowed,
+      run: (targets) => runHubApps(token, action, app.name, targets),
+    })
+  }
+
+  if (failure !== null && apps === null) {
+    return (
+      <Failure>
+        <span className="flex flex-col items-center gap-3">
+          <span>Could not load the App Store. {failure}</span>
+          <Button variant="primary" onClick={() => void load()}>
+            Retry
+          </Button>
+        </span>
+      </Failure>
+    )
+  }
+  if (apps === null) return <Loading>Loading the App Store…</Loading>
+  if (apps.length === 0) {
+    return (
+      <Empty
+        title="No apps in the store"
+        actions={
+          <Button variant="primary" onClick={() => void load()}>
+            Refresh
+          </Button>
+        }
+      >
+        The store did not list any app.
+      </Empty>
+    )
+  }
+
+  return (
+    <Panel title="DNS App Store" description="What the store offers and where it is installed" groups2>
+      <ul className="m-0 flex list-none flex-col p-0" aria-label="Store apps">
+        {apps.map((app) => {
+          const on = app.installedOn ?? []
+          return (
+            <li key={app.name} aria-label={app.name} className="flex flex-wrap items-start justify-between gap-4 border-b border-border px-4 py-3 last:border-b-0">
+              <div className="flex min-w-0 flex-1 flex-col gap-2">
+                <span className="flex flex-wrap items-center gap-2 text-sm font-medium">
+                  {app.name}
+                  <Tag>Version {app.version}</Tag>
+                  {on.map((id) => (
+                    <Tag key={id} tone="info">
+                      {id}
+                    </Tag>
+                  ))}
+                </span>
+                <span className="text-xs text-muted-foreground">{app.description}</span>
+              </div>
+              {canWrite && (
+                <div className="flex items-center gap-2">
+                  <Button size="sm" variant="primary" onClick={() => open(app, 'install-store', 'Install')}>
+                    Install…
+                  </Button>
+                  {on.length > 0 && (
+                    <Button size="sm" onClick={() => open(app, 'update-store', 'Update')}>
+                      Update…
+                    </Button>
+                  )}
+                </div>
+              )}
+            </li>
+          )
+        })}
+      </ul>
+      {request !== null && (
+        <RunFlow
+          request={request}
+          onClose={(changed) => {
+            setRequest(null)
+            if (changed) {
+              void load()
+              onChanged()
+            }
+          }}
+        />
+      )}
+    </Panel>
+  )
+}

src/screens/apps/ImportBaselines.tsx

@@ -0,0 +1,126 @@
+import { useEffect, useRef, useState } from 'react'
+import { importBaselines, type ImportPlan } from '../../api/apps'
+import { Alert } from '../../ui/Alert'
+import { Button } from '../../ui/Button'
+import { Dialog } from '../../ui/Dialog'
+import { LabeledInput } from '../../ui/Field'
+import { Loading } from '../../ui/Empty'
+import { Tag } from '../../ui/Tag'
+
+/*
+Loads a file of baselines (made by "Export baselines") into this hub. The file is checked first and
+the summary says, app by app, what is new, what changes and what is the same; nothing is written
+until Import is pressed. It only touches the hub's own baselines: no server gets a config from it.
+*/
+const STATUS = { new: 'New', changed: 'Changed', same: 'Same' } as const
+const TONE = { new: 'info', changed: 'warn', same: 'ok' } as const
+
+export function ImportBaselines({
+  open,
+  onOpenChange,
+  token,
+  onImported,
+}: {
+  open: boolean
+  onOpenChange: (o: boolean) => void
+  token: string | null
+  onImported: () => void
+}) {
+  const [text, setText] = useState<string | null>(null)
+  const [plan, setPlan] = useState<ImportPlan | null>(null)
+  const [problem, setProblem] = useState<string | null>(null)
+  const [busy, setBusy] = useState(false)
+  const [done, setDone] = useState(false)
+  const fileRef = useRef<HTMLInputElement>(null)
+
+  useEffect(() => {
+    if (open) {
+      setText(null)
+      setPlan(null)
+      setProblem(null)
+      setBusy(false)
+      setDone(false)
+    }
+  }, [open])
+
+  async function check() {
+    const file = fileRef.current?.files?.[0]
+    if (!file) {
+      setProblem('Please select a baselines file.')
+      return
+    }
+    setBusy(true)
+    setProblem(null)
+    const content = await file.text()
+    const outcome = await importBaselines(token, content, false)
+    setBusy(false)
+    if (outcome.kind !== 'ok') {
+      setProblem(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+      return
+    }
+    setText(content)
+    setPlan(outcome.data.response)
+  }
+
+  async function commit() {
+    if (text === null) return
+    setBusy(true)
+    const outcome = await importBaselines(token, text, true)
+    setBusy(false)
+    if (outcome.kind !== 'ok') {
+      setProblem(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+      return
+    }
+    setPlan(outcome.data.response)
+    setDone(true)
+    onImported()
+  }
+
+  const changes = plan?.apps.filter((a) => a.status !== 'same').length ?? 0
+
+  return (
+    <Dialog
+      open={open}
+      onOpenChange={onOpenChange}
+      title="Import baselines"
+      actions={
+        plan === null ? (
+          <Button variant="primary" disabled={busy} onClick={() => void check()}>
+            Check file
+          </Button>
+        ) : !done ? (
+          <Button variant="primary" disabled={busy || changes === 0} onClick={() => void commit()}>
+            Import
+          </Button>
+        ) : undefined
+      }
+    >
+      {problem !== null && <Alert type="danger" title="Error">{problem}</Alert>}
+      {busy && <Loading>Working…</Loading>}
+      {plan === null ? (
+        <>
+          <p className="m-0 text-sm text-muted-foreground">
+            Choose a file made by Export baselines. It is checked first; nothing is changed until you press Import, and no server is touched.
+          </p>
+          <LabeledInput label="Baselines file" type="file" accept="application/json,.json" ref={fileRef} />
+        </>
+      ) : (
+        <>
+          {done && <Alert type="success" title="Imported.">The baselines were saved. The replaced baselines are in each app's history.</Alert>}
+          <ul className="m-0 flex list-none flex-col gap-2 p-0" aria-label="Apps in the file">
+            {plan.apps.map((a) => (
+              <li key={a.app} className="flex flex-wrap items-center gap-2 text-sm">
+                <span className="font-medium">{a.app}</span>
+                <Tag tone={TONE[a.status]}>{STATUS[a.status]}</Tag>
+                {a.ignoredServers.length > 0 && (
+                  <span className="text-xs text-muted-foreground">Local values of {a.ignoredServers.join(', ')} are ignored: this hub has no such server.</span>
+                )}
+              </li>
+            ))}
+          </ul>
+          {!done && changes === 0 && <p className="m-0 text-sm text-muted-foreground">Everything in the file is already here.</p>}
+        </>
+      )}
+    </Dialog>
+  )
+}

src/screens/apps/InstallApp.tsx

@@ -17,11 +17,14 @@ export function InstallApp({
   onOpenChange,
   token,
   onInstalled,
+  node = '',
 }: {
   open: boolean
   onOpenChange: (o: boolean) => void
   token: string | null
   onInstalled: (name: string) => void
+  /** Behind a hub: the one server it goes to. */
+  node?: string
 }) {
   const [name, setName] = useState('')
   const [alert, setAlert] = useState<AlertState | null>(null)
@@ -53,7 +56,7 @@ export function InstallApp({
     }
 
     setBusy(true)
-    const outcome = await installApp(token, name, file)
+    const outcome = await installApp(token, name, file, node)
     setBusy(false)
 
     if (outcome.kind !== 'ok') {

src/screens/apps/Profiles.test.tsx

@@ -0,0 +1,223 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { render, screen, waitFor, within } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { Profiles } from './Profiles'
+import * as api from '../../api/hub-profiles'
+import type { ProfileSummary, ProfileTargetPreview } from '../../api/hub-profiles'
+
+afterEach(() => vi.restoreAllMocks())
+
+const SERVERS = [
+  { id: 'mdns', writable: true },
+  { id: 'ndns', writable: true },
+]
+
+const FRESH: ProfileSummary = { name: 'Fresh', apps: 2, configs: 1, settings: 12, lists: { blocked: 3, allowed: 1 }, updatedAt: '2026-11-01T00:00:00Z', updatedBy: 'admin' }
+
+const listOf = (profiles: ProfileSummary[]) => vi.spyOn(api, 'listProfiles').mockResolvedValue({ kind: 'ok', data: { response: { profiles } } } as never)
+
+const PREVIEW: ProfileTargetPreview = {
+  server: 'ndns',
+  writable: true,
+  ok: true,
+  apps: [
+    { name: 'Blocker', state: 'install', profileVersion: '2.0', storeVersion: '2.0' },
+    { name: 'Logs', state: 'version-differs', profileVersion: '2.0', installedVersion: '1.0' },
+  ],
+  configs: [{ name: 'Blocker', state: 'after-install' }],
+  settings: { fields: ['blockingType'], secretFields: ['tsigKeys'], unchanged: false, warnings: [], changes: { kind: 'json', changes: [{ path: 'blockingType', type: 'changed', a: 'AnyAddress', b: 'NxDomain' }] } },
+  lists: { blocked: { missing: 2, extra: 1 }, allowed: { missing: 0, extra: 0 }, status: 'differs' },
+  warnings: [],
+}
+
+const result = (steps: { step: 'apps' | 'configs' | 'settings' | 'lists'; status: 'ok' | 'failed' | 'skipped' | 'not-run'; message: string }[], ok: boolean) =>
+  ({ kind: 'ok', data: { response: { results: [{ server: 'ndns', ok, message: ok ? 'Profile applied.' : 'Stopped at settings: refused', steps }] } } }) as never
+
+describe('Profiles list', () => {
+  it('shows a skeleton while it loads, then each profile with its parts', async () => {
+    listOf([FRESH])
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={SERVERS} />)
+    expect(screen.getByRole('status', { name: 'Loading the profiles' })).toBeInTheDocument()
+    const row = await screen.findByRole('listitem', { name: 'Fresh' })
+    expect(within(row).getByText('2 apps')).toBeInTheDocument()
+    expect(within(row).getByText('1 config')).toBeInTheDocument()
+    expect(within(row).getByText('12 settings')).toBeInTheDocument()
+    expect(within(row).getByText('4 list domains')).toBeInTheDocument()
+    for (const name of ['View', 'Apply to servers…', 'Export', 'Delete']) expect(within(row).getByRole('button', { name })).toBeInTheDocument()
+  })
+
+  it('has an empty state whose action creates a profile from the current baselines', async () => {
+    listOf([])
+    const snapshot = vi.spyOn(api, 'snapshotProfile').mockResolvedValue({ kind: 'ok', data: { response: { profile: {} as never, replaced: false } } } as never)
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={SERVERS} />)
+    expect(await screen.findByText('No profiles yet')).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Create from current baselines' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Create a profile' })
+    await userEvent.type(within(dialog).getByLabelText('Profile name'), 'Fresh')
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Create' }))
+    expect(snapshot).toHaveBeenCalledWith('t', 'Fresh', { apps: undefined, settings: true, lists: true })
+  })
+
+  it('lets the admin leave a part out', async () => {
+    listOf([])
+    const snapshot = vi.spyOn(api, 'snapshotProfile').mockResolvedValue({ kind: 'ok', data: { response: { profile: {} as never, replaced: false } } } as never)
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={SERVERS} />)
+    await userEvent.click(await screen.findByRole('button', { name: 'Create from current baselines' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Create a profile' })
+    await userEvent.type(within(dialog).getByLabelText('Profile name'), 'Light')
+    await userEvent.click(within(dialog).getByRole('checkbox', { name: 'Apps' }))
+    await userEvent.click(within(dialog).getByRole('checkbox', { name: 'Blocked and Allowed lists' }))
+    await userEvent.click(within(dialog).getByRole('button', { name: 'Create' }))
+    expect(snapshot).toHaveBeenCalledWith('t', 'Light', { apps: [], settings: true, lists: false })
+  })
+
+  it('shows an error with Retry', async () => {
+    const spy = vi.spyOn(api, 'listProfiles').mockResolvedValueOnce({ kind: 'error', message: 'boom' } as never).mockResolvedValue({ kind: 'ok', data: { response: { profiles: [FRESH] } } } as never)
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={SERVERS} />)
+    expect(await screen.findByText(/boom/)).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Retry' }))
+    expect(await screen.findByRole('listitem', { name: 'Fresh' })).toBeInTheDocument()
+    expect(spy).toHaveBeenCalledTimes(2)
+  })
+
+  it('views a profile with the secrets masked and no config text', async () => {
+    listOf([FRESH])
+    vi.spyOn(api, 'getProfile').mockResolvedValue({
+      kind: 'ok',
+      data: {
+        response: {
+          name: 'Fresh', createdAt: '', updatedAt: '2026-11-01T00:00:00Z', updatedBy: 'admin',
+          apps: [{ name: 'Blocker', version: '2.0', hasConfig: true, bytes: 12 }],
+          settings: { tsigKeys: [{ keyName: 'k', sharedSecret: '<secret>' }] },
+          lists: { blocked: 3, allowed: 1 },
+        },
+      },
+    } as never)
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={SERVERS} />)
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Fresh' })).getByRole('button', { name: 'View' }))
+    const dialog = await screen.findByRole('dialog', { name: 'Profile: Fresh' })
+    expect(await within(dialog).findByText('With config')).toBeInTheDocument()
+    expect((within(dialog).getByLabelText('Shared settings (JSON)') as HTMLTextAreaElement).value).toContain('<secret>')
+    expect(within(dialog).getByText('3 blocked and 1 allowed domains.')).toBeInTheDocument()
+  })
+
+  it('deletes only after a confirmation', async () => {
+    const list = listOf([FRESH])
+    const removed = vi.spyOn(api, 'deleteProfile').mockResolvedValue({ kind: 'ok', data: { response: { deleted: true } } } as never)
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={SERVERS} />)
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Fresh' })).getByRole('button', { name: 'Delete' }))
+    expect(removed).not.toHaveBeenCalled()
+    list.mockResolvedValue({ kind: 'ok', data: { response: { profiles: [] } } } as never)
+    await userEvent.click(await screen.findByRole('button', { name: 'Delete' }))
+    expect(removed).toHaveBeenCalledWith('t', 'Fresh')
+    expect(await screen.findByText('No profiles yet')).toBeInTheDocument()
+  })
+})
+
+describe('applying a profile', () => {
+  it('previews the steps per server, applies, and shows the steps of each result', async () => {
+    listOf([FRESH])
+    const preview = vi.spyOn(api, 'previewProfile').mockResolvedValue({ kind: 'ok', data: { response: { profile: 'Fresh', targets: [PREVIEW] } } } as never)
+    const apply = vi.spyOn(api, 'applyProfile').mockResolvedValue(
+      result(
+        [
+          { step: 'apps', status: 'ok', message: 'Installed Blocker.' },
+          { step: 'configs', status: 'ok', message: '1 config(s) applied, 0 already in sync.' },
+          { step: 'settings', status: 'ok', message: 'Applied and verified (1 field).' },
+          { step: 'lists', status: 'ok', message: 'Added 2, removed 0.' },
+        ],
+        true,
+      ),
+    )
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={SERVERS} />)
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Fresh' })).getByRole('button', { name: 'Apply to servers…' }))
+    const picker = await screen.findByRole('dialog', { name: 'Apply the profile Fresh' })
+    await userEvent.click(within(picker).getByRole('checkbox', { name: 'ndns' }))
+    await userEvent.click(within(picker).getByRole('button', { name: 'Preview' }))
+    expect(preview).toHaveBeenCalledWith('t', 'Fresh', ['ndns'], false)
+
+    const review = await screen.findByRole('dialog', { name: 'Apply the profile Fresh: review' })
+    const steps = within(review).getByRole('list', { name: 'Steps on ndns' })
+    expect(within(steps).getByText('1. Apps')).toBeInTheDocument()
+    expect(within(steps).getByText('Will be installed')).toBeInTheDocument()
+    expect(within(steps).getByText('Other version installed')).toBeInTheDocument()
+    expect(within(steps).getByText('2. App configs')).toBeInTheDocument()
+    expect(within(steps).getByText(/tsigKeys: the secret values are sent but never shown/)).toBeInTheDocument()
+    expect(within(steps).getByText(/Blocked: 2 to add, 1 extra/)).toBeInTheDocument()
+    await userEvent.click(within(review).getByRole('button', { name: 'Apply' }))
+
+    expect(apply).toHaveBeenCalledWith('t', 'Fresh', ['ndns'], false, {})
+    const list = await screen.findByRole('list', { name: 'Steps on ndns' })
+    expect(within(list).getByText('Installed Blocker.')).toBeInTheDocument()
+    expect(within(list).getAllByText('Done')).toHaveLength(4)
+  })
+
+  it('removes list extras only when ticked in the review', async () => {
+    listOf([FRESH])
+    vi.spyOn(api, 'previewProfile').mockResolvedValue({ kind: 'ok', data: { response: { profile: 'Fresh', targets: [PREVIEW] } } } as never)
+    const apply = vi.spyOn(api, 'applyProfile').mockResolvedValue(result([{ step: 'lists', status: 'ok', message: 'done' }], true))
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={SERVERS} />)
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Fresh' })).getByRole('button', { name: 'Apply to servers…' }))
+    await userEvent.click(within(await screen.findByRole('dialog', { name: 'Apply the profile Fresh' })).getByRole('checkbox', { name: 'ndns' }))
+    await userEvent.click(screen.getByRole('button', { name: 'Preview' }))
+    await userEvent.click(await screen.findByRole('checkbox', { name: 'Also remove extras' }))
+    await userEvent.click(screen.getByRole('button', { name: 'Apply' }))
+    await waitFor(() => expect(apply).toHaveBeenCalledWith('t', 'Fresh', ['ndns'], true, {}))
+  })
+
+  it('says which step failed and retries from that step', async () => {
+    listOf([FRESH])
+    vi.spyOn(api, 'previewProfile').mockResolvedValue({ kind: 'ok', data: { response: { profile: 'Fresh', targets: [PREVIEW] } } } as never)
+    const apply = vi
+      .spyOn(api, 'applyProfile')
+      .mockResolvedValueOnce(
+        result(
+          [
+            { step: 'apps', status: 'ok', message: 'Installed Blocker.' },
+            { step: 'configs', status: 'ok', message: '1 config(s) applied, 0 already in sync.' },
+            { step: 'settings', status: 'failed', message: 'refused' },
+            { step: 'lists', status: 'not-run', message: 'An earlier step failed.' },
+          ],
+          false,
+        ),
+      )
+      .mockResolvedValue(
+        result(
+          [
+            { step: 'apps', status: 'skipped', message: 'Done by an earlier run.' },
+            { step: 'configs', status: 'skipped', message: 'Done by an earlier run.' },
+            { step: 'settings', status: 'ok', message: 'Applied and verified (1 field).' },
+            { step: 'lists', status: 'ok', message: 'Added 2, removed 0.' },
+          ],
+          true,
+        ),
+      )
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={SERVERS} />)
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Fresh' })).getByRole('button', { name: 'Apply to servers…' }))
+    await userEvent.click(within(await screen.findByRole('dialog', { name: 'Apply the profile Fresh' })).getByRole('checkbox', { name: 'ndns' }))
+    await userEvent.click(screen.getByRole('button', { name: 'Preview' }))
+    await userEvent.click(await screen.findByRole('button', { name: 'Apply' }))
+
+    const list = await screen.findByRole('list', { name: 'Steps on ndns' })
+    expect(within(list).getByText('Failed')).toBeInTheDocument()
+    expect(within(list).getByText('Not run')).toBeInTheDocument()
+    expect(screen.getByText(/Stopped at settings/)).toBeInTheDocument()
+
+    await userEvent.click(screen.getByRole('button', { name: 'Retry failed' }))
+    await waitFor(() => expect(apply).toHaveBeenCalledTimes(2))
+    expect(apply).toHaveBeenLastCalledWith('t', 'Fresh', ['ndns'], false, { ndns: 'settings' })
+    await waitFor(() => expect(within(screen.getByRole('list', { name: 'Steps on ndns' })).getAllByText('Done earlier')).toHaveLength(2))
+  })
+
+  it('lists a server with no admin token disabled, and shows a preview error', async () => {
+    listOf([FRESH])
+    vi.spyOn(api, 'previewProfile').mockResolvedValue({ kind: 'error', message: 'unreachable' } as never)
+    render(<Profiles open onOpenChange={() => {}} token="t" servers={[{ id: 'mdns', writable: false }, { id: 'ndns', writable: true }]} />)
+    await userEvent.click(within(await screen.findByRole('listitem', { name: 'Fresh' })).getByRole('button', { name: 'Apply to servers…' }))
+    const picker = await screen.findByRole('dialog', { name: 'Apply the profile Fresh' })
+    expect(within(picker).getByRole('checkbox', { name: 'mdns' })).toBeDisabled()
+    await userEvent.click(within(picker).getByRole('checkbox', { name: 'ndns' }))
+    await userEvent.click(within(picker).getByRole('button', { name: 'Preview' }))
+    expect(await screen.findByText('unreachable')).toBeInTheDocument()
+  })
+})

src/screens/apps/Profiles.tsx

@@ -0,0 +1,546 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import {
+  applyProfile,
+  deleteProfile,
+  exportProfiles,
+  getProfile,
+  importProfiles,
+  listProfiles,
+  previewProfile,
+  snapshotProfile,
+  type ProfileImportPlan,
+  type ProfileSummary,
+  type ProfileTargetPreview,
+  type ProfileView,
+} from '../../api/hub-profiles'
+import type { RunOutcome } from '../../api/apps'
+import { Alert } from '../../ui/Alert'
+import { Button } from '../../ui/Button'
+import { Check } from '../../ui/Check'
+import { Confirm } from '../../ui/Confirm'
+import { Dialog } from '../../ui/Dialog'
+import { Empty, Failure } from '../../ui/Empty'
+import { LabeledInput, LabeledTextarea } from '../../ui/Field'
+import { Tag } from '../../ui/Tag'
+import { Skeleton } from '@/components/ui/skeleton'
+import { DiffView } from './ConfigDiffView'
+import { RunFlow, pickerServers, type RunRequest } from './RunFlow'
+
+/*
+Behind a hub, for an admin: profiles, named bundles of store apps (with the hub's baseline config of
+each), the shared settings and the Blocked and Allowed lists. One is made from the hub's current
+baselines, can be exported to another hub, and is applied to one or several servers through the
+same picker, review and results as every other change here. A server's steps run in order (apps,
+app configs, settings, lists) and a failed one stops the rest for that server; "Retry failed" starts
+again at the failed step. Nothing here shows a secret or an app's config text.
+*/
+
+const sessionEnded = (kind: string, message?: string): string => (kind === 'error' ? (message ?? 'Failed.') : 'The session ended.')
+const when = (iso: string): string => (iso === '' ? '' : new Date(iso).toLocaleString())
+
+const APP_STATE = {
+  install: ['warn', 'Will be installed'],
+  present: ['ok', 'Already installed'],
+  'version-differs': ['info', 'Other version installed'],
+  'not-in-store': ['dan', 'Not in this store'],
+} as const
+
+const CONFIG_STATE = {
+  same: ['ok', 'Same'],
+  differs: ['warn', 'Will change'],
+  'after-install': ['info', 'Set after install'],
+  unreadable: ['dan', 'Cannot read'],
+} as const
+
+/* ---- Apply: the review of what each server would get ---- */
+
+function PreviewBody({ targets, opts }: { targets: ProfileTargetPreview[]; opts: { remove: boolean } }) {
+  const [remove, setRemove] = useState(opts.remove)
+  const extras = targets.reduce((n, t) => n + (t.lists?.blocked.extra ?? 0) + (t.lists?.allowed.extra ?? 0), 0)
+  return (
+    <div className="flex flex-col gap-6">
+      {targets.map((t) => (
+        <section key={t.server} aria-label={`Preview ${t.server}`} className="flex flex-col gap-3">
+          <h3 className="m-0 flex items-center gap-2 text-sm font-semibold">
+            {t.server}
+            {!t.ok && <Tag tone="dan">Cannot apply</Tag>}
+          </h3>
+          {t.error !== undefined && <Alert type="danger" title="Error">{t.error}</Alert>}
+          {t.warnings.map((w) => (
+            <Alert key={w} type="warning" title="Warning">
+              {w}
+            </Alert>
+          ))}
+          {t.ok && (
+            <ol className="m-0 flex list-none flex-col gap-3 p-0" aria-label={`Steps on ${t.server}`}>
+              {t.apps.length > 0 && (
+                <li className="flex flex-col gap-1 text-sm">
+                  <span className="font-medium">1. Apps</span>
+                  <ul className="m-0 flex list-none flex-col gap-1 p-0 pl-4">
+                    {t.apps.map((a) => (
+                      <li key={a.name} className="flex flex-wrap items-center gap-2 text-xs">
+                        <span className="font-medium">{a.name}</span>
+                        <Tag tone={APP_STATE[a.state][0]}>{APP_STATE[a.state][1]}</Tag>
+                        <span className="text-muted-foreground">
+                          {a.state === 'version-differs' ? `${a.installedVersion} here, ${a.profileVersion} in the profile; it is not updated` : a.state === 'install' ? `Store offers ${a.storeVersion}` : `Version ${a.profileVersion}`}
+                        </span>
+                      </li>
+                    ))}
+                  </ul>
+                </li>
+              )}
+              {t.configs.length > 0 && (
+                <li className="flex flex-col gap-1 text-sm">
+                  <span className="font-medium">2. App configs</span>
+                  <ul className="m-0 flex list-none flex-col gap-2 p-0 pl-4">
+                    {t.configs.map((c) => (
+                      <li key={c.name} className="flex flex-col gap-1 text-xs">
+                        <span className="flex flex-wrap items-center gap-2">
+                          <span className="font-medium">{c.name}</span>
+                          <Tag tone={CONFIG_STATE[c.state][0]}>{CONFIG_STATE[c.state][1]}</Tag>
+                          {c.error !== undefined && <span className="text-muted-foreground">{c.error}</span>}
+                        </span>
+                        {c.changes !== undefined && <DiffView diff={c.changes} from="Now" to="After apply" />}
+                      </li>
+                    ))}
+                  </ul>
+                </li>
+              )}
+              {t.settings !== null && (
+                <li className="flex flex-col gap-1 text-sm">
+                  <span className="flex items-center gap-2 font-medium">
+                    3. Settings
+                    {t.settings.error !== undefined ? <Tag tone="dan">Cannot read</Tag> : t.settings.unchanged ? <Tag tone="ok">Already in sync</Tag> : <Tag tone="warn">{`${t.settings.fields.length} field(s) will change`}</Tag>}
+                  </span>
+                  {t.settings.error !== undefined && <span className="pl-4 text-xs text-muted-foreground">{t.settings.error}</span>}
+                  {t.settings.secretFields.length > 0 && <span className="pl-4 text-xs text-muted-foreground">{t.settings.secretFields.join(', ')}: the secret values are sent but never shown.</span>}
+                  {t.settings.error === undefined && !t.settings.unchanged && (
+                    <div className="pl-4">
+                      <DiffView diff={t.settings.changes} from="Now" to="After apply" />
+                    </div>
+                  )}
+                </li>
+              )}
+              {t.lists !== null && (
+                <li className="flex flex-col gap-1 text-sm">
+                  <span className="flex items-center gap-2 font-medium">
+                    4. Blocked and Allowed lists
+                    {t.lists.error !== undefined ? <Tag tone="dan">Cannot read</Tag> : t.lists.status === 'same' ? <Tag tone="ok">Already in sync</Tag> : <Tag tone="warn">Will change</Tag>}
+                  </span>
+                  {t.lists.error !== undefined ? (
+                    <span className="pl-4 text-xs text-muted-foreground">{t.lists.error}</span>
+                  ) : (
+                    <span className="pl-4 text-xs text-muted-foreground">
+                      Blocked: {t.lists.blocked.missing} to add, {t.lists.blocked.extra} extra. Allowed: {t.lists.allowed.missing} to add, {t.lists.allowed.extra} extra.
+                    </span>
+                  )}
+                  {t.lists.tooMany !== undefined && <span className="pl-4 text-xs text-muted-foreground">{t.lists.tooMany}</span>}
+                </li>
+              )}
+            </ol>
+          )}
+        </section>
+      ))}
+      {/* Only when a server has list domains the profile does not: otherwise there is nothing to remove. */}
+      {extras > 0 && (
+      <Check
+        label="Also remove extras"
+        help={`Deletes the ${extras} list domain(s) a server has that the profile does not. The server's lists are saved to the hub's history first.`}
+        checked={remove}
+        onChange={(on) => {
+          opts.remove = on
+          setRemove(on)
+        }}
+      />
+      )}
+    </div>
+  )
+}
+
+/* The picker, the review and the results of applying one profile. */
+function applyRequestFor(token: string | null, name: string, servers: { id: string; writable: boolean }[]): RunRequest {
+  const opts = { remove: false }
+  // Where each server should start next time: the step that failed on it.
+  const from: Record<string, string> = {}
+  return {
+    title: `Apply the profile ${name}`,
+    intro: 'Each server goes through the steps in order: install missing apps from its own store, apply the app configs, the settings, then the lists. A step that fails stops the later ones for that server.',
+    confirmLabel: 'Preview',
+    servers: pickerServers(servers),
+    preselected: [],
+    review: async (targets) => {
+      opts.remove = false
+      const outcome = await previewProfile(token, name, targets, false)
+      if (outcome.kind !== 'ok') return { kind: 'error', message: sessionEnded(outcome.kind, outcome.kind === 'error' ? outcome.message : undefined) }
+      return { kind: 'ok', confirmLabel: 'Apply', context: undefined, body: <PreviewBody targets={outcome.data.response.targets} opts={opts} /> }
+    },
+    run: async (targets): Promise<RunOutcome> => {
+      const outcome = await applyProfile(token, name, targets, opts.remove, { ...from })
+      if (outcome.kind === 'ok') {
+        for (const r of outcome.data.response.results) {
+          const failed = r.steps?.find((s) => s.status === 'failed')
+          if (r.ok || failed === undefined) delete from[r.server]
+          else from[r.server] = failed.step
+        }
+      }
+      return outcome
+    },
+  }
+}
+
+/* ---- Make one ---- */
+
+function CreateProfile({ token, onClose, onDone }: { token: string | null; onClose: () => void; onDone: () => void }) {
+  const [name, setName] = useState('')
+  const [apps, setApps] = useState(true)
+  const [settings, setSettings] = useState(true)
+  const [lists, setLists] = useState(true)
+  const [busy, setBusy] = useState(false)
+  const [problem, setProblem] = useState<string | null>(null)
+
+  async function create() {
+    setBusy(true)
+    setProblem(null)
+    const outcome = await snapshotProfile(token, name.trim(), { apps: apps ? undefined : [], settings, lists })
+    setBusy(false)
+    if (outcome.kind !== 'ok') return setProblem(sessionEnded(outcome.kind, outcome.kind === 'error' ? outcome.message : undefined))
+    onDone()
+  }
+
+  return (
+    <Dialog
+      open
+      onOpenChange={(o) => {
+        if (!o) onClose()
+      }}
+      title="Create a profile"
+      close="Cancel"
+      actions={
+        <Button variant="primary" disabled={busy || name.trim() === '' || (!apps && !settings && !lists)} onClick={() => void create()}>
+          Create
+        </Button>
+      }
+    >
+      <p className="m-0 text-sm text-muted-foreground">
+        A profile is a copy of the hub&apos;s baselines as they are now. An existing profile with the same name is replaced.
+      </p>
+      {problem !== null && <Alert type="danger" title="Error">{problem}</Alert>}
+      <LabeledInput label="Profile name" value={name} maxLength={60} onChange={(e) => setName(e.target.value)} />
+      <div className="flex flex-col gap-3" role="group" aria-label="Parts">
+        <Check label="Apps" help="Every app that has a baseline, with its store version and its baseline config. Store apps only." checked={apps} onChange={setApps} />
+        <Check label="Settings" help="The shared fields of the settings baseline. Host fields stay out." checked={settings} onChange={setSettings} />
+        <Check label="Blocked and Allowed lists" help="The lists baseline." checked={lists} onChange={setLists} />
+      </div>
+    </Dialog>
+  )
+}
+
+/* ---- View one ---- */
+
+function ViewProfile({ token, name, onClose }: { token: string | null; name: string; onClose: () => void }) {
+  const [view, setView] = useState<ProfileView | null>(null)
+  const [failure, setFailure] = useState<string | null>(null)
+
+  const load = useCallback(async () => {
+    const outcome = await getProfile(token, name)
+    if (outcome.kind === 'ok') {
+      setView(outcome.data.response)
+      setFailure(null)
+    } else setFailure(sessionEnded(outcome.kind, outcome.kind === 'error' ? outcome.message : undefined))
+  }, [token, name])
+
+  useEffect(() => {
+    void load()
+  }, [load])
+
+  return (
+    <Dialog
+      open
+      onOpenChange={(o) => {
+        if (!o) onClose()
+      }}
+      size="wide"
+      title={`Profile: ${name}`}
+    >
+      {failure !== null ? (
+        <Failure>
+          <span className="flex flex-col items-center gap-3">
+            <span>Could not read the profile. {failure}</span>
+            <Button variant="primary" onClick={() => void load()}>
+              Retry
+            </Button>
+          </span>
+        </Failure>
+      ) : view === null ? (
+        <Skeleton className="h-24 w-full" role="status" aria-label="Loading the profile" />
+      ) : (
+        <div className="flex flex-col gap-4">
+          <p className="m-0 text-xs text-muted-foreground">
+            Made by {view.updatedBy} at {when(view.updatedAt)}. Secrets show as {'<secret>'}; an app&apos;s config text is not shown.
+          </p>
+          <section aria-label="Apps" className="flex flex-col gap-2">
+            <h3 className="m-0 text-sm font-semibold">Apps</h3>
+            {view.apps.length === 0 ? (
+              <Empty compact>No apps.</Empty>
+            ) : (
+              <ul className="m-0 flex list-none flex-col gap-1 p-0">
+                {view.apps.map((a) => (
+                  <li key={a.name} className="flex flex-wrap items-center gap-2 text-sm">
+                    <span className="font-medium">{a.name}</span>
+                    <span className="text-muted-foreground">{a.version}</span>
+                    <Tag tone={a.hasConfig ? 'info' : 'neutral'}>{a.hasConfig ? 'With config' : 'No config'}</Tag>
+                  </li>
+                ))}
+              </ul>
+            )}
+          </section>
+          <section aria-label="Settings" className="flex flex-col gap-2">
+            <h3 className="m-0 text-sm font-semibold">Settings</h3>
+            {view.settings === null ? <Empty compact>Not part of this profile.</Empty> : <LabeledTextarea label="Shared settings (JSON)" mono rows={10} readOnly spellCheck={false} value={JSON.stringify(view.settings, null, 2)} onChange={() => {}} />}
+          </section>
+          <section aria-label="Lists" className="flex flex-col gap-2">
+            <h3 className="m-0 text-sm font-semibold">Blocked and Allowed lists</h3>
+            {view.lists === null ? <Empty compact>Not part of this profile.</Empty> : <p className="m-0 text-sm">{view.lists.blocked} blocked and {view.lists.allowed} allowed domains.</p>}
+          </section>
+        </div>
+      )}
+    </Dialog>
+  )
+}
+
+/* ---- Import ---- */
+
+function ImportProfiles({ token, onClose, onDone }: { token: string | null; onClose: () => void; onDone: () => void }) {
+  const [text, setText] = useState<string | null>(null)
+  const [plan, setPlan] = useState<ProfileImportPlan | null>(null)
+  const [problem, setProblem] = useState<string | null>(null)
+  const [busy, setBusy] = useState(false)
+  const [done, setDone] = useState(false)
+  const fileRef = useRef<HTMLInputElement>(null)
+
+  async function check() {
+    const file = fileRef.current?.files?.[0]
+    if (!file) return setProblem('Please select a profiles file.')
+    setBusy(true)
+    setProblem(null)
+    const content = await file.text()
+    const outcome = await importProfiles(token, content, false)
+    setBusy(false)
+    if (outcome.kind !== 'ok') return setProblem(sessionEnded(outcome.kind, outcome.kind === 'error' ? outcome.message : undefined))
+    setText(content)
+    setPlan(outcome.data.response)
+  }
+
+  async function commit() {
+    if (text === null) return
+    setBusy(true)
+    const outcome = await importProfiles(token, text, true)
+    setBusy(false)
+    if (outcome.kind !== 'ok') return setProblem(sessionEnded(outcome.kind, outcome.kind === 'error' ? outcome.message : undefined))
+    setPlan(outcome.data.response)
+    setDone(true)
+    onDone()
+  }
+
+  const changes = plan?.profiles.filter((p) => p.status !== 'same').length ?? 0
+  const TONE = { new: 'info', changed: 'warn', same: 'ok' } as const
+  const LABEL = { new: 'New', changed: 'Changed', same: 'Same' } as const
+
+  return (
+    <Dialog
+      open
+      onOpenChange={(o) => {
+        if (!o) onClose()
+      }}
+      title="Import profiles"
+      actions={
+        plan === null ? (
+          <Button variant="primary" disabled={busy} onClick={() => void check()}>
+            Check file
+          </Button>
+        ) : !done ? (
+          <Button variant="primary" disabled={busy || changes === 0} onClick={() => void commit()}>
+            Import
+          </Button>
+        ) : undefined
+      }
+    >
+      {problem !== null && <Alert type="danger" title="Error">{problem}</Alert>}
+      {plan === null ? (
+        <>
+          <p className="m-0 text-sm text-muted-foreground">Choose a file made by Export. It is checked first; nothing is changed until you press Import, and no server is touched.</p>
+          <LabeledInput label="Profiles file" type="file" accept="application/json,.json" ref={fileRef} />
+        </>
+      ) : (
+        <>
+          {done && <Alert type="success" title="Imported.">The profiles were saved.</Alert>}
+          <ul className="m-0 flex list-none flex-col gap-2 p-0" aria-label="Profiles in the file">
+            {plan.profiles.map((p) => (
+              <li key={p.name} className="flex flex-wrap items-center gap-2 text-sm">
+                <span className="font-medium">{p.name}</span>
+                <Tag tone={TONE[p.status]}>{LABEL[p.status]}</Tag>
+                <span className="text-xs text-muted-foreground">{p.apps} app(s){p.settings ? ', settings' : ''}{p.lists ? ', lists' : ''}</span>
+              </li>
+            ))}
+          </ul>
+        </>
+      )}
+    </Dialog>
+  )
+}
+
+/* ---- The list ---- */
+
+export function Profiles({
+  open,
+  onOpenChange,
+  token,
+  servers,
+}: {
+  open: boolean
+  onOpenChange: (o: boolean) => void
+  token: string | null
+  /** The servers of the hub, and whether it holds an admin token for each. */
+  servers: { id: string; writable: boolean }[]
+}) {
+  const [profiles, setProfiles] = useState<ProfileSummary[] | null>(null)
+  const [failure, setFailure] = useState<string | null>(null)
+  const [creating, setCreating] = useState(false)
+  const [importing, setImporting] = useState(false)
+  const [viewing, setViewing] = useState<string | null>(null)
+  const [deleting, setDeleting] = useState<string | null>(null)
+  const [request, setRequest] = useState<RunRequest | null>(null)
+  const [problem, setProblem] = useState<string | null>(null)
+
+  const load = useCallback(async () => {
+    const outcome = await listProfiles(token)
+    if (outcome.kind === 'ok') {
+      setProfiles(outcome.data.response.profiles)
+      setFailure(null)
+    } else setFailure(sessionEnded(outcome.kind, outcome.kind === 'error' ? outcome.message : undefined))
+  }, [token])
+
+  useEffect(() => {
+    if (open) void load()
+  }, [open, load])
+
+  async function exportFile(name: string) {
+    setProblem(null)
+    const outcome = await exportProfiles(token, name)
+    if (outcome.kind !== 'ok') return setProblem(sessionEnded(outcome.kind, outcome.kind === 'error' ? outcome.message : undefined))
+    const url = URL.createObjectURL(new Blob([outcome.data.text], { type: 'application/json' }))
+    const link = document.createElement('a')
+    link.href = url
+    link.download = outcome.data.filename
+    link.click()
+    URL.revokeObjectURL(url)
+  }
+
+  async function remove(name: string) {
+    const outcome = await deleteProfile(token, name)
+    if (outcome.kind !== 'ok') setProblem(sessionEnded(outcome.kind, outcome.kind === 'error' ? outcome.message : undefined))
+    await load()
+  }
+
+  const create = (
+    <Button variant="primary" onClick={() => setCreating(true)}>
+      Create from current baselines
+    </Button>
+  )
+
+  return (
+    <>
+      <Dialog
+        open={open}
+        onOpenChange={onOpenChange}
+        size="wide"
+        title="Profiles"
+        actions={
+          <>
+            <Button onClick={() => setImporting(true)}>Import profiles</Button>
+            {profiles !== null && profiles.length > 0 && create}
+          </>
+        }
+      >
+        <p className="m-0 text-sm text-muted-foreground">
+          A profile brings a new or reset server in line: it installs the apps from that server&apos;s own store and applies their configs, the shared settings and the Blocked and Allowed lists.
+        </p>
+        {problem !== null && <Alert type="danger" title="Error">{problem}</Alert>}
+        {failure !== null && profiles === null ? (
+          <Failure>
+            <span className="flex flex-col items-center gap-3">
+              <span>Could not read the profiles. {failure}</span>
+              <Button variant="primary" onClick={() => void load()}>
+                Retry
+              </Button>
+            </span>
+          </Failure>
+        ) : profiles === null ? (
+          <div className="flex flex-col gap-3" role="status" aria-label="Loading the profiles">
+            <Skeleton className="h-12 w-full" />
+            <Skeleton className="h-12 w-full" />
+          </div>
+        ) : profiles.length === 0 ? (
+          <Empty title="No profiles yet" actions={create}>
+            Make one from the baselines the hub holds now, or import a file from another hub.
+          </Empty>
+        ) : (
+          <ul className="m-0 flex list-none flex-col gap-3 p-0" aria-label="Profiles">
+            {profiles.map((p) => (
+              <li key={p.name} aria-label={p.name} className="flex flex-wrap items-center gap-3 border-b border-border pb-3 text-sm last:border-b-0">
+                <span className="font-medium">{p.name}</span>
+                <Tag tone="neutral">{`${p.apps} app${p.apps === 1 ? '' : 's'}`}</Tag>
+                {p.configs > 0 && <Tag tone="neutral">{`${p.configs} config${p.configs === 1 ? '' : 's'}`}</Tag>}
+                {p.settings !== null && <Tag tone="neutral">{`${p.settings} settings`}</Tag>}
+                {p.lists !== null && <Tag tone="neutral">{`${p.lists.blocked + p.lists.allowed} list domains`}</Tag>}
+                <span className="text-xs text-muted-foreground">{p.updatedBy} · {when(p.updatedAt)}</span>
+                <span className="ml-auto flex flex-wrap gap-2">
+                  <Button size="sm" onClick={() => setViewing(p.name)}>
+                    View
+                  </Button>
+                  <Button size="sm" variant="primary" onClick={() => setRequest(applyRequestFor(token, p.name, servers))}>
+                    Apply to servers…
+                  </Button>
+                  <Button size="sm" onClick={() => void exportFile(p.name)}>
+                    Export
+                  </Button>
+                  <Button size="sm" variant="danger" onClick={() => setDeleting(p.name)}>
+                    Delete
+                  </Button>
+                </span>
+              </li>
+            ))}
+          </ul>
+        )}
+      </Dialog>
+
+      {creating && (
+        <CreateProfile
+          token={token}
+          onClose={() => setCreating(false)}
+          onDone={() => {
+            setCreating(false)
+            void load()
+          }}
+        />
+      )}
+      {importing && (
+        <ImportProfiles
+          token={token}
+          onClose={() => setImporting(false)}
+          onDone={() => void load()}
+        />
+      )}
+      {viewing !== null && <ViewProfile token={token} name={viewing} onClose={() => setViewing(null)} />}
+      <Confirm
+        open={deleting !== null}
+        title="Delete profile"
+        text={`Delete the profile ${deleting ?? ''}? Servers that already had it applied keep what they got.`}
+        label="Delete"
+        onClose={() => setDeleting(null)}
+        onConfirm={() => deleting !== null && remove(deleting)}
+      />
+      {request !== null && <RunFlow request={request} onClose={() => setRequest(null)} />}
+    </>
+  )
+}

src/screens/apps/RunFlow.tsx

@@ -0,0 +1,215 @@
+import { useState, type ReactNode } from 'react'
+import type { RunOutcome, TargetResult } from '../../api/apps'
+import { Button } from '../../ui/Button'
+import { Check } from '../../ui/Check'
+import { Dialog } from '../../ui/Dialog'
+import { Failure, Loading } from '../../ui/Empty'
+import { Tag } from '../../ui/Tag'
+
+/*
+Behind a hub, a change to the apps of several servers goes through three steps in one place:
+pick the servers, wait, read what happened on each. A server that cannot be chosen is still
+listed, disabled, with the reason, so nobody wonders where it went. "Retry failed" runs the
+same change again on the servers that failed and nowhere else.
+*/
+
+export interface PickerServer {
+  id: string
+  /** Why this server cannot be chosen; absent when it can. */
+  reason?: string
+}
+
+export interface RunRequest {
+  title: string
+  intro?: ReactNode
+  confirmLabel: string
+  servers: PickerServer[]
+  /** The servers ticked at the start. Only those without a reason count. */
+  preselected: string[]
+  /*
+  A review step between the picker and the change (the config sync uses it): it reads what
+  would happen on the picked servers and draws it. `context` is handed back to `run`.
+  */
+  review?: (targets: string[], setReady: (ready: boolean) => void) => Promise<{ kind: 'ok'; body: ReactNode; context: unknown; confirmLabel?: string } | { kind: 'error'; message: string }>
+  run: (targets: string[], context?: unknown) => Promise<RunOutcome>
+}
+
+/* What stops a server from being a target: it did not answer, or the hub has no admin token for it. */
+export function pickerServers(
+  servers: { id: string; writable: boolean; ok?: boolean }[],
+  extra: Record<string, string> = {},
+): PickerServer[] {
+  return servers.map((s) => {
+    const reason = s.ok === false ? 'Did not answer' : !s.writable ? 'No admin token configured on the hub' : extra[s.id]
+    return reason === undefined ? { id: s.id } : { id: s.id, reason }
+  })
+}
+
+/* The steps of a profile apply, drawn under the server they ran on. */
+const STEP_LABEL = { apps: 'Apps', configs: 'App configs', settings: 'Settings', lists: 'Lists' } as const
+const STEP_STATUS = { ok: 'Done', failed: 'Failed', skipped: 'Done earlier', 'not-run': 'Not run' } as const
+
+type Stage ='pick' | 'reviewing' | 'review' | 'running' | 'results'
+
+export function RunFlow({ request, onClose }: { request: RunRequest; onClose: (changed: boolean) => void }) {
+  const [stage, setStage] = useState<Stage>('pick')
+  const [picked, setPicked] = useState<string[]>(() =>
+    request.preselected.filter((id) => request.servers.some((s) => s.id === id && s.reason === undefined)),
+  )
+  const [results, setResults] = useState<TargetResult[]>([])
+  const [failure, setFailure] = useState<string | null>(null)
+  const [lastTargets, setLastTargets] = useState<string[]>([])
+  const [reviewed, setReviewed] = useState<{ body: ReactNode; context: unknown; confirmLabel?: string } | null>(null)
+  const [reviewFailure, setReviewFailure] = useState<string | null>(null)
+  /* A review can hold the confirm button back until the user has answered it (an Overwrite tick). */
+  const [ready, setReady] = useState(true)
+
+  async function next(targets: string[]) {
+    if (request.review === undefined) return go(targets)
+    setStage('reviewing')
+    setReviewFailure(null)
+    setReady(true)
+    const outcome = await request.review(targets, setReady)
+    if (outcome.kind === 'ok') setReviewed(outcome)
+    else {
+      setReviewed(null)
+      setReviewFailure(outcome.message)
+    }
+    setLastTargets(targets)
+    setStage('review')
+  }
+
+  async function go(targets: string[], context: unknown = reviewed?.context) {
+    setStage('running')
+    setFailure(null)
+    setLastTargets(targets)
+    const outcome = await request.run(targets, context)
+    if (outcome.kind === 'ok') {
+      const fresh = outcome.data.response.results
+      // A retry replaces only the rows it re-ran.
+      setResults((old) => [...old.filter((r) => !fresh.some((f) => f.server === r.server)), ...fresh])
+    } else {
+      setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+    }
+    setStage('results')
+  }
+
+  const failed = results.filter((r) => !r.ok).map((r) => r.server)
+  const changed = results.some((r) => r.ok)
+
+  if (stage === 'pick') {
+    return (
+      <Dialog
+        open
+        onOpenChange={() => onClose(false)}
+        title={request.title}
+        close="Cancel"
+        actions={
+          <Button variant="primary" disabled={picked.length === 0} onClick={() => void next(picked)}>
+            {request.confirmLabel}
+          </Button>
+        }
+      >
+        {request.intro != null && <div className="m-0 text-sm text-muted-foreground">{request.intro}</div>}
+        <div className="flex flex-col gap-3" role="group" aria-label="Servers">
+          {request.servers.map((s) => (
+            <Check
+              key={s.id}
+              label={s.id}
+              checked={picked.includes(s.id)}
+              disabled={s.reason !== undefined}
+              help={s.reason}
+              onChange={(on) => setPicked((old) => (on ? [...old, s.id] : old.filter((id) => id !== s.id)))}
+            />
+          ))}
+        </div>
+      </Dialog>
+    )
+  }
+
+  if (stage === 'reviewing') {
+    return (
+      <Dialog open onOpenChange={() => {}} title={request.title} close="Close">
+        <Loading>Reading {lastTargets.join(', ')}…</Loading>
+      </Dialog>
+    )
+  }
+
+  if (stage === 'review') {
+    return (
+      <Dialog
+        open
+        onOpenChange={() => onClose(false)}
+        title={`${request.title}: review`}
+        size="wide"
+        close="Cancel"
+        actions={
+          <>
+            <Button onClick={() => setStage('pick')}>Back</Button>
+            {reviewed !== null && (
+              <Button variant="primary" disabled={!ready} onClick={() => void go(lastTargets)}>
+                {reviewed.confirmLabel ?? request.confirmLabel}
+              </Button>
+            )}
+          </>
+        }
+      >
+        {reviewFailure !== null ? <Failure>{reviewFailure}</Failure> : reviewed?.body}
+      </Dialog>
+    )
+  }
+
+  if (stage === 'running') {
+    return (
+      <Dialog open onOpenChange={() => {}} title={request.title} close="Close">
+        <Loading>Working on {lastTargets.join(', ')}…</Loading>
+      </Dialog>
+    )
+  }
+
+  return (
+    <Dialog
+      open
+      onOpenChange={() => onClose(changed)}
+      title={`${request.title}: results`}
+      actions={
+        failure !== null ? (
+          <Button variant="primary" onClick={() => void go(lastTargets)}>
+            Retry
+          </Button>
+        ) : failed.length > 0 ? (
+          <Button variant="primary" onClick={() => void go(failed)}>
+            Retry failed
+          </Button>
+        ) : undefined
+      }
+    >
+      {failure !== null ? (
+        <Failure>{failure}</Failure>
+      ) : (
+        <ul className="m-0 flex list-none flex-col gap-3 p-0" aria-label="Results">
+          {results.map((r) => (
+            <li key={r.server} className="flex flex-col gap-2 text-sm">
+              <span className="flex flex-wrap items-center gap-2">
+                <span className="font-medium">{r.server}</span>
+                <Tag tone={r.ok ? 'ok' : 'dan'}>{r.ok ? 'Done' : 'Failed'}</Tag>
+                {r.message !== undefined && <span className="text-muted-foreground">{r.message}</span>}
+              </span>
+              {r.steps !== undefined && r.steps.length > 0 && (
+                <ul className="m-0 flex list-none flex-col gap-1 p-0 pl-4" aria-label={`Steps on ${r.server}`}>
+                  {r.steps.map((s) => (
+                    <li key={s.step} className="flex flex-wrap items-center gap-2 text-xs">
+                      <span className="font-medium">{STEP_LABEL[s.step]}</span>
+                      <Tag tone={s.status === 'ok' ? 'ok' : s.status === 'failed' ? 'dan' : 'neutral'}>{STEP_STATUS[s.status]}</Tag>
+                      <span className="text-muted-foreground">{s.message}</span>
+                    </li>
+                  ))}
+                </ul>
+              )}
+            </li>
+          ))}
+        </ul>
+      )}
+    </Dialog>
+  )
+}

src/screens/apps/StoreApps.tsx

@@ -30,8 +30,11 @@ and the freshly placed row is left without its "Store Update" button.
 export function StoreApps({
   token,
   onChanged,
+  node = '',
 }: {
   token: string | null
+  /** Behind a hub: the one server it acts on. */
+  node?: string
   onChanged: () => void
 }) {
   const [storeApps, setStoreApps] = useState<StoreApp[] | null>(null)
@@ -50,7 +53,7 @@ export function StoreApps({
   */
 
   const load = useCallback(async () => {
-    const outcome = await listStoreApps(token)
+    const outcome = await listStoreApps(token, node)
     if (outcome.kind === 'ok') {
       setStoreApps(outcome.data.response.storeApps)
       return
@@ -85,7 +88,7 @@ export function StoreApps({
         title: 'Store App Installed!',
         text: `DNS application '${app.name}' was installed successfully from DNS App Store.`,
       },
-      downloadAndInstall(token, app.name, app.url),
+      downloadAndInstall(token, app.name, app.url, node),
     )
   }
 
@@ -97,7 +100,7 @@ export function StoreApps({
         title: 'Store App Updated!',
         text: `DNS application '${app.name}' was updated successfully from DNS App Store.`,
       },
-      downloadAndUpdate(token, app.name, app.url),
+      downloadAndUpdate(token, app.name, app.url, node),
     )
   }
 
@@ -116,7 +119,7 @@ export function StoreApps({
         title: 'Store App Uninstalled!',
         text: `DNS application '${app.name}' was uninstalled successfully.`,
       },
-      uninstallApp(token, app.name),
+      uninstallApp(token, app.name, node),
     )
   }
 

src/screens/apps/UpdateApp.tsx

@@ -19,12 +19,15 @@ export function UpdateApp({
   token,
   name,
   onUpdated,
+  node = '',
 }: {
   open: boolean
   onOpenChange: (o: boolean) => void
   token: string | null
   name: string
   onUpdated: (name: string) => void
+  /** Behind a hub: the one server it goes to. */
+  node?: string
 }) {
   const [alert, setAlert] = useState<AlertState | null>(null)
   const [busy, setBusy] = useState(false)
@@ -49,7 +52,7 @@ export function UpdateApp({
     }
 
     setBusy(true)
-    const outcome = await updateApp(token, name, file)
+    const outcome = await updateApp(token, name, file, node)
     setBusy(false)
 
     if (outcome.kind !== 'ok') {

src/screens/apps/applyRequest.tsx

@@ -0,0 +1,90 @@
+import { applyConfig, getHistoryDiff, previewConfig, rollbackHistory, type HistoryEntry } from '../../api/apps'
+import { Alert } from '../../ui/Alert'
+import { Tag } from '../../ui/Tag'
+import { DiffView } from './ConfigDiffView'
+import type { PickerServer, RunRequest } from './RunFlow'
+
+/*
+The two changes to a server's config that go through the picker, the review and the results: bring
+it to the baseline (the sync), and put an earlier version back (a rollback). The Config dialog and
+the drift list both start them.
+*/
+
+const sessionEnded = (kind: string, message?: string): string => (kind === 'error' ? (message ?? 'Failed.') : 'The session ended.')
+
+/* The baseline plus each server's local values, to the chosen servers, after a preview. */
+export function applyRequest(token: string | null, name: string, servers: PickerServer[], preselected: string[]): RunRequest {
+  return {
+    title: `Apply the baseline of ${name}`,
+    intro: 'Each server gets the baseline plus its own local values. You see what changes before anything is written.',
+    confirmLabel: 'Preview',
+    servers,
+    preselected,
+    review: async (targets) => {
+      const outcome = await previewConfig(token, name, targets)
+      if (outcome.kind !== 'ok') return { kind: 'error', message: sessionEnded(outcome.kind, outcome.kind === 'error' ? outcome.message : undefined) }
+      const previews = outcome.data.response.targets
+      return {
+        kind: 'ok',
+        confirmLabel: 'Apply',
+        context: Object.fromEntries(previews.filter((p) => p.ok).map((p) => [p.server, p.currentHash])),
+        body: (
+          <div className="flex flex-col gap-6">
+            {previews.map((p) => (
+              <section key={p.server} aria-label={`Preview ${p.server}`} className="flex flex-col gap-3">
+                <h3 className="m-0 flex items-center gap-2 text-sm font-semibold">
+                  {p.server}
+                  {!p.ok ? <Tag tone="dan">Cannot apply</Tag> : p.unchanged ? <Tag tone="ok">Already in sync</Tag> : <Tag tone="warn">Will change</Tag>}
+                </h3>
+                {p.error !== undefined && <Alert type="danger" title="Error">{p.error}</Alert>}
+                {p.warnings.map((w) => (
+                  <Alert key={w} type="warning" title="Warning">
+                    {w}
+                  </Alert>
+                ))}
+                {p.ok && !p.unchanged && <DiffView diff={p.changes} from="Now" to="After apply" />}
+              </section>
+            ))}
+          </div>
+        ),
+      }
+    },
+    run: (targets, context) => applyConfig(token, name, targets, (context ?? {}) as Record<string, string>),
+  }
+}
+
+/* One saved version of a server's config, put back on the chosen servers after seeing what changes. */
+export function rollbackRequest(token: string | null, name: string, entry: HistoryEntry, servers: PickerServer[], preselected: string[]): RunRequest {
+  return {
+    title: `Roll back ${name}`,
+    intro: `Put the version saved at ${new Date(entry.at).toLocaleString()} back on the servers you choose. Their current configs are saved to the history first.`,
+    confirmLabel: 'Preview',
+    servers,
+    preselected,
+    review: async (targets) => {
+      const diffs = await Promise.all(targets.map((server) => getHistoryDiff(token, name, entry.id, server)))
+      const failed = diffs.find((d) => d.kind !== 'ok')
+      if (failed !== undefined) return { kind: 'error', message: sessionEnded(failed.kind, failed.kind === 'error' ? failed.message : undefined) }
+      const ok = diffs.flatMap((d) => (d.kind === 'ok' ? [d.data.response] : []))
+      return {
+        kind: 'ok',
+        confirmLabel: 'Roll back',
+        context: Object.fromEntries(ok.map((d) => [d.server, d.currentHash])),
+        body: (
+          <div className="flex flex-col gap-6">
+            {ok.map((d) => (
+              <section key={d.server} aria-label={`Preview ${d.server}`} className="flex flex-col gap-3">
+                <h3 className="m-0 flex items-center gap-2 text-sm font-semibold">
+                  {d.server}
+                  {d.unchanged ? <Tag tone="ok">Already this version</Tag> : <Tag tone="warn">Will change</Tag>}
+                </h3>
+                {!d.unchanged && <DiffView diff={d.changes} from="Now" to="After rollback" />}
+              </section>
+            ))}
+          </div>
+        ),
+      }
+    },
+    run: (targets, context) => rollbackHistory(token, name, entry.id, targets, (context ?? {}) as Record<string, string>),
+  }
+}

src/screens/blocking/AddDomainBar.tsx

@@ -1,9 +1,13 @@
-import { useId, useRef, useState } from 'react'
+import { useContext, useId, useRef, useState } from 'react'
+import { HubNode, nodeArg } from '../../lib/hub-write'
+import { HubAll } from '../../lib/hub-all'
 import { addDomain, type DomainList } from '../../api/zonelists'
 import { Panel, Body } from '../../ui/Panel'
 import { Field, Input } from '../../ui/Field'
 import { Loading } from '../../ui/Empty'
 import { PermissionButton } from '../../ui/PermissionButton'
+import { Menu } from '../../ui/Menu'
+import { LockedItem } from './Locked'
 import { loadQuickList, type QuickEntry } from '../../lib/quick-lists'
 import { noticeFromFailure, type Notice } from '../../lib/notice'
 import { checkDomain, verdictOf, type Verdict } from './check-domain'
@@ -51,6 +55,10 @@ export function AddDomainBar({
   onNotice: (n: Notice) => void
   onChanged: (list: DomainList, domain: string) => void
 }) {
+  /* Behind a hub: the server the section has chosen (empty elsewhere, which sends no `node`). */
+  const node = useContext(HubNode)
+  /* With "All servers" chosen the verbs open the flow that makes the change on the servers picked next. */
+  const all = useContext(HubAll)
   const [field, setField] = useState('')
   const [busy, setBusy] = useState(false)
   const entry = useRef<HTMLInputElement>(null)
@@ -82,7 +90,7 @@ export function AddDomainBar({
       checkDomain(token, domain, {
         settings: missing(permissions, 'Settings.canView'),
         allowed: missing(permissions, 'Allowed.canView'),
-      }),
+      }, ...nodeArg(node)),
       /* The names the Lists tab gives its rows: Technitium's catalogue, then ours. */
       loadQuickList('quick-block-lists').then((c) => [...c, ...moreLists(c)]),
     ])
@@ -119,8 +127,26 @@ export function AddDomainBar({
       return
     }
 
+    if (all !== null) {
+      const verb = list === 'allowed' ? 'Allowed' : 'Blocked'
+      all({
+        action: `${list}.add`,
+        params: { domain: domain.trim() },
+        title: `Add '${domain.trim()}' to ${verb} on several servers`,
+        confirmLabel: `Add to ${verb}`,
+        onDone: (changed) => {
+          if (changed) {
+            setField('')
+            forget()
+            onChanged(list, domain)
+          }
+        },
+      })
+      return
+    }
+
     setBusy(true)
-    const outcome = await addDomain(list, token, domain)
+    const outcome = await addDomain(list, token, domain, ...nodeArg(node))
     setBusy(false)
     if (outcome.kind !== 'ok') {
       onNotice(noticeFromFailure(outcome))
@@ -137,6 +163,30 @@ export function AddDomainBar({
     onChanged(list, domain)
   }
 
+  /* Removing a rule from Blocked or Allowed: only offered for several servers at once, since on one server the Rules tab does it. */
+  function remove(list: DomainList) {
+    const domain = field.trim()
+    if (domain === '') {
+      onNotice({ type: 'warning', title: 'Missing!', text: 'Please enter a domain name to remove.' })
+      entry.current?.focus()
+      return
+    }
+    const verb = list === 'allowed' ? 'Allowed' : 'Blocked'
+    all?.({
+      action: `${list}.delete`,
+      params: { domain },
+      title: `Remove '${domain}' from ${verb} on several servers`,
+      confirmLabel: `Remove from ${verb}`,
+      onDone: (changed) => {
+        if (changed) {
+          setField('')
+          forget()
+          onChanged(list, domain)
+        }
+      },
+    })
+  }
+
   return (
     <Panel title="Check, block or allow a domain">
       <Body>
@@ -189,6 +239,33 @@ export function AddDomainBar({
           >
             Check
           </PermissionButton>
+          {/* One menu, not two more buttons: the row has no room for five at desktop width. */}
+          {all !== null && (
+            <Menu label="Remove" text="Remove">
+              {(close) =>
+                (['blocked', 'allowed'] as const).map((list) => {
+                  const verb = list === 'allowed' ? 'Allowed' : 'Blocked'
+                  const need = missing(permissions, list === 'allowed' ? 'Allowed.canModify' : 'Blocked.canModify')
+                  return need != null ? (
+                    <LockedItem key={list} need={need} placement="left">
+                      From {verb}
+                    </LockedItem>
+                  ) : (
+                    <button
+                      key={list}
+                      type="button"
+                      onClick={() => {
+                        close()
+                        remove(list)
+                      }}
+                    >
+                      From {verb}
+                    </button>
+                  )
+                })
+              }
+            </Menu>
+          )}
         </form>
         {/* One live region for the whole answer: the wait and the verdict that
             replaces it are announced once each. */}

src/screens/blocking/AllServers.hub.test.tsx

@@ -0,0 +1,207 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { render, screen, within } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import * as dashboard from '../../api/dashboard'
+import * as settings from '../../api/settings'
+import * as apps from '../../api/apps'
+import * as hubRun from '../../api/hub-run'
+import { Overview } from './Overview'
+import { BlockLists } from './BlockLists'
+import { HubMode } from '../../ui/ClusterNodeSelect'
+import { HubNode, HubWritable } from '../../lib/hub-write'
+import { HubServerIds } from '../../lib/hub-all'
+import type { Permissions } from './permissions'
+import { checkUrls } from './AllServersActions'
+
+vi.mock('../dashboard/Chart', () => ({ Chart: ({ aria }: { aria: string }) => <div role="img" aria-label={aria} /> }))
+
+afterEach(() => {
+  vi.restoreAllMocks()
+  localStorage.clear()
+})
+
+const NODES = [
+  { name: 'mdns', type: 'Primary' },
+  { name: 'ndns', type: 'Secondary' },
+]
+
+/* An operator: blocking and cache yes, settings no (what the hub session says). */
+const OPERATOR: Permissions = {
+  Dashboard: { canView: true, canModify: false, canDelete: false },
+  Settings: { canView: true, canModify: false, canDelete: false },
+  Blocked: { canView: true, canModify: true, canDelete: true },
+  Allowed: { canView: true, canModify: true, canDelete: true },
+  DnsClient: { canView: true, canModify: false, canDelete: false },
+}
+
+const STATS = {
+  totalQueries: 100, totalNoError: 0, totalServerFailure: 0, totalNxDomain: 0, totalRefused: 0,
+  totalAuthoritative: 0, totalRecursive: 0, totalCached: 0, totalBlocked: 10, totalDropped: 0,
+  totalClients: 1, zones: 0, cachedEntries: 0, allowedZones: 0, blockedZones: 0, allowListZones: 0, blockListZones: 0,
+}
+const MAIN = { labels: ['a'], datasets: [{ label: 'Total', data: [1] }, { label: 'Blocked', data: [1] }] }
+
+function serve() {
+  vi.spyOn(settings, 'getSettings').mockResolvedValue({ enableBlocking: true } as never)
+  vi.spyOn(apps, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [] } } } as never)
+  vi.spyOn(dashboard, 'getDashboardStats').mockResolvedValue({
+    kind: 'ok',
+    data: { stats: STATS, mainChartData: MAIN, queryResponseChartData: MAIN, queryTypeChartData: MAIN, protocolTypeChartData: MAIN, topClients: [], topDomains: [], topBlockedDomains: [] },
+  } as never)
+}
+
+const results = (...r: { server: string; ok: boolean; message?: string }[]) => ({ kind: 'ok', data: { response: { results: r } } }) as never
+
+function hubOverview(writable = ['mdns', 'ndns'], permissions: Permissions = OPERATOR) {
+  serve()
+  return render(
+    <HubMode.Provider value>
+      <HubWritable.Provider value={writable}>
+        <HubServerIds.Provider value={['mdns', 'ndns']}>
+          <Overview token="t" permissions={permissions} nodes={NODES} clusterInitialised />
+        </HubServerIds.Provider>
+      </HubWritable.Provider>
+    </HubMode.Provider>,
+  )
+}
+
+describe('Blocking overview behind a hub, in All servers', () => {
+  it('says the change goes to the servers chosen next, and keeps Block and Allow enabled', async () => {
+    hubOverview()
+    expect(await screen.findByText(/goes to the servers you choose in the next step/)).toBeInTheDocument()
+    expect(screen.queryByText(/Choose a server to make changes/)).not.toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Block' })).toBeEnabled()
+    expect(screen.getByRole('button', { name: 'Allow' })).toBeEnabled()
+  })
+
+  it('Block opens the picker, runs on the ticked servers and retries only the failed', async () => {
+    const run = vi
+      .spyOn(hubRun, 'runOnServers')
+      .mockResolvedValueOnce(results({ server: 'mdns', ok: true, message: 'Added to Blocked.' }, { server: 'ndns', ok: false, message: 'ndns did not answer' }))
+      .mockResolvedValueOnce(results({ server: 'ndns', ok: true, message: 'Added to Blocked.' }))
+    hubOverview()
+    await userEvent.type(await screen.findByLabelText('Domain'), 'ads.example')
+    await userEvent.click(screen.getByRole('button', { name: 'Block' }))
+    const picker = await screen.findByRole('dialog')
+    expect(within(picker).getByRole('checkbox', { name: 'mdns' })).toBeChecked()
+    expect(within(picker).getByRole('checkbox', { name: 'ndns' })).toBeChecked()
+    await userEvent.click(within(picker).getByRole('button', { name: 'Add to Blocked' }))
+    expect(run).toHaveBeenNthCalledWith(1, 't', 'blocked.add', { domain: 'ads.example' }, ['mdns', 'ndns'])
+    const list = await screen.findByRole('list', { name: 'Results' })
+    expect(within(list).getByText('ndns did not answer')).toBeInTheDocument()
+    expect(within(list).getAllByText('Failed')).toHaveLength(1)
+    await userEvent.click(screen.getByRole('button', { name: 'Retry failed' }))
+    expect(run).toHaveBeenNthCalledWith(2, 't', 'blocked.add', { domain: 'ads.example' }, ['ndns'])
+    expect(await within(await screen.findByRole('list', { name: 'Results' })).findAllByText('Done')).toHaveLength(2)
+  })
+
+  it('disables a server the hub holds no admin token for, with the reason', async () => {
+    hubOverview(['mdns'])
+    await userEvent.type(await screen.findByLabelText('Domain'), 'ads.example')
+    await userEvent.click(screen.getByRole('button', { name: 'Allow' }))
+    const picker = await screen.findByRole('dialog')
+    expect(within(picker).getByRole('checkbox', { name: 'ndns' })).toBeDisabled()
+    expect(within(picker).getByText('No admin token configured on the hub')).toBeInTheDocument()
+    expect(within(picker).getByRole('checkbox', { name: 'mdns' })).toBeChecked()
+  })
+
+  it('removes a domain from Blocked on several servers', async () => {
+    const run = vi.spyOn(hubRun, 'runOnServers').mockResolvedValue(results({ server: 'mdns', ok: true }, { server: 'ndns', ok: true }))
+    hubOverview()
+    await userEvent.type(await screen.findByLabelText('Domain'), 'ads.example')
+    await userEvent.click(screen.getByRole('button', { name: 'Remove' }))
+    await userEvent.click(await screen.findByRole('menuitem', { name: 'From Blocked' }))
+    await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Remove from Blocked' }))
+    expect(run).toHaveBeenCalledWith('t', 'blocked.delete', { domain: 'ads.example' }, ['mdns', 'ndns'])
+  })
+
+  it('lets an operator use the blocking switch, and runs a temporary disable on all servers', async () => {
+    const run = vi.spyOn(hubRun, 'runOnServers').mockResolvedValue(results({ server: 'mdns', ok: true }, { server: 'ndns', ok: true }))
+    hubOverview()
+    const trigger = await screen.findByRole('button', { name: 'Blocking options' })
+    expect(trigger).toBeEnabled()
+    await userEvent.click(trigger)
+    await userEvent.click(await screen.findByRole('menuitem', { name: 'Disable Blocking For 15 Minutes' }))
+    await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Disable blocking' }))
+    expect(run).toHaveBeenCalledWith('t', 'blocking.disable', { minutes: '15' }, ['mdns', 'ndns'])
+  })
+
+  it('updates the lists on all servers', async () => {
+    const run = vi.spyOn(hubRun, 'runOnServers').mockResolvedValue(results({ server: 'mdns', ok: true }, { server: 'ndns', ok: true }))
+    hubOverview()
+    await userEvent.click(await screen.findByRole('button', { name: 'Update lists on all servers' }))
+    await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Update lists' }))
+    expect(run).toHaveBeenCalledWith('t', 'blocking.updateLists', {}, ['mdns', 'ndns'])
+  })
+
+  it('edits the list URLs on all servers, refusing an address that is not http or https', async () => {
+    const run = vi.spyOn(hubRun, 'runOnServers').mockResolvedValue(results({ server: 'mdns', ok: true }, { server: 'ndns', ok: true }))
+    hubOverview()
+    await userEvent.click(await screen.findByRole('button', { name: 'Edit list URLs on all servers' }))
+    const ask = await screen.findByRole('dialog')
+    const field = within(ask).getByLabelText('Block list URLs')
+    await userEvent.type(field, 'ftp://bad.example/x')
+    await userEvent.click(within(ask).getByRole('button', { name: 'Choose servers' }))
+    expect(await within(ask).findByRole('alert')).toHaveTextContent(/Not a valid http, https or file address/)
+    expect(run).not.toHaveBeenCalled()
+    await userEvent.clear(field)
+    await userEvent.type(field, 'https://a.example/hosts.txt')
+    await userEvent.click(within(ask).getByRole('button', { name: 'Choose servers' }))
+    await userEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Replace URLs' }))
+    expect(run).toHaveBeenCalledWith('t', 'blocking.setListUrls', { urls: 'https://a.example/hosts.txt' }, ['mdns', 'ndns'])
+  })
+
+  it('offers nothing of this to a role that may not change blocking', async () => {
+    hubOverview(['mdns', 'ndns'], {
+      ...OPERATOR,
+      Blocked: { canView: true, canModify: false, canDelete: false },
+      Allowed: { canView: true, canModify: false, canDelete: false },
+    })
+    await screen.findByLabelText('Domain')
+    expect(screen.queryByRole('button', { name: 'Update lists on all servers' })).not.toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Remove' })).not.toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Block' })).toBeDisabled()
+  })
+})
+
+describe('the blocking switch outside a hub', () => {
+  it('still asks for Settings: Modify, and offers nothing for several servers', async () => {
+    serve()
+    render(<Overview token="t" permissions={OPERATOR} />)
+    expect(await screen.findByRole('button', { name: 'Disable' })).toBeDisabled()
+    expect(screen.queryByRole('button', { name: 'Update lists on all servers' })).not.toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Remove' })).not.toBeInTheDocument()
+  })
+})
+
+describe('block lists behind a hub', () => {
+  it('lets an operator update now and offers the actions for all servers, with the URLs filled in', async () => {
+    vi.spyOn(settings, 'readSettings').mockResolvedValue({ kind: 'ok', data: { blockListUrls: ['https://a.example/x', '# note'], blockListNextUpdatedOn: null } } as never)
+    vi.spyOn(dashboard, 'getDashboardStats').mockResolvedValue({ kind: 'ok', data: { stats: STATS } } as never)
+    render(
+      <HubMode.Provider value>
+        <HubWritable.Provider value={['mdns']}>
+          <HubServerIds.Provider value={['mdns', 'ndns']}>
+            <HubNode.Provider value="mdns">
+              <BlockLists token="t" permissions={OPERATOR} allServers />
+            </HubNode.Provider>
+          </HubServerIds.Provider>
+        </HubWritable.Provider>
+      </HubMode.Provider>,
+    )
+    expect(await screen.findByRole('button', { name: 'Update lists on all servers' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Update Now' })).toBeEnabled()
+    await userEvent.click(screen.getByRole('button', { name: 'Edit list URLs on all servers' }))
+    expect(await screen.findByLabelText('Block list URLs')).toHaveValue('https://a.example/x\n# note')
+  })
+})
+
+describe('the list dialog check', () => {
+  it('takes allow lines, switched-off lines and headings, and refuses a bad scheme after the bang or a comma', () => {
+    expect(checkUrls('# Privacy\nhttps://a.example/x\n!https://b.example/allow\n#https://c.example/off\nfile:///x.txt')).toBeNull()
+    expect(checkUrls('!ftp://b.example/allow')).toMatch(/Not a valid/)
+    expect(checkUrls('# a, b')).toMatch(/comma/)
+    expect(checkUrls('https://a.example/x,y')).toMatch(/comma/)
+    expect(checkUrls('  \n ')).toMatch(/at least one/)
+  })
+})

src/screens/blocking/AllServersActions.tsx

@@ -0,0 +1,93 @@
+import { useContext, useState } from 'react'
+import { useAllServersFlow } from '../../lib/hub-all'
+import { Button } from '../../ui/Button'
+import { AskDialog } from '../../ui/AskDialog'
+import { HubMode } from '../../ui/ClusterNodeSelect'
+
+/*
+Behind a hub: the block list actions that go to several servers. "Update lists" asks every chosen
+server to download its lists now; "Edit list URLs" REPLACES the list of URLs on every chosen
+server with the one typed here (one per line), so the dialog says so. The
+caller shows this only to a role that may change blocking. Drawn nowhere outside a hub.
+*/
+
+/*
+What the dialog starts with: the lines of the chosen server's list as it holds them, `!` allow
+lists, `#` switched-off lists and headings included, so saving it back changes nothing.
+*/
+export function plainUrls(lines: readonly string[] | null | undefined): string {
+  return (lines ?? []).map((l) => l.trim()).filter((l) => l !== '').join('\n')
+}
+
+/* The hub's rules (hub/servers-run.ts `cleanUrls`), so the dialog and the hub give the same answer. */
+export function checkUrls(text: string): string | null {
+  const lines = text.split(/\r?\n/).map((l) => l.trim()).filter((l) => l !== '')
+  if (lines.length === 0) return 'Give at least one block list URL.'
+  if (lines.length > 300) return 'At most 300 lines.'
+  for (const line of lines) {
+    if (line.includes(',')) return `A line cannot contain a comma: ${line.slice(0, 60)}`
+    if (line.startsWith('#')) {
+      if (line.length > 200) return 'A comment line is too long.'
+      continue
+    }
+    const body = line.startsWith('!') ? line.slice(1).trim() : line
+    let ok = line.length <= 2000 && !/\s/.test(body)
+    if (ok) {
+      try {
+        ok = ['http:', 'https:', 'file:'].includes(new URL(body).protocol)
+      } catch {
+        ok = false
+      }
+    }
+    if (!ok) return `Not a valid http, https or file address: ${line.slice(0, 60)}`
+  }
+  return null
+}
+
+export function AllServersActions({ token, initialUrls }: { token: string | null; initialUrls?: readonly string[] | null }) {
+  const hub = useContext(HubMode)
+  const { start, flow } = useAllServersFlow(token)
+  const [editing, setEditing] = useState(false)
+  if (!hub) return null
+  return (
+    <>
+      <Button
+        onClick={() =>
+          start({
+            action: 'blocking.updateLists',
+            title: 'Update block lists on several servers',
+            intro: 'Each chosen server downloads its block lists now.',
+            confirmLabel: 'Update lists',
+          })
+        }
+      >
+        Update lists on all servers
+      </Button>
+      <Button onClick={() => setEditing(true)}>Edit list URLs on all servers</Button>
+      {editing && (
+        <AskDialog
+          title="Edit list URLs on all servers"
+          label="Block list URLs"
+          intro="One address per line: a list, `!` before an allow list, `#` before a switched-off list or a heading. This replaces the whole list of block list URLs on every server you choose next."
+          initial={plainUrls(initialUrls)}
+          multiline
+          placeholder="https://example.com/hosts.txt"
+          confirmLabel="Choose servers"
+          check={checkUrls}
+          onClose={() => setEditing(false)}
+          onSubmit={(text) => {
+            setEditing(false)
+            start({
+              action: 'blocking.setListUrls',
+              params: { urls: text },
+              title: 'Replace block list URLs on several servers',
+              intro: 'The list of URLs on each chosen server is replaced by the one you typed.',
+              confirmLabel: 'Replace URLs',
+            })
+          }}
+        />
+      )}
+      {flow}
+    </>
+  )
+}

src/screens/blocking/BlockLists.tsx

@@ -1,4 +1,7 @@
-import { useCallback, useEffect, useId, useRef, useState, type FormEvent, type ReactNode } from 'react'
+import { useCallback, useContext, useEffect, useId, useRef, useState, type FormEvent, type ReactNode } from 'react'
+import { HubNode, nodeArg } from '../../lib/hub-write'
+import { HubMode } from '../../ui/ClusterNodeSelect'
+import { AllServersActions } from './AllServersActions'
 import { readSettings, setSettings, forceUpdateBlockLists } from '../../api/settings'
 import { getDashboardStats } from '../../api/dashboard'
 import { loadQuickList, type QuickEntry } from '../../lib/quick-lists'
@@ -24,7 +27,7 @@ import {
 } from './list-lines'
 import { Locked } from './Locked'
 import { hostsOf, moreLists } from './extra-lists'
-import { missing, type Permissions } from './permissions'
+import { missing, switchNeed, type Permissions } from './permissions'
 import shared from './Blocking.module.css'
 import styles from './BlockLists.module.css'
 import { UrlText } from '../../ui/UrlText'
@@ -201,17 +204,25 @@ export function BlockLists({
   permissions,
   clusterInitialised = false,
   onDirty,
+  allServers = false,
 }: {
   tabs?: ReactNode
   token: string | null
   permissions: Permissions
   clusterInitialised?: boolean
+  /** Behind a hub: the role may change blocking on several servers at once, so the actions for that are offered. */
+  allServers?: boolean
   /** Told whether there are unsaved changes, for the tab's mark; false on leaving. */
   onDirty?: (dirty: boolean) => void
 }) {
-  const node = clusterInitialised ? 'cluster' : ''
+  /* Behind a hub the lists are the chosen server's own (no cluster); elsewhere the cluster's. */
+  const hubNode = useContext(HubNode)
+  const node = hubNode !== '' ? hubNode : clusterInitialised ? 'cluster' : ''
   const viewNeed = missing(permissions, 'Settings.canView')
   const modifyNeed = missing(permissions, 'Settings.canModify')
+  /* Behind a hub the list update is a blocking action (an operator's), unlike saving the URLs, which is a settings change. */
+  const hub = useContext(HubMode)
+  const updateNeed = switchNeed(permissions, hub)
   const statsNeed = missing(permissions, 'Dashboard.canView')
   const invalidId = useId()
 
@@ -461,7 +472,7 @@ export function BlockLists({
     // known if the read fails.
     const before = await readSettings(token, node)
     const from = before.kind !== 'ok' ? serverNext.current : (before.data.blockListNextUpdatedOn ?? null)
-    const ok = await forceUpdateBlockLists(token)
+    const ok = await forceUpdateBlockLists(token, ...nodeArg(hubNode))
     setBusy(false)
     if (!ok) return
     // main.js:2356. The label becomes "Updating Now" without reloading the settings.
@@ -621,6 +632,7 @@ export function BlockLists({
             </>
           )}
           <Panel className={shared.centred}><Body>
+            <div className="flex flex-col gap-4">
             <div className={shared.kpi}>
               <div className={shared.kpiText}>
                 <span className={shared.kpiValue}>{loading ? '—' : nextUpdateText(next)}</span>
@@ -630,13 +642,19 @@ export function BlockLists({
                 </span>
               </div>
               <PermissionButton
-                permission={modifyNeed}
+                permission={updateNeed}
                 disabled={busy || !hasSaved}
                 onClick={() => setAskUpdate(true)}
               >
                 Update Now
               </PermissionButton>
             </div>
+            {hub && allServers && (
+              <div className="flex flex-wrap items-center gap-2">
+                <AllServersActions token={token} initialUrls={saved?.map((l) => l.raw)} />
+              </div>
+            )}
+            </div>
           </Body></Panel>
         </div>
 

src/screens/blocking/Blocking.tsx

@@ -1,9 +1,13 @@
-import { useEffect, useState } from 'react'
+import { useContext, useEffect, useState } from 'react'
+import { ClusterNodeSelect, HubMode } from '../../ui/ClusterNodeSelect'
+import { CompareServersButton } from '../settings/SettingsCompare'
+import { HubNode, useHubDefaultNode, useHubWrite } from '../../lib/hub-write'
+import { HubWriteHint } from '../../ui/HubWriteHint'
 import { getSettings } from '../../api/settings'
 import { getDashboardStats } from '../../api/dashboard'
 import { BlockingMenu } from '../dashboard/BlockingMenu'
 import { Tag } from '../../ui/Tag'
-import { missing } from './permissions'
+import { missing, switchNeed, withoutWrites } from './permissions'
 import { untilText, useBlockingState } from './blocking-state'
 import { SubTabs } from '../../ui/SubTabs'
 import { ruleSearch } from './rules-model'
@@ -36,12 +40,12 @@ function dropRule(): void {
 
 /* What the tabs count: the rules (blocked and allowed zones) and the lists. Each is read
    once and only when the session may read it; a failed read leaves the tab without one. */
-function useCounts(token: string | null, permissions: Permissions) {
+function useCounts(token: string | null, permissions: Permissions, node = '') {
   const [counts, setCounts] = useState<{ Rules?: number; Lists?: number }>({})
   useEffect(() => {
     let alive = true
     if (missing(permissions, 'Dashboard.canView') == null) {
-      void getDashboardStats(token).then((o) => {
+      void getDashboardStats(token, undefined, undefined, node).then((o) => {
         if (alive && o.kind === 'ok') {
           const s = o.data.stats
           setCounts((c) => ({ ...c, Rules: s.blockedZones + s.allowedZones }))
@@ -49,7 +53,7 @@ function useCounts(token: string | null, permissions: Permissions) {
       })
     }
     if (missing(permissions, 'Settings.canView') == null) {
-      void getSettings(token).then((s) => {
+      void getSettings(token, node).then((s) => {
         if (alive && s != null) setCounts((c) => ({ ...c, Lists: s.blockListUrls?.length ?? 0 }))
       })
     }
@@ -58,16 +62,17 @@ function useCounts(token: string | null, permissions: Permissions) {
     }
     // The permission object is read once: the counts are a hint, not a live figure.
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [token])
+  }, [token, node])
   return counts
 }
 
 /* The state strip of the canvas, above the tabs. Overview carries the full status block
    itself, so the strip is for the other two. */
-function Banner({ token, permissions }: { token: string | null; permissions: Permissions }) {
-  const { state } = useBlockingState(token, permissions)
+function Banner({ token, permissions, node }: { token: string | null; permissions: Permissions; node: string }) {
+  const { state } = useBlockingState(token, permissions, true, node)
+  const hub = useContext(HubMode)
   if (state == null || state === 'failed') return null
-  const canModify = missing(permissions, 'Settings.canModify') == null
+  const canModify = switchNeed(permissions, hub) == null
   return (
     <div className={shared.banner} role="status">
       {state.kind === 'on' && <Tag tone="ok">Enabled</Tag>}
@@ -82,7 +87,7 @@ function Banner({ token, permissions }: { token: string | null; permissions: Per
       </span>
       {canModify && (
         <span className={shared.bannerAct}>
-          <BlockingMenu token={token} text={state.kind === 'on' ? 'Temporarily Disable Blocking' : 'Enable Blocking'} onNotice={() => {}} />
+          <BlockingMenu token={token} text={state.kind === 'on' ? 'Temporarily Disable Blocking' : 'Enable Blocking'} onNotice={() => {}} node={node} />
         </span>
       )}
     </div>
@@ -131,11 +136,35 @@ export function Blocking({
 
   const [listsDirty, setListsDirty] = useState(false)
 
-  const counts = useCounts(token, permissions)
+  /*
+  Behind a hub the Rules and Lists tabs act on ONE server, chosen here (the first at the start;
+  Overview keeps its own selector, which also offers "All servers" for the figures). Where the
+  hub cannot write (no admin token) the writes are switched off and the line under the selector
+  says why. Outside a hub none of this exists: `hubNode` is empty and `permissions` are the role's.
+  */
+  const hub = useContext(HubMode)
+  const defaultNode = useHubDefaultNode(nodes)
+  const [chosen, setChosen] = useState(defaultNode)
+  const hubNode = hub ? chosen : ''
+  const write = useHubWrite(hubNode)
+  const effective = write.allowed ? permissions : withoutWrites(permissions)
+
+  const counts = useCounts(token, permissions, hubNode)
 
   const tabs = (
     <>
-    {active !== 'Overview' && <Banner token={token} permissions={permissions} />}
+    {hub && permissions?.Settings?.canModify === true && (
+      <div className="flex flex-wrap items-center gap-2">
+        <CompareServersButton token={token} scope="blocking" />
+      </div>
+    )}
+    {hub && active !== 'Overview' && (
+      <>
+        <ClusterNodeSelect nodes={nodes} initialised={clusterInitialised} value={chosen} onChange={setChosen} />
+        <HubWriteHint write={write} />
+      </>
+    )}
+    {active !== 'Overview' && <Banner token={token} permissions={effective} node={hubNode} />}
     <SubTabs
       label="Blocking sections"
       section="blocking"
@@ -149,17 +178,24 @@ export function Blocking({
   )
 
   if (active === 'Rules') {
-    return <Rules tabs={tabs} token={token} permissions={permissions} nodes={nodes} clusterInitialised={clusterInitialised} />
+    return (
+      <HubNode.Provider value={hubNode}>
+        <Rules tabs={tabs} token={token} permissions={effective} nodes={nodes} clusterInitialised={clusterInitialised} />
+      </HubNode.Provider>
+    )
   }
   if (active === 'Lists') {
     return (
+      <HubNode.Provider value={hubNode}>
       <BlockLists
         tabs={tabs}
         token={token}
-        permissions={permissions}
+        permissions={effective}
         clusterInitialised={clusterInitialised}
         onDirty={setListsDirty}
+        allServers={hub && missing(permissions, 'Blocked.canModify', 'Allowed.canModify') == null}
       />
+      </HubNode.Provider>
     )
   }
   return (

src/screens/blocking/Overview.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from 'react'
+import { useCallback, useContext, useEffect, useMemo, useRef, useState, type ReactNode } from 'react'
 import {
   getDashboardStats, RANGES, RANGE_LABEL,
   type ChartData, type DashboardStats, type Range,
@@ -6,7 +6,7 @@ import {
 import { percentage, hasData, Legend, useHidden } from '../dashboard/Dashboard'
 import { Chart } from '../dashboard/Chart'
 import { InChrome } from '../../app/ChromeSlot'
-import { ClusterNodeSelect, AGGREGATE, type ClusterNode } from '../../ui/ClusterNodeSelect'
+import { ClusterNodeSelect, AGGREGATE, HubMode, type ClusterNode } from '../../ui/ClusterNodeSelect'
 import { SectionHeader } from '../../ui/SectionHeader'
 import { Segmented } from '../../ui/Segmented'
 import { Panel, Body } from '../../ui/Panel'
@@ -20,7 +20,10 @@ import { TopTable } from './TopTable'
 import { RecentBlocked } from './RecentBlocked'
 import { LAST_HOUR_REFRESH_MS, useLiveView } from '../dashboard/useLive'
 import { Locked } from './Locked'
-import { missing, type Permissions } from './permissions'
+import { missing, withoutWrites, type Permissions } from './permissions'
+import { HubNode, useHubWrite } from '../../lib/hub-write'
+import { HubAll, useAllServersFlow } from '../../lib/hub-all'
+import { HubWriteHint } from '../../ui/HubWriteHint'
 import shared from './Blocking.module.css'
 import styles from './Overview.module.css'
 
@@ -181,7 +184,7 @@ function Gap({ failure, loading, announce = true }: { failure: boolean; loading:
 export function Overview({
   tabs,
   token,
-  permissions,
+  permissions: rolePermissions,
   nodes = [],
   clusterInitialised = false,
   serverDomain,
@@ -193,9 +196,23 @@ export function Overview({
   clusterInitialised?: boolean
   serverDomain?: string
 }) {
-  const statsNeed = missing(permissions, 'Dashboard.canView')
+  const statsNeed = missing(rolePermissions, 'Dashboard.canView')
   const [range, setRange] = useState<Period>('LastHour')
   const [node, setNode] = useState<string>(() => localStorage.getItem('dashboardClusterNode') || AGGREGATE)
+  /* Behind a hub, a change is allowed only on a chosen server the hub can write to: with "All servers"
+     the controls are switched off like a missing permission, and the line under the selector says why. */
+  const hub = useContext(HubMode)
+  const write = useHubWrite(node)
+  const permissions = write.allowed ? rolePermissions : withoutWrites(rolePermissions)
+  /*
+  With "All servers" chosen a role that may change blocking still can: the controls of the status
+  block and the add bar open a flow that picks the servers (`HubAll`). The tables and the recent
+  list below keep the switched-off permissions, since a row acts on one server.
+  */
+  const all = useAllServersFlow(token)
+  const allServers =
+    hub && node === AGGREGATE && missing(rolePermissions, 'Blocked.canModify', 'Allowed.canModify') == null
+  const barPermissions = allServers ? rolePermissions : permissions
   const [data, setData] = useState<DashboardStats | null>(null)
   /** The first read of this period and node failed: there is nothing to show. */
   const [failure, setFailure] = useState(false)
@@ -318,7 +335,9 @@ export function Overview({
   const loading = data == null && !failure
 
   return (
-    <>
+    <HubNode.Provider value={hub ? node : ''}>
+    <HubAll.Provider value={allServers ? all.start : null}>
+      {all.flow}
       <InChrome>
         <ClusterNodeSelect
           nodes={nodes}
@@ -328,6 +347,7 @@ export function Overview({
           onChange={setNode}
         />
       </InChrome>
+      <HubWriteHint write={write} allServers={allServers} />
 
       <SectionHeader
         title="Blocking"
@@ -371,10 +391,10 @@ export function Overview({
         )}
 
         <div className={`${shared.row2} ${styles.checkRow}`}>
-          <StatusPanel token={token} permissions={permissions} onNotice={setNotice} />
+          <StatusPanel token={token} permissions={barPermissions} onNotice={setNotice} />
           <AddDomainBar
             token={token}
-            permissions={permissions}
+            permissions={barPermissions}
             onNotice={setNotice}
             onChanged={() => void load()}
           />
@@ -465,6 +485,7 @@ export function Overview({
           refreshOn={s?.totalBlocked}
         />
       </div>
-    </>
+    </HubAll.Provider>
+    </HubNode.Provider>
   )
 }

src/screens/blocking/Rules.tsx

@@ -1,4 +1,5 @@
-import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from 'react'
+import { useCallback, useContext, useEffect, useMemo, useRef, useState, type ReactNode } from 'react'
+import { HubNode, nodeArg } from '../../lib/hub-write'
 import { readRuleExport } from '../../api/blocking'
 import { addDomain, deleteDomain, exportDomains, flushList, type DomainList } from '../../api/zonelists'
 import { SectionHeader } from '../../ui/SectionHeader'
@@ -89,6 +90,8 @@ export function Rules({
   nodes?: { name: string; type: string }[]
   clusterInitialised?: boolean
 }) {
+  /* Behind a hub: the server the section has chosen (empty elsewhere). */
+  const hubNode = useContext(HubNode)
   const viewBlocked = missing(permissions, 'Blocked.canView') == null
   const viewAllowed = missing(permissions, 'Allowed.canView') == null
   /* Neither list can be read: there is nothing to count, so nothing is counted;
@@ -162,7 +165,8 @@ export function Rules({
      change, the PRIMARY, where the change was made (other-zones.js:269, 434): a
      secondary still answers the old list until the cluster syncs. */
   const load = useCallback(async (fromPrimary = false) => {
-    const node = fromPrimary ? primary.current : ''
+    /* Behind a hub every read and change is on the server the section has chosen. */
+    const node = hubNode !== '' ? hubNode : fromPrimary ? primary.current : ''
     const mine = ++asked.current
     const read = (list: DomainList, may: boolean) => (may ? readRuleExport(list, token, node) : null)
     const [b, a] = await Promise.all([read('blocked', viewBlocked), read('allowed', viewAllowed)])
@@ -181,7 +185,7 @@ export function Rules({
     setFailed(false)
     setStale(false)
     setLastGood(new Date().toISOString())
-  }, [token, viewBlocked, viewAllowed])
+  }, [token, viewBlocked, viewAllowed, hubNode])
 
   useEffect(() => {
     void load()
@@ -252,7 +256,7 @@ export function Rules({
       label: 'Delete',
       action: () =>
         mutate(
-          () => deleteDomain(rule.list, token, rule.domain),
+          () => deleteDomain(rule.list, token, rule.domain, ...nodeArg(hubNode)),
           isAllowed
             ? { type: 'success', title: 'Deleted!', text: `Domain '${rule.domain}' was deleted from Allowed Zone successfully.` }
             : { type: 'success', title: 'Deleted!', text: `Blocked zone '${rule.domain}' was deleted successfully.` },
@@ -268,7 +272,7 @@ export function Rules({
       return
     }
     setBusy(true)
-    const outcome = await deleteDomain(rule.list, token, rule.domain)
+    const outcome = await deleteDomain(rule.list, token, rule.domain, ...nodeArg(hubNode))
     if (!mounted.current) return
     setBusy(false)
     if (outcome.kind !== 'ok') {
@@ -285,7 +289,7 @@ export function Rules({
     const deleted = undo.rules
     setUndo({ rules: deleted, phase: 'restoring' })
     setBusy(true)
-    const result = await restoreRules(deleted, (r) => addDomain(r.list, token, r.domain))
+    const result = await restoreRules(deleted, (r) => addDomain(r.list, token, r.domain, ...nodeArg(hubNode)))
     if (!mounted.current) return
     setBusy(false)
     changed()
@@ -309,7 +313,7 @@ export function Rules({
       label: 'Flush',
       action: () =>
         mutate(
-          () => flushList(list, token),
+          () => flushList(list, token, ...nodeArg(hubNode)),
           isAllowed
             ? { type: 'success', title: 'Flushed!', text: 'Allowed zone was flushed successfully.' }
             : { type: 'success', title: 'Flushed!', text: 'Blocked zone was flushed successfully.' },
@@ -319,7 +323,7 @@ export function Rules({
 
   async function runExport(list: DomainList) {
     setBusy(true)
-    const r = await exportDomains(list, token)
+    const r = await exportDomains(list, token, ...nodeArg(hubNode))
     setBusy(false)
     if (!r.ok) return
     setNotice({
@@ -454,7 +458,7 @@ export function Rules({
                  are framed boxes of their own, and flush they doubled the panel's edge. */
               <div className={styles.tree}>
                 <Lists
-                  key={`${treeList}:${generation}`}
+                  key={`${treeList}:${generation}:${hubNode}`}
                   list={treeList}
                   token={token}
                   nodes={nodes}
@@ -617,6 +621,7 @@ export function Rules({
           list={importing}
           open
           token={token}
+          node={hubNode}
           onClose={() => setImporting(null)}
           onDone={(n) => {
             setNotice(n)

src/screens/blocking/StatusPanel.tsx

@@ -1,3 +1,7 @@
+import { useContext } from 'react'
+import { HubNode } from '../../lib/hub-write'
+import { HubAll } from '../../lib/hub-all'
+import { HubMode } from '../../ui/ClusterNodeSelect'
 import { BlockingMenu } from '../dashboard/BlockingMenu'
 import { Panel, Body } from '../../ui/Panel'
 import { PermissionButton } from '../../ui/PermissionButton'
@@ -7,7 +11,8 @@ import { Icon } from '../../ui/Icon'
 import { Tag } from '../../ui/Tag'
 import type { Notice } from '../../lib/notice'
 import { Locked } from './Locked'
-import { missing, type Permissions } from './permissions'
+import { missing, switchNeed, type Permissions } from './permissions'
+import { AllServersActions } from './AllServersActions'
 import { untilText, useBlockingState } from './blocking-state'
 import shared from './Blocking.module.css'
 import styles from './Overview.module.css'
@@ -44,11 +49,17 @@ export function StatusPanel({
   /* The reads (when, from where, and the one when a pause ends) are
      `useBlockingState`'s, shared with the sidebar's mark. A change made from the
      menu below reaches it as every settings write does (`lib/settings-signal.ts`). */
-  const { state, retry } = useBlockingState(token, permissions)
+  /* Behind a hub: the server the section has chosen. */
+  const node = useContext(HubNode)
+  const { state, retry } = useBlockingState(token, permissions, true, node)
+  /* Behind a hub the switch is a blocking change (Blocked and Allowed), not a settings one. With "All
+     servers" chosen it opens the flow, and the list actions for several servers sit beside it. */
+  const hub = useContext(HubMode)
+  const all = useContext(HubAll)
 
   if (viewNeed != null) return <Locked title="Blocking" need={viewNeed} />
 
-  const modifyNeed = missing(permissions, 'Settings.canModify')
+  const modifyNeed = switchNeed(permissions, hub)
 
   if (state === 'failed') {
     return (
@@ -73,6 +84,7 @@ export function StatusPanel({
   return (
     <Panel className={shared.centred}>
       <Body>
+        <div className="flex flex-col gap-4">
         {state == null ? (
           <Loading compact />
         ) : (
@@ -109,10 +121,16 @@ export function StatusPanel({
             {modifyNeed != null ? (
               <PermissionButton permission={modifyNeed}>{verb}</PermissionButton>
             ) : (
-              <BlockingMenu token={token} text={verb} onNotice={onNotice} />
+              <BlockingMenu token={token} text={verb} onNotice={onNotice} node={node} />
             )}
           </div>
         )}
+        {state != null && all !== null && modifyNeed == null && (
+          <div className="flex flex-wrap items-center gap-2">
+            <AllServersActions token={token} />
+          </div>
+        )}
+        </div>
       </Body>
     </Panel>
   )

src/screens/blocking/blocking-state.ts

@@ -2,7 +2,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'
 import { getSettings } from '../../api/settings'
 import { minuteStamp } from '../../lib/dates'
 import { onSettingsChanged } from '../../lib/settings-signal'
-import { missing, type Permissions } from './permissions'
+import { missing, switchNeed, type Permissions } from './permissions'
 
 /*
 Whether blocking is on, read where the Overview's status block reads it
@@ -62,6 +62,8 @@ export function useBlockingState(
   /** False reads nothing, as a missing `Settings.canView` does: the sidebar without
       a Blocking entry to mark. */
   enabled = true,
+  /** Behind a hub: the server read (a hub has no settings of its own). Empty elsewhere. */
+  node = '',
 ): { state: BlockingRead; reload: () => void; retry: () => void } {
   const viewNeed = enabled ? missing(permissions, 'Settings.canView') : 'Settings.canView'
   const [state, setState] = useState<BlockingRead>(null)
@@ -70,11 +72,11 @@ export function useBlockingState(
   const reload = useCallback(() => {
     if (viewNeed != null) return
     const mine = ++asked.current
-    void getSettings(token).then((s) => {
+    void getSettings(token, node).then((s) => {
       if (mine !== asked.current) return
       setState(s == null ? 'failed' : stateOf(s.enableBlocking, s.temporaryDisableBlockingTill))
     })
-  }, [token, viewNeed])
+  }, [token, viewNeed, node])
 
   useEffect(() => {
     // A read already travelling must not land after the permission went.
@@ -126,10 +128,10 @@ export interface BlockingMark {
   tone: 'acc' | 'neutral'
 }
 
-export function blockingMark(state: BlockingRead, permissions: Permissions, now = new Date()): BlockingMark | null {
+export function blockingMark(state: BlockingRead, permissions: Permissions, now = new Date(), hub = false): BlockingMark | null {
   if (state == null || state === 'failed' || state.kind === 'on') return null
   if (state.kind === 'off') return { text: 'Off', label: 'Blocking, disabled', tone: 'neutral' }
-  const canLift = missing(permissions, 'Settings.canModify') == null
+  const canLift = switchNeed(permissions, hub) == null
   return {
     text: 'Paused',
     label: `Blocking, paused ${untilText(state.till, now).replace(/^Until/, 'until')}`,
@@ -138,7 +140,7 @@ export function blockingMark(state: BlockingRead, permissions: Permissions, now
 }
 
 /** The mark for the sidebar's Blocking entry; `visible` false when there is no entry. */
-export function useBlockingMark(token: string | null, permissions: Permissions, visible: boolean): BlockingMark | null {
+export function useBlockingMark(token: string | null, permissions: Permissions, visible: boolean, hub = false): BlockingMark | null {
   const { state } = useBlockingState(token, permissions, visible)
-  return blockingMark(state, permissions)
+  return blockingMark(state, permissions, new Date(), hub)
 }

src/screens/blocking/check-domain.ts

@@ -2,6 +2,7 @@ import type { ApiOutcome } from '../../api/client'
 import { readRuleExport } from '../../api/blocking'
 import { resolve } from '../../api/dnsclient'
 import { readSettings } from '../../api/settings'
+import { nodeArg } from '../../lib/hub-write'
 import type { QuickEntry } from '../../lib/quick-lists'
 import { listName } from './list-lines'
 
@@ -321,11 +322,13 @@ export async function checkDomain(
   token: string | null,
   domain: string,
   needs: { settings?: string; allowed?: string },
+  /** Behind a hub: the server asked (a hub has no server of its own to ask). */
+  node = '',
 ): Promise<ApiOutcome<CheckInputs>> {
   const [answer, settings, allowed] = await Promise.all([
-    resolve(token, { server: 'this-server', domain, type: 'A', protocol: 'UDP', dnssec: false, eDnsClientSubnet: '' }),
-    needs.settings != null ? { need: needs.settings } : readSettings(token),
-    needs.allowed != null ? { need: needs.allowed } : readRuleExport('allowed', token),
+    resolve(token, { server: 'this-server', domain, type: 'A', protocol: 'UDP', dnssec: false, eDnsClientSubnet: '', ...(node === '' ? {} : { node }) }),
+    needs.settings != null ? { need: needs.settings } : readSettings(token, ...nodeArg(node)),
+    needs.allowed != null ? { need: needs.allowed } : readRuleExport('allowed', token, ...nodeArg(node)),
   ])
   if (answer.kind !== 'ok') return answer
   return {

src/screens/blocking/permissions.ts

@@ -19,6 +19,30 @@ export function missing(perms: Permissions, ...needs: Need[]): Need | undefined
   return undefined
 }
 
+/*
+What the blocking switch (the temporary disable, enable, and the block list update) asks for.
+Upstream asks `Settings: Modify` of the call. Behind a hub those calls are blocking, which an
+operator may do (`Blocked` and `Allowed` Modify), while `Settings: Modify` is an admin's: the
+switch follows the blocking permission there, and is unchanged anywhere else.
+*/
+export function switchNeed(perms: Permissions, hub: boolean): Need | undefined {
+  return hub ? missing(perms, 'Blocked.canModify', 'Allowed.canModify') : missing(perms, 'Settings.canModify')
+}
+
+/*
+Behind a hub, when the chosen node is not one a change can be made on ("All servers", or a server
+the hub holds no admin token for): the same permissions with every write switched off. The
+controls then render as they do for a role without the permission.
+*/
+export function withoutWrites(perms: Permissions): Permissions {
+  if (perms == null) return perms
+  const out: Record<string, Permission> = { ...perms }
+  for (const section of ['Allowed', 'Blocked', 'Settings']) {
+    out[section] = { canView: perms[section]?.canView ?? true, canModify: false, canDelete: false }
+  }
+  return out
+}
+
 /** The same sentence `PermissionButton` draws: `Requires Logs: View`. */
 export function requiresText(need: string): string {
   const [section, action] = need.split('.')

src/screens/dashboard/BlockingMenu.tsx

@@ -1,7 +1,11 @@
-import { useState } from 'react'
+import { useContext, useState } from 'react'
 import { getSettings, setSettings, temporaryDisableBlocking } from '../../api/settings'
 import { Confirm } from '../../ui/Confirm'
 import { Menu } from '../../ui/Menu'
+import { Button } from '../../ui/Button'
+import { Tooltip } from '../../ui/Tooltip'
+import { nodeArg, useHubWrite } from '../../lib/hub-write'
+import { HubAll } from '../../lib/hub-all'
 import type { AlertType } from '../../ui/Alert'
 import { noticeFromFailure } from '../../lib/notice'
 
@@ -53,8 +57,14 @@ export function BlockingMenu({
   onNotice,
   text = 'Blocking',
   onChanged,
+  node = '',
+  allowed = true,
 }: {
   token: string | null
+  /** Behind a hub: the server the change is made on (empty or "cluster": none chosen, so it is switched off). */
+  node?: string
+  /** Behind a hub: what the role may do. */
+  allowed?: boolean
   onNotice: (a: { type: AlertType; title: string; text: string }) => void
   /** The trigger text. The Dashboard keeps upstream's "Blocking". */
   text?: string
@@ -64,10 +74,12 @@ export function BlockingMenu({
   const [active, setActive] = useState<boolean | null>(null)
   const [pending, setPending] = useState<Pending | null>(null)
   const [busy, setBusy] = useState(false)
+  /* Behind a hub with "All servers" chosen: the choice opens the flow that makes it on the servers picked next. */
+  const all = useContext(HubAll)
 
   async function loadState() {
     setActive(null)
-    const s = await getSettings(token)
+    const s = await getSettings(token, ...nodeArg(node))
     if (s != null) setActive(s.enableBlocking)
   }
 
@@ -78,7 +90,7 @@ export function BlockingMenu({
       label: turnOn ? 'Enable' : 'Disable',
       variant: turnOn ? 'primary' : 'danger',
       perform: async () => {
-        const r = await setSettings(token, { enableBlocking: String(turnOn) })
+        const r = await setSettings(token, { enableBlocking: String(turnOn), ...(node ? { node } : {}) })
         // The same text as the other thirty-six: it was the only one that said
         // a bare "Session expired." for this very condition.
         if (r.kind !== 'ok') throw new Error(noticeFromFailure(r).text)
@@ -99,7 +111,7 @@ export function BlockingMenu({
       label: 'Disable',
       variant: 'danger',
       perform: async () => {
-        const till = await temporaryDisableBlocking(token, String(minutes))
+        const till = await temporaryDisableBlocking(token, String(minutes), ...nodeArg(node))
         if (till == null) throw new Error('The request failed.')
         setActive(false)
         onNotice({
@@ -126,6 +138,52 @@ export function BlockingMenu({
     }
   }
 
+  /* Behind a hub, only on a chosen server the hub can write to; the tooltip says why not. */
+  const write = useHubWrite(node, allowed)
+  if (all !== null) {
+    return (
+      <Menu label="Blocking options" text={text}>
+        {(close) => (
+          <>
+            <button
+              role="menuitem"
+              onClick={() => {
+                close()
+                all({ action: 'blocking.enable', title: 'Enable blocking on several servers', confirmLabel: 'Enable blocking' })
+              }}
+            >
+              Enable Blocking
+            </button>
+            {DURATIONS.map((p) => (
+              <button
+                key={p.minutes}
+                role="menuitem"
+                onClick={() => {
+                  close()
+                  all({
+                    action: 'blocking.disable',
+                    params: { minutes: String(p.minutes) },
+                    title: `Disable blocking for ${p.minutes} minute(s) on several servers`,
+                    confirmLabel: 'Disable blocking',
+                  })
+                }}
+              >
+                {p.text}
+              </button>
+            ))}
+          </>
+        )}
+      </Menu>
+    )
+  }
+  if (!write.allowed) {
+    return (
+      <Tooltip text={write.reason ?? 'Requires Settings: Modify'}>
+        <Button disabled>{text}</Button>
+      </Tooltip>
+    )
+  }
+
   return (
     <>
       <Menu label="Blocking options" text={text} onOpen={() => void loadState()}>

src/screens/dashboard/Dashboard.tsx

@@ -1,4 +1,5 @@
-import { useEffect, useMemo, useRef, useState } from 'react'
+import { useContext, useEffect, useMemo, useRef, useState } from 'react'
+import { HubNode } from '../../lib/hub-write'
 import {
   RANGE_LABEL, RANGES, getDashboardStats,
   type DashboardStats, type Range, type Stats, type TopKind,
@@ -13,7 +14,7 @@ import { Heatmap } from './Heatmap.tsx'
 import {
   bucketWindow, periodFilters, protocolFilter, queryTypeFilter, responseTypeFilter,
 } from './jump'
-import { ClusterNodeSelect, AGGREGATE, type ClusterNode } from '../../ui/ClusterNodeSelect'
+import { ClusterNodeSelect, AGGREGATE, HubMode, type ClusterNode } from '../../ui/ClusterNodeSelect'
 import { InChrome } from '../../app/ChromeSlot'
 import { TopStats } from './TopStats'
 import { TopRowMenu } from './TopRowMenu'
@@ -52,11 +53,15 @@ export function Dashboard({
   token,
   nodes = [],
   clusterInitialised = false,
+  canBlock = true,
 }: {
   token: string | null
   nodes?: ClusterNode[]
   clusterInitialised?: boolean
+  /** Behind a hub: whether the role may switch blocking or allow and block a domain. */
+  canBlock?: boolean
 }) {
+  const hub = useContext(HubMode)
   /*
   Which node this screen is reading. Upstream offers the aggregate here and
   remembers the choice per screen (`cluster.js`, key `dashboardClusterNode`).
@@ -96,7 +101,7 @@ export function Dashboard({
   const [notice, setNotice] = useState<{ type: AlertType; title: string; text: string } | null>(null)
   /* Allowing or blocking from a top list reports on the page (main.js:2825 and
      2853 pass no alert placeholder). */
-  const action = useDomainAction(token, setNotice)
+  const action = useDomainAction(token, setNotice, hub ? node : '')
   /*
   The custom range. `start`/`end` are what is typed into the two fields;
   `requested` is the last thing "Show" was pressed with, which is what triggers the
@@ -490,7 +495,7 @@ export function Dashboard({
           rows={view?.topBlockedDomains ?? []}
           onMore={() => setTop('TopBlockedDomains')}
           failure={failure}
-          beforeMore={<BlockingMenu token={token} onNotice={setNotice} />}
+          beforeMore={<BlockingMenu token={token} onNotice={setNotice} node={hub ? node : ''} allowed={canBlock} />}
           menu={(f) => <TopRowMenu kind="TopBlockedDomains" name={f.name} node={node} action={action} />}
         />
         <Breakdown
@@ -505,7 +510,9 @@ export function Dashboard({
         />
       </div>
 
-      <TopStats type={top} range={range} token={token} node={node} onClose={() => setTop(null)} />
+      <HubNode.Provider value={hub ? node : ''}>
+        <TopStats type={top} range={range} token={token} node={node} onClose={() => setTop(null)} />
+      </HubNode.Provider>
     </>
   )
 }

src/screens/dhcp/Dhcp.hub.test.tsx

@@ -0,0 +1,68 @@
+import { describe, expect, it, vi, afterEach } from 'vitest'
+import { render, screen } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { Dhcp } from './Dhcp'
+import * as api from '../../api/dhcp'
+import { HubMode } from '../../ui/ClusterNodeSelect'
+import { HubWritable } from '../../lib/hub-write'
+
+afterEach(() => vi.restoreAllMocks())
+
+/* Behind a hub the screen starts on the first server, and changes only where the hub can write. */
+
+const NODES = [
+  { name: 'mdns', type: 'Primary' },
+  { name: 'ndns', type: 'Secondary' },
+]
+
+function hubDhcp(writable: string[], canModify = true) {
+  vi.spyOn(api, 'listScopes').mockResolvedValue({ kind: 'ok', data: [] })
+  return render(
+    <HubMode.Provider value>
+      <HubWritable.Provider value={writable}>
+        <Dhcp token="t" sub="Scopes" nodes={NODES} clusterInitialised canModify={canModify} canDelete={canModify} />
+      </HubWritable.Provider>
+    </HubMode.Provider>,
+  )
+}
+
+describe('DHCP behind a hub', () => {
+  it('starts on the first server, named in the request and shown in the selector', async () => {
+    hubDhcp(['mdns', 'ndns'])
+    await screen.findByText('No Scope Found')
+    expect(api.listScopes).toHaveBeenCalledWith('t', 'mdns')
+    expect(screen.getByRole('combobox', { name: 'Server' })).toHaveTextContent('mdns')
+    expect(screen.queryByRole('option', { name: 'All servers' })).not.toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Add Scope' })).toBeInTheDocument()
+    expect(screen.queryByRole('status')).not.toBeInTheDocument()
+  })
+
+  it('switches changes off on a server the hub holds no admin token for, and says why', async () => {
+    const user = userEvent.setup()
+    hubDhcp(['mdns'])
+    await screen.findByText('No Scope Found')
+    await user.click(screen.getByRole('combobox', { name: 'Server' }))
+    await user.click(await screen.findByRole('option', { name: 'ndns' }))
+
+    expect(await screen.findByText(/No admin token configured on the hub/)).toBeInTheDocument()
+    expect(api.listScopes).toHaveBeenLastCalledWith('t', 'ndns')
+    expect(screen.queryByRole('button', { name: 'Add Scope' })).not.toBeInTheDocument()
+  })
+
+  it('says nothing about the server when the role is what forbids the change', async () => {
+    hubDhcp(['mdns'], false)
+    await screen.findByText('No Scope Found')
+    expect(screen.queryByRole('button', { name: 'Add Scope' })).not.toBeInTheDocument()
+    expect(screen.queryByText(/admin token|Choose a server/)).not.toBeInTheDocument()
+  })
+})
+
+describe('DHCP outside a hub', () => {
+  it('keeps the empty node and no hint', async () => {
+    vi.spyOn(api, 'listScopes').mockResolvedValue({ kind: 'ok', data: [] })
+    render(<Dhcp token="t" sub="Scopes" />)
+    await screen.findByText('No Scope Found')
+    expect(api.listScopes).toHaveBeenCalledWith('t', '')
+    expect(screen.getByRole('button', { name: 'Add Scope' })).toBeInTheDocument()
+  })
+})

src/screens/dhcp/Dhcp.tsx

@@ -3,6 +3,8 @@ import { SubTabs } from '../../ui/SubTabs'
 import { ClusterNodeSelect } from '../../ui/ClusterNodeSelect'
 import { Leases } from './Leases'
 import { Scopes } from './Scopes'
+import { useHubDefaultNode, useHubWrite } from '../../lib/hub-write'
+import { HubWriteHint } from '../../ui/HubWriteHint'
 
 /*
 DHCP. Two sub-tabs, with upstream's literal labels (index.html:2496-2497):
@@ -48,12 +50,17 @@ export function Dhcp({
   clusterInitialised = false,
   token,
   sub,
-  canModify = true,
-  canDelete = true,
+  canModify: mayModify = true,
+  canDelete: mayDelete = true,
 }: DhcpProps) {
   /* Which node this screen reads. Upstream mounts a selector here
      and the parameter was already travelling empty. */
-  const [node, setNode] = useState<string>(() => '')
+  const defaultNode = useHubDefaultNode(nodes)
+  const [node, setNode] = useState<string>(defaultNode)
+  /* Behind a hub, a change is allowed only on a chosen server the hub can write to. */
+  const write = useHubWrite(node, mayModify)
+  const canModify = write.allowed
+  const canDelete = useHubWrite(node, mayDelete).allowed
   const [nodeLocked, setNodeLocked] = useState(false)
   const requested = (sub ?? 'Leases') as SubTab
   const active: SubTab = SUB_TABS.includes(requested) ? requested : 'Leases'
@@ -85,6 +92,7 @@ export function Dhcp({
         disabled={nodeLocked}
         label="Cluster Node"
       />
+      <HubWriteHint write={write} />
       {active === 'Scopes' ? (
         <Scopes
           tabs={tabs}