/* Settings and blocking compare across the servers (stage 7): the baseline kept in the hub, the fields that stay local, a preview, an apply with history and read-back, and the custom Blocked and Allowed lists compared as sets. The settings are read with each server's ADMIN token when the hub has one (a read-only token may see blanked secrets, and a baseline imported from those would blank them elsewhere). Apply sends ONLY the shared fields that differ, through the same checks as a per-server `settings/set` (guards.ts), after saving the server's settings to the history, and reads them back. Nothing here puts a secret value, or a hash of one, in an answer; the audit lines carry field names only (see settings-sync.ts). */ import { createHash } from 'node:crypto' import type { HubConfig, HubServer } from './config.ts' import { WEB_SERVICE_MESSAGE, isForbiddenSettingsField, webServiceChanges } from './guards.ts' import { BLOCKING_FIELDS, LOCAL_DEFAULTS, MAX_DELETES, MAX_LIST, changedFields, compareSettings, diffSets, hasSecret, isBuiltInLocal, isFieldName, maskSettings, maskValue, normalize, parseDomains, prepare, restoreSecrets, sameValue, toParams, type Comparison, } from './settings-sync.ts' import { cleanDomain } from './servers-run.ts' import { UserError } from './apps-config.ts' import type { FieldChange } from './apps-sync.ts' import { MAX_CONFIG } from './apps-sync.ts' import { listHistoryIn, readHistoryIn, readListsBaseline, readSettingsBaseline, readSettingsOverrides, settingsHistoryDir, writeHistoryIn, writeListsBaseline, writeSettingsBaseline, writeSettingsOverrides, type HistoryEntry, type ListsBaseline, type SettingsBaseline, } from './store.ts' import { Upstream } from './upstream.ts' import { readCapped } from './zones-sync.ts' const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed') const hashOf = (value: unknown): string => createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 16) export const MAX_ADDS = 1000 const LIST_TIMEOUT_MS = 30_000 const LIST_BYTES = 8 * 1024 * 1024 export interface TargetResult { server: string ok: boolean message?: string } export interface SettingsTargetPreview { server: string ok: boolean error?: string writable: boolean /* Tells the apply whether the server changed since this preview. Built from masked values. */ currentHash: string /* `{kind:'json'}` so the console's field table draws it as it does a config. */ changes: { kind: 'json'; changes: FieldChange[] } /* Fields whose change is a secret: shown as "secret changed", never as values. */ secretFields: string[] unchanged: boolean warnings: string[] } export interface ListsServerView { id: string ok: boolean error?: string writable: boolean blocked: { count: number; missing: number; extra: number; missingSample: string[]; extraSample: string[] } allowed: { count: number; missing: number; extra: number; missingSample: string[]; extraSample: string[] } status: 'same' | 'differs' | 'unknown' } const SAMPLE = 100 export class SettingsConfig { #config: HubConfig #upstream: Upstream #now: () => number constructor(config: HubConfig, upstream: Upstream, now: () => number = Date.now) { this.#config = config this.#upstream = upstream this.#now = now } #server(id: string): HubServer { const found = this.#config.servers.find((s) => s.id === id) if (found === undefined) throw new UserError(`Unknown server: ${id}.`) return found } #targets(ids: string[]): HubServer[] { const unique = [...new Set(ids)] if (unique.length === 0) throw new UserError('Choose at least one server.') return unique.map((id) => this.#server(id)) } /* Fresh `settings/get`, with the admin token when there is one. */ async #read(server: HubServer): Promise { this.#upstream.invalidate(server.id) return this.#upstream.call(server, 'settings/get', {}, server.adminToken ?? server.token) } async #readAll(): Promise> { const entries = await Promise.all( this.#config.servers.map(async (s): Promise<[string, { settings: unknown } | { error: string }]> => { try { return [s.id, { settings: await this.#read(s) }] } catch (e) { return [s.id, { error: reason(e) }] } }), ) return Object.fromEntries(entries) } getBaseline(): SettingsBaseline | null { return readSettingsBaseline(this.#config.dataDir) } #extraLocal(): string[] { return readSettingsOverrides(this.#config.dataDir).paths } /* The baseline as the console may see it: secrets masked. */ baselineView(): { settings: Record; importedFrom?: string; updatedAt: string; updatedBy: string } | null { const b = this.getBaseline() return b === null ? null : { ...b, settings: maskSettings(b.settings) } } async compare(): Promise { const baseline = this.getBaseline() const extra = this.#extraLocal() const live = await this.#readAll() const comparison: Comparison = compareSettings({ baseline: baseline?.settings ?? null, live, extraLocal: extra }) return { baseline: this.baselineView(), localFields: { builtIn: [...LOCAL_DEFAULTS], builtInPatterns: ['webService*', 'user*, sso*, ldap*, auth*'], extra }, blockingFields: [...BLOCKING_FIELDS], servers: comparison.servers.map((s) => ({ ...s, writable: this.#config.servers.find((x) => x.id === s.id)?.adminToken !== undefined })), fields: comparison.fields, withBaseline: comparison.withBaseline, } } /* Writes the baseline; the one it replaces goes to the history first. */ #setBaseline(settings: Record, importedFrom: string | undefined, user: string): void { const old = this.getBaseline() if (old !== null && JSON.stringify(old.settings) !== JSON.stringify(settings)) { try { writeHistoryIn(settingsHistoryDir(this.#config.dataDir), 'baseline', JSON.stringify(old.settings), 'before-save', this.#now()) } catch (e) { throw new UserError(`The old baseline could not be saved first (${reason(e)}); nothing was changed.`) } } writeSettingsBaseline(this.#config.dataDir, { settings, ...(importedFrom === undefined ? {} : { importedFrom }), updatedAt: new Date(this.#now()).toISOString(), updatedBy: user, }) } /* Takes a server's settings as the baseline; with `group=blocking` only its blocking fields replace the baseline's. */ async importBaseline(serverId: string, group: string | undefined, user: string): Promise<{ detail: string }> { const server = this.#server(serverId) let raw: unknown try { raw = await this.#read(server) } catch (e) { throw new UserError(`${serverId}: ${reason(e)}`) } const read = normalize(raw) if (Object.keys(read).length === 0) throw new UserError(`${serverId} sent no settings.`) let next = read if (group === 'blocking') { const old = this.getBaseline()?.settings ?? {} const kept: Record = { ...old } for (const name of BLOCKING_FIELDS) if (Object.hasOwn(read, name)) kept[name] = read[name] next = kept } else if (group !== undefined && group !== '') throw new UserError('group can only be "blocking".') this.#setBaseline(next, serverId, user) return { detail: `from ${serverId}${group === 'blocking' ? ' (blocking fields)' : ''}, ${Object.keys(next).length} fields` } } /* The baseline from the editor's JSON. A secret shown as the placeholder keeps the stored one. */ saveBaseline(textBody: string, user: string): { detail: string } { if (textBody.length > MAX_CONFIG) throw new UserError('The baseline is larger than 1 MB.') let parsed: unknown try { parsed = JSON.parse(textBody) } catch (e) { throw new UserError(`The baseline is not valid JSON: ${reason(e)}`) } if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) throw new UserError('The baseline must be a JSON object of settings.') const named = Object.keys(parsed as object) const bad = named.find((k) => !isFieldName(k)) if (bad !== undefined) throw new UserError(`"${bad.slice(0, 40)}" is not a setting name.`) let settings: Record try { settings = restoreSecrets(normalize(parsed), this.getBaseline()?.settings ?? null) } catch (e) { throw new UserError(reason(e)) } this.#setBaseline(settings, this.getBaseline()?.importedFrom, user) return { detail: `${Object.keys(settings).length} fields` } } saveOverrides(pathsRaw: string[]): { detail: string } { const seen = new Set() if (pathsRaw.length > 50) throw new UserError('At most 50 extra local fields.') for (const p of pathsRaw) { if (!isFieldName(p)) throw new UserError(`"${p.slice(0, 40)}" is not a setting name.`) if (seen.has(p)) throw new UserError(`"${p}" is listed twice.`) seen.add(p) } writeSettingsOverrides(this.#config.dataDir, { paths: pathsRaw }) return { detail: `${pathsRaw.length} extra local field(s)` } } /* ---- Preview and apply ---- */ #fingerprint(current: unknown, baseline: Record, extra: string[]): string { const { settings } = prepare(current) const shared = Object.keys(settings) .filter((k) => !isBuiltInLocal(k) && !extra.includes(k) && Object.hasOwn(baseline, k)) .sort() .map((k) => [k, maskValue(k, settings[k])]) return hashOf(shared) } async preview(ids: string[], only?: readonly string[], override?: SettingsBaseline): Promise<{ targets: SettingsTargetPreview[] }> { const targets = this.#targets(ids) const baseline = override ?? this.getBaseline() if (baseline === null) throw new UserError('There is no settings baseline yet. Accept one from a server first.') const extra = this.#extraLocal() const base = normalize(baseline.settings) const out = await Promise.all( targets.map(async (s): Promise => { const empty = { kind: 'json' as const, changes: [] } const writable = s.adminToken !== undefined let current: unknown try { current = await this.#read(s) } catch (e) { return { server: s.id, ok: false, error: reason(e), writable, currentHash: '', changes: empty, secretFields: [], unchanged: false, warnings: [] } } const names = changedFields(baseline.settings, current, extra, only) const live = prepare(current) const warnings: string[] = [] if (!writable) warnings.push('The hub has no admin token for this server, so it cannot write to it.') if (live.paused) warnings.push('Blocking is paused for a while on this server; its blocking switch is left alone.') const changes: FieldChange[] = names.map((name) => { const [a, b] = [maskValue(name, live.settings[name]), maskValue(name, base[name])] return { path: name, type: a === undefined ? 'added' : b === undefined ? 'removed' : 'changed', a, b } }) return { server: s.id, ok: true, writable, currentHash: this.#fingerprint(current, base, extra), changes: { kind: 'json', changes }, secretFields: names.filter(hasSecret), unchanged: names.length === 0, warnings, } }), ) return { targets: out } } async #applyOne(s: HubServer, baseline: SettingsBaseline, extra: string[], expect: string | undefined, only?: readonly string[]): Promise<{ result: TargetResult; detail?: string }> { const fail = (message: string): { result: TargetResult } => ({ result: { server: s.id, ok: false, message } }) if (s.adminToken === undefined) return fail('no admin token configured') try { let current: unknown try { current = await this.#read(s) } catch (e) { return fail(reason(e)) } const base = normalize(baseline.settings) if (expect !== undefined && expect !== this.#fingerprint(current, base, extra)) return fail('the settings changed on the server since the preview; run the preview again') const names = changedFields(baseline.settings, current, extra, only) if (names.length === 0) return { result: { server: s.id, ok: true, message: 'Already in sync; nothing was written.' } } const params: Record = {} for (const name of names) { const encoded = toParams(name, base[name]) if ('error' in encoded) return fail(`${name}: ${encoded.error}; nothing was changed`) Object.assign(params, encoded.params) } // The same checks as a per-server settings/set (guards.ts). if (Object.keys(params).some((k) => isForbiddenSettingsField(k) || k.startsWith('webService'))) return fail('a setting the hub does not change was in the list; nothing was changed') const web = webServiceChanges(params, current) if (web.length > 0) return fail(`${WEB_SERVICE_MESSAGE} nothing was changed`) try { writeHistoryIn(settingsHistoryDir(this.#config.dataDir), s.id, JSON.stringify(normalize(current)), 'before-apply', this.#now()) } catch (e) { return fail(`the old settings could not be saved first (${reason(e)}); nothing was changed`) } await this.#upstream.write(s, 'settings/set', params) const detail = names.join(', ') let after: unknown try { after = prepare(await this.#read(s)).settings } catch (e) { return { result: { server: s.id, ok: false, message: `the server accepted the change but could not be read back (${reason(e)})` }, detail } } const wrong = names.filter((n) => !sameValue(n, base[n], (after as Record)[n])) if (wrong.length > 0) return { result: { server: s.id, ok: false, message: `the server accepted the change but reads back different values for: ${wrong.join(', ')}` }, detail } return { result: { server: s.id, ok: true, message: `Applied and verified (${names.length} field${names.length === 1 ? '' : 's'}).` }, detail } } catch (e) { return fail(reason(e)) } } /* `override` is a baseline that is not the hub's own (a profile's settings). */ async apply(ids: string[], expect: Record, only?: readonly string[], override?: SettingsBaseline): Promise<{ results: TargetResult[]; details: Record }> { const targets = this.#targets(ids) const baseline = override ?? this.getBaseline() if (baseline === null) throw new UserError('There is no settings baseline yet. Accept one from a server first.') const extra = this.#extraLocal() const done = await Promise.all(targets.map((s) => this.#applyOne(s, baseline, extra, expect[s.id], only))) const details: Record = {} for (const d of done) if (d.detail !== undefined) details[d.result.server] = d.detail return { results: done.map((d) => d.result), details } } /* The extra local field names (for the profiles, which keep shared fields only). */ extraLocal(): string[] { return this.#extraLocal() } /* For the drift check: which servers differ from the settings baseline and from the lists baseline. Nothing is read for a baseline that does not exist. Field names only, never a value. */ async driftFindings(): Promise<{ kind: 'settings' | 'lists'; server: string; status: 'differs' | 'unreadable'; fields: string[] }[]> { const out: { kind: 'settings' | 'lists'; server: string; status: 'differs' | 'unreadable'; fields: string[] }[] = [] const baseline = this.getBaseline() const lists = this.getListsBaseline() if (baseline === null && lists === null) return out const extra = this.#extraLocal() await Promise.all( this.#config.servers.map(async (s) => { if (baseline !== null) { try { const fields = changedFields(baseline.settings, await this.#read(s), extra) if (fields.length > 0) out.push({ kind: 'settings', server: s.id, status: 'differs', fields }) } catch { out.push({ kind: 'settings', server: s.id, status: 'unreadable', fields: [] }) } } if (lists !== null) { try { const live = await this.#readLists(s) const fields = (['blocked', 'allowed'] as const).filter((l) => { const d = diffSets(lists[l], live[l]) return d.missing.length + d.extra.length > 0 }) if (fields.length > 0) out.push({ kind: 'lists', server: s.id, status: 'differs', fields }) } catch { out.push({ kind: 'lists', server: s.id, status: 'unreadable', fields: [] }) } } }), ) return out } /* ---- History: list and view (no rollback; apply the baseline again, or accept an old one by hand) ---- */ historyList(): HistoryEntry[] { return listHistoryIn(settingsHistoryDir(this.#config.dataDir)) } historyGet(id: string): { entry: HistoryEntry; settings?: Record; lists?: { blocked: number; allowed: number } } { const entry = this.historyList().find((e) => e.id === id) const text = entry === undefined ? null : readHistoryIn(settingsHistoryDir(this.#config.dataDir), id) if (entry === undefined || text === null) throw new UserError('There is no such history entry.') let parsed: unknown try { parsed = JSON.parse(text) } catch { throw new UserError('That history entry cannot be read.') } if (entry.kind === 'before-lists') { const p = parsed as { blocked?: unknown[]; allowed?: unknown[] } return { entry, lists: { blocked: p.blocked?.length ?? 0, allowed: p.allowed?.length ?? 0 } } } return { entry, settings: maskSettings(normalize(parsed)) } } /* ---- Blocked and Allowed lists ---- */ async #readList(server: HubServer, list: 'blocked' | 'allowed'): Promise { const file = await this.#upstream.raw(server, `${list}/export`, {}) return parseDomains(await readCapped(file, LIST_BYTES, LIST_TIMEOUT_MS, `The ${list} list`)) } async #readLists(server: HubServer): Promise<{ blocked: string[]; allowed: string[] }> { const [blocked, allowed] = await Promise.all([this.#readList(server, 'blocked'), this.#readList(server, 'allowed')]) return { blocked, allowed } } getListsBaseline(): ListsBaseline | null { return readListsBaseline(this.#config.dataDir) } async listsCompare(): Promise { const baseline = this.getListsBaseline() const reads = await Promise.all( this.#config.servers.map(async (s) => { try { return { s, lists: await this.#readLists(s), error: undefined } } catch (e) { return { s, lists: null, error: reason(e) } } }), ) const reference = baseline ?? reads.find((r) => r.lists !== null)?.lists ?? null const refId = baseline === null ? reads.find((r) => r.lists !== null)?.s.id : undefined const view = (live: string[], ref: string[], own: boolean) => { const d = own ? { missing: [], extra: [] } : diffSets(ref, live) return { count: live.length, missing: d.missing.length, extra: d.extra.length, missingSample: d.missing.slice(0, SAMPLE), extraSample: d.extra.slice(0, SAMPLE) } } const servers: ListsServerView[] = reads.map((r) => { const writable = r.s.adminToken !== undefined if (r.lists === null || reference === null) { const none = { count: 0, missing: 0, extra: 0, missingSample: [], extraSample: [] } return { id: r.s.id, ok: false, error: r.error, writable, blocked: none, allowed: none, status: 'unknown' } } const own = r.s.id === refId const blocked = view(r.lists.blocked, reference.blocked, own) const allowed = view(r.lists.allowed, reference.allowed, own) const differs = blocked.missing + blocked.extra + allowed.missing + allowed.extra > 0 return { id: r.s.id, ok: true, writable, blocked, allowed, status: differs ? 'differs' : 'same' } }) // Without a baseline the first server is the reference; it differs when any other does. if (baseline === null) { const first = servers.find((s) => s.id === refId) if (first !== undefined && servers.some((s) => s.id !== first.id && s.status === 'differs')) first.status = 'differs' } return { baseline: baseline === null ? null : { blocked: baseline.blocked.length, allowed: baseline.allowed.length, importedFrom: baseline.importedFrom, updatedAt: baseline.updatedAt, updatedBy: baseline.updatedBy }, withBaseline: baseline !== null, servers, } } async listsImport(serverId: string, user: string): Promise<{ detail: string }> { const server = this.#server(serverId) let lists: { blocked: string[]; allowed: string[] } try { lists = await this.#readLists(server) } catch (e) { throw new UserError(`${serverId}: ${reason(e)}`) } if (lists.blocked.length > MAX_LIST || lists.allowed.length > MAX_LIST) { throw new UserError(`A list is longer than ${MAX_LIST} domains. The hub compares custom lists, not whole block lists.`) } writeListsBaseline(this.#config.dataDir, { ...lists, importedFrom: serverId, updatedAt: new Date(this.#now()).toISOString(), updatedBy: user }) return { detail: `from ${serverId}: ${lists.blocked.length} blocked, ${lists.allowed.length} allowed` } } async listsPreview(ids: string[], removeExtras: boolean, override?: ListsBaseline): Promise<{ targets: (ListsServerView & { tooMany?: string })[] }> { const targets = this.#targets(ids) const baseline = override ?? this.getListsBaseline() if (baseline === null) throw new UserError('There is no lists baseline yet. Accept one from a server first.') const out = await Promise.all( targets.map(async (s) => { const writable = s.adminToken !== undefined const none = { count: 0, missing: 0, extra: 0, missingSample: [], extraSample: [] } try { const live = await this.#readLists(s) const mk = (l: string[], b: string[]) => { const d = diffSets(b, l) return { count: l.length, missing: d.missing.length, extra: d.extra.length, missingSample: d.missing.slice(0, SAMPLE), extraSample: d.extra.slice(0, SAMPLE) } } const blocked = mk(live.blocked, baseline.blocked) const allowed = mk(live.allowed, baseline.allowed) const adds = blocked.missing + allowed.missing const deletes = removeExtras ? blocked.extra + allowed.extra : 0 const tooMany = adds > MAX_ADDS ? `${adds} domains to add; at most ${MAX_ADDS} at a time.` : deletes > MAX_DELETES ? `${deletes} domains to remove; at most ${MAX_DELETES} at a time.` : undefined return { id: s.id, ok: true, writable, blocked, allowed, status: adds + (removeExtras ? blocked.extra + allowed.extra : 0) > 0 ? ('differs' as const) : ('same' as const), ...(tooMany === undefined ? {} : { tooMany }) } } catch (e) { return { id: s.id, ok: false, error: reason(e), writable, blocked: none, allowed: none, status: 'unknown' as const } } }), ) return { targets: out } } /* Adds the missing domains and, only when asked, deletes the extras; one call per domain, a few at a time. */ async #applyLists(s: HubServer, baseline: ListsBaseline, removeExtras: boolean): Promise<{ result: TargetResult; detail?: string }> { const fail = (message: string): { result: TargetResult } => ({ result: { server: s.id, ok: false, message } }) if (s.adminToken === undefined) return fail('no admin token configured') try { const live = await this.#readLists(s) const plan = (['blocked', 'allowed'] as const).map((list) => { const d = diffSets(baseline[list], live[list]) return { list, add: d.missing, remove: removeExtras ? d.extra : [] } }) const adds = plan.reduce((n, p) => n + p.add.length, 0) const removals = plan.reduce((n, p) => n + p.remove.length, 0) if (adds + removals === 0) return { result: { server: s.id, ok: true, message: 'Already in sync; nothing was written.' } } if (adds > MAX_ADDS) return fail(`${adds} domains to add; at most ${MAX_ADDS} at a time, so nothing was changed`) if (removals > MAX_DELETES) return fail(`${removals} domains to remove; at most ${MAX_DELETES} at a time, so nothing was changed`) const calls: { path: string; domain: string }[] = [] let invalid = 0 for (const p of plan) { for (const d of p.add) (cleanDomain(d) === null ? invalid++ : calls.push({ path: `${p.list}/add`, domain: d })) for (const d of p.remove) (cleanDomain(d) === null ? invalid++ : calls.push({ path: `${p.list}/delete`, domain: d })) } if (removals > 0) { try { writeHistoryIn(settingsHistoryDir(this.#config.dataDir), s.id, JSON.stringify(live), 'before-lists', this.#now()) } catch (e) { return fail(`the old lists could not be saved first (${reason(e)}); nothing was changed`) } } let failed = 0 let firstError = '' for (let i = 0; i < calls.length; i += 10) { await Promise.all( calls.slice(i, i + 10).map(async (c) => { try { await this.#upstream.write(s, c.path, { domain: c.domain }) } catch (e) { failed++ if (firstError === '') firstError = reason(e) } }), ) } const detail = `${adds} to add, ${removals} to remove` if (failed > 0 || invalid > 0) { return { result: { server: s.id, ok: false, message: `${failed} change(s) failed${firstError === '' ? '' : ` (${firstError})`}${invalid > 0 ? `, ${invalid} domain(s) skipped as invalid` : ''}` }, detail } } const after = await this.#readLists(s) const left = (['blocked', 'allowed'] as const).reduce((n, l) => n + diffSets(baseline[l], after[l]).missing.length + (removeExtras ? diffSets(baseline[l], after[l]).extra.length : 0), 0) if (left > 0) return { result: { server: s.id, ok: false, message: `the changes were accepted but ${left} difference(s) remain when read back` }, detail } return { result: { server: s.id, ok: true, message: `Added ${adds}, removed ${removals}.` }, detail } } catch (e) { return fail(reason(e)) } } async listsApply(ids: string[], removeExtras: boolean, override?: ListsBaseline): Promise<{ results: TargetResult[]; details: Record }> { const targets = this.#targets(ids) const baseline = override ?? this.getListsBaseline() if (baseline === null) throw new UserError('There is no lists baseline yet. Accept one from a server first.') const done = await Promise.all(targets.map((s) => this.#applyLists(s, baseline, removeExtras))) const details: Record = {} for (const d of done) if (d.detail !== undefined) details[d.result.server] = d.detail return { results: done.map((d) => d.result), details } } }