feat: harden firewall policies telemetry and operations

Implement reliable agent apply, telemetry v2, analytics retention, guarded rollouts, enrollment, labels and webhooks. Improve ReUI frontend data safety and bounded list loading, and add Docker smoke, recovery and resource checks. Document local verification and pending native acceptance.

scripts/bench/bundle.mjs

@@ -0,0 +1,18 @@
+import { readdir, readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { gzipSync } from 'node:zlib'
+const directory = process.argv[2] || 'apps/web/dist'
+const files = []
+async function walk(path) {
+  for (const entry of await readdir(path, { withFileTypes: true })) {
+    const name = join(path, entry.name)
+    if (entry.isDirectory()) await walk(name)
+    else if (/\.(js|css)$/.test(name)) {
+      const content = await readFile(name)
+      files.push({ file: name, bytes: content.length, gzipBytes: gzipSync(content, { level: 9 }).length })
+    }
+  }
+}
+await walk(directory)
+files.sort((a, b) => b.bytes - a.bytes)
+console.log(JSON.stringify({ node: process.version, directory, totalBytes: files.reduce((n, f) => n + f.bytes, 0), totalGzipBytes: files.reduce((n, f) => n + f.gzipBytes, 0), files }, null, 2))

scripts/bench/fixture.mjs

@@ -0,0 +1,33 @@
+// Pipe into `docker exec -i CONTAINER node --input-type=module` on a disposable DB.
+import { createRequire } from 'node:module'
+import { createHash } from 'node:crypto'
+import { fileURLToPath } from 'node:url'
+import { resolve } from 'node:path'
+export const benchmarkPrefix = (i, first = 10) => `${first}.${i >>> 16}.${(i >>> 8) & 255}.${i & 255}/32`
+export function seedFixture(db) {
+  db.pragma('foreign_keys = ON')
+  db.transaction(() => {
+  for (const table of ['agents', 'ip_lists', 'agent_telemetry_state', 'agent_stats_samples', 'agent_stats_rollups']) {
+    if (db.prepare(`SELECT count(*) AS n FROM ${table}`).get().n !== 0) throw new Error(`Fresh benchmark database required: ${table} is not empty`)
+  }
+  db.prepare('INSERT INTO ip_lists(id,name,type) VALUES (?,?,?)').run('bench-list', 'Benchmark 50k', 'static')
+  db.prepare('INSERT INTO policy_sets(id,name) VALUES (?,?)').run('bench-set', 'Benchmark')
+  db.prepare('INSERT INTO policy_rules(id,set_id,priority,action,list_id) VALUES (?,?,?,?,?)').run('bench-rule', 'bench-set', 10, 'deny', 'bench-list')
+  const entry = db.prepare('INSERT INTO ip_list_entries(id,list_id,cidr) VALUES (?,?,?)')
+  for (let i = 0; i < 50_000; i++) entry.run(`bench-entry-${i}`, 'bench-list', benchmarkPrefix(i))
+  const agent = db.prepare('INSERT INTO agents(id,name,token_prefix,token_hash,status) VALUES (?,?,?,?,?)')
+  const assign = db.prepare('INSERT INTO agent_policy_sets(agent_id,set_id) VALUES (?,?)')
+  for (let i = 0; i < 100; i++) {
+    const token = `bench-agent-token-${i}`
+    agent.run(`bench-agent-${i}`, `Benchmark ${i}`, token.slice(0, 12), createHash('sha256').update(token).digest('hex'), 'approved')
+    assign.run(`bench-agent-${i}`, 'bench-set')
+  }
+  })()
+  return { fixture: { agents: 100, prefixes: 50_000 }, integrity: db.pragma('integrity_check', { simple: true }) }
+}
+if (!process.argv[1] || resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const require = createRequire('/app/node_modules/@evofw/db/package.json')
+  const Database = require('better-sqlite3')
+  const db = new Database('/data/app.db')
+  try { console.log(JSON.stringify(seedFixture(db))) } finally { db.close() }
+}

scripts/bench/load.mjs

@@ -0,0 +1,147 @@
+import { performance } from 'node:perf_hooks'
+import assert from 'node:assert/strict'
+import { randomUUID } from 'node:crypto'
+import { setTimeout as delay } from 'node:timers/promises'
+import { benchmarkPrefix } from './fixture.mjs'
+const base = process.env.BENCH_URL || 'http://127.0.0.1:8080'
+const concurrency = Number(process.env.BENCH_CONCURRENCY || 10)
+if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 100) throw new Error('BENCH_CONCURRENCY must be 1..100')
+const steadyConcurrency = Number(process.env.BENCH_STEADY_CONCURRENCY || 2)
+const steadyIntervalMs = Number(process.env.BENCH_STEADY_INTERVAL_MS || 1200)
+const p95LimitMs = process.env.BENCH_P95_LIMIT_MS ? Number(process.env.BENCH_P95_LIMIT_MS) : null
+if (!Number.isInteger(steadyConcurrency) || steadyConcurrency < 1 || steadyConcurrency > 10) throw new Error('BENCH_STEADY_CONCURRENCY must be 1..10')
+if (!Number.isInteger(steadyIntervalMs) || steadyIntervalMs < 0 || steadyIntervalMs > 5000) throw new Error('BENCH_STEADY_INTERVAL_MS must be 0..5000')
+if (p95LimitMs !== null && (!Number.isFinite(p95LimitMs) || p95LimitMs <= 0)) throw new Error('BENCH_P95_LIMIT_MS must be positive')
+const measurements = []
+let phase = 'setup'
+const runId = randomUUID()
+const observedEnd = Math.floor(Date.now() / 60_000) * 60_000
+const range = new URLSearchParams({ from: new Date(observedEnd - 5 * 60_000).toISOString(), to: new Date(observedEnd + 60_000).toISOString(), resolution: '1m' })
+const timeseriesPath = `/api/v1/stats/timeseries?${range}`
+async function request(path, options = {}, metadata = {}) {
+  const started = performance.now()
+  const response = await fetch(`${base}${path}`, { ...options, signal: AbortSignal.timeout(120_000) })
+  const body = await response.arrayBuffer()
+  const bytes = body.byteLength
+  measurements.push({ phase, ...metadata, path, status: response.status, ms: performance.now() - started, bytes })
+  if (!response.ok) throw new Error(`HTTP ${response.status}: ${path}`)
+  return JSON.parse(new TextDecoder().decode(body))
+}
+const summary = records => {
+  const sorted = records.map(r => r.ms).sort((a, b) => a - b)
+  return { count: records.length, failures: records.filter(r => r.status >= 400).length, p50Ms: sorted[Math.floor(sorted.length * 0.5)], p95Ms: sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))], maxMs: sorted.at(-1) }
+}
+const profile = name => {
+  const records = measurements.filter(row => row.phase === name)
+  return { requests: records.length, policy: summary(records.filter(r => r.path.endsWith('/policy'))), heartbeat: summary(records.filter(r => r.path.endsWith('/heartbeat'))), reports: summary(records.filter(r => r.path.endsWith('/apply-report'))), ingestion: summary(records.filter(r => r.reportAttempt === 'new')), duplicateRetries: summary(records.filter(r => r.reportAttempt === 'duplicate')), summaries: summary(records.filter(r => r.path.endsWith('/summary'))), timeseries: summary(records.filter(r => r.path.startsWith('/api/v1/stats/timeseries'))) }
+}
+async function submitReport(i, sequence) {
+  const report = { telemetry_version: 2, report_id: `${runId}-${i}-${sequence}`, sequence, boot_id: runId, observed_at: new Date(observedEnd - (4 - sequence) * 60_000).toISOString(), interval_ms: 60_000,
+    counter_epoch: { traffic: runId }, capabilities: ['policy_ipv4'], quality: { partial: false, truncated: false, unsupported: [], lost_reports: 0 }, status: 'ok', packets_dropped: sequence * 100, packets_accepted: sequence * 200,
+    detail_counts: { ip_found: 0, ip_sent: 0, port_found: 0, port_sent: 0 } }
+  for (let retry = 0; retry < 2; retry++) {
+    const ack = await request('/v1/agent/apply-report', { method: 'POST', headers: { authorization: `Bearer bench-agent-token-${i}`, 'content-type': 'application/json' }, body: JSON.stringify(report) }, { reportAttempt: retry === 0 ? 'new' : 'duplicate' })
+    assert.equal(ack.ok, true)
+    assert.equal(ack.report_id, report.report_id)
+    assert.equal(ack.sequence, sequence)
+    assert.equal(ack.accepted, retry === 0, 'New report must be accepted once')
+    assert.equal(ack.duplicate, retry === 1, 'Exact retry must be acknowledged as duplicate')
+  }
+}
+async function verifyTotals(reports, dropped, accepted) {
+  // Raw API filters receive-time, whereas timeseries filters observation-time.
+  // A paced sweep can cross several minute boundaries after observedEnd.
+  const rawRange = new URLSearchParams({ from: new Date(observedEnd - 5 * 60_000).toISOString(), to: new Date(Date.now() + 60_000).toISOString() })
+  const agents = await request('/api/v1/agents')
+  assert.equal(agents.total, 100)
+  assert.equal(agents.items.length, 100)
+  for (const agent of agents.items) {
+    assert.equal(agent.total_packets_dropped, dropped, 'Duplicate retry changed dropped totals')
+    assert.equal(agent.total_packets_accepted, accepted, 'Duplicate retry changed accepted totals')
+    const raw = await request(`/api/v1/agents/${agent.id}/stats/raw?${rawRange}&limit=10`)
+    assert.equal(raw.items.length, reports, 'Each report must be persisted exactly once')
+    assert.equal(raw.has_more, false, 'Unexpected extra telemetry rows')
+    assert.deepEqual(raw.items.map(row => row.sequence).sort((a, b) => a - b), Array.from({ length: reports }, (_, i) => i + 1))
+    assert.ok(raw.items.every(row => row.report_id.startsWith(runId)))
+    assert.equal(raw.items.reduce((n, row) => n + (row.delta_dropped ?? 0), 0), dropped)
+    assert.equal(raw.items.reduce((n, row) => n + (row.delta_accepted ?? 0), 0), accepted)
+  }
+  const aggregate = await request(timeseriesPath)
+  assert.equal(aggregate.agent_ids.length, 100)
+  assert.equal(aggregate.items.reduce((n, row) => n + (row.packets_dropped ?? 0), 0), dropped * 100)
+  assert.equal(aggregate.items.reduce((n, row) => n + (row.packets_accepted ?? 0), 0), accepted * 100)
+}
+// All agents share the same originating host/NAT. Exercise the complete burst,
+// exact report retries and concurrent UI summary reads, including >300 requests.
+let before, after, failure
+try {
+before = await request('/api/v1/diagnostics')
+const initial = await request('/api/v1/agents')
+assert.equal(initial.total, 100, 'Fresh benchmark fixture must contain exactly 100 agents')
+assert.equal(initial.items.length, 100)
+for (let i = 0; i < 100; i++) {
+  const agent = initial.items.find(a => a.id === `bench-agent-${i}`)
+  assert.ok(agent, `Missing benchmark agent ${i}`)
+  assert.equal(agent.total_packets_dropped, 0, 'Fresh fixture required')
+  assert.equal(agent.total_packets_accepted, 0, 'Fresh fixture required')
+  assert.equal(agent.diagnostics.telemetry_version, null, 'Refusing reused telemetry watermark')
+  assert.equal(agent.diagnostics.last_telemetry_at, null, 'Refusing reused history')
+}
+let next = 0
+phase = 'burst'
+await Promise.all(Array.from({ length: concurrency }, async () => {
+  while (next < 100) {
+    const i = next++
+    const headers = { authorization: `Bearer bench-agent-token-${i}` }
+    await request('/v1/agent/policy', { headers })
+    await request('/v1/agent/heartbeat', { method: 'POST', headers })
+    for (let sequence = 1; sequence <= 3; sequence++) {
+      await submitReport(i, sequence)
+    }
+    await request('/api/v1/ui/summary')
+    await request(timeseriesPath)
+  }
+}))
+phase = 'burst-verification'
+await verifyTotals(3, 200, 400)
+phase = 'warmup'
+await request('/api/v1/ui/summary')
+await request(timeseriesPath)
+phase = 'steady'
+next = 0
+await Promise.all(Array.from({ length: steadyConcurrency }, async () => {
+  while (next < 100) {
+    const i = next++, started = performance.now()
+    // Fresh ingestion, not replay-only timing. The second POST checks exact ACK.
+    await submitReport(i, 4)
+    await request('/api/v1/ui/summary')
+    await request(timeseriesPath)
+    await delay(Math.max(0, steadyIntervalMs - (performance.now() - started)))
+  }
+}))
+phase = 'steady-verification'
+await verifyTotals(4, 300, 600)
+phase = 'import'
+const list = await request('/api/v1/lists', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'Benchmark import 100k', type: 'static' }) })
+const values = Array.from({ length: 100_000 }, (_, i) => benchmarkPrefix(i, 11))
+const importBody = JSON.stringify({ values })
+const imported = await request(`/api/v1/lists/${list.id}/entries?return=summary`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: importBody })
+assert.equal(imported.count, 100_000)
+assert.equal(imported.changed, true)
+assert.match(imported.content_hash, /^sha256:[0-9a-f]{64}$/)
+assert.ok(measurements.at(-1).bytes < 256, 'Import summary must stay bounded')
+after = await request('/api/v1/diagnostics')
+if (p95LimitMs !== null) {
+  const steady = profile('steady')
+  for (const category of ['ingestion', 'timeseries']) {
+    assert.ok(steady[category].count > 0, `No steady ${category} samples`)
+    assert.ok(steady[category].p95Ms <= p95LimitMs, `Steady ${category} p95 ${steady[category].p95Ms.toFixed(2)}ms exceeds ${p95LimitMs}ms`)
+  }
+}
+} catch (error) { failure = error.message; process.exitCode = 1 }
+const growth = before && after ? Object.fromEntries(['db_bytes', 'wal_bytes'].map(key => [key, before.database[key] === null || after.database[key] === null ? null : after.database[key] - before.database[key]])) : null
+console.log(JSON.stringify({ node: process.version, runId, freshFixtureRequired: true, sameNatAgents: 100, requests: measurements.length, rateLimited: measurements.filter(r => r.status === 429).length,
+  profiles: { burst: { concurrency, expectedRequests: 1000, ...profile('burst') }, steady: { concurrency: steadyConcurrency, intervalPerWorkerMs: steadyIntervalMs, expectedRequests: 400, p95LimitMs, ...profile('steady') } },
+  assertions: failure ? { passed: false, error: failure } : { passed: true, burst: { reportsPerAgent: 3, droppedPerAgent: 200, acceptedPerAgent: 400, aggregateDropped: 20_000, aggregateAccepted: 40_000 }, final: { reportsPerAgent: 4, droppedPerAgent: 300, acceptedPerAgent: 600, aggregateDropped: 30_000, aggregateAccepted: 60_000 }, duplicateAcksVerified: true, importedPrefixes: 100_000, importSummaryBoundBytes: 256 },
+  database: { before: before?.database, after: after?.database, growth }, measurements }, null, 2))
+if (measurements.some(r => r.status >= 400)) process.exitCode = 1

scripts/bench/runtime.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env sh
+# Linux/Docker host, isolated fixture only. Pass the same image to repeat a run.
+set -eu
+image="${1:?Usage: runtime.sh IMAGE OUTPUT_DIR}"
+out="${2:?OUTPUT_DIR required}"
+[ ! -e "$out" ] || { echo 'Use a new output directory for each baseline run' >&2; exit 1; }
+mkdir -p "$out"
+id="evofw-bench-$(date +%s)-$$"
+volume="$id-data"
+cleanup() {
+  docker logs "$id" >"$out/container.log" 2>&1 || true
+  [ -z "${sampler:-}" ] || { kill "$sampler" 2>/dev/null || true; wait "$sampler" 2>/dev/null || true; }
+  docker rm -f "$id" >/dev/null 2>&1 || true
+  docker volume rm "$volume" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT HUP INT TERM
+docker version >"$out/docker-version.txt"
+docker info >"$out/docker-info.txt"
+docker image inspect "$image" >"$out/image.json"
+if docker volume inspect "$volume" >/dev/null 2>&1; then echo 'Refusing reused benchmark volume' >&2; exit 1; fi
+docker volume create "$volume" >/dev/null
+started="$(date +%s)"
+docker run -d --name "$id" --cpus 2 --memory 4g --publish 127.0.0.1::8080 \
+  --mount "type=volume,src=$volume,dst=/data" -e EVOFW_ALLOW_UNSAFE=true -e AUTH_REQUIRED=false "$image" >/dev/null
+attempts=0
+until docker exec "$id" node -e "fetch('http://127.0.0.1:8080/ready').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" >/dev/null 2>&1; do
+  attempts=$((attempts + 1)); [ "$attempts" -lt 120 ] || exit 1; sleep 1
+done
+printf '{"coldReadySeconds":%s,"cpus":2,"memoryBytes":4294967296}\n' "$(( $(date +%s) - started ))" >"$out/startup.json"
+docker exec -i "$id" node --input-type=module <scripts/bench/fixture.mjs >"$out/fixture.json"
+# Includes CPU, memory, block IO and PIDs. Metrics are raw observations, no claimed gains.
+(while docker inspect -f '{{.State.Running}}' "$id" 2>/dev/null | grep -q true; do docker stats --no-stream --format '{{json .}}' "$id"; sleep 1; done) >"$out/stats.jsonl" &
+sampler=$!
+port="$(docker port "$id" 8080/tcp | sed 's/.*://')"
+BENCH_P95_LIMIT_MS="${BENCH_P95_LIMIT_MS:-500}" BENCH_URL="http://127.0.0.1:$port" node scripts/bench/load.mjs >"$out/load.json" || load_failed=true
+cat "$out/load.json"
+docker exec "$id" sh -c 'ls -ln /data/app.db*; cat /proc/1/status; cat /proc/1/io' >"$out/db-process.txt"
+sh deploy/docker/backup.sh "$id" "$out/backup.db"
+[ "${load_failed:-false}" = false ]