feat: harden firewall policies telemetry and operations

Implement reliable agent apply, telemetry v2, analytics retention, guarded rollouts, enrollment, labels and webhooks. Improve ReUI frontend data safety and bounded list loading, and add Docker smoke, recovery and resource checks. Document local verification and pending native acceptance.

scripts/bench/load.mjs

@@ -0,0 +1,147 @@
+import { performance } from 'node:perf_hooks'
+import assert from 'node:assert/strict'
+import { randomUUID } from 'node:crypto'
+import { setTimeout as delay } from 'node:timers/promises'
+import { benchmarkPrefix } from './fixture.mjs'
+const base = process.env.BENCH_URL || 'http://127.0.0.1:8080'
+const concurrency = Number(process.env.BENCH_CONCURRENCY || 10)
+if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 100) throw new Error('BENCH_CONCURRENCY must be 1..100')
+const steadyConcurrency = Number(process.env.BENCH_STEADY_CONCURRENCY || 2)
+const steadyIntervalMs = Number(process.env.BENCH_STEADY_INTERVAL_MS || 1200)
+const p95LimitMs = process.env.BENCH_P95_LIMIT_MS ? Number(process.env.BENCH_P95_LIMIT_MS) : null
+if (!Number.isInteger(steadyConcurrency) || steadyConcurrency < 1 || steadyConcurrency > 10) throw new Error('BENCH_STEADY_CONCURRENCY must be 1..10')
+if (!Number.isInteger(steadyIntervalMs) || steadyIntervalMs < 0 || steadyIntervalMs > 5000) throw new Error('BENCH_STEADY_INTERVAL_MS must be 0..5000')
+if (p95LimitMs !== null && (!Number.isFinite(p95LimitMs) || p95LimitMs <= 0)) throw new Error('BENCH_P95_LIMIT_MS must be positive')
+const measurements = []
+let phase = 'setup'
+const runId = randomUUID()
+const observedEnd = Math.floor(Date.now() / 60_000) * 60_000
+const range = new URLSearchParams({ from: new Date(observedEnd - 5 * 60_000).toISOString(), to: new Date(observedEnd + 60_000).toISOString(), resolution: '1m' })
+const timeseriesPath = `/api/v1/stats/timeseries?${range}`
+async function request(path, options = {}, metadata = {}) {
+  const started = performance.now()
+  const response = await fetch(`${base}${path}`, { ...options, signal: AbortSignal.timeout(120_000) })
+  const body = await response.arrayBuffer()
+  const bytes = body.byteLength
+  measurements.push({ phase, ...metadata, path, status: response.status, ms: performance.now() - started, bytes })
+  if (!response.ok) throw new Error(`HTTP ${response.status}: ${path}`)
+  return JSON.parse(new TextDecoder().decode(body))
+}
+const summary = records => {
+  const sorted = records.map(r => r.ms).sort((a, b) => a - b)
+  return { count: records.length, failures: records.filter(r => r.status >= 400).length, p50Ms: sorted[Math.floor(sorted.length * 0.5)], p95Ms: sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))], maxMs: sorted.at(-1) }
+}
+const profile = name => {
+  const records = measurements.filter(row => row.phase === name)
+  return { requests: records.length, policy: summary(records.filter(r => r.path.endsWith('/policy'))), heartbeat: summary(records.filter(r => r.path.endsWith('/heartbeat'))), reports: summary(records.filter(r => r.path.endsWith('/apply-report'))), ingestion: summary(records.filter(r => r.reportAttempt === 'new')), duplicateRetries: summary(records.filter(r => r.reportAttempt === 'duplicate')), summaries: summary(records.filter(r => r.path.endsWith('/summary'))), timeseries: summary(records.filter(r => r.path.startsWith('/api/v1/stats/timeseries'))) }
+}
+async function submitReport(i, sequence) {
+  const report = { telemetry_version: 2, report_id: `${runId}-${i}-${sequence}`, sequence, boot_id: runId, observed_at: new Date(observedEnd - (4 - sequence) * 60_000).toISOString(), interval_ms: 60_000,
+    counter_epoch: { traffic: runId }, capabilities: ['policy_ipv4'], quality: { partial: false, truncated: false, unsupported: [], lost_reports: 0 }, status: 'ok', packets_dropped: sequence * 100, packets_accepted: sequence * 200,
+    detail_counts: { ip_found: 0, ip_sent: 0, port_found: 0, port_sent: 0 } }
+  for (let retry = 0; retry < 2; retry++) {
+    const ack = await request('/v1/agent/apply-report', { method: 'POST', headers: { authorization: `Bearer bench-agent-token-${i}`, 'content-type': 'application/json' }, body: JSON.stringify(report) }, { reportAttempt: retry === 0 ? 'new' : 'duplicate' })
+    assert.equal(ack.ok, true)
+    assert.equal(ack.report_id, report.report_id)
+    assert.equal(ack.sequence, sequence)
+    assert.equal(ack.accepted, retry === 0, 'New report must be accepted once')
+    assert.equal(ack.duplicate, retry === 1, 'Exact retry must be acknowledged as duplicate')
+  }
+}
+async function verifyTotals(reports, dropped, accepted) {
+  // Raw API filters receive-time, whereas timeseries filters observation-time.
+  // A paced sweep can cross several minute boundaries after observedEnd.
+  const rawRange = new URLSearchParams({ from: new Date(observedEnd - 5 * 60_000).toISOString(), to: new Date(Date.now() + 60_000).toISOString() })
+  const agents = await request('/api/v1/agents')
+  assert.equal(agents.total, 100)
+  assert.equal(agents.items.length, 100)
+  for (const agent of agents.items) {
+    assert.equal(agent.total_packets_dropped, dropped, 'Duplicate retry changed dropped totals')
+    assert.equal(agent.total_packets_accepted, accepted, 'Duplicate retry changed accepted totals')
+    const raw = await request(`/api/v1/agents/${agent.id}/stats/raw?${rawRange}&limit=10`)
+    assert.equal(raw.items.length, reports, 'Each report must be persisted exactly once')
+    assert.equal(raw.has_more, false, 'Unexpected extra telemetry rows')
+    assert.deepEqual(raw.items.map(row => row.sequence).sort((a, b) => a - b), Array.from({ length: reports }, (_, i) => i + 1))
+    assert.ok(raw.items.every(row => row.report_id.startsWith(runId)))
+    assert.equal(raw.items.reduce((n, row) => n + (row.delta_dropped ?? 0), 0), dropped)
+    assert.equal(raw.items.reduce((n, row) => n + (row.delta_accepted ?? 0), 0), accepted)
+  }
+  const aggregate = await request(timeseriesPath)
+  assert.equal(aggregate.agent_ids.length, 100)
+  assert.equal(aggregate.items.reduce((n, row) => n + (row.packets_dropped ?? 0), 0), dropped * 100)
+  assert.equal(aggregate.items.reduce((n, row) => n + (row.packets_accepted ?? 0), 0), accepted * 100)
+}
+// All agents share the same originating host/NAT. Exercise the complete burst,
+// exact report retries and concurrent UI summary reads, including >300 requests.
+let before, after, failure
+try {
+before = await request('/api/v1/diagnostics')
+const initial = await request('/api/v1/agents')
+assert.equal(initial.total, 100, 'Fresh benchmark fixture must contain exactly 100 agents')
+assert.equal(initial.items.length, 100)
+for (let i = 0; i < 100; i++) {
+  const agent = initial.items.find(a => a.id === `bench-agent-${i}`)
+  assert.ok(agent, `Missing benchmark agent ${i}`)
+  assert.equal(agent.total_packets_dropped, 0, 'Fresh fixture required')
+  assert.equal(agent.total_packets_accepted, 0, 'Fresh fixture required')
+  assert.equal(agent.diagnostics.telemetry_version, null, 'Refusing reused telemetry watermark')
+  assert.equal(agent.diagnostics.last_telemetry_at, null, 'Refusing reused history')
+}
+let next = 0
+phase = 'burst'
+await Promise.all(Array.from({ length: concurrency }, async () => {
+  while (next < 100) {
+    const i = next++
+    const headers = { authorization: `Bearer bench-agent-token-${i}` }
+    await request('/v1/agent/policy', { headers })
+    await request('/v1/agent/heartbeat', { method: 'POST', headers })
+    for (let sequence = 1; sequence <= 3; sequence++) {
+      await submitReport(i, sequence)
+    }
+    await request('/api/v1/ui/summary')
+    await request(timeseriesPath)
+  }
+}))
+phase = 'burst-verification'
+await verifyTotals(3, 200, 400)
+phase = 'warmup'
+await request('/api/v1/ui/summary')
+await request(timeseriesPath)
+phase = 'steady'
+next = 0
+await Promise.all(Array.from({ length: steadyConcurrency }, async () => {
+  while (next < 100) {
+    const i = next++, started = performance.now()
+    // Fresh ingestion, not replay-only timing. The second POST checks exact ACK.
+    await submitReport(i, 4)
+    await request('/api/v1/ui/summary')
+    await request(timeseriesPath)
+    await delay(Math.max(0, steadyIntervalMs - (performance.now() - started)))
+  }
+}))
+phase = 'steady-verification'
+await verifyTotals(4, 300, 600)
+phase = 'import'
+const list = await request('/api/v1/lists', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'Benchmark import 100k', type: 'static' }) })
+const values = Array.from({ length: 100_000 }, (_, i) => benchmarkPrefix(i, 11))
+const importBody = JSON.stringify({ values })
+const imported = await request(`/api/v1/lists/${list.id}/entries?return=summary`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: importBody })
+assert.equal(imported.count, 100_000)
+assert.equal(imported.changed, true)
+assert.match(imported.content_hash, /^sha256:[0-9a-f]{64}$/)
+assert.ok(measurements.at(-1).bytes < 256, 'Import summary must stay bounded')
+after = await request('/api/v1/diagnostics')
+if (p95LimitMs !== null) {
+  const steady = profile('steady')
+  for (const category of ['ingestion', 'timeseries']) {
+    assert.ok(steady[category].count > 0, `No steady ${category} samples`)
+    assert.ok(steady[category].p95Ms <= p95LimitMs, `Steady ${category} p95 ${steady[category].p95Ms.toFixed(2)}ms exceeds ${p95LimitMs}ms`)
+  }
+}
+} catch (error) { failure = error.message; process.exitCode = 1 }
+const growth = before && after ? Object.fromEntries(['db_bytes', 'wal_bytes'].map(key => [key, before.database[key] === null || after.database[key] === null ? null : after.database[key] - before.database[key]])) : null
+console.log(JSON.stringify({ node: process.version, runId, freshFixtureRequired: true, sameNatAgents: 100, requests: measurements.length, rateLimited: measurements.filter(r => r.status === 429).length,
+  profiles: { burst: { concurrency, expectedRequests: 1000, ...profile('burst') }, steady: { concurrency: steadyConcurrency, intervalPerWorkerMs: steadyIntervalMs, expectedRequests: 400, p95LimitMs, ...profile('steady') } },
+  assertions: failure ? { passed: false, error: failure } : { passed: true, burst: { reportsPerAgent: 3, droppedPerAgent: 200, acceptedPerAgent: 400, aggregateDropped: 20_000, aggregateAccepted: 40_000 }, final: { reportsPerAgent: 4, droppedPerAgent: 300, acceptedPerAgent: 600, aggregateDropped: 30_000, aggregateAccepted: 60_000 }, duplicateAcksVerified: true, importedPrefixes: 100_000, importSummaryBoundBytes: 256 },
+  database: { before: before?.database, after: after?.database, growth }, measurements }, null, 2))
+if (measurements.some(r => r.status >= 400)) process.exitCode = 1