feat: harden firewall policies telemetry and operations

Implement reliable agent apply, telemetry v2, analytics retention, guarded rollouts, enrollment, labels and webhooks. Improve ReUI frontend data safety and bounded list loading, and add Docker smoke, recovery and resource checks. Document local verification and pending native acceptance.

scripts/bench/bundle.mjs

@@ -0,0 +1,18 @@
+import { readdir, readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { gzipSync } from 'node:zlib'
+const directory = process.argv[2] || 'apps/web/dist'
+const files = []
+async function walk(path) {
+  for (const entry of await readdir(path, { withFileTypes: true })) {
+    const name = join(path, entry.name)
+    if (entry.isDirectory()) await walk(name)
+    else if (/\.(js|css)$/.test(name)) {
+      const content = await readFile(name)
+      files.push({ file: name, bytes: content.length, gzipBytes: gzipSync(content, { level: 9 }).length })
+    }
+  }
+}
+await walk(directory)
+files.sort((a, b) => b.bytes - a.bytes)
+console.log(JSON.stringify({ node: process.version, directory, totalBytes: files.reduce((n, f) => n + f.bytes, 0), totalGzipBytes: files.reduce((n, f) => n + f.gzipBytes, 0), files }, null, 2))

scripts/bench/fixture.mjs

@@ -0,0 +1,33 @@
+// Pipe into `docker exec -i CONTAINER node --input-type=module` on a disposable DB.
+import { createRequire } from 'node:module'
+import { createHash } from 'node:crypto'
+import { fileURLToPath } from 'node:url'
+import { resolve } from 'node:path'
+export const benchmarkPrefix = (i, first = 10) => `${first}.${i >>> 16}.${(i >>> 8) & 255}.${i & 255}/32`
+export function seedFixture(db) {
+  db.pragma('foreign_keys = ON')
+  db.transaction(() => {
+  for (const table of ['agents', 'ip_lists', 'agent_telemetry_state', 'agent_stats_samples', 'agent_stats_rollups']) {
+    if (db.prepare(`SELECT count(*) AS n FROM ${table}`).get().n !== 0) throw new Error(`Fresh benchmark database required: ${table} is not empty`)
+  }
+  db.prepare('INSERT INTO ip_lists(id,name,type) VALUES (?,?,?)').run('bench-list', 'Benchmark 50k', 'static')
+  db.prepare('INSERT INTO policy_sets(id,name) VALUES (?,?)').run('bench-set', 'Benchmark')
+  db.prepare('INSERT INTO policy_rules(id,set_id,priority,action,list_id) VALUES (?,?,?,?,?)').run('bench-rule', 'bench-set', 10, 'deny', 'bench-list')
+  const entry = db.prepare('INSERT INTO ip_list_entries(id,list_id,cidr) VALUES (?,?,?)')
+  for (let i = 0; i < 50_000; i++) entry.run(`bench-entry-${i}`, 'bench-list', benchmarkPrefix(i))
+  const agent = db.prepare('INSERT INTO agents(id,name,token_prefix,token_hash,status) VALUES (?,?,?,?,?)')
+  const assign = db.prepare('INSERT INTO agent_policy_sets(agent_id,set_id) VALUES (?,?)')
+  for (let i = 0; i < 100; i++) {
+    const token = `bench-agent-token-${i}`
+    agent.run(`bench-agent-${i}`, `Benchmark ${i}`, token.slice(0, 12), createHash('sha256').update(token).digest('hex'), 'approved')
+    assign.run(`bench-agent-${i}`, 'bench-set')
+  }
+  })()
+  return { fixture: { agents: 100, prefixes: 50_000 }, integrity: db.pragma('integrity_check', { simple: true }) }
+}
+if (!process.argv[1] || resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const require = createRequire('/app/node_modules/@evofw/db/package.json')
+  const Database = require('better-sqlite3')
+  const db = new Database('/data/app.db')
+  try { console.log(JSON.stringify(seedFixture(db))) } finally { db.close() }
+}

scripts/bench/load.mjs

@@ -0,0 +1,147 @@
+import { performance } from 'node:perf_hooks'
+import assert from 'node:assert/strict'
+import { randomUUID } from 'node:crypto'
+import { setTimeout as delay } from 'node:timers/promises'
+import { benchmarkPrefix } from './fixture.mjs'
+const base = process.env.BENCH_URL || 'http://127.0.0.1:8080'
+const concurrency = Number(process.env.BENCH_CONCURRENCY || 10)
+if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 100) throw new Error('BENCH_CONCURRENCY must be 1..100')
+const steadyConcurrency = Number(process.env.BENCH_STEADY_CONCURRENCY || 2)
+const steadyIntervalMs = Number(process.env.BENCH_STEADY_INTERVAL_MS || 1200)
+const p95LimitMs = process.env.BENCH_P95_LIMIT_MS ? Number(process.env.BENCH_P95_LIMIT_MS) : null
+if (!Number.isInteger(steadyConcurrency) || steadyConcurrency < 1 || steadyConcurrency > 10) throw new Error('BENCH_STEADY_CONCURRENCY must be 1..10')
+if (!Number.isInteger(steadyIntervalMs) || steadyIntervalMs < 0 || steadyIntervalMs > 5000) throw new Error('BENCH_STEADY_INTERVAL_MS must be 0..5000')
+if (p95LimitMs !== null && (!Number.isFinite(p95LimitMs) || p95LimitMs <= 0)) throw new Error('BENCH_P95_LIMIT_MS must be positive')
+const measurements = []
+let phase = 'setup'
+const runId = randomUUID()
+const observedEnd = Math.floor(Date.now() / 60_000) * 60_000
+const range = new URLSearchParams({ from: new Date(observedEnd - 5 * 60_000).toISOString(), to: new Date(observedEnd + 60_000).toISOString(), resolution: '1m' })
+const timeseriesPath = `/api/v1/stats/timeseries?${range}`
+async function request(path, options = {}, metadata = {}) {
+  const started = performance.now()
+  const response = await fetch(`${base}${path}`, { ...options, signal: AbortSignal.timeout(120_000) })
+  const body = await response.arrayBuffer()
+  const bytes = body.byteLength
+  measurements.push({ phase, ...metadata, path, status: response.status, ms: performance.now() - started, bytes })
+  if (!response.ok) throw new Error(`HTTP ${response.status}: ${path}`)
+  return JSON.parse(new TextDecoder().decode(body))
+}
+const summary = records => {
+  const sorted = records.map(r => r.ms).sort((a, b) => a - b)
+  return { count: records.length, failures: records.filter(r => r.status >= 400).length, p50Ms: sorted[Math.floor(sorted.length * 0.5)], p95Ms: sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))], maxMs: sorted.at(-1) }
+}
+const profile = name => {
+  const records = measurements.filter(row => row.phase === name)
+  return { requests: records.length, policy: summary(records.filter(r => r.path.endsWith('/policy'))), heartbeat: summary(records.filter(r => r.path.endsWith('/heartbeat'))), reports: summary(records.filter(r => r.path.endsWith('/apply-report'))), ingestion: summary(records.filter(r => r.reportAttempt === 'new')), duplicateRetries: summary(records.filter(r => r.reportAttempt === 'duplicate')), summaries: summary(records.filter(r => r.path.endsWith('/summary'))), timeseries: summary(records.filter(r => r.path.startsWith('/api/v1/stats/timeseries'))) }
+}
+async function submitReport(i, sequence) {
+  const report = { telemetry_version: 2, report_id: `${runId}-${i}-${sequence}`, sequence, boot_id: runId, observed_at: new Date(observedEnd - (4 - sequence) * 60_000).toISOString(), interval_ms: 60_000,
+    counter_epoch: { traffic: runId }, capabilities: ['policy_ipv4'], quality: { partial: false, truncated: false, unsupported: [], lost_reports: 0 }, status: 'ok', packets_dropped: sequence * 100, packets_accepted: sequence * 200,
+    detail_counts: { ip_found: 0, ip_sent: 0, port_found: 0, port_sent: 0 } }
+  for (let retry = 0; retry < 2; retry++) {
+    const ack = await request('/v1/agent/apply-report', { method: 'POST', headers: { authorization: `Bearer bench-agent-token-${i}`, 'content-type': 'application/json' }, body: JSON.stringify(report) }, { reportAttempt: retry === 0 ? 'new' : 'duplicate' })
+    assert.equal(ack.ok, true)
+    assert.equal(ack.report_id, report.report_id)
+    assert.equal(ack.sequence, sequence)
+    assert.equal(ack.accepted, retry === 0, 'New report must be accepted once')
+    assert.equal(ack.duplicate, retry === 1, 'Exact retry must be acknowledged as duplicate')
+  }
+}
+async function verifyTotals(reports, dropped, accepted) {
+  // Raw API filters receive-time, whereas timeseries filters observation-time.
+  // A paced sweep can cross several minute boundaries after observedEnd.
+  const rawRange = new URLSearchParams({ from: new Date(observedEnd - 5 * 60_000).toISOString(), to: new Date(Date.now() + 60_000).toISOString() })
+  const agents = await request('/api/v1/agents')
+  assert.equal(agents.total, 100)
+  assert.equal(agents.items.length, 100)
+  for (const agent of agents.items) {
+    assert.equal(agent.total_packets_dropped, dropped, 'Duplicate retry changed dropped totals')
+    assert.equal(agent.total_packets_accepted, accepted, 'Duplicate retry changed accepted totals')
+    const raw = await request(`/api/v1/agents/${agent.id}/stats/raw?${rawRange}&limit=10`)
+    assert.equal(raw.items.length, reports, 'Each report must be persisted exactly once')
+    assert.equal(raw.has_more, false, 'Unexpected extra telemetry rows')
+    assert.deepEqual(raw.items.map(row => row.sequence).sort((a, b) => a - b), Array.from({ length: reports }, (_, i) => i + 1))
+    assert.ok(raw.items.every(row => row.report_id.startsWith(runId)))
+    assert.equal(raw.items.reduce((n, row) => n + (row.delta_dropped ?? 0), 0), dropped)
+    assert.equal(raw.items.reduce((n, row) => n + (row.delta_accepted ?? 0), 0), accepted)
+  }
+  const aggregate = await request(timeseriesPath)
+  assert.equal(aggregate.agent_ids.length, 100)
+  assert.equal(aggregate.items.reduce((n, row) => n + (row.packets_dropped ?? 0), 0), dropped * 100)
+  assert.equal(aggregate.items.reduce((n, row) => n + (row.packets_accepted ?? 0), 0), accepted * 100)
+}
+// All agents share the same originating host/NAT. Exercise the complete burst,
+// exact report retries and concurrent UI summary reads, including >300 requests.
+let before, after, failure
+try {
+before = await request('/api/v1/diagnostics')
+const initial = await request('/api/v1/agents')
+assert.equal(initial.total, 100, 'Fresh benchmark fixture must contain exactly 100 agents')
+assert.equal(initial.items.length, 100)
+for (let i = 0; i < 100; i++) {
+  const agent = initial.items.find(a => a.id === `bench-agent-${i}`)
+  assert.ok(agent, `Missing benchmark agent ${i}`)
+  assert.equal(agent.total_packets_dropped, 0, 'Fresh fixture required')
+  assert.equal(agent.total_packets_accepted, 0, 'Fresh fixture required')
+  assert.equal(agent.diagnostics.telemetry_version, null, 'Refusing reused telemetry watermark')
+  assert.equal(agent.diagnostics.last_telemetry_at, null, 'Refusing reused history')
+}
+let next = 0
+phase = 'burst'
+await Promise.all(Array.from({ length: concurrency }, async () => {
+  while (next < 100) {
+    const i = next++
+    const headers = { authorization: `Bearer bench-agent-token-${i}` }
+    await request('/v1/agent/policy', { headers })
+    await request('/v1/agent/heartbeat', { method: 'POST', headers })
+    for (let sequence = 1; sequence <= 3; sequence++) {
+      await submitReport(i, sequence)
+    }
+    await request('/api/v1/ui/summary')
+    await request(timeseriesPath)
+  }
+}))
+phase = 'burst-verification'
+await verifyTotals(3, 200, 400)
+phase = 'warmup'
+await request('/api/v1/ui/summary')
+await request(timeseriesPath)
+phase = 'steady'
+next = 0
+await Promise.all(Array.from({ length: steadyConcurrency }, async () => {
+  while (next < 100) {
+    const i = next++, started = performance.now()
+    // Fresh ingestion, not replay-only timing. The second POST checks exact ACK.
+    await submitReport(i, 4)
+    await request('/api/v1/ui/summary')
+    await request(timeseriesPath)
+    await delay(Math.max(0, steadyIntervalMs - (performance.now() - started)))
+  }
+}))
+phase = 'steady-verification'
+await verifyTotals(4, 300, 600)
+phase = 'import'
+const list = await request('/api/v1/lists', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'Benchmark import 100k', type: 'static' }) })
+const values = Array.from({ length: 100_000 }, (_, i) => benchmarkPrefix(i, 11))
+const importBody = JSON.stringify({ values })
+const imported = await request(`/api/v1/lists/${list.id}/entries?return=summary`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: importBody })
+assert.equal(imported.count, 100_000)
+assert.equal(imported.changed, true)
+assert.match(imported.content_hash, /^sha256:[0-9a-f]{64}$/)
+assert.ok(measurements.at(-1).bytes < 256, 'Import summary must stay bounded')
+after = await request('/api/v1/diagnostics')
+if (p95LimitMs !== null) {
+  const steady = profile('steady')
+  for (const category of ['ingestion', 'timeseries']) {
+    assert.ok(steady[category].count > 0, `No steady ${category} samples`)
+    assert.ok(steady[category].p95Ms <= p95LimitMs, `Steady ${category} p95 ${steady[category].p95Ms.toFixed(2)}ms exceeds ${p95LimitMs}ms`)
+  }
+}
+} catch (error) { failure = error.message; process.exitCode = 1 }
+const growth = before && after ? Object.fromEntries(['db_bytes', 'wal_bytes'].map(key => [key, before.database[key] === null || after.database[key] === null ? null : after.database[key] - before.database[key]])) : null
+console.log(JSON.stringify({ node: process.version, runId, freshFixtureRequired: true, sameNatAgents: 100, requests: measurements.length, rateLimited: measurements.filter(r => r.status === 429).length,
+  profiles: { burst: { concurrency, expectedRequests: 1000, ...profile('burst') }, steady: { concurrency: steadyConcurrency, intervalPerWorkerMs: steadyIntervalMs, expectedRequests: 400, p95LimitMs, ...profile('steady') } },
+  assertions: failure ? { passed: false, error: failure } : { passed: true, burst: { reportsPerAgent: 3, droppedPerAgent: 200, acceptedPerAgent: 400, aggregateDropped: 20_000, aggregateAccepted: 40_000 }, final: { reportsPerAgent: 4, droppedPerAgent: 300, acceptedPerAgent: 600, aggregateDropped: 30_000, aggregateAccepted: 60_000 }, duplicateAcksVerified: true, importedPrefixes: 100_000, importSummaryBoundBytes: 256 },
+  database: { before: before?.database, after: after?.database, growth }, measurements }, null, 2))
+if (measurements.some(r => r.status >= 400)) process.exitCode = 1

scripts/bench/runtime.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env sh
+# Linux/Docker host, isolated fixture only. Pass the same image to repeat a run.
+set -eu
+image="${1:?Usage: runtime.sh IMAGE OUTPUT_DIR}"
+out="${2:?OUTPUT_DIR required}"
+[ ! -e "$out" ] || { echo 'Use a new output directory for each baseline run' >&2; exit 1; }
+mkdir -p "$out"
+id="evofw-bench-$(date +%s)-$$"
+volume="$id-data"
+cleanup() {
+  docker logs "$id" >"$out/container.log" 2>&1 || true
+  [ -z "${sampler:-}" ] || { kill "$sampler" 2>/dev/null || true; wait "$sampler" 2>/dev/null || true; }
+  docker rm -f "$id" >/dev/null 2>&1 || true
+  docker volume rm "$volume" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT HUP INT TERM
+docker version >"$out/docker-version.txt"
+docker info >"$out/docker-info.txt"
+docker image inspect "$image" >"$out/image.json"
+if docker volume inspect "$volume" >/dev/null 2>&1; then echo 'Refusing reused benchmark volume' >&2; exit 1; fi
+docker volume create "$volume" >/dev/null
+started="$(date +%s)"
+docker run -d --name "$id" --cpus 2 --memory 4g --publish 127.0.0.1::8080 \
+  --mount "type=volume,src=$volume,dst=/data" -e EVOFW_ALLOW_UNSAFE=true -e AUTH_REQUIRED=false "$image" >/dev/null
+attempts=0
+until docker exec "$id" node -e "fetch('http://127.0.0.1:8080/ready').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" >/dev/null 2>&1; do
+  attempts=$((attempts + 1)); [ "$attempts" -lt 120 ] || exit 1; sleep 1
+done
+printf '{"coldReadySeconds":%s,"cpus":2,"memoryBytes":4294967296}\n' "$(( $(date +%s) - started ))" >"$out/startup.json"
+docker exec -i "$id" node --input-type=module <scripts/bench/fixture.mjs >"$out/fixture.json"
+# Includes CPU, memory, block IO and PIDs. Metrics are raw observations, no claimed gains.
+(while docker inspect -f '{{.State.Running}}' "$id" 2>/dev/null | grep -q true; do docker stats --no-stream --format '{{json .}}' "$id"; sleep 1; done) >"$out/stats.jsonl" &
+sampler=$!
+port="$(docker port "$id" 8080/tcp | sed 's/.*://')"
+BENCH_P95_LIMIT_MS="${BENCH_P95_LIMIT_MS:-500}" BENCH_URL="http://127.0.0.1:$port" node scripts/bench/load.mjs >"$out/load.json" || load_failed=true
+cat "$out/load.json"
+docker exec "$id" sh -c 'ls -ln /data/app.db*; cat /proc/1/status; cat /proc/1/io' >"$out/db-process.txt"
+sh deploy/docker/backup.sh "$id" "$out/backup.db"
+[ "${load_failed:-false}" = false ]

scripts/ci/benchmark-fixture.test.mjs

@@ -0,0 +1,23 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { createMemoryDb, runMigrations } from '../../packages/db/dist/index.js'
+import { canonicalCidr } from '../../packages/shared/dist/index.js'
+import { seedFixture, benchmarkPrefix } from '../bench/fixture.mjs'
+
+test('benchmark fixture satisfies migrated constraints at full 100-agent/50k scale', () => {
+  const { sqlite } = createMemoryDb()
+  try {
+    runMigrations(sqlite)
+    assert.equal(seedFixture(sqlite).integrity, 'ok')
+    for (const [table, expected] of [['agents', 100], ['agent_policy_sets', 100], ['ip_list_entries', 50_000]]) {
+      assert.equal(sqlite.prepare(`SELECT count(*) AS n FROM ${table}`).get().n, expected)
+    }
+    assert.equal(sqlite.prepare('SELECT priority FROM policy_rules WHERE id=?').get('bench-rule').priority, 10)
+    assert.deepEqual(sqlite.pragma('foreign_key_check'), [])
+    assert.throws(() => seedFixture(sqlite), /Fresh benchmark database required/)
+    assert.equal(sqlite.prepare('SELECT count(*) AS n FROM agents').get().n, 100)
+    const prefixes = Array.from({ length: 100_000 }, (_, i) => benchmarkPrefix(i, 11))
+    assert.equal(new Set(prefixes).size, 100_000)
+    for (const cidr of prefixes) assert.equal(canonicalCidr(cidr), cidr)
+  } finally { sqlite.close() }
+})

scripts/ci/generate-openapi-schemas.mjs

@@ -0,0 +1,53 @@
+// Schema definitions come from the compiled shared Zod contracts, with no new deps.
+import { createRequire } from 'node:module'
+import { readFileSync, writeFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+import * as shared from '../../packages/shared/dist/index.js'
+const require = createRequire(resolve('packages/shared/package.json'))
+const { z } = require('zod')
+const definitions = {
+  Agent: 'agentSchema', AgentDiagnostics: 'agentDiagnosticsSchema', AgentPolicy: 'agentPolicySchema',
+  ApplyReport: 'applyReportBodySchema', ApplyReportV2: 'applyReportV2BodySchema', TelemetryQuality: 'telemetryQualitySchema', ApplyError: 'applyErrorSchema',
+  TimeseriesResponse: 'timeseriesResponseSchema', UiSummary: 'uiSummarySchema', ExplainResponse: 'explainResponseSchema',
+  CreateOverride: 'createOverrideBodySchema', IpOverride: 'ipOverrideSchema', IpList: 'ipListSchema',
+  CreateWebhook: 'webhookBodySchema', PatchWebhook: 'webhookPatchSchema',
+  EnrollWithCredential: 'enrollmentCredentialBodySchema', TokenRotationExchange: 'tokenRotationExchangeBodySchema',
+  CredentialStatus: 'credentialStatusSchema', TokenRotationAck: 'tokenRotationAckSchema',
+  AdminCredentialStatus: 'adminCredentialStatusSchema', ListImportSummary: 'listImportSummarySchema',
+  CreatePolicySnapshot: 'createPolicySnapshotSchema', PolicySnapshotDiffRequest: 'policySnapshotDiffSchema',
+  CreatePolicyRollout: 'createPolicyRolloutSchema', PolicyRolloutTransition: 'policyRolloutTransitionSchema', ReleasePolicyPin: 'releasePolicyPinSchema',
+  CreateAgentLabel: 'agentLabelBodySchema', PatchAgentLabel: 'patchAgentLabelSchema', PutAgentLabels: 'putAgentLabelsSchema',
+  PreviewLabelAssignment: 'previewLabelAssignmentSchema', ApplyLabelAssignment: 'applyLabelAssignmentSchema',
+}
+const schemas = Object.fromEntries(Object.entries(definitions).map(([name, exported]) => [name, z.toJSONSchema(shared[exported], { target: 'openapi-3.0', io: 'input', unrepresentable: 'any' })]))
+const schemaRef = name => ({ $ref: `#/components/schemas/${name}` })
+const operation = (summary, requestSchema, responseSchema, status = '200', security = 'bearerAuth') => ({
+  summary, tags: ['ops'], security: [{ [security]: [] }],
+  ...(requestSchema ? { requestBody: { required: true, content: { 'application/json': { schema: schemaRef(requestSchema) } } } } : {}),
+  responses: { [status]: { description: summary, ...(responseSchema ? { content: { 'application/json': { schema: schemaRef(responseSchema) } } } : {}) }, '400': { description: 'Invalid request' }, '401': { description: 'Authentication required' }, '403': { description: 'Insufficient permission' }, '409': { description: 'Stale revision, conflicting operation or resource still referenced' } },
+})
+const idParameter = [{ name: 'id', in: 'path', required: true, schema: { type: 'string', minLength: 1, maxLength: 128 } }]
+const pageParameters = [{ name: 'limit', in: 'query', schema: { type: 'integer', minimum: 1, maximum: 50, default: 20 } }, { name: 'offset', in: 'query', schema: { type: 'integer', minimum: 0, maximum: 10000, default: 0 } }]
+const paths = {
+  '/v1/agent/credentials': { get: operation('Agent credential status; no-store; approved current token only', null, 'CredentialStatus', '200', 'agentToken') },
+  '/v1/agent/token-rotation': { post: operation('Atomic token exchange; old-token retry only for exact completed exchange within 24 hours; no-store', 'TokenRotationExchange', 'TokenRotationAck', '200', 'agentToken') },
+  '/api/v1/agents/{id}/token-rotation': { parameters: idParameter, get: operation('Operator credential rotation status; agents read permission; no-store', null, 'AdminCredentialStatus'), post: operation('Request rotation for a supported approved client; agents write permission', null, null, '202') },
+  '/api/v1/policies/snapshots': { get: { ...operation('List immutable snapshot metadata; policies read permission'), parameters: pageParameters }, post: operation('Capture immutable effective policy for source agent; policies write permission', 'CreatePolicySnapshot', null, '201') },
+  '/api/v1/policies/snapshots/{id}': { parameters: idParameter, get: operation('Get immutable snapshot and payload; policies read permission'), delete: operation('Delete unreferenced snapshot; policies write permission') },
+  '/api/v1/policies/snapshots/{id}/diff': { parameters: idParameter, post: operation('Read-only bounded diff for up to 100 agents; policies write permission', 'PolicySnapshotDiffRequest') },
+  '/api/v1/policies/rollouts': { get: { ...operation('List rollout metadata; policies read permission'), parameters: pageParameters }, post: operation('Start explicitly selected canary rollout with generation checks; policies write permission', 'CreatePolicyRollout', null, '201') },
+  '/api/v1/policies/rollouts/{id}': { parameters: idParameter, get: operation('Get rollout and per-target acknowledgement; policies read permission'), delete: operation('Delete finished rollout after all pins released; policies write permission') },
+  '/api/v1/policies/rollouts/{id}/confirm': { parameters: idParameter, post: operation('Confirm rollout at expected version after fresh exact canary acknowledgement; policies write permission', 'PolicyRolloutTransition') },
+  '/api/v1/policies/rollouts/{id}/rollback': { parameters: idParameter, post: operation('Rollback to captured per-agent immutable policies; policies write permission', 'PolicyRolloutTransition') },
+  '/api/v1/agents/{id}/policy-pin/release': { parameters: idParameter, post: operation('Release selected rollout pin with expected generation; agents write permission', 'ReleasePolicyPin') },
+  '/api/v1/agent-labels': { get: { ...operation('List labels; agents read permission'), parameters: pageParameters }, post: operation('Create unique agent label; agents write permission', 'CreateAgentLabel', null, '201') },
+  '/api/v1/agent-labels/{id}': { parameters: idParameter, patch: operation('Update label; agents write permission', 'PatchAgentLabel'), delete: operation('Delete label without deleting assigned agents; agents write permission') },
+  '/api/v1/agents/{id}/labels': { parameters: idParameter, get: operation('Read agent label IDs; agents read permission'), put: operation('Replace agent labels atomically; agents write permission', 'PutAgentLabels') },
+  '/api/v1/agent-labels/preview-assignment': { post: operation('Preview add/replace policy-set assignment from explicit agent IDs or labels; policies read permission', 'PreviewLabelAssignment') },
+  '/api/v1/agent-labels/assign-policy-sets': { post: operation('Apply previously reviewed assignment with exact per-agent generations; policies write permission', 'ApplyLabelAssignment') },
+}
+const output = JSON.stringify({ 'x-generated-by': 'scripts/ci/generate-openapi-schemas.mjs; rebuild @evofw/shared first', components: { schemas }, paths }, null, 2) + '\n'
+const path = resolve('docs/openapi.schemas.json')
+if (process.argv.includes('--check')) {
+  if (readFileSync(path, 'utf8') !== output) throw new Error('OpenAPI schemas differ from shared contracts; rebuild shared and regenerate')
+} else writeFileSync(path, output)

scripts/ci/image-smoke.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env sh
+# Runs only disposable named volumes. IMAGE must already be loaded locally.
+set -eu
+image="${1:?Usage: image-smoke.sh IMAGE [PREVIOUS_IMAGE]}"
+previous="${2:-}"
+if [ -n "$previous" ]; then
+  printf '%s' "$previous" | grep -Eq '@sha256:[0-9a-f]{64}$' || { echo 'PREVIOUS_IMAGE must be pinned by digest' >&2; exit 1; }
+elif [ "${REQUIRE_UPGRADE:-false}" = true ] && [ "${FIRST_RELEASE:-false}" != true ]; then
+  echo 'Release requires a digest-pinned previous image; set FIRST_RELEASE=true only for an explicitly approved first release' >&2
+  exit 1
+fi
+id="evofw-smoke-$(date +%s)-$$"
+volume="$id-data"
+restore_volume="$id-restore"
+container="$id"
+tmp="$(mktemp -d)"
+cleanup() {
+  result=$?
+  docker logs "$container" >>"$tmp/container.log" 2>&1 || true
+  if [ "$result" -ne 0 ]; then cat "$tmp/container.log" >&2; fi
+  docker rm -f "$container" >/dev/null 2>&1 || true
+  docker volume rm "$volume" "$restore_volume" >/dev/null 2>&1 || true
+  if [ -n "${SMOKE_ARTIFACT_DIR:-}" ]; then mkdir -p "$SMOKE_ARTIFACT_DIR"; cp "$tmp/container.log" "$SMOKE_ARTIFACT_DIR/"; fi
+  rm -rf "$tmp"
+}
+trap cleanup EXIT HUP INT TERM
+docker volume create "$volume" >/dev/null
+docker volume create "$restore_volume" >/dev/null
+start() {
+  docker run -d --name "$container" --mount "type=volume,src=$2,dst=/data" \
+    -e EVOFW_ALLOW_UNSAFE=true -e AUTH_REQUIRED=false "$1" >/dev/null
+  attempts=0
+  until docker exec "$container" node -e "fetch('http://127.0.0.1:8080/ready',{signal:AbortSignal.timeout(1000)}).then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"; do
+    attempts=$((attempts + 1))
+    [ "$attempts" -lt 120 ] || { docker logs "$container"; exit 1; }
+    [ "$(docker inspect -f '{{.State.Running}}' "$container")" = true ] || { docker logs "$container"; exit 1; }
+    sleep 1
+  done
+}
+marker() {
+  docker exec "$container" node -e "const D=require('node:module').createRequire('/app/node_modules/@evofw/db/package.json')('better-sqlite3'); const db=new D('/data/app.db'); if(process.argv[1]==='write') db.prepare('INSERT OR REPLACE INTO settings(key,value) VALUES (?,?)').run('ci-smoke','survives'); else if(db.prepare('SELECT value FROM settings WHERE key=?').get('ci-smoke')?.value!=='survives')process.exitCode=1; db.close()" "$1"
+}
+stop() {
+  docker stop -t 30 "$container" >/dev/null
+  [ "$(docker inspect -f '{{.State.ExitCode}}' "$container")" = 0 ]
+  [ "$(docker inspect -f '{{.State.OOMKilled}}' "$container")" = false ]
+  docker logs "$container" >>"$tmp/container.log" 2>&1
+  docker rm "$container" >/dev/null
+}
+if [ -n "$previous" ]; then
+  # Explicit previous version/digest is preferable to a mutable latest tag.
+  docker pull "$previous"
+  start "$previous" "$volume"
+  marker write
+  stop
+  start "$image" "$volume"
+  marker read
+  echo 'upgrade: passed'
+else
+  if [ "${FIRST_RELEASE:-false}" = true ]; then
+    echo 'upgrade: first-release exception explicitly selected (no previous deployment)'
+  else
+    echo 'upgrade: NOT RUN (development smoke only; not release validation)'
+  fi
+  start "$image" "$volume"
+fi
+docker exec -i "$container" node --input-type=module <<'NODE'
+import assert from 'node:assert/strict'
+import { existsSync, readdirSync } from 'node:fs'
+for (const path of ['/health', '/ready', '/api/v1/agents']) {
+  const response = await fetch(`http://127.0.0.1:8080${path}`)
+  assert.equal(response.status, 200, path)
+  await response.json()
+}
+for (const path of ['/', '/agents']) {
+  const response = await fetch(`http://127.0.0.1:8080${path}`)
+  assert.equal(response.status, 200, path)
+  assert.match(await response.text(), /<html/i)
+}
+for (const path of ['/v1/agent/install.sh', '/v1/agent/uninstall.sh', '/v1/agent/sync-script', '/v1/agent/mikrotik-install.rsc']) {
+  const response = await fetch(`http://127.0.0.1:8080${path}`)
+  assert.equal(response.status, 200, path)
+  assert.ok((await response.text()).length > 100, path)
+}
+assert.ok(existsSync('/app/node_modules/@evofw/db/migrations'))
+assert.ok(existsSync('/app/dist/agent-scripts/install.sh'))
+for (const name of ['src', 'test', '.turbo', 'pnpm-lock.yaml']) assert.ok(!existsSync(`/app/${name}`), name)
+assert.ok(!readdirSync('/app/dist').some(name => name.endsWith('.sh') || name.endsWith('.rsc')), 'no duplicate root scripts')
+NODE
+marker write
+sh deploy/docker/backup.sh "$container" "$tmp/backup.db"
+stop
+start "$image" "$volume"
+marker read
+stop
+sh deploy/docker/restore.sh "$image" "$restore_volume" "$tmp/backup.db"
+start "$image" "$restore_volume"
+marker read
+stop
+echo 'image smoke: passed (migration, native SQLite, API, SPA, scripts, SIGTERM, persistence, online backup and restore)'

scripts/ci/linux-agent-fixture.py

@@ -0,0 +1,60 @@
+"""Isolated namespace-only control-plane fixture for the shipped Linux agent."""
+import http.server
+import json
+import pathlib
+import sys
+import socket
+import threading
+
+root = pathlib.Path(sys.argv[1]) if __name__ == "__main__" else pathlib.Path(".")
+policy = {
+    "hash": "sha256:" + "1" * 64, "generation": 1, "apply_version": 2,
+    "default_action": "accept", "deny_cidrs": ["198.18.0.2/32", "2001:db8:1::2/128"],
+    "allow_cidrs": [], "port_rules": [], "sync_interval_sec": 60,
+}
+
+
+def acknowledgement(report):
+    """The shipped durable sender only dequeues an exact ID+sequence ACK."""
+    return {"ok": True, "accepted": True, "report_id": report.get("report_id"), "sequence": report.get("sequence")}
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == "/v1/agent/policy":
+            result = dict(policy)
+            if (root / "invalid").exists():
+                result.update(hash="sha256:" + "2" * 64, deny_cidrs=["invalid-prefix"])
+        else:
+            result = {"ok": True}
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(json.dumps(result).encode())
+
+    def do_POST(self):
+        length = int(self.headers.get("Content-Length", 0))
+        if length > 1024 * 1024:
+            self.send_error(413)
+            return
+        report = json.loads(self.rfile.read(length))
+        with (root / "reports.jsonl").open("a") as handle:
+            handle.write(json.dumps(report) + "\n")
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(json.dumps(acknowledgement(report)).encode())
+
+    def log_message(self, *_):
+        pass
+
+
+class IPv6Server(http.server.ThreadingHTTPServer):
+    address_family = socket.AF_INET6
+
+
+if __name__ == "__main__":
+    server = http.server.ThreadingHTTPServer(("0.0.0.0", 0), Handler)
+    server6 = IPv6Server(("::", 0), Handler)
+    (root / "port6").write_text(str(server6.server_port))
+    threading.Thread(target=server6.serve_forever, daemon=True).start()
+    (root / "port").write_text(str(server.server_port))
+    server.serve_forever()

scripts/ci/linux-agent-fixture.test.py

@@ -0,0 +1,21 @@
+"""Pure contract regression; no sockets or firewall access."""
+import importlib.util
+import pathlib
+import unittest
+
+spec = importlib.util.spec_from_file_location("fixture", pathlib.Path(__file__).with_name("linux-agent-fixture.py"))
+fixture = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(fixture)
+
+
+class AcknowledgementTest(unittest.TestCase):
+    def test_exact_report_retry_has_stable_id_and_sequence_ack(self):
+        report = {"telemetry_version": 2, "report_id": "report-123", "sequence": 17}
+        expected = {"ok": True, "accepted": True, "report_id": "report-123", "sequence": 17}
+        self.assertEqual(fixture.acknowledgement(report), expected)
+        self.assertEqual(fixture.acknowledgement(dict(report)), expected)
+        self.assertNotEqual(fixture.acknowledgement({**report, "sequence": 18}), expected)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/ci/linux-netns-smoke.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# All firewall mutations run only after unshare creates BOTH a network and mount
+# namespace. Never installs/runs the agent against the runner's host firewall.
+set -Eeuo pipefail
+[[ $(uname -s) == Linux ]] || { echo 'Linux namespace runner required' >&2; exit 1; }
+for command in unshare ip nft ipset iptables ip6tables python3 curl mount flock timeout; do command -v "$command" >/dev/null; done
+temporary=$(mktemp -d)
+trap 'rm -rf -- "$temporary"' EXIT
+cp apps/api/src/agent-scripts/evofw-firewall.sh "$temporary/agent.sh"
+cp scripts/ci/linux-agent-fixture.py "$temporary/server.py"
+parent_net=$(readlink /proc/self/ns/net)
+parent_mount=$(readlink /proc/self/ns/mnt)
+for backend in nft ipset; do
+rm -f "$temporary/port" "$temporary/port6" "$temporary/invalid"
+unshare --net --mount --pid --fork --mount-proc env EVOFW_TEST_PARENT_NET="$parent_net" EVOFW_TEST_PARENT_MOUNT="$parent_mount" \
+  bash -s -- "$temporary" "$backend" <<'CHILD'
+set -Eeuo pipefail
+root=$1
+backend=$2
+unset http_proxy HTTP_PROXY https_proxy HTTPS_PROXY ALL_PROXY all_proxy
+export NO_PROXY='*'
+[[ $(readlink /proc/self/ns/net) != "$EVOFW_TEST_PARENT_NET" ]]
+[[ $(readlink /proc/self/ns/mnt) != "$EVOFW_TEST_PARENT_MOUNT" ]]
+mount --make-rprivate /
+# Agent hardcoded config/state/log paths are private mounts, never host writes.
+for directory in /etc /var/lib /var/log /run; do mount -t tmpfs tmpfs "$directory"; done
+mkdir -p /etc/evofw /var/lib/evofw
+ip link set lo up
+ip netns add client
+ip link add server0 type veth peer name client0
+ip link set client0 netns client
+ip addr add 198.18.0.1/24 dev server0
+ip -6 addr add 2001:db8:1::1/64 dev server0 nodad
+ip link set server0 up
+ip -n client addr add 198.18.0.2/24 dev client0
+ip -n client -6 addr add 2001:db8:1::2/64 dev client0 nodad
+ip -n client link set client0 up
+ip -n client link set lo up
+python3 "$root/server.py" "$root" &
+server_pid=$!
+trap 'kill "$server_pid" 2>/dev/null || true; ip netns del client 2>/dev/null || true' EXIT
+for attempt in {1..100}; do [[ ! -f "$root/port" ]] || break; sleep .05; done
+port=$(cat "$root/port")
+port6=$(cat "$root/port6")
+cat > /etc/evofw/agent.conf <<CONF
+EVOFW_CP_URL=http://127.0.0.1:$port
+CLIENT_TOKEN=namespace-test-only
+KERNEL_BACKEND=$backend
+EVOFW_SKIP_SELF_UPDATE=1
+CONF
+nft add table inet unrelated
+nft 'add chain inet unrelated input { type filter hook input priority 10; policy accept; }'
+# A fresh connection through a veth verifies actual packet enforcement.
+ip netns exec client curl --noproxy '*' --max-time 2 -fsS "http://198.18.0.1:$port/echo" >/dev/null
+ip netns exec client curl --noproxy '*' --max-time 2 -gfsS "http://[2001:db8:1::1]:$port6/echo" >/dev/null
+bash "$root/agent.sh"
+python3 -c 'import json; assert json.load(open("/var/lib/evofw/telemetry-state.json"))["pending"] == [], "fixture ACK did not drain durable telemetry"'
+[[ $(cat /var/lib/evofw/last_hash) == "sha256:$(printf '1%.0s' {1..64})" ]]
+nft list table inet unrelated >/dev/null
+if ip netns exec client curl --noproxy '*' --max-time 2 -fsS "http://198.18.0.1:$port/echo" >/dev/null; then echo 'deny did not block packet' >&2; exit 1; fi
+if ip netns exec client curl --noproxy '*' --max-time 2 -gfsS "http://[2001:db8:1::1]:$port6/echo" >/dev/null; then echo 'IPv6 deny did not block packet' >&2; exit 1; fi
+# Same hash must reconcile missing owned state, retaining unrelated rules.
+if [[ "$backend" == nft ]]; then nft delete table inet evofw; else ipset flush evofw_deny_v4; fi
+bash "$root/agent.sh"
+if [[ "$backend" == nft ]]; then nft list table inet evofw >/dev/null; else ipset test evofw_deny_v4 198.18.0.2; fi
+nft list table inet unrelated >/dev/null
+touch "$root/invalid"
+if bash "$root/agent.sh"; then echo 'invalid candidate unexpectedly applied' >&2; exit 1; fi
+[[ $(cat /var/lib/evofw/last_hash) == "sha256:$(printf '1%.0s' {1..64})" ]]
+if [[ "$backend" == nft ]]; then
+  nft list set inet evofw deny_v4 | grep -q '198.18.0.2'
+  nft list set inet evofw deny_v6 | grep -q '2001:db8:1::2'
+else
+  ipset test evofw_deny_v4 198.18.0.2
+  ipset test evofw_deny_v6 2001:db8:1::2
+fi
+nft list table inet unrelated >/dev/null
+echo "Linux namespace smoke passed ($backend): IPv4+IPv6 apply, real packet deny, drift repair, invalid-candidate last-good, unrelated table preservation"
+CHILD
+done

scripts/ci/manifest-platform.mjs

@@ -0,0 +1,12 @@
+import { readFileSync } from 'node:fs'
+const [file, digest, platform] = process.argv.slice(2)
+const [os, architecture, variant] = platform.split('/')
+const manifest = JSON.parse(readFileSync(file, 'utf8'))
+let selected = digest
+if (Array.isArray(manifest.manifests)) {
+  const matches = manifest.manifests.filter(entry => entry.platform?.os === os && entry.platform?.architecture === architecture && (entry.platform?.variant || undefined) === variant)
+  if (matches.length !== 1) throw new Error(`Expected one ${platform} manifest, found ${matches.length}`)
+  selected = matches[0].digest
+}
+if (!/^sha256:[0-9a-f]{64}$/.test(selected)) throw new Error('Invalid platform digest')
+process.stdout.write(selected)

scripts/ci/manifest-platform.test.mjs

@@ -0,0 +1,23 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { spawnSync } from 'node:child_process'
+test('mirror verification selects exactly one platform and refuses absent/ambiguous descriptors', () => {
+  const directory = mkdtempSync(join(tmpdir(), 'evofw-manifest-'))
+  const file = join(directory, 'index.json')
+  const digest = `sha256:${'a'.repeat(64)}`
+  const descriptor = { digest, platform: { os: 'linux', architecture: 'amd64' } }
+  const run = (manifest) => {
+    writeFileSync(file, JSON.stringify(manifest))
+    return spawnSync(process.execPath, ['scripts/ci/manifest-platform.mjs', file, digest, 'linux/amd64'], { encoding: 'utf8' })
+  }
+  try {
+    assert.equal(run({ manifests: [descriptor, { ...descriptor, platform: { os: 'linux', architecture: 'arm64' } }] }).stdout, digest)
+    assert.notEqual(run({ manifests: [] }).status, 0)
+    assert.notEqual(run({ manifests: [descriptor, descriptor] }).status, 0)
+    assert.equal(run({ schemaVersion: 2 }).stdout, digest)
+    assert.notEqual(run({ manifests: [{ ...descriptor, digest: 'invalid' }] }).status, 0)
+  } finally { rmSync(directory, { recursive: true, force: true }) }
+})

scripts/ci/prepare-builder.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+# Each workflow uses a fresh run-scoped builder, created by setup-buildx-action.
+# Never remove another job's builder from a shared Docker daemon.
+set -eu
+name="${1:-evofw}"
+config="${2:-deploy/docker/buildkitd.toml}"
+docker buildx inspect "$name" --bootstrap >/dev/null
+container="$(docker ps -q --filter "name=buildx_buildkit_${name}0")"
+[ -n "$container" ] || { echo 'BuildKit container missing' >&2; exit 1; }
+effective="$(mktemp)"
+trap 'rm -f "$effective"' EXIT HUP INT TERM
+docker exec "$container" cat /etc/buildkit/buildkitd.toml >"$effective"
+for expected in 'reservedSpace = "2GB"' 'maxUsedSpace = "12GB"' 'minFreeSpace = "5GB"' 'keepDuration = "48h"'; do
+  grep -F "$expected" "$effective" >/dev/null || { echo 'BuildKit effective GC configuration mismatch' >&2; exit 1; }
+done

scripts/ci/sqlite-backup.test.mjs

@@ -0,0 +1,50 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { createRequire } from 'node:module'
+import { mkdtempSync, rmSync, writeFileSync, existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { resolve, join } from 'node:path'
+import { spawnSync } from 'node:child_process'
+const root = resolve(import.meta.dirname, '../..')
+const Database = createRequire(join(root, 'packages/db/package.json'))('better-sqlite3')
+const script = join(root, 'deploy/docker/evofw/sqlite-backup.mjs')
+
+test('online backup includes committed WAL and restore verifies before offline replacement', () => {
+  const directory = mkdtempSync(join(tmpdir(), 'evofw-backup-test-'))
+  let db
+  const run = (...args) => spawnSync(process.execPath, [script, ...args], { cwd: directory, encoding: 'utf8', env: { ...process.env, NODE_PATH: join(root, 'packages/db/node_modules') } })
+  try {
+    const source = join(directory, 'source.db')
+    const backup = join(directory, 'backup.db')
+    const restored = join(directory, 'restored.db')
+    db = new Database(source)
+    db.pragma('journal_mode = WAL')
+    db.exec('CREATE TABLE records(id INTEGER PRIMARY KEY, value TEXT); INSERT INTO records VALUES(1,\'committed WAL\')')
+    assert.ok(existsSync(`${source}-wal`))
+    const result = run('backup', source, backup)
+    assert.equal(result.status, 0, result.stderr)
+    const snapshot = new Database(backup, { readonly: true })
+    assert.equal(snapshot.prepare('SELECT value FROM records').get().value, 'committed WAL')
+    snapshot.close()
+    assert.notEqual(run('backup', source, backup).status, 0, 'never overwrite an existing backup')
+    assert.notEqual(run('restore', backup, restored).status, 0, 'offline declaration required')
+    writeFileSync(restored, 'old offline data')
+    writeFileSync(`${restored}-wal`, 'stale')
+    writeFileSync(`${restored}-shm`, 'stale')
+    const recovery = run('restore', backup, restored, '--offline')
+    assert.equal(recovery.status, 0, recovery.stderr)
+    assert.ok(!existsSync(`${restored}-wal`))
+    const recovered = new Database(restored, { readonly: true })
+    assert.equal(recovered.prepare('SELECT value FROM records').get().value, 'committed WAL')
+    recovered.close()
+    const invalid = join(directory, 'invalid.db')
+    writeFileSync(invalid, 'not sqlite')
+    assert.notEqual(run('restore', invalid, restored, '--offline').status, 0)
+    const preserved = new Database(restored, { readonly: true })
+    assert.equal(preserved.prepare('SELECT count(*) AS n FROM records').get().n, 1)
+    preserved.close()
+  } finally {
+    db?.close()
+    rmSync(directory, { recursive: true, force: true })
+  }
+})