fix(api): bound analytics backfill and reduce SQLite write amplification
quality / changes (push) Successful in 7s
quality / openapi (push) Skipped
quality / web (push) Skipped
quality / commitlint (push) Skipped
quality / api (push) Successful in 1m12s
quality / linux-agent-smoke (push) Successful in 22s
quality / docker-check (push) Successful in 5m25s
CD / quality (push) Successful in 7m8s
CD / publish (push) Successful in 2m49s

This commit is contained in:
Denozordec committed 2026-10-26 15:04:01 +07:00
1 parent ce90fa9701
commit 7699e8b809
6 files changed
+213 -34

No files matched your search

+12
View File
@@ -6,6 +6,18 @@ export type StatsRollupInsert = typeof agentStatsRollups.$inferInsert
/** Called inside the ingest transaction; replay checks precede this additive write. */
export function addStatsRollups(db: Db, rows: StatsRollupInsert[]) {
// Backfill observations often share hour/day buckets. Merge them before
// UPSERT rather than rewriting the same SQLite pages per observation.
const merged = new Map<string, StatsRollupInsert>()
for (const input of rows) {
const key = `${input.agentId}\0${input.resolution}\0${input.bucketStart}`
const row = merged.get(key)
if (!row) { merged.set(key, { ...input }); continue }
for (const field of ['packetsDropped', 'packetsAccepted', 'droppedCoverageMs', 'acceptedCoverageMs', 'lostReports'] as const) row[field] = (row[field] ?? 0) + (input[field] ?? 0)
for (const field of ['partial', 'truncated', 'estimated'] as const) row[field] = Math.max(row[field] ?? 0, input[field] ?? 0)
row.unsupportedJson = JSON.stringify([...new Set([...(JSON.parse(row.unsupportedJson ?? '[]') as string[]), ...(JSON.parse(input.unsupportedJson ?? '[]') as string[])])].slice(0, 16))
}
rows = [...merged.values()]
for (let offset = 0; offset < rows.length; offset += 100) {
db.insert(agentStatsRollups).values(rows.slice(offset, offset + 100)).onConflictDoUpdate({
target: [agentStatsRollups.agentId, agentStatsRollups.resolution, agentStatsRollups.bucketStart],