fix(api): bound analytics backfill and reduce SQLite write amplification
quality / changes (push) Successful in 7s
quality / openapi (push) Skipped
quality / web (push) Skipped
quality / commitlint (push) Skipped
quality / api (push) Successful in 1m12s
quality / linux-agent-smoke (push) Successful in 22s
quality / docker-check (push) Successful in 5m25s
CD / quality (push) Successful in 7m8s
CD / publish (push) Successful in 2m49s
quality / changes (push) Successful in 7s
quality / openapi (push) Skipped
quality / web (push) Skipped
quality / commitlint (push) Skipped
quality / api (push) Successful in 1m12s
quality / linux-agent-smoke (push) Successful in 22s
quality / docker-check (push) Successful in 5m25s
CD / quality (push) Successful in 7m8s
CD / publish (push) Successful in 2m49s
This commit is contained in:
1 parent
ce90fa9701
commit
7699e8b809
6 files changed
+213
-34
No files matched your search
@@ -6,6 +6,18 @@ export type StatsRollupInsert = typeof agentStatsRollups.$inferInsert
|
||||
|
||||
/** Called inside the ingest transaction; replay checks precede this additive write. */
|
||||
export function addStatsRollups(db: Db, rows: StatsRollupInsert[]) {
|
||||
// Backfill observations often share hour/day buckets. Merge them before
|
||||
// UPSERT rather than rewriting the same SQLite pages per observation.
|
||||
const merged = new Map<string, StatsRollupInsert>()
|
||||
for (const input of rows) {
|
||||
const key = `${input.agentId}\0${input.resolution}\0${input.bucketStart}`
|
||||
const row = merged.get(key)
|
||||
if (!row) { merged.set(key, { ...input }); continue }
|
||||
for (const field of ['packetsDropped', 'packetsAccepted', 'droppedCoverageMs', 'acceptedCoverageMs', 'lostReports'] as const) row[field] = (row[field] ?? 0) + (input[field] ?? 0)
|
||||
for (const field of ['partial', 'truncated', 'estimated'] as const) row[field] = Math.max(row[field] ?? 0, input[field] ?? 0)
|
||||
row.unsupportedJson = JSON.stringify([...new Set([...(JSON.parse(row.unsupportedJson ?? '[]') as string[]), ...(JSON.parse(input.unsupportedJson ?? '[]') as string[])])].slice(0, 16))
|
||||
}
|
||||
rows = [...merged.values()]
|
||||
for (let offset = 0; offset < rows.length; offset += 100) {
|
||||
db.insert(agentStatsRollups).values(rows.slice(offset, offset + 100)).onConflictDoUpdate({
|
||||
target: [agentStatsRollups.agentId, agentStatsRollups.resolution, agentStatsRollups.bucketStart],
|
||||
|
||||
Reference in new issue
Block a user