fix(alerts): не подтверждать события устаревшими GRE BGP снимками

This commit is contained in:
Denozordec committed 2026-10-12 12:03:11 +07:00
1 parent 7ad66f178b
commit 13618e7efe
14 files changed
+1174 -24

No files matched your search

@@ -5,20 +5,34 @@ import { collectGreSignalsFromRuns } from "./alert-engine/gre-signals.js"
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
const old = { v: SCHEDULER_RUN_SNAPSHOT_VERSION, job: "gre_bgp", sampledAt: "2026-10-12T00:00:00Z", bgpPeers: [{ key: "old-peer", state: "Idle" }], greTunnels: [{ targetLabel: "old-gre", status: "down" }], greWritten: 1, bgpWritten: 1 }
const row = (resultJson: unknown) => ({ resultJson, finishedAt: "2026-10-12T00:01:00Z" })
const clock = () => Date.parse("2026-10-12T00:01:00Z")
for (const broken of [null, "{", { job: "wrong" }, { ...old, bgpPeers: [null] }, { ...old, greTunnels: "bad" }]) {
const snapshots = await readGreBgpRunSnapshots(async (sql, params) => { assert.match(sql, /status IN \('ok', 'error'\)/); assert.deepEqual(params, ["gre_bgp", 8]); return [row(broken), row(old)] })
const snapshots = await readGreBgpRunSnapshots(async (sql, params) => { assert.match(sql, /status IN \('ok', 'error'\)/); assert.deepEqual(params, ["gre_bgp", 8]); return [row(broken), row(old)] }, clock)
assert(snapshots[0].fatalError)
assert.deepEqual(collectBgpSignalsFromRuns(snapshots), [])
assert.deepEqual(collectGreSignalsFromRuns(snapshots), [])
}
const current = { ...old, bgpPeers: [{ key: "current-peer", state: "Established" }], greTunnels: [], errors: ["GRE: source refused"] }
const partial = await readGreBgpRunSnapshots(async () => [row(JSON.stringify(current)), row(old)])
const partial = await readGreBgpRunSnapshots(async () => [row(JSON.stringify(current)), row(old)], clock)
assert.deepEqual(collectBgpSignalsFromRuns(partial), [{ key: "current-peer", state: "Established", prevState: null }])
assert.deepEqual(collectGreSignalsFromRuns(partial), [])
const empty = await readGreBgpRunSnapshots(async () => [row({ ...old, bgpPeers: [], greTunnels: [] }), row(old)])
const empty = await readGreBgpRunSnapshots(async () => [row({ ...old, bgpPeers: [], greTunnels: [] }), row(old)], clock)
assert.deepEqual(collectBgpSignalsFromRuns(empty), []); assert.deepEqual(collectGreSignalsFromRuns(empty), [])
const duplicate = { ...old, greTunnels: [{ targetLabel: "old-gre", status: "up" as const }, { targetLabel: "old-gre", status: "down" as const }] }
assert.deepEqual(collectGreSignalsFromRuns([duplicate]), [])
const history = await readGreBgpRunSnapshots(async () => [row({ ...old, greTunnels: [{ targetLabel: "old-gre", status: "up" }] }), row(duplicate), row(old)])
const history = await readGreBgpRunSnapshots(async () => [row({ ...old, greTunnels: [{ targetLabel: "old-gre", status: "up" }] }), row(duplicate), row(old)], clock)
assert.deepEqual(collectGreSignalsFromRuns(history), [{ targetLabel: "old-gre", status: "up", prevStatus: null }])
for (const latest of [
{ ...row(old), finishedAt: "2026-10-11T23:59:00Z" },
{ ...row(old), finishedAt: "2026-10-12T00:03:00Z" },
row({ ...old, sampledAt: "2026-10-11T23:59:00Z" }),
row({ ...old, sampledAt: "2026-10-12T00:03:00Z" }),
row({ ...old, sampledAt: "invalid" }),
row({ ...old, sampledAt: { toString: "invalid" } }),
]) {
const stale = await readGreBgpRunSnapshots(async () => [latest, row(old)], clock)
assert(stale[0].fatalError); assert.deepEqual(collectBgpSignalsFromRuns(stale), []); assert.deepEqual(collectGreSignalsFromRuns(stale), [])
}
const boundary = await readGreBgpRunSnapshots(async () => [row({ ...old, sampledAt: "2026-10-11T23:59:30Z" }), row(old)], clock)
assert.equal(boundary[0].fatalError, undefined)
console.log("GRE/BGP history: latest completed failures retained, malformed/null barrier, partial BGP preserved, empty/deleted peers never resurrected; no SQL query")
@@ -1,23 +1,28 @@
import { dbAll } from "../db/index.js"
import { SCHEDULER_RUN_SNAPSHOT_VERSION, type GreBgpSnapshotRunSnapshot } from "../types/scheduler-run-snapshot.js"
import { GRE_BGP_INTERVAL_MS, GRE_BGP_MAX_SAMPLE_AGE_MS } from "./gre-bgp-timing.js"
interface RunRow { resultJson: unknown; finishedAt: Date | string }
type ReadRows = (sql: string, params: unknown[]) => Promise<RunRow[]>
/** Completed failures are evidence of missing current data, not permission to reuse an older success. */
export async function readGreBgpRunSnapshots(read: ReadRows = (sql, params) => dbAll<RunRow>(sql, params)): Promise<GreBgpSnapshotRunSnapshot[]> {
export async function readGreBgpRunSnapshots(read: ReadRows = (sql, params) => dbAll<RunRow>(sql, params), clock: () => number = Date.now): Promise<GreBgpSnapshotRunSnapshot[]> {
const rows = await read(`SELECT result_json AS "resultJson", finished_at AS "finishedAt"
FROM scheduler_runs WHERE job_key = ? AND status IN ('ok', 'error')
ORDER BY finished_at DESC LIMIT ?`, ["gre_bgp", 8])
return rows.map(row => {
const now = clock()
const recent = (time: number) => Number.isFinite(time) && time >= now - GRE_BGP_MAX_SAMPLE_AGE_MS && time <= now + GRE_BGP_INTERVAL_MS
return rows.map((row, index) => {
let parsed: unknown = row.resultJson
if (typeof parsed === "string") { try { parsed = JSON.parse(parsed) } catch { parsed = null } }
if (parsed && typeof parsed === "object") {
const s = parsed as GreBgpSnapshotRunSnapshot
const validBgp = s.bgpPeers === undefined || (Array.isArray(s.bgpPeers) && s.bgpPeers.every(p => p && typeof p.key === "string" && typeof p.state === "string"))
const validGre = s.greTunnels === undefined || (Array.isArray(s.greTunnels) && s.greTunnels.every(p => p && typeof p.targetLabel === "string" && ["up", "down", "degraded"].includes(p.status)))
if (s.v === SCHEDULER_RUN_SNAPSHOT_VERSION && s.job === "gre_bgp" && validBgp && validGre) return s
const sampled = typeof s.sampledAt === "string" ? Date.parse(s.sampledAt) : NaN
const fresh = index !== 0 || (recent(new Date(row.finishedAt).getTime()) && recent(sampled))
if (s.v === SCHEDULER_RUN_SNAPSHOT_VERSION && s.job === "gre_bgp" && validBgp && validGre && fresh) return s
}
const date = new Date(row.finishedAt)
return { v: SCHEDULER_RUN_SNAPSHOT_VERSION, job: "gre_bgp", sampledAt: Number.isFinite(date.getTime()) ? date.toISOString() : new Date(0).toISOString(), greWritten: 0, bgpWritten: 0, greTunnels: [], bgpPeers: [], fatalError: "Недоступен результат последнего опроса GRE/BGP" }
return { v: SCHEDULER_RUN_SNAPSHOT_VERSION, job: "gre_bgp", sampledAt: Number.isFinite(date.getTime()) ? date.toISOString() : new Date(0).toISOString(), greWritten: 0, bgpWritten: 0, greTunnels: [], bgpPeers: [], fatalError: "Последний опрос GRE/BGP недоступен или устарел" }
})
}
@@ -38,7 +38,13 @@ else {
await insert("ok", { ...old, greTunnels: [{ targetLabel: "duplicate", status: "up" }, { targetLabel: "duplicate", status: "down" }], bgpPeers: [{ key: "duplicate", state: "Established" }, { key: "duplicate", state: "Idle" }] })
signals = await buildSignalSnapshotFromCollectors()
assert.deepEqual(signals.bgpSessions, [{ key: "duplicate", state: "Unknown", prevState: null }]); assert.deepEqual(signals.greTunnels, [])
console.log("GRE/BGP owned SQL: 6 real rows; newest error/partial, NULL/malformed barrier, empty/deleted and duplicate guards passed")
for (const sampledAt of [new Date(Date.now() - 120_000).toISOString(), new Date(Date.now() + 120_000).toISOString()]) {
await insert("ok", { ...old, sampledAt })
assert((await readGreBgpRunSnapshots())[0].fatalError)
signals = await buildSignalSnapshotFromCollectors()
assert.deepEqual(signals.bgpSessions, []); assert.deepEqual(signals.greTunnels, [])
}
console.log("GRE/BGP owned SQL: 8 real rows; newest error/partial, NULL/malformed, empty/duplicate and stale/future guards passed")
} finally {
await pool.query("DELETE FROM scheduler_runs WHERE id LIKE $1", [prefix + "%"])
assert.equal((await pool.query("SELECT count(*)::int AS n FROM scheduler_runs WHERE id LIKE $1", [prefix + "%"])).rows[0].n, 0)
+2
View File
@@ -0,0 +1,2 @@
export const GRE_BGP_INTERVAL_MS = 30_000
export const GRE_BGP_MAX_SAMPLE_AGE_MS = 3 * GRE_BGP_INTERVAL_MS
+3 -2
View File
@@ -32,6 +32,7 @@ import {
} from "./uptime-collector.js"
import { runScheduledSpeedProbesOnce } from "./uptime-speed-service.js"
import { collectGreBgpSnapshotOnce } from "./gre-bgp-snapshot-collector.js"
import { GRE_BGP_INTERVAL_MS } from "./gre-bgp-timing.js"
import { runAlertEngineOnce } from "./alert-engine/run-once.js"
import {
clearAlertCollectorHooksTimer,
@@ -310,7 +311,7 @@ export async function refreshScheduler(): Promise<void> {
)
}
const greBgpMs = 30_000
const greBgpMs = GRE_BGP_INTERVAL_MS
void executeSchedulerJob("gre_bgp").catch(() => {})
timers.set(
"gre_bgp",
@@ -416,7 +417,7 @@ export async function getSchedulerStatus() {
uptime_ping: { enabled: pingOn, intervalSec: uptime.probeIntervalSec },
uptime_speed: { enabled: spdOn, intervalSec: uptime.speedIntervalSec },
internet_path: { enabled: internetPath.enabled, intervalSec: internetPath.intervalSec },
gre_bgp: { enabled: true, intervalSec: 30 },
gre_bgp: { enabled: true, intervalSec: GRE_BGP_INTERVAL_MS / 1000 },
certificates_renew: { enabled: certRenew.enabled, intervalSec: certRenew.intervalSec },
backups: { enabled: backupSchedule.enabled, intervalSec: 60 },
geoip_update: { enabled: geoip.enabled, intervalSec: geoip.updateIntervalSec },