fix(alerts): не подтверждать события устаревшими GRE BGP снимками
This commit is contained in:
1 parent
7ad66f178b
commit
13618e7efe
14 files changed
+1174
-24
No files matched your search
@@ -5,20 +5,34 @@ import { collectGreSignalsFromRuns } from "./alert-engine/gre-signals.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
const old = { v: SCHEDULER_RUN_SNAPSHOT_VERSION, job: "gre_bgp", sampledAt: "2026-10-12T00:00:00Z", bgpPeers: [{ key: "old-peer", state: "Idle" }], greTunnels: [{ targetLabel: "old-gre", status: "down" }], greWritten: 1, bgpWritten: 1 }
|
||||
const row = (resultJson: unknown) => ({ resultJson, finishedAt: "2026-10-12T00:01:00Z" })
|
||||
const clock = () => Date.parse("2026-10-12T00:01:00Z")
|
||||
for (const broken of [null, "{", { job: "wrong" }, { ...old, bgpPeers: [null] }, { ...old, greTunnels: "bad" }]) {
|
||||
const snapshots = await readGreBgpRunSnapshots(async (sql, params) => { assert.match(sql, /status IN \('ok', 'error'\)/); assert.deepEqual(params, ["gre_bgp", 8]); return [row(broken), row(old)] })
|
||||
const snapshots = await readGreBgpRunSnapshots(async (sql, params) => { assert.match(sql, /status IN \('ok', 'error'\)/); assert.deepEqual(params, ["gre_bgp", 8]); return [row(broken), row(old)] }, clock)
|
||||
assert(snapshots[0].fatalError)
|
||||
assert.deepEqual(collectBgpSignalsFromRuns(snapshots), [])
|
||||
assert.deepEqual(collectGreSignalsFromRuns(snapshots), [])
|
||||
}
|
||||
const current = { ...old, bgpPeers: [{ key: "current-peer", state: "Established" }], greTunnels: [], errors: ["GRE: source refused"] }
|
||||
const partial = await readGreBgpRunSnapshots(async () => [row(JSON.stringify(current)), row(old)])
|
||||
const partial = await readGreBgpRunSnapshots(async () => [row(JSON.stringify(current)), row(old)], clock)
|
||||
assert.deepEqual(collectBgpSignalsFromRuns(partial), [{ key: "current-peer", state: "Established", prevState: null }])
|
||||
assert.deepEqual(collectGreSignalsFromRuns(partial), [])
|
||||
const empty = await readGreBgpRunSnapshots(async () => [row({ ...old, bgpPeers: [], greTunnels: [] }), row(old)])
|
||||
const empty = await readGreBgpRunSnapshots(async () => [row({ ...old, bgpPeers: [], greTunnels: [] }), row(old)], clock)
|
||||
assert.deepEqual(collectBgpSignalsFromRuns(empty), []); assert.deepEqual(collectGreSignalsFromRuns(empty), [])
|
||||
const duplicate = { ...old, greTunnels: [{ targetLabel: "old-gre", status: "up" as const }, { targetLabel: "old-gre", status: "down" as const }] }
|
||||
assert.deepEqual(collectGreSignalsFromRuns([duplicate]), [])
|
||||
const history = await readGreBgpRunSnapshots(async () => [row({ ...old, greTunnels: [{ targetLabel: "old-gre", status: "up" }] }), row(duplicate), row(old)])
|
||||
const history = await readGreBgpRunSnapshots(async () => [row({ ...old, greTunnels: [{ targetLabel: "old-gre", status: "up" }] }), row(duplicate), row(old)], clock)
|
||||
assert.deepEqual(collectGreSignalsFromRuns(history), [{ targetLabel: "old-gre", status: "up", prevStatus: null }])
|
||||
for (const latest of [
|
||||
{ ...row(old), finishedAt: "2026-10-11T23:59:00Z" },
|
||||
{ ...row(old), finishedAt: "2026-10-12T00:03:00Z" },
|
||||
row({ ...old, sampledAt: "2026-10-11T23:59:00Z" }),
|
||||
row({ ...old, sampledAt: "2026-10-12T00:03:00Z" }),
|
||||
row({ ...old, sampledAt: "invalid" }),
|
||||
row({ ...old, sampledAt: { toString: "invalid" } }),
|
||||
]) {
|
||||
const stale = await readGreBgpRunSnapshots(async () => [latest, row(old)], clock)
|
||||
assert(stale[0].fatalError); assert.deepEqual(collectBgpSignalsFromRuns(stale), []); assert.deepEqual(collectGreSignalsFromRuns(stale), [])
|
||||
}
|
||||
const boundary = await readGreBgpRunSnapshots(async () => [row({ ...old, sampledAt: "2026-10-11T23:59:30Z" }), row(old)], clock)
|
||||
assert.equal(boundary[0].fatalError, undefined)
|
||||
console.log("GRE/BGP history: latest completed failures retained, malformed/null barrier, partial BGP preserved, empty/deleted peers never resurrected; no SQL query")
|
||||
@@ -1,23 +1,28 @@
|
||||
import { dbAll } from "../db/index.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION, type GreBgpSnapshotRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { GRE_BGP_INTERVAL_MS, GRE_BGP_MAX_SAMPLE_AGE_MS } from "./gre-bgp-timing.js"
|
||||
interface RunRow { resultJson: unknown; finishedAt: Date | string }
|
||||
type ReadRows = (sql: string, params: unknown[]) => Promise<RunRow[]>
|
||||
|
||||
/** Completed failures are evidence of missing current data, not permission to reuse an older success. */
|
||||
export async function readGreBgpRunSnapshots(read: ReadRows = (sql, params) => dbAll<RunRow>(sql, params)): Promise<GreBgpSnapshotRunSnapshot[]> {
|
||||
export async function readGreBgpRunSnapshots(read: ReadRows = (sql, params) => dbAll<RunRow>(sql, params), clock: () => number = Date.now): Promise<GreBgpSnapshotRunSnapshot[]> {
|
||||
const rows = await read(`SELECT result_json AS "resultJson", finished_at AS "finishedAt"
|
||||
FROM scheduler_runs WHERE job_key = ? AND status IN ('ok', 'error')
|
||||
ORDER BY finished_at DESC LIMIT ?`, ["gre_bgp", 8])
|
||||
return rows.map(row => {
|
||||
const now = clock()
|
||||
const recent = (time: number) => Number.isFinite(time) && time >= now - GRE_BGP_MAX_SAMPLE_AGE_MS && time <= now + GRE_BGP_INTERVAL_MS
|
||||
return rows.map((row, index) => {
|
||||
let parsed: unknown = row.resultJson
|
||||
if (typeof parsed === "string") { try { parsed = JSON.parse(parsed) } catch { parsed = null } }
|
||||
if (parsed && typeof parsed === "object") {
|
||||
const s = parsed as GreBgpSnapshotRunSnapshot
|
||||
const validBgp = s.bgpPeers === undefined || (Array.isArray(s.bgpPeers) && s.bgpPeers.every(p => p && typeof p.key === "string" && typeof p.state === "string"))
|
||||
const validGre = s.greTunnels === undefined || (Array.isArray(s.greTunnels) && s.greTunnels.every(p => p && typeof p.targetLabel === "string" && ["up", "down", "degraded"].includes(p.status)))
|
||||
if (s.v === SCHEDULER_RUN_SNAPSHOT_VERSION && s.job === "gre_bgp" && validBgp && validGre) return s
|
||||
const sampled = typeof s.sampledAt === "string" ? Date.parse(s.sampledAt) : NaN
|
||||
const fresh = index !== 0 || (recent(new Date(row.finishedAt).getTime()) && recent(sampled))
|
||||
if (s.v === SCHEDULER_RUN_SNAPSHOT_VERSION && s.job === "gre_bgp" && validBgp && validGre && fresh) return s
|
||||
}
|
||||
const date = new Date(row.finishedAt)
|
||||
return { v: SCHEDULER_RUN_SNAPSHOT_VERSION, job: "gre_bgp", sampledAt: Number.isFinite(date.getTime()) ? date.toISOString() : new Date(0).toISOString(), greWritten: 0, bgpWritten: 0, greTunnels: [], bgpPeers: [], fatalError: "Недоступен результат последнего опроса GRE/BGP" }
|
||||
return { v: SCHEDULER_RUN_SNAPSHOT_VERSION, job: "gre_bgp", sampledAt: Number.isFinite(date.getTime()) ? date.toISOString() : new Date(0).toISOString(), greWritten: 0, bgpWritten: 0, greTunnels: [], bgpPeers: [], fatalError: "Последний опрос GRE/BGP недоступен или устарел" }
|
||||
})
|
||||
}
|
||||
@@ -38,7 +38,13 @@ else {
|
||||
await insert("ok", { ...old, greTunnels: [{ targetLabel: "duplicate", status: "up" }, { targetLabel: "duplicate", status: "down" }], bgpPeers: [{ key: "duplicate", state: "Established" }, { key: "duplicate", state: "Idle" }] })
|
||||
signals = await buildSignalSnapshotFromCollectors()
|
||||
assert.deepEqual(signals.bgpSessions, [{ key: "duplicate", state: "Unknown", prevState: null }]); assert.deepEqual(signals.greTunnels, [])
|
||||
console.log("GRE/BGP owned SQL: 6 real rows; newest error/partial, NULL/malformed barrier, empty/deleted and duplicate guards passed")
|
||||
for (const sampledAt of [new Date(Date.now() - 120_000).toISOString(), new Date(Date.now() + 120_000).toISOString()]) {
|
||||
await insert("ok", { ...old, sampledAt })
|
||||
assert((await readGreBgpRunSnapshots())[0].fatalError)
|
||||
signals = await buildSignalSnapshotFromCollectors()
|
||||
assert.deepEqual(signals.bgpSessions, []); assert.deepEqual(signals.greTunnels, [])
|
||||
}
|
||||
console.log("GRE/BGP owned SQL: 8 real rows; newest error/partial, NULL/malformed, empty/duplicate and stale/future guards passed")
|
||||
} finally {
|
||||
await pool.query("DELETE FROM scheduler_runs WHERE id LIKE $1", [prefix + "%"])
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM scheduler_runs WHERE id LIKE $1", [prefix + "%"])).rows[0].n, 0)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
export const GRE_BGP_INTERVAL_MS = 30_000
|
||||
export const GRE_BGP_MAX_SAMPLE_AGE_MS = 3 * GRE_BGP_INTERVAL_MS
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
} from "./uptime-collector.js"
|
||||
import { runScheduledSpeedProbesOnce } from "./uptime-speed-service.js"
|
||||
import { collectGreBgpSnapshotOnce } from "./gre-bgp-snapshot-collector.js"
|
||||
import { GRE_BGP_INTERVAL_MS } from "./gre-bgp-timing.js"
|
||||
import { runAlertEngineOnce } from "./alert-engine/run-once.js"
|
||||
import {
|
||||
clearAlertCollectorHooksTimer,
|
||||
@@ -310,7 +311,7 @@ export async function refreshScheduler(): Promise<void> {
|
||||
)
|
||||
}
|
||||
|
||||
const greBgpMs = 30_000
|
||||
const greBgpMs = GRE_BGP_INTERVAL_MS
|
||||
void executeSchedulerJob("gre_bgp").catch(() => {})
|
||||
timers.set(
|
||||
"gre_bgp",
|
||||
@@ -416,7 +417,7 @@ export async function getSchedulerStatus() {
|
||||
uptime_ping: { enabled: pingOn, intervalSec: uptime.probeIntervalSec },
|
||||
uptime_speed: { enabled: spdOn, intervalSec: uptime.speedIntervalSec },
|
||||
internet_path: { enabled: internetPath.enabled, intervalSec: internetPath.intervalSec },
|
||||
gre_bgp: { enabled: true, intervalSec: 30 },
|
||||
gre_bgp: { enabled: true, intervalSec: GRE_BGP_INTERVAL_MS / 1000 },
|
||||
certificates_renew: { enabled: certRenew.enabled, intervalSec: certRenew.intervalSec },
|
||||
backups: { enabled: backupSchedule.enabled, intervalSec: 60 },
|
||||
geoip_update: { enabled: geoip.enabled, intervalSec: geoip.updateIntervalSec },
|
||||
|
||||
Reference in new issue
Block a user