chore: synchronize pending app/backend updates and repository hygiene
Includes current frontend and backend work in progress and removes generated artifacts from tracking to keep the repository clean for дальнейшая разработка. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"build": "tsc",
|
||||
"test:alert-engine": "tsx src/services/alert-engine/groups.test.ts && tsx src/services/alert-engine/evaluate-rule.test.ts && tsx src/services/alert-engine/confirm-stability.test.ts",
|
||||
"start": "node dist/index.js",
|
||||
"db:generate": "drizzle-kit generate",
|
||||
"db:migrate": "drizzle-kit migrate",
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
/**
|
||||
* Порог среднего RTT (мс) для ICMP-проб: выше — статус `warn` при потерях ≤1%.
|
||||
* 60 мс было слишком жёстко для WAN/CDN (например ~65–80 мс до 1.1.1.1 из РФ — нормальная география).
|
||||
* Фронт: держите то же значение в `lib/ping-probe.ts`.
|
||||
*/
|
||||
export const PING_PROBE_WARN_RTT_MS = 120
|
||||
@@ -1,4 +1,6 @@
|
||||
import Database from "better-sqlite3"
|
||||
|
||||
type SqliteHandle = InstanceType<typeof Database>
|
||||
import { drizzle } from "drizzle-orm/better-sqlite3"
|
||||
import { env } from "../config.js"
|
||||
import * as schema from "./schema.js"
|
||||
@@ -190,6 +192,60 @@ CREATE TABLE IF NOT EXISTS uptime_speed_test_runs (
|
||||
CREATE INDEX IF NOT EXISTS idx_uptime_speed_test_runs_created_at
|
||||
ON uptime_speed_test_runs(created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scheduler_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_key TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
error TEXT,
|
||||
duration_ms INTEGER NOT NULL DEFAULT 0,
|
||||
result_json TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduler_runs_job_time
|
||||
ON scheduler_runs(job_key, finished_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS servers_api_ping_settings (
|
||||
id INTEGER PRIMARY KEY,
|
||||
enabled INTEGER NOT NULL DEFAULT 0,
|
||||
interval_sec INTEGER NOT NULL DEFAULT 120,
|
||||
last_collected_at TEXT,
|
||||
last_duration_ms INTEGER,
|
||||
last_error TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS servers_rest_ping_samples (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
server_id INTEGER NOT NULL,
|
||||
sampled_at TEXT NOT NULL,
|
||||
ok INTEGER NOT NULL DEFAULT 0,
|
||||
latency_ms INTEGER,
|
||||
error TEXT,
|
||||
FOREIGN KEY (server_id) REFERENCES servers(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_servers_rest_ping_samples_server_id
|
||||
ON servers_rest_ping_samples(server_id, id DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_gre_tunnel_samples (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sampled_at TEXT NOT NULL,
|
||||
target_label TEXT NOT NULL,
|
||||
status TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_alert_gre_tunnel_samples_label_id
|
||||
ON alert_gre_tunnel_samples(target_label, id DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_bgp_peer_samples (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sampled_at TEXT NOT NULL,
|
||||
peer_key TEXT NOT NULL,
|
||||
state TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_alert_bgp_peer_samples_key_id
|
||||
ON alert_bgp_peer_samples(peer_key, id DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS evobgp_settings (
|
||||
id INTEGER PRIMARY KEY,
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
@@ -197,6 +253,120 @@ CREATE TABLE IF NOT EXISTS evobgp_settings (
|
||||
enabled INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_telegram_settings (
|
||||
id INTEGER PRIMARY KEY,
|
||||
bot_token TEXT NOT NULL DEFAULT '',
|
||||
chat_id TEXT NOT NULL DEFAULT '',
|
||||
message_thread_id INTEGER,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_rules (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
target TEXT NOT NULL,
|
||||
condition TEXT NOT NULL,
|
||||
severity TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
cooldown TEXT NOT NULL DEFAULT '5м',
|
||||
rule_chat_id TEXT NOT NULL DEFAULT '',
|
||||
recovery_mode TEXT NOT NULL DEFAULT 'always',
|
||||
recovery_stability_sec INTEGER,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_history (
|
||||
id TEXT PRIMARY KEY,
|
||||
rule_id TEXT,
|
||||
rule_name TEXT NOT NULL,
|
||||
severity TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
sent_ok INTEGER NOT NULL DEFAULT 1,
|
||||
fired_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_alert_history_fired_at ON alert_history(fired_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_alert_history_rule_id ON alert_history(rule_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_groups (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
combine_mode TEXT NOT NULL DEFAULT 'any',
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
cooldown_override TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_rule_targets (
|
||||
id TEXT PRIMARY KEY,
|
||||
rule_id TEXT NOT NULL,
|
||||
target TEXT NOT NULL,
|
||||
sort_index INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_alert_rule_targets_rule ON alert_rule_targets(rule_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_rule_conditions (
|
||||
id TEXT PRIMARY KEY,
|
||||
rule_id TEXT NOT NULL,
|
||||
condition_line TEXT NOT NULL,
|
||||
sort_index INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_alert_rule_conditions_rule ON alert_rule_conditions(rule_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_engine_state (
|
||||
scope_key TEXT PRIMARY KEY,
|
||||
last_fired_at TEXT NOT NULL DEFAULT '',
|
||||
last_payload_hash TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_engine_prev_live (
|
||||
kind TEXT PRIMARY KEY,
|
||||
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_engine_confirm_pending (
|
||||
rule_id TEXT PRIMARY KEY,
|
||||
payload_hash TEXT NOT NULL,
|
||||
since_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_destinations (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL DEFAULT 'telegram',
|
||||
label TEXT NOT NULL DEFAULT '',
|
||||
telegram_chat_id TEXT NOT NULL DEFAULT '',
|
||||
message_thread_id INTEGER,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_outbox (
|
||||
id TEXT PRIMARY KEY,
|
||||
dedupe_key TEXT NOT NULL,
|
||||
channel TEXT NOT NULL DEFAULT 'telegram',
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
max_retries INTEGER NOT NULL DEFAULT 3,
|
||||
next_attempt_at TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
sent_at TEXT,
|
||||
last_error TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_alert_outbox_status_next_attempt
|
||||
ON alert_outbox(status, next_attempt_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_alert_outbox_dedupe
|
||||
ON alert_outbox(dedupe_key);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_engine_cursor (
|
||||
id INTEGER PRIMARY KEY,
|
||||
last_source_finished_at TEXT,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
`)
|
||||
|
||||
// Lightweight schema evolution for existing databases without migrations
|
||||
@@ -215,6 +385,10 @@ const hasShowOnDashboardColumn = uptimeProbeCols.some((c) => c.name === "show_on
|
||||
if (!hasShowOnDashboardColumn) {
|
||||
sqlite.exec(`ALTER TABLE uptime_probes ADD COLUMN show_on_dashboard INTEGER NOT NULL DEFAULT 0`)
|
||||
}
|
||||
const hasProbeIntervalSecColumn = uptimeProbeCols.some((c) => c.name === "interval_sec")
|
||||
if (!hasProbeIntervalSecColumn) {
|
||||
sqlite.exec(`ALTER TABLE uptime_probes ADD COLUMN interval_sec INTEGER NOT NULL DEFAULT 0`)
|
||||
}
|
||||
|
||||
const uptimeSettingsCols = sqlite.prepare(`PRAGMA table_info('uptime_settings')`).all() as Array<{ name?: string }>
|
||||
const hasProbeIntervalColumn = uptimeSettingsCols.some((c) => c.name === "probe_interval_sec")
|
||||
@@ -226,6 +400,16 @@ if (!hasSpeedIntervalColumn) {
|
||||
sqlite.exec(`ALTER TABLE uptime_settings ADD COLUMN speed_interval_sec INTEGER NOT NULL DEFAULT 60`)
|
||||
}
|
||||
|
||||
const hasUptimeJobFlags = uptimeSettingsCols.some((c) => c.name === "resources_enabled")
|
||||
if (!hasUptimeJobFlags) {
|
||||
sqlite.exec(`ALTER TABLE uptime_settings ADD COLUMN resources_enabled INTEGER NOT NULL DEFAULT 1`)
|
||||
sqlite.exec(`ALTER TABLE uptime_settings ADD COLUMN ping_enabled INTEGER NOT NULL DEFAULT 1`)
|
||||
sqlite.exec(`ALTER TABLE uptime_settings ADD COLUMN speed_enabled INTEGER NOT NULL DEFAULT 1`)
|
||||
sqlite.exec(
|
||||
`UPDATE uptime_settings SET resources_enabled = enabled, ping_enabled = enabled, speed_enabled = enabled WHERE id = 1`,
|
||||
)
|
||||
}
|
||||
|
||||
const uptimeSpeedProbeCols = sqlite.prepare(`PRAGMA table_info('uptime_speed_probes')`).all() as Array<{ name?: string }>
|
||||
const ensureSpeedProbeCol = (name: string, ddl: string) => {
|
||||
if (!uptimeSpeedProbeCols.some((c) => c.name === name)) sqlite.exec(ddl)
|
||||
@@ -240,6 +424,11 @@ ensureSpeedProbeCol("last_ping_loss_pct", `ALTER TABLE uptime_speed_probes ADD C
|
||||
ensureSpeedProbeCol("last_ping_at", `ALTER TABLE uptime_speed_probes ADD COLUMN last_ping_at TEXT`)
|
||||
ensureSpeedProbeCol("last_ping_error", `ALTER TABLE uptime_speed_probes ADD COLUMN last_ping_error TEXT`)
|
||||
|
||||
const schedulerRunCols = sqlite.prepare(`PRAGMA table_info('scheduler_runs')`).all() as Array<{ name?: string }>
|
||||
if (!schedulerRunCols.some((c) => c.name === "result_json")) {
|
||||
sqlite.exec(`ALTER TABLE scheduler_runs ADD COLUMN result_json TEXT`)
|
||||
}
|
||||
|
||||
const serverCols = sqlite.prepare(`PRAGMA table_info('servers')`).all() as Array<{ name?: string }>
|
||||
if (!serverCols.some((c) => c.name === "lan_subnet")) {
|
||||
sqlite.exec(`ALTER TABLE servers ADD COLUMN lan_subnet TEXT NOT NULL DEFAULT ''`)
|
||||
@@ -248,6 +437,44 @@ if (!serverCols.some((c) => c.name === "wan_uplinks")) {
|
||||
sqlite.exec(`ALTER TABLE servers ADD COLUMN wan_uplinks TEXT NOT NULL DEFAULT '[]'`)
|
||||
}
|
||||
|
||||
const alertTgCols = sqlite.prepare(`PRAGMA table_info('alert_telegram_settings')`).all() as Array<{ name?: string }>
|
||||
if (!alertTgCols.some((c) => c.name === "message_thread_id")) {
|
||||
sqlite.exec(`ALTER TABLE alert_telegram_settings ADD COLUMN message_thread_id INTEGER`)
|
||||
}
|
||||
|
||||
const alertRulesCols = sqlite.prepare(`PRAGMA table_info('alert_rules')`).all() as Array<{ name?: string }>
|
||||
if (!alertRulesCols.some((c) => c.name === "group_id")) {
|
||||
sqlite.exec(`ALTER TABLE alert_rules ADD COLUMN group_id TEXT`)
|
||||
}
|
||||
if (!alertRulesCols.some((c) => c.name === "confirm_stability_sec")) {
|
||||
sqlite.exec(`ALTER TABLE alert_rules ADD COLUMN confirm_stability_sec INTEGER`)
|
||||
}
|
||||
if (!alertRulesCols.some((c) => c.name === "recovery_mode")) {
|
||||
sqlite.exec(`ALTER TABLE alert_rules ADD COLUMN recovery_mode TEXT NOT NULL DEFAULT 'always'`)
|
||||
}
|
||||
if (!alertRulesCols.some((c) => c.name === "recovery_stability_sec")) {
|
||||
sqlite.exec(`ALTER TABLE alert_rules ADD COLUMN recovery_stability_sec INTEGER`)
|
||||
}
|
||||
|
||||
const alertHistoryCols = sqlite.prepare(`PRAGMA table_info('alert_history')`).all() as Array<{ name?: string }>
|
||||
if (!alertHistoryCols.some((c) => c.name === "group_id")) {
|
||||
sqlite.exec(`ALTER TABLE alert_history ADD COLUMN group_id TEXT`)
|
||||
}
|
||||
|
||||
/** Одна строка target на правило из legacy-колонки `alert_rules.target` */
|
||||
sqlite.exec(`
|
||||
INSERT OR IGNORE INTO alert_rule_targets (id, rule_id, target, sort_index)
|
||||
SELECT 'rt-' || id || '-0', id, target, 0 FROM alert_rules
|
||||
WHERE id NOT IN (SELECT rule_id FROM alert_rule_targets)
|
||||
`)
|
||||
|
||||
/** Одна строка условия из legacy `alert_rules.condition` */
|
||||
sqlite.exec(`
|
||||
INSERT OR IGNORE INTO alert_rule_conditions (id, rule_id, condition_line, sort_index)
|
||||
SELECT 'rc-' || id || '-0', id, condition, 0 FROM alert_rules
|
||||
WHERE id NOT IN (SELECT rule_id FROM alert_rule_conditions)
|
||||
`)
|
||||
|
||||
sqlite.exec(`
|
||||
INSERT INTO traffic_settings (id, enabled, interval_sec, retention_days)
|
||||
SELECT 1, 1, 30, 14
|
||||
@@ -266,4 +493,25 @@ SELECT 1, '', '', 0
|
||||
WHERE NOT EXISTS (SELECT 1 FROM evobgp_settings WHERE id = 1);
|
||||
`)
|
||||
|
||||
sqlite.exec(`
|
||||
INSERT INTO servers_api_ping_settings (id, enabled, interval_sec)
|
||||
SELECT 1, 0, 120
|
||||
WHERE NOT EXISTS (SELECT 1 FROM servers_api_ping_settings WHERE id = 1);
|
||||
`)
|
||||
|
||||
sqlite.exec(`
|
||||
INSERT INTO alert_telegram_settings (id, bot_token, chat_id)
|
||||
SELECT 1, '', ''
|
||||
WHERE NOT EXISTS (SELECT 1 FROM alert_telegram_settings WHERE id = 1);
|
||||
`)
|
||||
|
||||
sqlite.exec(`
|
||||
INSERT INTO alert_engine_cursor (id, last_source_finished_at)
|
||||
SELECT 1, NULL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM alert_engine_cursor WHERE id = 1);
|
||||
`)
|
||||
|
||||
export const db = drizzle(sqlite, { schema })
|
||||
|
||||
/** Прямой доступ к better-sqlite3 для сложных read-only запросов (напр. /api/alerts). */
|
||||
export const sqliteDatabase: SqliteHandle = sqlite
|
||||
|
||||
@@ -115,6 +115,46 @@ export const trafficSettings = sqliteTable("traffic_settings", {
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Настройки фоновой проверки доступности RouterOS REST API по серверам каталога. */
|
||||
export const serversApiPingSettings = sqliteTable("servers_api_ping_settings", {
|
||||
id: integer("id").primaryKey(),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(false),
|
||||
intervalSec: integer("interval_sec").notNull().default(120),
|
||||
lastCollectedAt: text("last_collected_at"),
|
||||
lastDurationMs: integer("last_duration_ms"),
|
||||
lastError: text("last_error"),
|
||||
createdAt: text("created_at").notNull().default(sql`(datetime('now'))`),
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Сырые пробы REST `/system/identity` по серверам — для UI сбора и для `buildSignalSnapshot` (алерты). */
|
||||
export const serversRestPingSamples = sqliteTable("servers_rest_ping_samples", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
serverId: integer("server_id")
|
||||
.notNull()
|
||||
.references(() => servers.id, { onDelete: "cascade" }),
|
||||
sampledAt: text("sampled_at").notNull(),
|
||||
ok: integer("ok", { mode: "boolean" }).notNull(),
|
||||
latencyMs: integer("latency_ms"),
|
||||
error: text("error"),
|
||||
})
|
||||
|
||||
/** Сырые снимки GRE для алертов — пишет джоба `gre_bgp`. */
|
||||
export const alertGreTunnelSamples = sqliteTable("alert_gre_tunnel_samples", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
sampledAt: text("sampled_at").notNull(),
|
||||
targetLabel: text("target_label").notNull(),
|
||||
status: text("status").notNull(),
|
||||
})
|
||||
|
||||
/** Сырые снимки BGP-сессий для алертов — пишет джоба `gre_bgp`. */
|
||||
export const alertBgpPeerSamples = sqliteTable("alert_bgp_peer_samples", {
|
||||
id: integer("id").primaryKey({ autoIncrement: true }),
|
||||
sampledAt: text("sampled_at").notNull(),
|
||||
peerKey: text("peer_key").notNull(),
|
||||
state: text("state").notNull(),
|
||||
})
|
||||
|
||||
// ── raw traffic samples (per server/interface/timepoint) ──────────────────────
|
||||
|
||||
export const trafficSamples = sqliteTable("traffic_samples", {
|
||||
@@ -136,7 +176,11 @@ export const trafficSamples = sqliteTable("traffic_samples", {
|
||||
|
||||
export const uptimeSettings = sqliteTable("uptime_settings", {
|
||||
id: integer("id").primaryKey(),
|
||||
/** Устаревший агрегат: синхронизируется как resources ∨ ping ∨ speed (для совместимости). */
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
resourcesEnabled: integer("resources_enabled", { mode: "boolean" }).notNull().default(true),
|
||||
pingEnabled: integer("ping_enabled", { mode: "boolean" }).notNull().default(true),
|
||||
speedEnabled: integer("speed_enabled", { mode: "boolean" }).notNull().default(true),
|
||||
intervalSec: integer("interval_sec").notNull().default(15),
|
||||
probeIntervalSec: integer("probe_interval_sec").notNull().default(15),
|
||||
speedIntervalSec: integer("speed_interval_sec").notNull().default(60),
|
||||
@@ -158,6 +202,8 @@ export const uptimeProbes = sqliteTable("uptime_probes", {
|
||||
target: text("target").notNull(),
|
||||
probeFilter: text("probe_filter").notNull().default("—"),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
/** 0 — брать глобальный probe_interval_sec из uptime_settings */
|
||||
intervalSec: integer("interval_sec").notNull().default(0),
|
||||
/** Выводить пробу в блоке «Активные пробы» на дашборде */
|
||||
showOnDashboard: integer("show_on_dashboard", { mode: "boolean" }).notNull().default(false),
|
||||
sortOrder: integer("sort_order").notNull().default(0),
|
||||
@@ -231,6 +277,155 @@ export const evobgpSettings = sqliteTable("evobgp_settings", {
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Единственная строка id=1: токен и чат Telegram для оповещений. */
|
||||
export const alertTelegramSettings = sqliteTable("alert_telegram_settings", {
|
||||
id: integer("id").primaryKey(),
|
||||
botToken: text("bot_token").notNull().default(""),
|
||||
chatId: text("chat_id").notNull().default(""),
|
||||
/** Тема супергруппы (forum): `message_thread_id` в Bot API; NULL — общий чат */
|
||||
messageThreadId: integer("message_thread_id"),
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Группы правил: ANY = хотя бы одно; ALL = все одновременно в окне тика. */
|
||||
export const alertGroups = sqliteTable("alert_groups", {
|
||||
id: text("id").primaryKey(),
|
||||
name: text("name").notNull(),
|
||||
combineMode: text("combine_mode", { enum: ["any", "all"] }).notNull().default("any"),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
/** Переопределение cooldown для агрегата группы; NULL — по умолчанию 5м */
|
||||
cooldownOverride: text("cooldown_override", {
|
||||
enum: ["1м", "5м", "15м", "1ч", "4ч", "24ч"],
|
||||
}),
|
||||
createdAt: text("created_at").notNull().default(sql`(datetime('now'))`),
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Правила оповещений (UI /alerts). */
|
||||
export const alertRules = sqliteTable("alert_rules", {
|
||||
id: text("id").primaryKey(),
|
||||
name: text("name").notNull(),
|
||||
type: text("type", {
|
||||
enum: ["gre-tunnel", "bgp-peer", "bgp-prefix", "gre-client", "server", "rtt", "loss", "traffic"],
|
||||
}).notNull(),
|
||||
/** Сводная подпись (первый target или join); для совместимости и поиска */
|
||||
target: text("target").notNull(),
|
||||
condition: text("condition").notNull(),
|
||||
severity: text("severity", { enum: ["critical", "warning", "info"] }).notNull(),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
cooldown: text("cooldown", {
|
||||
enum: ["1м", "5м", "15м", "1ч", "4ч", "24ч"],
|
||||
}).notNull().default("5м"),
|
||||
chatId: text("rule_chat_id").notNull().default(""),
|
||||
/** NULL / 0 — выкл. Секунды: уведомление только если условие держится столько времени без «отмены» (восстановление сигнала). */
|
||||
confirmStabilitySec: integer("confirm_stability_sec"),
|
||||
/** Политика recovery-уведомления для правила. */
|
||||
recoveryMode: text("recovery_mode", {
|
||||
enum: ["always", "never", "conditional"],
|
||||
}).notNull().default("always"),
|
||||
/** Доп. задержка подтверждения для recovery при `conditional`; NULL — без доп. ожидания. */
|
||||
recoveryStabilitySec: integer("recovery_stability_sec"),
|
||||
/** NULL — правило вне группы (отдельные отправки по cooldown правила) */
|
||||
groupId: text("group_id"),
|
||||
createdAt: text("created_at").notNull().default(sql`(datetime('now'))`),
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Несколько объектов на правило; срабатывание по OR (хотя бы один). */
|
||||
export const alertRuleTargets = sqliteTable("alert_rule_targets", {
|
||||
id: text("id").primaryKey(),
|
||||
ruleId: text("rule_id").notNull(),
|
||||
target: text("target").notNull(),
|
||||
sortIndex: integer("sort_index").notNull().default(0),
|
||||
})
|
||||
|
||||
/** Несколько условий на правило; срабатывание по OR (любое из выбранных). */
|
||||
export const alertRuleConditions = sqliteTable("alert_rule_conditions", {
|
||||
id: text("id").primaryKey(),
|
||||
ruleId: text("rule_id").notNull(),
|
||||
conditionLine: text("condition_line").notNull(),
|
||||
sortIndex: integer("sort_index").notNull().default(0),
|
||||
})
|
||||
|
||||
/** Состояние движка: cooldown / дедуп по ключу rule:id или group:id */
|
||||
export const alertEngineState = sqliteTable("alert_engine_state", {
|
||||
scopeKey: text("scope_key").primaryKey(),
|
||||
lastFiredAt: text("last_fired_at").notNull().default(""),
|
||||
lastPayloadHash: text("last_payload_hash"),
|
||||
})
|
||||
|
||||
/** Снимок прошлого тика для live-сигналов (GRE/BGP): переходы «offline» / «восстановился». */
|
||||
export const alertEnginePrevLive = sqliteTable("alert_engine_prev_live", {
|
||||
kind: text("kind", { enum: ["gre", "bgp"] }).primaryKey(),
|
||||
payloadJson: text("payload_json").notNull().default("{}"),
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Ожидание стабильности срабатывания по правилу (антидребезг). */
|
||||
export const alertEngineConfirmPending = sqliteTable("alert_engine_confirm_pending", {
|
||||
ruleId: text("rule_id").primaryKey(),
|
||||
payloadHash: text("payload_hash").notNull(),
|
||||
sinceAt: text("since_at").notNull(),
|
||||
})
|
||||
|
||||
/** Задел: несколько получателей Telegram (пока UI не подключён) */
|
||||
export const alertDestinations = sqliteTable("alert_destinations", {
|
||||
id: text("id").primaryKey(),
|
||||
kind: text("kind", { enum: ["telegram"] }).notNull().default("telegram"),
|
||||
label: text("label").notNull().default(""),
|
||||
telegramChatId: text("telegram_chat_id").notNull().default(""),
|
||||
messageThreadId: integer("message_thread_id"),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
createdAt: text("created_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Журнал отправок / срабатываний (персист + при необходимости пополняется воркером). */
|
||||
export const alertHistory = sqliteTable("alert_history", {
|
||||
id: text("id").primaryKey(),
|
||||
ruleId: text("rule_id"),
|
||||
groupId: text("group_id"),
|
||||
ruleName: text("rule_name").notNull(),
|
||||
severity: text("severity", { enum: ["critical", "warning", "info"] }).notNull(),
|
||||
message: text("message").notNull(),
|
||||
sentOk: integer("sent_ok", { mode: "boolean" }).notNull().default(true),
|
||||
firedAt: text("fired_at").notNull(),
|
||||
})
|
||||
|
||||
/** Outbox доставки уведомлений: ретраи и идемпотентность отправки каналов. */
|
||||
export const alertOutbox = sqliteTable("alert_outbox", {
|
||||
id: text("id").primaryKey(),
|
||||
dedupeKey: text("dedupe_key").notNull(),
|
||||
channel: text("channel", { enum: ["telegram"] }).notNull().default("telegram"),
|
||||
status: text("status", { enum: ["pending", "sent", "failed"] }).notNull().default("pending"),
|
||||
retryCount: integer("retry_count").notNull().default(0),
|
||||
maxRetries: integer("max_retries").notNull().default(3),
|
||||
nextAttemptAt:text("next_attempt_at").notNull(),
|
||||
payloadJson: text("payload_json").notNull(),
|
||||
createdAt: text("created_at").notNull().default(sql`(datetime('now'))`),
|
||||
sentAt: text("sent_at"),
|
||||
lastError: text("last_error"),
|
||||
})
|
||||
|
||||
/** Cursor движка: watermark последней обработанной точки snapshot-источников. */
|
||||
export const alertEngineCursor = sqliteTable("alert_engine_cursor", {
|
||||
id: integer("id").primaryKey(),
|
||||
lastSourceFinishedAt: text("last_source_finished_at"),
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Журнал прогонов планировщика (аудит). */
|
||||
export const schedulerRuns = sqliteTable("scheduler_runs", {
|
||||
id: text("id").primaryKey(),
|
||||
jobKey: text("job_key").notNull(),
|
||||
startedAt: text("started_at").notNull(),
|
||||
finishedAt: text("finished_at").notNull(),
|
||||
status: text("status", { enum: ["ok", "error"] }).notNull(),
|
||||
error: text("error"),
|
||||
durationMs: integer("duration_ms").notNull().default(0),
|
||||
/** JSON: см. `SchedulerRunSnapshot` в types/scheduler-run-snapshot.ts */
|
||||
resultJson: text("result_json"),
|
||||
})
|
||||
|
||||
export const uptimeSpeedTestRuns = sqliteTable("uptime_speed_test_runs", {
|
||||
id: text("id").primaryKey(),
|
||||
probeId: text("probe_id"),
|
||||
@@ -264,6 +459,7 @@ export type SnapshotInsert = typeof serverSnapshots.$inferInsert
|
||||
export type FilterRuleRow = typeof filterRules.$inferSelect
|
||||
export type RecursiveRouteRow = typeof recursiveRoutes.$inferSelect
|
||||
export type TrafficSettingsRow = typeof trafficSettings.$inferSelect
|
||||
export type ServersApiPingSettingsRow = typeof serversApiPingSettings.$inferSelect
|
||||
export type TrafficSampleRow = typeof trafficSamples.$inferSelect
|
||||
export type UptimeSettingsRow = typeof uptimeSettings.$inferSelect
|
||||
export type UptimeProbeRow = typeof uptimeProbes.$inferSelect
|
||||
@@ -271,4 +467,14 @@ export type UptimeProbeSampleRow = typeof uptimeProbeSamples.$inferSelect
|
||||
export type UptimeResourceSampleRow = typeof uptimeResourceSamples.$inferSelect
|
||||
export type UptimeSpeedProbeRow = typeof uptimeSpeedProbes.$inferSelect
|
||||
export type UptimeSpeedTestRunRow = typeof uptimeSpeedTestRuns.$inferSelect
|
||||
export type SchedulerRunRow = typeof schedulerRuns.$inferSelect
|
||||
export type EvobgpSettingsRow = typeof evobgpSettings.$inferSelect
|
||||
export type AlertTelegramSettingsRow = typeof alertTelegramSettings.$inferSelect
|
||||
export type AlertGroupRow = typeof alertGroups.$inferSelect
|
||||
export type AlertRuleRow = typeof alertRules.$inferSelect
|
||||
export type AlertRuleTargetRow = typeof alertRuleTargets.$inferSelect
|
||||
export type AlertEngineStateRow = typeof alertEngineState.$inferSelect
|
||||
export type AlertDestinationRow = typeof alertDestinations.$inferSelect
|
||||
export type AlertHistoryRow = typeof alertHistory.$inferSelect
|
||||
export type AlertOutboxRow = typeof alertOutbox.$inferSelect
|
||||
export type AlertEngineCursorRow = typeof alertEngineCursor.$inferSelect
|
||||
|
||||
+11
-8
@@ -9,12 +9,15 @@ import execRoutes from "./routes/exec.js"
|
||||
import filtersRoutes from "./routes/filters.js"
|
||||
import recursiveRoutes from "./routes/recursive-routes.js"
|
||||
import trafficRoutes from "./routes/traffic.js"
|
||||
import serversApiPingRoutes from "./routes/servers-api-ping.js"
|
||||
import uptimeRoutes from "./routes/uptime.js"
|
||||
import networkRoutes from "./routes/network.js"
|
||||
import evobgpRoutes from "./routes/evobgp.js"
|
||||
import probesRoutes from "./routes/probes.js"
|
||||
import { collectTrafficOnce, restartTrafficCollector, stopTrafficCollector } from "./services/traffic-collector.js"
|
||||
import { collectUptimeOnce, restartUptimeCollector, stopUptimeCollector } from "./services/uptime-collector.js"
|
||||
import schedulerRoutes from "./routes/scheduler.js"
|
||||
import sidebarCountsRoutes from "./routes/sidebar-counts.js"
|
||||
import alertsRoutes from "./routes/alerts.js"
|
||||
import { refreshScheduler, stopScheduler } from "./services/scheduler.js"
|
||||
|
||||
// ── app factory ────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -49,18 +52,18 @@ await app.register(execRoutes, { prefix: "/api" })
|
||||
await app.register(filtersRoutes, { prefix: "/api" })
|
||||
await app.register(recursiveRoutes, { prefix: "/api" })
|
||||
await app.register(trafficRoutes, { prefix: "/api" })
|
||||
await app.register(serversApiPingRoutes, { prefix: "/api" })
|
||||
await app.register(uptimeRoutes, { prefix: "/api" })
|
||||
await app.register(networkRoutes, { prefix: "/api" })
|
||||
await app.register(evobgpRoutes, { prefix: "/api" })
|
||||
await app.register(probesRoutes, { prefix: "/api" })
|
||||
await app.register(schedulerRoutes, { prefix: "/api" })
|
||||
await app.register(sidebarCountsRoutes, { prefix: "/api" })
|
||||
await app.register(alertsRoutes, { prefix: "/api" })
|
||||
|
||||
restartTrafficCollector()
|
||||
void collectTrafficOnce()
|
||||
restartUptimeCollector()
|
||||
void collectUptimeOnce()
|
||||
refreshScheduler()
|
||||
app.addHook("onClose", async () => {
|
||||
stopTrafficCollector()
|
||||
stopUptimeCollector()
|
||||
stopScheduler()
|
||||
})
|
||||
|
||||
// ── start ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import { z } from "zod"
|
||||
import {
|
||||
getAlertsMeta,
|
||||
getTelegramBotToken,
|
||||
getTelegramMessageThreadIdForApi,
|
||||
getTelegramPublic,
|
||||
listAlertGroups,
|
||||
listAlertRules,
|
||||
listMergedHistory,
|
||||
replaceAlertsConfig,
|
||||
formatAlertRuleTestTelegramText,
|
||||
sendTelegramAlertMessage,
|
||||
updateTelegramSettings,
|
||||
} from "../services/alerts-service.js"
|
||||
|
||||
const AlertTypeSchema = z.enum([
|
||||
"gre-tunnel",
|
||||
"bgp-peer",
|
||||
"bgp-prefix",
|
||||
"gre-client",
|
||||
"server",
|
||||
"rtt",
|
||||
"loss",
|
||||
"traffic",
|
||||
])
|
||||
const AlertSeveritySchema = z.enum(["critical", "warning", "info"])
|
||||
const AlertCooldownSchema = z.enum(["1м", "5м", "15м", "1ч", "4ч", "24ч"])
|
||||
const RecoveryModeSchema = z.enum(["always", "never", "conditional"])
|
||||
|
||||
const AlertGroupBodySchema = z.object({
|
||||
id: z.string().min(1),
|
||||
name: z.string().min(1),
|
||||
combineMode: z.enum(["any", "all"]),
|
||||
enabled: z.boolean(),
|
||||
cooldownOverride: z.union([AlertCooldownSchema, z.null()]),
|
||||
})
|
||||
|
||||
const AlertRuleBodySchema = z
|
||||
.object({
|
||||
id: z.string().min(1),
|
||||
name: z.string().min(1),
|
||||
type: AlertTypeSchema,
|
||||
target: z.string().optional(),
|
||||
targets: z.array(z.string().min(1)).optional(),
|
||||
groupId: z.string().nullable().optional(),
|
||||
/** Сводка; если переданы `conditions`, может дублировать первую строку */
|
||||
condition: z.string().optional(),
|
||||
/** Несколько условий (OR); приоритет над одной строкой `condition` */
|
||||
conditions: z.array(z.string().min(1)).optional(),
|
||||
severity: AlertSeveritySchema,
|
||||
enabled: z.boolean(),
|
||||
cooldown: AlertCooldownSchema,
|
||||
/** Секунды подтверждения стабильности; 0 / null / не передано — выкл. */
|
||||
confirmStabilitySec: z.union([z.number().int().min(0).max(86400), z.null()]).optional(),
|
||||
recoveryMode: RecoveryModeSchema.optional(),
|
||||
recoveryStabilitySec: z.union([z.number().int().min(0).max(86400), z.null()]).optional(),
|
||||
chatId: z.string(),
|
||||
})
|
||||
.refine((r) => (r.targets != null && r.targets.length > 0) || Boolean(r.target?.trim()), {
|
||||
message: "Нужен хотя бы один объект: targets или target",
|
||||
path: ["targets"],
|
||||
})
|
||||
.refine(
|
||||
(r) =>
|
||||
(r.conditions != null && r.conditions.length > 0) || Boolean(r.condition?.trim()),
|
||||
{ message: "Нужно хотя бы одно условие: conditions или condition", path: ["conditions"] },
|
||||
)
|
||||
|
||||
const PutRulesSchema = z.object({
|
||||
rules: z.array(AlertRuleBodySchema),
|
||||
groups: z.array(AlertGroupBodySchema).optional(),
|
||||
})
|
||||
|
||||
const PutTelegramSchema = z.object({
|
||||
/** undefined — не менять; null или "" — очистить токен */
|
||||
token: z.union([z.string(), z.null()]).optional(),
|
||||
chatId: z.string().optional(),
|
||||
/** undefined — не менять; null — сбросить (общий чат супергруппы) */
|
||||
messageThreadId: z.union([z.number().int().positive(), z.null()]).optional(),
|
||||
})
|
||||
|
||||
const RulePreviewTelegramSchema = z.object({
|
||||
name: z.string().min(1),
|
||||
targets: z.array(z.string().min(1)).min(1),
|
||||
conditionLine: z.string().min(1),
|
||||
severity: AlertSeveritySchema,
|
||||
cooldown: AlertCooldownSchema,
|
||||
})
|
||||
|
||||
const TestTelegramSchema = z.object({
|
||||
/** Если не передан — взять сохранённый токен из БД */
|
||||
token: z.string().optional(),
|
||||
chatId: z.string().optional(),
|
||||
/** Если не передан — из БД; для проверки черновика до сохранения */
|
||||
messageThreadId: z.coerce.number().int().positive().optional(),
|
||||
/** Если задан — вместо короткого текста отправляется предпросмотр правила (создание / правка). */
|
||||
rulePreview: RulePreviewTelegramSchema.optional(),
|
||||
})
|
||||
|
||||
const alertsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
app.get("/alerts", async (_req, reply) => {
|
||||
const tg = getTelegramPublic()
|
||||
const connected = tg.tokenConfigured && Boolean(tg.chatId?.trim())
|
||||
return reply.send({
|
||||
telegram: { ...tg, connected },
|
||||
groups: listAlertGroups(),
|
||||
rules: listAlertRules(),
|
||||
history: listMergedHistory(),
|
||||
meta: getAlertsMeta(),
|
||||
})
|
||||
})
|
||||
|
||||
app.put("/alerts/rules", async (req, reply) => {
|
||||
const parsed = PutRulesSchema.safeParse(req.body)
|
||||
if (!parsed.success) {
|
||||
return reply.status(400).send({ error: "Некорректное тело запроса", details: parsed.error.flatten() })
|
||||
}
|
||||
const groups =
|
||||
parsed.data.groups ??
|
||||
listAlertGroups().map((g) => ({
|
||||
id: g.id,
|
||||
name: g.name,
|
||||
combineMode: g.combineMode,
|
||||
enabled: g.enabled,
|
||||
cooldownOverride: g.cooldownOverride,
|
||||
}))
|
||||
const rules = parsed.data.rules.map((r) => ({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
type: r.type,
|
||||
target: r.target,
|
||||
targets: r.targets,
|
||||
groupId: r.groupId ?? null,
|
||||
condition: r.condition,
|
||||
conditions:
|
||||
r.conditions && r.conditions.length > 0
|
||||
? r.conditions.map((c) => c.trim()).filter(Boolean)
|
||||
: r.condition?.trim()
|
||||
? [r.condition.trim()]
|
||||
: [],
|
||||
severity: r.severity,
|
||||
enabled: r.enabled,
|
||||
cooldown: r.cooldown,
|
||||
confirmStabilitySec: r.confirmStabilitySec ?? null,
|
||||
recoveryMode: r.recoveryMode ?? "always",
|
||||
recoveryStabilitySec: r.recoveryStabilitySec ?? null,
|
||||
chatId: r.chatId,
|
||||
}))
|
||||
replaceAlertsConfig({ groups, rules })
|
||||
return reply.send({ ok: true, groups: listAlertGroups(), rules: listAlertRules() })
|
||||
})
|
||||
|
||||
app.put("/alerts/telegram", async (req, reply) => {
|
||||
const parsed = PutTelegramSchema.safeParse(req.body)
|
||||
if (!parsed.success) {
|
||||
return reply.status(400).send({ error: "Некорректное тело запроса", details: parsed.error.flatten() })
|
||||
}
|
||||
const next = updateTelegramSettings({
|
||||
token: parsed.data.token,
|
||||
chatId: parsed.data.chatId,
|
||||
messageThreadId: parsed.data.messageThreadId,
|
||||
})
|
||||
const connected = next.tokenConfigured && Boolean(next.chatId?.trim())
|
||||
return reply.send({ ok: true, telegram: { ...next, connected } })
|
||||
})
|
||||
|
||||
app.post("/alerts/telegram/test", async (req, reply) => {
|
||||
const parsed = TestTelegramSchema.safeParse(req.body ?? {})
|
||||
if (!parsed.success) {
|
||||
return reply.status(400).send({ error: "Некорректное тело запроса", details: parsed.error.flatten() })
|
||||
}
|
||||
const fromDb = getTelegramBotToken()
|
||||
const token = (parsed.data.token?.trim() || fromDb).trim()
|
||||
const pub = getTelegramPublic()
|
||||
const chatId = (parsed.data.chatId?.trim() || pub.chatId || "").trim()
|
||||
if (!token) {
|
||||
return reply.status(400).send({ error: "Не задан Bot Token (сохраните в БД или передайте в запросе)" })
|
||||
}
|
||||
if (!chatId) {
|
||||
return reply.status(400).send({ error: "Не задан Chat ID" })
|
||||
}
|
||||
const threadId =
|
||||
parsed.data.messageThreadId != null && parsed.data.messageThreadId >= 1
|
||||
? parsed.data.messageThreadId
|
||||
: getTelegramMessageThreadIdForApi()
|
||||
const text = parsed.data.rulePreview
|
||||
? formatAlertRuleTestTelegramText(parsed.data.rulePreview)
|
||||
: "MikroTik Manager — тест оповещений (Telegram)."
|
||||
const send = await sendTelegramAlertMessage({
|
||||
text,
|
||||
token: parsed.data.token?.trim() || undefined,
|
||||
chatId,
|
||||
messageThreadId: threadId,
|
||||
})
|
||||
if (!send.ok) {
|
||||
return reply.status(502).send({ ok: false, error: send.error })
|
||||
}
|
||||
return reply.send({ ok: true })
|
||||
})
|
||||
}
|
||||
|
||||
export default alertsRoutes
|
||||
+4
-148
@@ -2,154 +2,10 @@ import { eq } from "drizzle-orm"
|
||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers } from "../db/schema.js"
|
||||
import { parseBgpSessions } from "../services/bgp-parse-sessions.js"
|
||||
import { MikrotikClient } from "../services/mikrotik.js"
|
||||
import { ServerIdParamSchema } from "../types/server.js"
|
||||
import type { RosBgpSession, BgpSessionRead } from "../types/server.js"
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect
|
||||
|
||||
/** Parse RouterOS duration like "1m30s", "1d5h13m23s580ms", "30s" → seconds */
|
||||
function parseDuration(s: string | undefined): number {
|
||||
if (!s) return 0
|
||||
let total = 0
|
||||
const matches = s.matchAll(/(\d+)(w|d|h|m(?!s)|s|ms)/g)
|
||||
for (const m of matches) {
|
||||
const n = parseInt(m[1])
|
||||
switch (m[2]) {
|
||||
case "w": total += n * 604800; break
|
||||
case "d": total += n * 86400; break
|
||||
case "h": total += n * 3600; break
|
||||
case "m": total += n * 60; break
|
||||
case "s": total += n; break
|
||||
// ms ignored for seconds precision
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
/**
|
||||
* Map RouterOS 7.x short capability codes to human-readable names.
|
||||
* Both local and remote capability strings are comma-separated short codes.
|
||||
*/
|
||||
function parseCapabilities(capStr: string | undefined): string[] {
|
||||
if (!capStr) return []
|
||||
const codeMap: Record<string, string> = {
|
||||
mp: "MP-BGP",
|
||||
rr: "Route Refresh",
|
||||
gr: "Graceful Restart",
|
||||
as4: "4-byte-AS",
|
||||
enhe: "Extended Next-Hop",
|
||||
role: "BGP Role",
|
||||
err: "Extended Route Refresh",
|
||||
llgr: "Long-Lived GR",
|
||||
"add-path": "ADD-PATH",
|
||||
}
|
||||
return capStr.split(",")
|
||||
.map(c => c.trim().toLowerCase())
|
||||
.filter(Boolean)
|
||||
.map(c => codeMap[c] ?? c)
|
||||
.filter((v, i, a) => a.indexOf(v) === i) // deduplicate
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine BGP session state from a RouterOS 7.x session object.
|
||||
*
|
||||
* RouterOS 7.x does NOT include a "state" field.
|
||||
* Instead: "established": "true" means the session is up.
|
||||
* For non-established sessions the "state" field may appear (RouterOS 6.x compat),
|
||||
* or we fall back by checking other indicators.
|
||||
*/
|
||||
function detectState(s: RosBgpSession): string {
|
||||
// RouterOS 7.x: explicit established flag
|
||||
if (s["established"] === "true") return "Established"
|
||||
|
||||
// RouterOS 6.x / some 7.x builds: "state" field
|
||||
if (s["state"]) {
|
||||
const map: Record<string, string> = {
|
||||
established: "Established",
|
||||
active: "Active",
|
||||
idle: "Idle",
|
||||
connect: "Connect",
|
||||
opensent: "OpenSent",
|
||||
openconfirm: "OpenConfirm",
|
||||
"open sent": "OpenSent",
|
||||
"open confirm": "OpenConfirm",
|
||||
}
|
||||
const key = s["state"].trim().toLowerCase()
|
||||
if (map[key]) return map[key]
|
||||
// partial match
|
||||
for (const [k, v] of Object.entries(map)) {
|
||||
if (key.startsWith(k)) return v
|
||||
}
|
||||
return key.charAt(0).toUpperCase() + key.slice(1)
|
||||
}
|
||||
|
||||
// Has uptime but no established flag → assume degraded Active
|
||||
if (s["uptime"]) return "Active"
|
||||
|
||||
return "Idle"
|
||||
}
|
||||
|
||||
function parseSessions(server: ServerRow, raw: RosBgpSession[]): BgpSessionRead[] {
|
||||
return raw.map((s, idx) => {
|
||||
const remoteAs = parseInt(s["remote.as"] ?? "0") || 0
|
||||
const localAs = parseInt(s["local.as"] ?? "0") || 0
|
||||
const type: "eBGP" | "iBGP" = (localAs > 0 && localAs === remoteAs) ? "iBGP" : "eBGP"
|
||||
|
||||
// Capabilities: prefer local caps (what we negotiated); merge with remote
|
||||
const localCaps = parseCapabilities(s["local.capabilities"])
|
||||
const remoteCaps = parseCapabilities(s["remote.capabilities"])
|
||||
// Old-style boolean capability fields (RouterOS 6.x)
|
||||
const legacyCaps: string[] = []
|
||||
if (s["4-octet-as-capability"] === "true") legacyCaps.push("4-byte-AS")
|
||||
if (s["as4-capability"] === "true") legacyCaps.push("4-byte-AS")
|
||||
if (s["refresh-capability"] === "true") legacyCaps.push("Route Refresh")
|
||||
if (s["add-path-capability"] === "true") legacyCaps.push("ADD-PATH")
|
||||
if (s["graceful-restart-capability"] === "true") legacyCaps.push("Graceful Restart")
|
||||
if (s["extended-message-capability"] === "true") legacyCaps.push("Extended Messages")
|
||||
const caps = [...new Set([...localCaps, ...remoteCaps, ...legacyCaps])]
|
||||
|
||||
// Hold time: RouterOS 7 uses "hold-time" ("1m30s"), RouterOS 6 uses "active-holdtime" (seconds)
|
||||
const holdTime = parseDuration(s["hold-time"])
|
||||
|| parseInt(s["active-holdtime"] ?? "0")
|
||||
|| 90
|
||||
|
||||
// Keepalive: may have "s" suffix
|
||||
const keepalive = parseDuration(s["keepalive-time"])
|
||||
|| 30
|
||||
|
||||
// Message counts: RouterOS 7 uses "remote.messages" / "local.messages"
|
||||
const inputMessages = parseInt(s["remote.messages"] ?? s["total-messages-received"] ?? "0") || 0
|
||||
const outputMessages = parseInt(s["local.messages"] ?? s["total-messages-sent"] ?? "0") || 0
|
||||
|
||||
const peerIp = (s["remote.address"] ?? "").replace(/\/\d+$/, "")
|
||||
|
||||
return {
|
||||
id: s[".id"] ?? String(idx),
|
||||
serverId: server.id,
|
||||
serverName: server.name || server.host,
|
||||
serverSite: server.site,
|
||||
serverCountry: server.country,
|
||||
name: s["name"] ?? peerIp,
|
||||
peerIp,
|
||||
remoteAs,
|
||||
localAs,
|
||||
localId: s["local.id"] ?? "",
|
||||
remoteId: s["remote.id"] ?? "",
|
||||
state: detectState(s),
|
||||
type,
|
||||
uptime: s["uptime"] || null,
|
||||
holdTime,
|
||||
keepalive,
|
||||
prefixesRx: parseInt(s["prefix-count"] ?? s["total-updates-received"] ?? "0") || 0,
|
||||
prefixesTx: parseInt(s["total-updates-sent"] ?? "0") || 0,
|
||||
inputMessages,
|
||||
outputMessages,
|
||||
capabilities: caps,
|
||||
lastError: s["last-notification"] || null,
|
||||
}
|
||||
})
|
||||
}
|
||||
import type { BgpSessionRead } from "../types/server.js"
|
||||
|
||||
const bgpRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
|
||||
@@ -176,7 +32,7 @@ const bgpRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(server)
|
||||
const raw = await client.getBgpSessions()
|
||||
return parseSessions(server, raw)
|
||||
return parseBgpSessions(server, raw)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
@@ -198,7 +54,7 @@ const bgpRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(server)
|
||||
const raw = await client.getBgpSessions()
|
||||
return reply.send(parseSessions(server, raw))
|
||||
return reply.send(parseBgpSessions(server, raw))
|
||||
} catch {
|
||||
return reply.send([])
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { eq, sql } from "drizzle-orm"
|
||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import { z } from "zod"
|
||||
import { db } from "../db/index.js"
|
||||
import { evobgpSettings } from "../db/schema.js"
|
||||
import { evobgpSettings, filterRules } from "../db/schema.js"
|
||||
|
||||
const PutEvobgpSchema = z.object({
|
||||
baseUrl: z.string().optional(),
|
||||
@@ -48,7 +48,14 @@ interface EvoCatalogRaw {
|
||||
asns: {
|
||||
items: Array<{
|
||||
module_id: string
|
||||
entry: { id: string; asn: number; community_id?: string | null }
|
||||
entry: {
|
||||
id: string
|
||||
asn: number
|
||||
community_id?: string | null
|
||||
asn_name?: string | null
|
||||
prefix_count?: number | null
|
||||
asn_resolved_at?: string | null
|
||||
}
|
||||
}>
|
||||
}
|
||||
ip_ranges: {
|
||||
@@ -62,6 +69,60 @@ interface EvoCatalogRaw {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* По UUID community в каталоге EvoBGP: сумма «маршрутов» —
|
||||
* 1 на домен, 1 на CIDR в ip_ranges, prefix_count на ASN.
|
||||
*/
|
||||
function aggregateCommunityRouteCounts(catalog: EvoCatalogRaw): Map<string, number> {
|
||||
const routesPerComm = new Map<string, number>()
|
||||
function bump(commId: string, routeAdd: number) {
|
||||
routesPerComm.set(commId, (routesPerComm.get(commId) ?? 0) + routeAdd)
|
||||
}
|
||||
|
||||
for (const rowItem of catalog.domains?.items ?? []) {
|
||||
const cid = rowItem.entry.community_id
|
||||
if (cid) bump(cid, 1)
|
||||
}
|
||||
for (const rowItem of catalog.ip_ranges?.items ?? []) {
|
||||
const cid = rowItem.entry.community_id
|
||||
if (cid) bump(cid, 1)
|
||||
}
|
||||
for (const rowItem of catalog.asns?.items ?? []) {
|
||||
const cid = rowItem.entry.community_id
|
||||
if (!cid) continue
|
||||
const pc = rowItem.entry.prefix_count
|
||||
const n = typeof pc === "number" && Number.isFinite(pc) && pc >= 0 ? Math.trunc(pc) : 0
|
||||
bump(cid, n)
|
||||
}
|
||||
|
||||
return routesPerComm
|
||||
}
|
||||
|
||||
/**
|
||||
* Уникальные серверы из локальной БД, у которых в BGP-фильтре указан community (строка AS:NNN).
|
||||
* Ключ — значение `filter_rules.community` после trim (как в EvoBGP `community`).
|
||||
*/
|
||||
function distinctServersByFilterCommunityValue(): Map<string, number> {
|
||||
const rows = db
|
||||
.select({ serverId: filterRules.serverId, community: filterRules.community })
|
||||
.from(filterRules)
|
||||
.all()
|
||||
const byVal = new Map<string, Set<number>>()
|
||||
for (const r of rows) {
|
||||
const v = (r.community ?? "").trim()
|
||||
if (!v) continue
|
||||
let set = byVal.get(v)
|
||||
if (!set) {
|
||||
set = new Set()
|
||||
byVal.set(v, set)
|
||||
}
|
||||
set.add(r.serverId)
|
||||
}
|
||||
const out = new Map<string, number>()
|
||||
for (const [v, set] of byVal) out.set(v, set.size)
|
||||
return out
|
||||
}
|
||||
|
||||
function problemMessage(text: string, status: number): string {
|
||||
try {
|
||||
const j = JSON.parse(text) as { detail?: string; title?: string }
|
||||
@@ -236,32 +297,48 @@ const evobgpRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
})
|
||||
|
||||
const asns = (catalog.asns?.items ?? []).map((rowItem) => {
|
||||
const mod = modMap.get(rowItem.module_id)
|
||||
const cid = rowItem.entry.community_id
|
||||
const e = rowItem.entry
|
||||
const name = (e.asn_name ?? "").trim()
|
||||
const pc = e.prefix_count
|
||||
const prefixes =
|
||||
typeof pc === "number" && Number.isFinite(pc) && pc >= 0 ? Math.trunc(pc) : 0
|
||||
let updated = fmtTime
|
||||
const rawResolved = (e.asn_resolved_at ?? "").trim()
|
||||
if (rawResolved) {
|
||||
const d = new Date(rawResolved)
|
||||
if (!Number.isNaN(d.getTime())) updated = d.toLocaleString("ru-RU")
|
||||
}
|
||||
return {
|
||||
id: `${rowItem.module_id}:${rowItem.entry.id}`,
|
||||
asn: `AS${rowItem.entry.asn}`,
|
||||
org: "—",
|
||||
id: `${rowItem.module_id}:${e.id}`,
|
||||
asn: `AS${e.asn}`,
|
||||
org: name || "—",
|
||||
country: "—",
|
||||
prefixes: 0,
|
||||
prefixes,
|
||||
filter: cid ? (commLabel.get(cid) ?? cid) : "—",
|
||||
updated: fmtTime,
|
||||
updated,
|
||||
enabled: true,
|
||||
}
|
||||
})
|
||||
|
||||
const communities = commItems.map((c) => ({
|
||||
id: c.id,
|
||||
value: c.community ?? c.id,
|
||||
name: c.title ?? c.community ?? c.id,
|
||||
description: "",
|
||||
type: "custom" as const,
|
||||
filterIds: [] as string[],
|
||||
serverCount: 0,
|
||||
prefixCount: 0,
|
||||
action: "permit" as const,
|
||||
enabled: true,
|
||||
}))
|
||||
const routeCountsByCommUuid = aggregateCommunityRouteCounts(catalog)
|
||||
const serversByCommunityStr = distinctServersByFilterCommunityValue()
|
||||
|
||||
const communities = commItems.map((c) => {
|
||||
const valueStr = (c.community ?? "").trim()
|
||||
return {
|
||||
id: c.id,
|
||||
value: c.community ?? c.id,
|
||||
name: c.title ?? c.community ?? c.id,
|
||||
description: "",
|
||||
type: "custom" as const,
|
||||
filterIds: [] as string[],
|
||||
serverCount: valueStr ? (serversByCommunityStr.get(valueStr) ?? 0) : 0,
|
||||
prefixCount: routeCountsByCommUuid.get(c.id) ?? 0,
|
||||
action: "permit" as const,
|
||||
enabled: true,
|
||||
}
|
||||
})
|
||||
|
||||
return reply.send({
|
||||
fetchedAt,
|
||||
|
||||
@@ -3,18 +3,10 @@ import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import { db } from "../db/index.js"
|
||||
import { filterRules, recursiveRoutes, servers } from "../db/schema.js"
|
||||
import { MikrotikClient } from "../services/mikrotik.js"
|
||||
import { parseDbServerId } from "../utils/server-id.js"
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect
|
||||
|
||||
function parseServerId(raw: unknown): number | null {
|
||||
const direct = Number.parseInt(String(raw ?? ""), 10)
|
||||
if (Number.isFinite(direct)) return direct
|
||||
const fromLegacy = String(raw ?? "").match(/(\d+)/)?.[1]
|
||||
if (!fromLegacy) return null
|
||||
const parsed = Number.parseInt(fromLegacy, 10)
|
||||
return Number.isFinite(parsed) ? parsed : null
|
||||
}
|
||||
|
||||
interface RosGre {
|
||||
".id"?: string
|
||||
name?: string
|
||||
@@ -95,23 +87,55 @@ function parseInnerIps(rule: string | undefined): { localInnerIp: string; remote
|
||||
|
||||
function parseFilterRule(raw: RosFilterRule): ApiFilterRule[] {
|
||||
const text = raw.rule ?? ""
|
||||
// RouterOS rule formats vary across versions/config styles:
|
||||
// - bgp-communities.has("AS:NNN")
|
||||
// - bgp-communities includes AS:NNN
|
||||
const communities = [
|
||||
...text.matchAll(/bgp-communities\.has\("([^"]+)"\)/gi),
|
||||
...text.matchAll(/bgp-communities\s+includes\s+([0-9]+:[0-9]+)/gi),
|
||||
].map(m => m[1]).filter(Boolean)
|
||||
const inlineComment = text.match(/#\s*([^\n]+)/)?.[1]?.trim()
|
||||
const out: ApiFilterRule[] = []
|
||||
|
||||
const extractCommunities = (src: string): string[] => {
|
||||
const communities = [
|
||||
...src.matchAll(/bgp-communities\.has\("([^"]+)"\)/gi),
|
||||
...src.matchAll(/bgp-communities\s+includes\s+([0-9]+:[0-9]+)/gi),
|
||||
].map(m => m[1]).filter(Boolean)
|
||||
return [...new Set(communities)]
|
||||
}
|
||||
|
||||
// Парсим по веткам if/else if, чтобы action применялся к "своим" communities.
|
||||
const branches = [...text.matchAll(/(?:if|else\s+if)\s*\(([\s\S]*?)\)\s*\{([\s\S]*?)\}/gi)]
|
||||
if (branches.length > 0) {
|
||||
for (const b of branches) {
|
||||
const condition = b[1] ?? ""
|
||||
const body = b[2] ?? ""
|
||||
const comms = extractCommunities(condition)
|
||||
if (comms.length === 0) continue
|
||||
|
||||
const isBlackhole =
|
||||
/set\s+type\s+blackhole/i.test(body) ||
|
||||
/set\s+blackhole\s+(?:yes|true)/i.test(body)
|
||||
const gwToken = body.match(/set\s+gw(?:ateway)?\s+([^\s;]+)/i)?.[1] ?? ""
|
||||
const outIface = body.match(/set\s+out-interface\s+([^\s;]+)/i)?.[1] ?? ""
|
||||
|
||||
for (const community of comms) {
|
||||
out.push({
|
||||
id: `${raw[".id"] ?? "live"}-${out.length}`,
|
||||
community,
|
||||
action: isBlackhole ? "blackhole" : "route",
|
||||
gateway: isBlackhole ? "" : gwToken,
|
||||
gatewayTunnelId: isBlackhole ? "" : outIface,
|
||||
description: inlineComment || raw.comment?.trim() || "",
|
||||
})
|
||||
}
|
||||
}
|
||||
if (out.length > 0) return out
|
||||
}
|
||||
|
||||
// Fallback для старого/плоского синтаксиса без явных блоков.
|
||||
const communities = extractCommunities(text)
|
||||
if (communities.length === 0) return []
|
||||
|
||||
const uniqueCommunities = [...new Set(communities)]
|
||||
const isBlackhole = /set\s+type\s+blackhole/i.test(text)
|
||||
const isBlackhole =
|
||||
/set\s+type\s+blackhole/i.test(text) ||
|
||||
/set\s+blackhole\s+(?:yes|true)/i.test(text)
|
||||
const gwToken = text.match(/set\s+gw(?:ateway)?\s+([^\s;]+)/i)?.[1] ?? ""
|
||||
const outIface = text.match(/set\s+out-interface\s+([^\s;]+)/i)?.[1] ?? ""
|
||||
const inlineComment = text.match(/#\s*([^\n]+)/)?.[1]?.trim()
|
||||
|
||||
return uniqueCommunities.map((community, idx) => ({
|
||||
return communities.map((community, idx) => ({
|
||||
id: `${raw[".id"] ?? "live"}-${idx}`,
|
||||
community,
|
||||
action: isBlackhole ? "blackhole" : "route",
|
||||
@@ -331,7 +355,7 @@ const filtersRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
/** Сравнение правил в БД с живым bgp-in на MikroTik (один запрос API к роутеру) */
|
||||
app.get("/filters/router-compare", async (req, reply) => {
|
||||
const q = req.query as { serverId?: string | number }
|
||||
const serverId = parseServerId(q.serverId)
|
||||
const serverId = parseDbServerId(q.serverId)
|
||||
if (serverId === null) {
|
||||
return reply.status(400).send({ error: "serverId is required" })
|
||||
}
|
||||
@@ -369,7 +393,7 @@ const filtersRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
/** GRE с роутеров: один сервер (?serverId) или все включённые (без query) — для /gre, карты сети */
|
||||
app.get("/filters/gre-tunnels", async (req, reply) => {
|
||||
const q = req.query as { serverId?: string | number }
|
||||
const sid = parseServerId(q.serverId)
|
||||
const sid = parseDbServerId(q.serverId)
|
||||
const enabledServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
|
||||
if (sid !== null) {
|
||||
@@ -423,7 +447,7 @@ const filtersRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
app.post("/filters/sync/from-router", async (_req, reply) => {
|
||||
const body = _req.body as { serverId?: string | number } | undefined
|
||||
const rawServerId = body?.serverId
|
||||
const serverId = parseServerId(rawServerId)
|
||||
const serverId = parseDbServerId(rawServerId)
|
||||
if (serverId === null) {
|
||||
return reply.status(400).send({ error: "serverId is required" })
|
||||
}
|
||||
|
||||
+400
-2
@@ -1,7 +1,7 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import { and, desc, eq } from "drizzle-orm"
|
||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers } from "../db/schema.js"
|
||||
import { serverSnapshots, servers, trafficSamples, uptimeSpeedProbes } from "../db/schema.js"
|
||||
import { MikrotikClient } from "../services/mikrotik.js"
|
||||
import { ServerIdParamSchema } from "../types/server.js"
|
||||
import type {
|
||||
@@ -9,8 +9,15 @@ import type {
|
||||
RosBfdSession,
|
||||
OspfNeighborRead, OspfInterfaceRead, OspfInstanceRead, BfdSessionRead,
|
||||
} from "../types/server.js"
|
||||
import { z } from "zod"
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect
|
||||
type TrafficSampleRow = typeof trafficSamples.$inferSelect
|
||||
type SpeedProbeRow = typeof uptimeSpeedProbes.$inferSelect
|
||||
|
||||
const OspfOptimizeBodySchema = z.object({
|
||||
pingWeight: z.number().int().min(0).max(100).default(60),
|
||||
})
|
||||
|
||||
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -193,6 +200,328 @@ function parseInstances(
|
||||
}))
|
||||
}
|
||||
|
||||
function calcRouteScore(pingMs: number, dlMbps: number, ulMbps: number, pingWeight: number) {
|
||||
const pingScore = Math.max(0, 100 - pingMs * 0.6)
|
||||
const speedScore = Math.min(100, (dlMbps + ulMbps) / 18)
|
||||
const w = pingWeight / 100
|
||||
return Math.round(w * pingScore + (1 - w) * speedScore)
|
||||
}
|
||||
|
||||
function getLatestSnapshotLatencyMs(serverId: number): number {
|
||||
const latest = db
|
||||
.select()
|
||||
.from(serverSnapshots)
|
||||
.where(eq(serverSnapshots.serverId, serverId))
|
||||
.orderBy(desc(serverSnapshots.polledAt))
|
||||
.limit(1)
|
||||
.all()
|
||||
return latest.length > 0 ? Math.max(1, Math.round(latest[0].latencyMs ?? 100)) : 100
|
||||
}
|
||||
|
||||
function latestTrafficByInterface(serverId: number): Map<string, TrafficSampleRow> {
|
||||
const rows = db
|
||||
.select()
|
||||
.from(trafficSamples)
|
||||
.where(eq(trafficSamples.serverId, serverId))
|
||||
.all()
|
||||
const map = new Map<string, TrafficSampleRow>()
|
||||
for (const row of rows) {
|
||||
const prev = map.get(row.interfaceName)
|
||||
if (!prev || row.sampledAt > prev.sampledAt) map.set(row.interfaceName, row)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
function normalizeSpeedMbps(row: TrafficSampleRow | undefined): { dlMbps: number; ulMbps: number } {
|
||||
if (!row) return { dlMbps: 10, ulMbps: 10 }
|
||||
const dl = Math.max(1, Math.round((row.rxBps ?? 0) / 1_000_000))
|
||||
const ul = Math.max(1, Math.round((row.txBps ?? 0) / 1_000_000))
|
||||
return { dlMbps: dl, ulMbps: ul }
|
||||
}
|
||||
|
||||
function isRefInterfaceName(name: string): boolean {
|
||||
return /^\(ref\s+\*.+\)$/.test(name.trim())
|
||||
}
|
||||
|
||||
function parsePingTimeMs(raw: string | undefined): number | null {
|
||||
if (!raw) return null
|
||||
const s = String(raw).trim().toLowerCase().replace(",", ".")
|
||||
const us = s.match(/^(\d+(?:\.\d+)?)\s*us$/)
|
||||
if (us) return Number.parseFloat(us[1]) / 1000
|
||||
const ms = s.match(/^(\d+(?:\.\d+)?)\s*ms$/)
|
||||
if (ms) return Number.parseFloat(ms[1])
|
||||
const sec = s.match(/^(\d+(?:\.\d+)?)\s*s$/)
|
||||
if (sec) return Number.parseFloat(sec[1]) * 1000
|
||||
const clock = s.match(/^(\d+):(\d+):(\d+(?:\.\d+)?)$/)
|
||||
if (clock) {
|
||||
const h = Number.parseFloat(clock[1])
|
||||
const m = Number.parseFloat(clock[2])
|
||||
const sc = Number.parseFloat(clock[3])
|
||||
return (h * 3600 + m * 60 + sc) * 1000
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function avgPingMsFromResults(rows: Array<{ time?: string; status?: string }>): number | null {
|
||||
const ok = rows.filter((r) => r.time && String(r.status ?? "").toLowerCase() !== "timeout")
|
||||
const vals = ok
|
||||
.map((r) => parsePingTimeMs(r.time))
|
||||
.filter((v): v is number => v != null && Number.isFinite(v))
|
||||
if (!vals.length) return null
|
||||
return Math.max(1, Math.round(vals.reduce((a, b) => a + b, 0) / vals.length))
|
||||
}
|
||||
|
||||
async function measurePingByInterface(
|
||||
client: MikrotikClient,
|
||||
ifaceName: string,
|
||||
targetAddr: string | undefined,
|
||||
fallbackMs: number,
|
||||
): Promise<number> {
|
||||
if (!targetAddr) return fallbackMs
|
||||
try {
|
||||
const rows = await client.ping(targetAddr, 2, ifaceName, { interval: "0.2s" })
|
||||
return avgPingMsFromResults(rows) ?? fallbackMs
|
||||
} catch {
|
||||
return fallbackMs
|
||||
}
|
||||
}
|
||||
|
||||
type OspfRankedInterface = {
|
||||
id: string
|
||||
interface: string
|
||||
currentCost: number
|
||||
pingMs: number
|
||||
dlMbps: number
|
||||
ulMbps: number
|
||||
score: number
|
||||
optimalCost: number
|
||||
}
|
||||
|
||||
async function buildOspfOptimizationPlan(server: ServerRow, pingWeight: number): Promise<OspfRankedInterface[]> {
|
||||
const { ifaceTemplates, areas, instances, neighbors } = await fetchServerOspf(server)
|
||||
const areaMap = buildAreaMap(areas)
|
||||
const parsed = parseInterfaces(server, ifaceTemplates, areas, instances, areaMap)
|
||||
const editable = parsed.filter((i) => !i.disabled && !isRefInterfaceName(i.interface))
|
||||
const localLatencyMs = getLatestSnapshotLatencyMs(server.id)
|
||||
const latestTraffic = latestTrafficByInterface(server.id)
|
||||
|
||||
// 1) Сопоставляем remote OSPF router-id -> сервер из каталога (чтобы взять унифицированный ping как в карте /route-optimizer).
|
||||
const neededRouterIds = new Set(neighbors.map((n) => String(n["router-id"] ?? "")).filter((x) => x.length > 0))
|
||||
|
||||
async function buildRouterIdToServerMap(needed: Set<string>): Promise<Map<string, ServerRow["id"]>> {
|
||||
const enabledServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
const out = new Map<string, ServerRow["id"]>()
|
||||
const neededLeft = new Set(needed)
|
||||
|
||||
for (const s of enabledServers) {
|
||||
if (neededLeft.size === 0) break
|
||||
try {
|
||||
const c = MikrotikClient.fromServer(s)
|
||||
const inst = await c.getOspfInstances()
|
||||
for (const i of inst) {
|
||||
const rid = String(i["router-id"] ?? "").trim()
|
||||
if (!rid) continue
|
||||
if (neededLeft.has(rid)) {
|
||||
out.set(rid, s.id)
|
||||
neededLeft.delete(rid)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore per-server errors, preview should still work
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const routerIdToServerId = await buildRouterIdToServerMap(neededRouterIds)
|
||||
|
||||
// 2) Сначала выбираем remote serverId для каждого OSPF интерфейса (по neighbor router-id).
|
||||
function pickBestNeighborForInterface(iface: string) {
|
||||
const cand = neighbors.filter((n) => n.interface === iface)
|
||||
if (!cand.length) return null
|
||||
const rank = (st: string) => (st === "Full" ? 100 : st === "2-Way" ? 60 : 30)
|
||||
return cand.sort((a, b) => rank(b.state) - rank(a.state))[0] ?? null
|
||||
}
|
||||
|
||||
const remoteServerIds = new Set<number>()
|
||||
const remoteForIface = new Map<string, number>()
|
||||
for (const iface of editable) {
|
||||
const n = pickBestNeighborForInterface(iface.interface)
|
||||
if (!n) continue
|
||||
const remoteServerId = routerIdToServerId.get(String(n["router-id"] ?? "").trim())
|
||||
if (remoteServerId != null) {
|
||||
remoteServerIds.add(remoteServerId)
|
||||
remoteForIface.set(iface.interface, remoteServerId)
|
||||
}
|
||||
}
|
||||
|
||||
// 3) Предгружаем uptimeSpeedProbes: оттуда берём и ping, и скорость (единый источник как у карты).
|
||||
const allSourceProbes = db
|
||||
.select()
|
||||
.from(uptimeSpeedProbes)
|
||||
.where(eq(uptimeSpeedProbes.srcServerId, server.id))
|
||||
.all()
|
||||
.filter((r: SpeedProbeRow) => r.enabled !== false)
|
||||
const speedProbesByDest = new Map<number, SpeedProbeRow[]>()
|
||||
for (const p of allSourceProbes) {
|
||||
const arr = speedProbesByDest.get(p.dstServerId) ?? []
|
||||
arr.push(p)
|
||||
speedProbesByDest.set(p.dstServerId, arr)
|
||||
}
|
||||
|
||||
function normIface(s: string) {
|
||||
return s.trim().toLowerCase()
|
||||
}
|
||||
|
||||
function pickBestSpeedProbe(probes: SpeedProbeRow[], ifaceName: string): SpeedProbeRow | undefined {
|
||||
if (probes.length === 0) return undefined
|
||||
const scoreProbeForIface = (p: SpeedProbeRow, targetIface: string): number => {
|
||||
const got = normIface(p.srcInterface)
|
||||
const target = normIface(targetIface)
|
||||
// Сверяемся с /gre: если у пробы есть srcInterface и у OSPF iface есть имя,
|
||||
// допускаем только точное совпадение интерфейса.
|
||||
if (got && target && got !== target) return -1
|
||||
const ifaceScore =
|
||||
got && target && got === target ? 100
|
||||
: (!got && target) ? 35
|
||||
: (!target && got) ? 15
|
||||
: 5
|
||||
const freshness =
|
||||
p.lastPingAt ? Math.max(0, 20 - Math.floor((Date.now() - Date.parse(p.lastPingAt)) / (60 * 60 * 1000))) : 0
|
||||
return ifaceScore
|
||||
+ (p.lastPingRttMs != null ? 25 : 0)
|
||||
+ (p.lastTxAvgMbps != null ? 12 : 0)
|
||||
+ (p.lastRxAvgMbps != null ? 12 : 0)
|
||||
+ freshness
|
||||
}
|
||||
|
||||
let best: SpeedProbeRow | undefined
|
||||
let bestScore = -1
|
||||
for (const p of probes) {
|
||||
const s = scoreProbeForIface(p, ifaceName)
|
||||
if (s > bestScore) {
|
||||
bestScore = s
|
||||
best = p
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
function assignProbesUniquely(
|
||||
ifaceNames: string[],
|
||||
probes: SpeedProbeRow[],
|
||||
): Map<string, SpeedProbeRow> {
|
||||
const assigned = new Map<string, SpeedProbeRow>()
|
||||
if (ifaceNames.length === 0 || probes.length === 0) return assigned
|
||||
|
||||
const scoreProbeForIface = (p: SpeedProbeRow, ifaceName: string): number => {
|
||||
const got = normIface(p.srcInterface)
|
||||
const target = normIface(ifaceName)
|
||||
if (got && target && got !== target) return -1
|
||||
const ifaceScore =
|
||||
got && target && got === target ? 100
|
||||
: (!got && target) ? 35
|
||||
: (!target && got) ? 15
|
||||
: 5
|
||||
const freshness =
|
||||
p.lastPingAt ? Math.max(0, 20 - Math.floor((Date.now() - Date.parse(p.lastPingAt)) / (60 * 60 * 1000))) : 0
|
||||
return ifaceScore
|
||||
+ (p.lastPingRttMs != null ? 25 : 0)
|
||||
+ (p.lastTxAvgMbps != null ? 12 : 0)
|
||||
+ (p.lastRxAvgMbps != null ? 12 : 0)
|
||||
+ freshness
|
||||
}
|
||||
|
||||
const leftIfaces = new Set(ifaceNames)
|
||||
const usedProbeIds = new Set<string>()
|
||||
|
||||
while (leftIfaces.size > 0) {
|
||||
let bestIface: string | null = null
|
||||
let bestProbe: SpeedProbeRow | null = null
|
||||
let bestScore = -1
|
||||
|
||||
for (const iface of leftIfaces) {
|
||||
for (const p of probes) {
|
||||
if (usedProbeIds.has(p.id)) continue
|
||||
const s = scoreProbeForIface(p, iface)
|
||||
if (s > bestScore) {
|
||||
bestScore = s
|
||||
bestIface = iface
|
||||
bestProbe = p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!bestIface || !bestProbe) break
|
||||
assigned.set(bestIface, bestProbe)
|
||||
leftIfaces.delete(bestIface)
|
||||
usedProbeIds.add(bestProbe.id)
|
||||
}
|
||||
|
||||
// Если проб меньше чем интерфейсов — добираем лучшими совпадениями (reuse допустим).
|
||||
for (const iface of leftIfaces) {
|
||||
const fallback = pickBestSpeedProbe(probes, iface)
|
||||
if (fallback) assigned.set(iface, fallback)
|
||||
}
|
||||
|
||||
return assigned
|
||||
}
|
||||
|
||||
const assignedProbeByIface = new Map<string, SpeedProbeRow>()
|
||||
const ifacesByDst = new Map<number | null, string[]>()
|
||||
for (const iface of editable) {
|
||||
const dst = remoteForIface.get(iface.interface) ?? null
|
||||
const arr = ifacesByDst.get(dst) ?? []
|
||||
arr.push(iface.interface)
|
||||
ifacesByDst.set(dst, arr)
|
||||
}
|
||||
for (const [dst, ifaceList] of ifacesByDst) {
|
||||
const probesByDst = dst != null ? speedProbesByDest.get(dst) ?? [] : []
|
||||
const pool = probesByDst.length > 0 ? probesByDst : allSourceProbes
|
||||
const assigned = assignProbesUniquely(ifaceList, pool)
|
||||
for (const [iface, probe] of assigned.entries()) assignedProbeByIface.set(iface, probe)
|
||||
}
|
||||
|
||||
return editable
|
||||
.map((iface) => {
|
||||
const t = latestTraffic.get(iface.interface)
|
||||
const speed = normalizeSpeedMbps(t)
|
||||
|
||||
const dstServerId = remoteForIface.get(iface.interface)
|
||||
const bestProbe = assignedProbeByIface.get(iface.interface)
|
||||
|
||||
const pingMs =
|
||||
bestProbe?.lastPingRttMs != null
|
||||
? Math.max(1, Math.round(bestProbe.lastPingRttMs))
|
||||
: dstServerId != null
|
||||
? Math.min(995, Math.round(localLatencyMs + getLatestSnapshotLatencyMs(dstServerId)))
|
||||
: localLatencyMs
|
||||
|
||||
const dlMbps =
|
||||
bestProbe?.lastTxAvgMbps != null
|
||||
? Math.max(1, Math.round(bestProbe.lastTxAvgMbps))
|
||||
: speed.dlMbps
|
||||
|
||||
const ulMbps =
|
||||
bestProbe?.lastRxAvgMbps != null
|
||||
? Math.max(1, Math.round(bestProbe.lastRxAvgMbps))
|
||||
: speed.ulMbps
|
||||
|
||||
return {
|
||||
id: iface.id,
|
||||
interface: iface.interface,
|
||||
currentCost: iface.cost,
|
||||
pingMs,
|
||||
dlMbps,
|
||||
ulMbps,
|
||||
score: calcRouteScore(pingMs, dlMbps, ulMbps, pingWeight),
|
||||
optimalCost: 0,
|
||||
}
|
||||
})
|
||||
.sort((a, b) => b.score - a.score || a.interface.localeCompare(b.interface))
|
||||
.map((row, idx) => ({ ...row, optimalCost: (idx + 1) * 10 }))
|
||||
}
|
||||
|
||||
// ── route plugin ──────────────────────────────────────────────────────────────
|
||||
|
||||
const ospfRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
@@ -322,6 +651,75 @@ const ospfRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
return reply.status(500).send({ error: String(err) })
|
||||
}
|
||||
})
|
||||
|
||||
// POST /api/servers/:id/ospf/optimize — compute + apply optimal OSPF cost for active templates
|
||||
app.post(
|
||||
"/servers/:id/ospf/optimize/preview",
|
||||
{ schema: { params: ServerIdParamSchema, body: OspfOptimizeBodySchema } },
|
||||
async (req, reply) => {
|
||||
const server = db
|
||||
.select().from(servers)
|
||||
.where(eq(servers.id, req.params.id))
|
||||
.limit(1).all()[0]
|
||||
|
||||
if (!server) return reply.status(404).send({ error: "Server not found" })
|
||||
|
||||
try {
|
||||
const pingWeight = req.body.pingWeight
|
||||
const ranked = await buildOspfOptimizationPlan(server, pingWeight)
|
||||
const changed = ranked.filter((r) => r.currentCost !== r.optimalCost)
|
||||
return reply.send({
|
||||
serverId: server.id,
|
||||
serverName: server.name || server.host,
|
||||
pingWeight,
|
||||
interfacesTotal: ranked.length,
|
||||
changedCount: changed.length,
|
||||
unchangedCount: ranked.length - changed.length,
|
||||
interfaces: ranked,
|
||||
changes: changed,
|
||||
})
|
||||
} catch (err) {
|
||||
return reply.status(500).send({ error: String(err) })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
app.post(
|
||||
"/servers/:id/ospf/optimize",
|
||||
{ schema: { params: ServerIdParamSchema, body: OspfOptimizeBodySchema } },
|
||||
async (req, reply) => {
|
||||
const server = db
|
||||
.select().from(servers)
|
||||
.where(eq(servers.id, req.params.id))
|
||||
.limit(1).all()[0]
|
||||
|
||||
if (!server) return reply.status(404).send({ error: "Server not found" })
|
||||
|
||||
try {
|
||||
const pingWeight = req.body.pingWeight
|
||||
const ranked = await buildOspfOptimizationPlan(server, pingWeight)
|
||||
|
||||
const client = MikrotikClient.fromServer(server)
|
||||
const changed = ranked.filter((r) => r.currentCost !== r.optimalCost)
|
||||
const applied: Array<{ interface: string; from: number; to: number }> = []
|
||||
for (const item of changed) {
|
||||
await client.setOspfInterfaceTemplateCost(item.id, item.optimalCost)
|
||||
applied.push({ interface: item.interface, from: item.currentCost, to: item.optimalCost })
|
||||
}
|
||||
|
||||
return reply.send({
|
||||
serverId: server.id,
|
||||
serverName: server.name || server.host,
|
||||
pingWeight,
|
||||
optimizedCount: changed.length,
|
||||
applied,
|
||||
interfaces: ranked,
|
||||
})
|
||||
} catch (err) {
|
||||
return reply.status(500).send({ error: String(err) })
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
export default ospfRoutes
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import { db } from "../db/index.js"
|
||||
import { recursiveRoutes, servers } from "../db/schema.js"
|
||||
import { MikrotikClient } from "../services/mikrotik.js"
|
||||
import { parseDbServerId } from "../utils/server-id.js"
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect
|
||||
|
||||
@@ -46,15 +47,6 @@ interface RecursiveRouteDto {
|
||||
disabled: boolean
|
||||
}
|
||||
|
||||
function parseServerId(raw: unknown): number | null {
|
||||
const direct = Number.parseInt(String(raw ?? ""), 10)
|
||||
if (Number.isFinite(direct)) return direct
|
||||
const legacy = String(raw ?? "").match(/(\d+)/)?.[1]
|
||||
if (!legacy) return null
|
||||
const parsed = Number.parseInt(legacy, 10)
|
||||
return Number.isFinite(parsed) ? parsed : null
|
||||
}
|
||||
|
||||
function isIpGateway(gw: string): boolean {
|
||||
return /^\d{1,3}(\.\d{1,3}){3}(?:%\S+)?$/.test(gw.trim())
|
||||
}
|
||||
@@ -150,7 +142,7 @@ async function replaceDbRoutes(serverId: number, routes: RecursiveRouteDto[]) {
|
||||
const recursiveRoutesPlugin: FastifyPluginAsyncZod = async (app) => {
|
||||
app.get("/recursive-routes/gateways", async (req, reply) => {
|
||||
const q = req.query as { serverId?: string | number }
|
||||
const serverId = parseServerId(q.serverId)
|
||||
const serverId = parseDbServerId(q.serverId)
|
||||
if (serverId === null) return reply.status(400).send({ error: "serverId is required" })
|
||||
const server = db.select().from(servers).where(eq(servers.id, serverId)).limit(1).all()[0]
|
||||
if (!server) return reply.status(404).send({ error: "Server not found" })
|
||||
@@ -180,7 +172,7 @@ const recursiveRoutesPlugin: FastifyPluginAsyncZod = async (app) => {
|
||||
|
||||
app.get("/recursive-routes", async (req, reply) => {
|
||||
const q = req.query as { serverId?: string | number }
|
||||
const serverId = parseServerId(q.serverId)
|
||||
const serverId = parseDbServerId(q.serverId)
|
||||
if (serverId === null) return reply.status(400).send({ error: "serverId is required" })
|
||||
const server = db.select().from(servers).where(eq(servers.id, serverId)).limit(1).all()[0]
|
||||
if (!server) return reply.status(404).send({ error: "Server not found" })
|
||||
@@ -189,7 +181,7 @@ const recursiveRoutesPlugin: FastifyPluginAsyncZod = async (app) => {
|
||||
|
||||
app.put("/recursive-routes", async (req, reply) => {
|
||||
const body = req.body as { serverId?: string | number; routes?: RecursiveRouteDto[] }
|
||||
const serverId = parseServerId(body.serverId)
|
||||
const serverId = parseDbServerId(body.serverId)
|
||||
if (serverId === null) return reply.status(400).send({ error: "serverId is required" })
|
||||
const server = db.select().from(servers).where(eq(servers.id, serverId)).limit(1).all()[0]
|
||||
if (!server) return reply.status(404).send({ error: "Server not found" })
|
||||
@@ -199,7 +191,7 @@ const recursiveRoutesPlugin: FastifyPluginAsyncZod = async (app) => {
|
||||
|
||||
app.post("/recursive-routes/sync/from-router", async (req, reply) => {
|
||||
const body = req.body as { serverId?: string | number } | undefined
|
||||
const serverId = parseServerId(body?.serverId)
|
||||
const serverId = parseDbServerId(body?.serverId)
|
||||
if (serverId === null) return reply.status(400).send({ error: "serverId is required" })
|
||||
const server: ServerRow | undefined = db
|
||||
.select().from(servers)
|
||||
@@ -235,7 +227,7 @@ const recursiveRoutesPlugin: FastifyPluginAsyncZod = async (app) => {
|
||||
|
||||
app.post("/recursive-routes/sync/to-router", async (req, reply) => {
|
||||
const body = req.body as { serverId?: string | number } | undefined
|
||||
const serverId = parseServerId(body?.serverId)
|
||||
const serverId = parseDbServerId(body?.serverId)
|
||||
if (serverId === null) return reply.status(400).send({ error: "serverId is required" })
|
||||
const server = db.select().from(servers).where(eq(servers.id, serverId)).limit(1).all()[0]
|
||||
if (!server) return reply.status(404).send({ error: "Server not found" })
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import {
|
||||
getSchedulerStatus,
|
||||
JOB_KEYS,
|
||||
listSchedulerRuns,
|
||||
refreshScheduler,
|
||||
runSchedulerJobNow,
|
||||
} from "../services/scheduler.js"
|
||||
|
||||
function parseJobKey(raw: string | undefined): typeof JOB_KEYS[number] | null {
|
||||
const s = String(raw ?? "").trim()
|
||||
return (JOB_KEYS as readonly string[]).includes(s) ? (s as typeof JOB_KEYS[number]) : null
|
||||
}
|
||||
|
||||
const schedulerRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
app.get("/scheduler/status", async (_req, reply) => {
|
||||
return reply.send(getSchedulerStatus())
|
||||
})
|
||||
|
||||
app.get("/scheduler/runs", async (req, reply) => {
|
||||
const q = req.query as { jobKey?: string; limit?: string; offset?: string }
|
||||
const jobKey = q.jobKey ? parseJobKey(q.jobKey) : undefined
|
||||
if (q.jobKey && jobKey == null) {
|
||||
return reply.status(400).send({ error: "Некорректный jobKey" })
|
||||
}
|
||||
const limit = Math.min(200, Math.max(1, Number.parseInt(String(q.limit ?? "50"), 10) || 50))
|
||||
const offset = Math.max(0, Number.parseInt(String(q.offset ?? "0"), 10) || 0)
|
||||
return reply.send(listSchedulerRuns({ jobKey: jobKey ?? undefined, limit, offset }))
|
||||
})
|
||||
|
||||
app.post("/scheduler/refresh", async (_req, reply) => {
|
||||
refreshScheduler()
|
||||
return reply.send({ ok: true, status: getSchedulerStatus() })
|
||||
})
|
||||
|
||||
app.post("/scheduler/jobs/:jobKey/run-now", async (req, reply) => {
|
||||
const jobKey = parseJobKey(String((req.params as { jobKey?: string }).jobKey ?? ""))
|
||||
if (!jobKey) return reply.status(400).send({ error: "Некорректный jobKey" })
|
||||
try {
|
||||
await runSchedulerJobNow(jobKey)
|
||||
return reply.send({ ok: true, status: getSchedulerStatus() })
|
||||
} catch (e) {
|
||||
const statusCode = (e as Error & { statusCode?: number }).statusCode
|
||||
if (statusCode === 409) {
|
||||
return reply.status(409).send({ error: "Задача уже выполняется" })
|
||||
}
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
return reply.status(500).send({ error: msg })
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export default schedulerRoutes
|
||||
@@ -0,0 +1,62 @@
|
||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import { scheduleAlertEngineAfterDataCollectors } from "../services/alert-collector-hooks.js"
|
||||
import { refreshScheduler } from "../services/scheduler.js"
|
||||
import {
|
||||
collectServersRestPingOnce,
|
||||
getServersApiPingSettings,
|
||||
getServersRestPingCollectorState,
|
||||
updateServersApiPingSettings,
|
||||
} from "../services/servers-rest-ping-collector.js"
|
||||
|
||||
const serversApiPingRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
app.get("/servers-api-ping/settings", async (_req, reply) => {
|
||||
const settings = getServersApiPingSettings()
|
||||
const state = getServersRestPingCollectorState()
|
||||
return reply.send({
|
||||
enabled: settings.enabled,
|
||||
intervalSec: settings.intervalSec,
|
||||
lastCollectedAt: settings.lastCollectedAt ?? null,
|
||||
lastDurationMs: settings.lastDurationMs ?? null,
|
||||
lastError: settings.lastError || null,
|
||||
collectorRunning: state.running,
|
||||
})
|
||||
})
|
||||
|
||||
app.put("/servers-api-ping/settings", async (req, reply) => {
|
||||
const body = req.body as {
|
||||
enabled?: boolean
|
||||
intervalSec?: number | string
|
||||
}
|
||||
const intervalSec =
|
||||
body.intervalSec == null ? undefined : Math.max(10, Number.parseInt(String(body.intervalSec), 10) || 120)
|
||||
const updated = updateServersApiPingSettings({
|
||||
enabled: body.enabled,
|
||||
intervalSec,
|
||||
})
|
||||
refreshScheduler()
|
||||
return reply.send({
|
||||
ok: true,
|
||||
settings: {
|
||||
enabled: updated.enabled,
|
||||
intervalSec: updated.intervalSec,
|
||||
lastCollectedAt: updated.lastCollectedAt ?? null,
|
||||
lastDurationMs: updated.lastDurationMs ?? null,
|
||||
lastError: updated.lastError || null,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
app.post("/servers-api-ping/collect-now", async (_req, reply) => {
|
||||
await collectServersRestPingOnce()
|
||||
scheduleAlertEngineAfterDataCollectors()
|
||||
const row = getServersApiPingSettings()
|
||||
return reply.send({
|
||||
ok: true,
|
||||
lastCollectedAt: row.lastCollectedAt ?? null,
|
||||
lastDurationMs: row.lastDurationMs ?? null,
|
||||
lastError: row.lastError || null,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export default serversApiPingRoutes
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import { db } from "../db/index.js"
|
||||
import {
|
||||
filterRules,
|
||||
recursiveRoutes,
|
||||
servers,
|
||||
uptimeProbes,
|
||||
uptimeSpeedProbes,
|
||||
} from "../db/schema.js"
|
||||
|
||||
const sidebarCountsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
app.get("/sidebar-counts", async (_req, reply) => {
|
||||
const [
|
||||
serversTotal,
|
||||
filterRulesTotal,
|
||||
uptimeProbesTotal,
|
||||
uptimeSpeedProbesTotal,
|
||||
recursiveRoutesTotal,
|
||||
] = [
|
||||
db.select().from(servers).all().length,
|
||||
db.select().from(filterRules).all().length,
|
||||
db.select().from(uptimeProbes).all().length,
|
||||
db.select().from(uptimeSpeedProbes).all().length,
|
||||
db.select().from(recursiveRoutes).all().length,
|
||||
]
|
||||
|
||||
return reply.send({
|
||||
servers: serversTotal,
|
||||
filterRules: filterRulesTotal,
|
||||
uptimeProbes: uptimeProbesTotal,
|
||||
uptimeSpeedProbes: uptimeSpeedProbesTotal,
|
||||
monitoringItems: uptimeProbesTotal + uptimeSpeedProbesTotal,
|
||||
recursiveRoutes: recursiveRoutesTotal,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export default sidebarCountsRoutes
|
||||
@@ -9,6 +9,8 @@ import {
|
||||
readServerSamplesInRange,
|
||||
updateTrafficSettings,
|
||||
} from "../services/traffic-collector.js"
|
||||
import { scheduleAlertEngineAfterDataCollectors } from "../services/alert-collector-hooks.js"
|
||||
import { refreshScheduler } from "../services/scheduler.js"
|
||||
|
||||
type SnapshotRow = typeof serverSnapshots.$inferSelect
|
||||
|
||||
@@ -189,6 +191,7 @@ const trafficRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
intervalSec,
|
||||
retentionDays,
|
||||
})
|
||||
refreshScheduler()
|
||||
return reply.send({
|
||||
ok: true,
|
||||
settings: {
|
||||
@@ -204,6 +207,7 @@ const trafficRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
|
||||
app.post("/traffic/collect-now", async (_req, reply) => {
|
||||
await collectTrafficOnce()
|
||||
scheduleAlertEngineAfterDataCollectors()
|
||||
const updated = getTrafficSettings()
|
||||
return reply.send({
|
||||
ok: true,
|
||||
|
||||
+125
-245
@@ -3,10 +3,11 @@ import { asc, desc, eq } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers, uptimeSpeedProbes, uptimeSpeedTestRuns } from "../db/schema.js"
|
||||
import { MikrotikClient } from "../services/mikrotik.js"
|
||||
import type { RosIpAddress } from "../types/server.js"
|
||||
import { scheduleAlertEngineAfterDataCollectors } from "../services/alert-collector-hooks.js"
|
||||
import { refreshScheduler, getSchedulerStatus } from "../services/scheduler.js"
|
||||
import {
|
||||
collectPingForProbeIds,
|
||||
collectUptimeOnce,
|
||||
parsePing,
|
||||
readProbeRows,
|
||||
readProbeSamplesSince,
|
||||
readResourceSamplesSince,
|
||||
@@ -15,6 +16,7 @@ import {
|
||||
updateProbeShowOnDashboard,
|
||||
updateUptimeSettings,
|
||||
} from "../services/uptime-collector.js"
|
||||
import { recordSpeedTestFailure, runBandwidthSpeedTest } from "../services/uptime-speed-service.js"
|
||||
|
||||
function rangeToMinutes(range: string | undefined): number {
|
||||
switch ((range ?? "1h").toLowerCase()) {
|
||||
@@ -34,69 +36,33 @@ function toSeries(values: number[], target = 40): number[] {
|
||||
return [...Array(target - values.length).fill(values[0] ?? 0), ...values]
|
||||
}
|
||||
|
||||
/** Сэмпл ресурсов из SQLite (readResourceSamplesSince). */
|
||||
type ResourceSampleRow = ReturnType<typeof readResourceSamplesSince>[number]
|
||||
|
||||
/**
|
||||
* Один «рабочий» IPv4/IPv6 для интерфейса: пропускаем записи /ip/address с disabled/invalid,
|
||||
* среди оставшихся предпочитаем статические адреса динамическим (иначе БТ может цепляться к отключённому IP).
|
||||
* Последний сэмпл по времени может быть offline после сбоя опроса — тогда UI «пустой».
|
||||
* Берём последний успешный (online + RAM), только если он не слишком старый относительно последней попытки,
|
||||
* иначе снова показываем устаревший uptime/метрики как раньше.
|
||||
*/
|
||||
function firstUsableIpOnInterface(rows: RosIpAddress[], iface: string): string {
|
||||
const name = iface.trim()
|
||||
const candidates = rows.filter((a) => {
|
||||
if (String(a.interface ?? "").trim() !== name) return false
|
||||
if (String(a.disabled ?? "false").toLowerCase() === "true") return false
|
||||
if (String(a.invalid ?? "false").toLowerCase() === "true") return false
|
||||
const addr = String(a.address ?? "").trim()
|
||||
return addr.includes("/")
|
||||
})
|
||||
candidates.sort((a, b) => {
|
||||
const ad = String(a.dynamic ?? "false").toLowerCase() === "true" ? 1 : 0
|
||||
const bd = String(b.dynamic ?? "false").toLowerCase() === "true" ? 1 : 0
|
||||
return ad - bd
|
||||
})
|
||||
const raw = candidates[0]?.address
|
||||
return raw ? String(raw).split("/")[0]?.trim() ?? "" : ""
|
||||
}
|
||||
function pickResourceDisplayRow(rows: ResourceSampleRow[], maxGapMs: number): {
|
||||
row: ResourceSampleRow | undefined
|
||||
hasData: boolean
|
||||
} {
|
||||
if (rows.length === 0) return { row: undefined, hasData: false }
|
||||
const last = rows[rows.length - 1]!
|
||||
const lastTs = Date.parse(last.sampledAt)
|
||||
const validOnline = (r: ResourceSampleRow) => r.status === "online" && r.totalMemory > 0
|
||||
|
||||
function parseRateToMbps(raw: unknown): number {
|
||||
const txt = String(raw ?? "").trim().toLowerCase()
|
||||
if (!txt) return 0
|
||||
const m = txt.match(/^(\d+(?:\.\d+)?)\s*(g|m|k)?(?:bps|bit\/s|bits\/s)?$/)
|
||||
if (m) {
|
||||
const n = Number.parseFloat(m[1])
|
||||
const u = m[2] ?? ""
|
||||
if (!Number.isFinite(n)) return 0
|
||||
if (u === "g") return n * 1000
|
||||
if (u === "m") return n
|
||||
if (u === "k") return n / 1000
|
||||
return n / 1_000_000
|
||||
if (validOnline(last)) {
|
||||
return { row: last, hasData: true }
|
||||
}
|
||||
const n = Number.parseFloat(txt.replace(/[^\d.]/g, ""))
|
||||
if (!Number.isFinite(n)) return 0
|
||||
return n > 10000 ? n / 1_000_000 : n
|
||||
}
|
||||
|
||||
const btestNodeLocks = new Map<number, Promise<void>>()
|
||||
const fallback = [...rows].reverse().find(validOnline)
|
||||
if (!fallback) return { row: undefined, hasData: false }
|
||||
|
||||
async function withBtestNodeLocks<T>(nodeIds: number[], fn: () => Promise<T>): Promise<T> {
|
||||
const ids = [...new Set(nodeIds.filter((id) => Number.isFinite(id)))].sort((a, b) => a - b)
|
||||
const releases: Array<() => void> = []
|
||||
try {
|
||||
for (const id of ids) {
|
||||
const prev = btestNodeLocks.get(id)
|
||||
let resolveCurrent!: () => void
|
||||
const current = new Promise<void>((resolve) => {
|
||||
resolveCurrent = resolve
|
||||
})
|
||||
btestNodeLocks.set(id, current)
|
||||
if (prev) await prev
|
||||
releases.push(() => {
|
||||
if (btestNodeLocks.get(id) === current) btestNodeLocks.delete(id)
|
||||
resolveCurrent()
|
||||
})
|
||||
}
|
||||
return await fn()
|
||||
} finally {
|
||||
for (let i = releases.length - 1; i >= 0; i -= 1) releases[i]()
|
||||
}
|
||||
const fbTs = Date.parse(fallback.sampledAt)
|
||||
if (lastTs - fbTs <= maxGapMs) return { row: fallback, hasData: true }
|
||||
return { row: undefined, hasData: false }
|
||||
}
|
||||
|
||||
const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
@@ -104,19 +70,28 @@ const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
const s = readUptimeSettings()
|
||||
return reply.send({
|
||||
enabled: s.enabled,
|
||||
resourcesEnabled: s.resourcesEnabled ?? s.enabled,
|
||||
pingEnabled: s.pingEnabled ?? s.enabled,
|
||||
speedEnabled: s.speedEnabled ?? s.enabled,
|
||||
/** Интервал сбора метрик ресурсов (сек). */
|
||||
intervalSec: s.intervalSec,
|
||||
probeIntervalSec: s.probeIntervalSec ?? s.intervalSec,
|
||||
/** Интервал тика ping и значение по умолчанию для проб без своего interval. */
|
||||
probeIntervalSec: s.probeIntervalSec,
|
||||
speedIntervalSec: s.speedIntervalSec ?? 60,
|
||||
retentionDays: s.retentionDays,
|
||||
lastCollectedAt: s.lastCollectedAt ?? null,
|
||||
lastDurationMs: s.lastDurationMs ?? null,
|
||||
lastError: s.lastError || null,
|
||||
scheduler: getSchedulerStatus(),
|
||||
})
|
||||
})
|
||||
|
||||
app.put("/uptime/settings", async (req, reply) => {
|
||||
const body = req.body as {
|
||||
enabled?: boolean
|
||||
resourcesEnabled?: boolean
|
||||
pingEnabled?: boolean
|
||||
speedEnabled?: boolean
|
||||
intervalSec?: number | string
|
||||
probeIntervalSec?: number | string
|
||||
speedIntervalSec?: number | string
|
||||
@@ -124,28 +99,37 @@ const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
}
|
||||
const updated = updateUptimeSettings({
|
||||
enabled: body.enabled,
|
||||
intervalSec: body.intervalSec == null ? undefined : Math.max(5, Number.parseInt(String(body.intervalSec), 10) || 15),
|
||||
resourcesEnabled: body.resourcesEnabled,
|
||||
pingEnabled: body.pingEnabled,
|
||||
speedEnabled: body.speedEnabled,
|
||||
intervalSec: body.intervalSec == null ? undefined : Math.max(5, Number.parseInt(String(body.intervalSec), 10) || 300),
|
||||
probeIntervalSec: body.probeIntervalSec == null ? undefined : Math.max(5, Number.parseInt(String(body.probeIntervalSec), 10) || 15),
|
||||
speedIntervalSec: body.speedIntervalSec == null ? undefined : Math.max(10, Number.parseInt(String(body.speedIntervalSec), 10) || 60),
|
||||
retentionDays: body.retentionDays == null ? undefined : Math.max(1, Number.parseInt(String(body.retentionDays), 10) || 14),
|
||||
})
|
||||
refreshScheduler()
|
||||
return reply.send({
|
||||
ok: true,
|
||||
settings: {
|
||||
enabled: updated.enabled,
|
||||
resourcesEnabled: updated.resourcesEnabled ?? updated.enabled,
|
||||
pingEnabled: updated.pingEnabled ?? updated.enabled,
|
||||
speedEnabled: updated.speedEnabled ?? updated.enabled,
|
||||
intervalSec: updated.intervalSec,
|
||||
probeIntervalSec: updated.probeIntervalSec ?? updated.intervalSec,
|
||||
probeIntervalSec: updated.probeIntervalSec,
|
||||
speedIntervalSec: updated.speedIntervalSec ?? 60,
|
||||
retentionDays: updated.retentionDays,
|
||||
lastCollectedAt: updated.lastCollectedAt ?? null,
|
||||
lastDurationMs: updated.lastDurationMs ?? null,
|
||||
lastError: updated.lastError || null,
|
||||
scheduler: getSchedulerStatus(),
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
app.post("/uptime/collect-now", async (_req, reply) => {
|
||||
await collectUptimeOnce()
|
||||
scheduleAlertEngineAfterDataCollectors()
|
||||
const s = readUptimeSettings()
|
||||
return reply.send({
|
||||
ok: true,
|
||||
@@ -155,8 +139,42 @@ const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
})
|
||||
})
|
||||
|
||||
/** Ручной ping по списку id проб (группа «сервер + назначение» на UI); без ожидания interval_sec. */
|
||||
app.post("/uptime/probes/collect-group", async (req, reply) => {
|
||||
const body = req.body as { probeIds?: unknown }
|
||||
const raw = body.probeIds
|
||||
const probeIds = Array.isArray(raw) ? raw.filter((x): x is string => typeof x === "string") : []
|
||||
try {
|
||||
const { polled } = await collectPingForProbeIds(probeIds)
|
||||
if (polled > 0) scheduleAlertEngineAfterDataCollectors()
|
||||
return reply.send({ ok: true, polled })
|
||||
} catch (e) {
|
||||
const status = (e as Error & { statusCode?: number }).statusCode
|
||||
if (status === 409) {
|
||||
return reply.status(409).send({ error: e instanceof Error ? e.message : "busy" })
|
||||
}
|
||||
throw e
|
||||
}
|
||||
})
|
||||
|
||||
app.get("/uptime/probes", async (_req, reply) => {
|
||||
return reply.send({ probes: readProbeRows() })
|
||||
const rows = readProbeRows()
|
||||
return reply.send({
|
||||
probes: rows.map((p) => ({
|
||||
id: p.id,
|
||||
srcServerId: String(p.srcServerId),
|
||||
srcInterface: p.srcInterface || "",
|
||||
name: p.name,
|
||||
target: p.target,
|
||||
probeFilter: p.probeFilter || "—",
|
||||
enabled: p.enabled,
|
||||
intervalSec: p.intervalSec ?? 0,
|
||||
showOnDashboard: p.showOnDashboard,
|
||||
sortOrder: p.sortOrder,
|
||||
createdAt: p.createdAt,
|
||||
updatedAt: p.updatedAt,
|
||||
})),
|
||||
})
|
||||
})
|
||||
|
||||
app.get("/uptime/speed-probes", async (_req, reply) => {
|
||||
@@ -275,6 +293,7 @@ const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
createdAt: now,
|
||||
}).run()
|
||||
}
|
||||
refreshScheduler()
|
||||
return reply.send({ ok: true })
|
||||
})
|
||||
|
||||
@@ -289,6 +308,7 @@ const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
filter?: string
|
||||
enabled?: boolean
|
||||
showOnDashboard?: boolean
|
||||
intervalSec?: number | string
|
||||
}>
|
||||
}
|
||||
const normalized = (body.probes ?? [])
|
||||
@@ -301,9 +321,13 @@ const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
probeFilter: String(p.filter ?? "—"),
|
||||
enabled: p.enabled !== false,
|
||||
showOnDashboard: p.showOnDashboard === true,
|
||||
intervalSec: p.intervalSec == null || p.intervalSec === ""
|
||||
? undefined
|
||||
: Number.parseInt(String(p.intervalSec), 10),
|
||||
}))
|
||||
.filter((p) => Number.isFinite(p.srcServerId) && p.name && p.target)
|
||||
replaceProbes(normalized)
|
||||
refreshScheduler()
|
||||
return reply.send({ ok: true })
|
||||
})
|
||||
|
||||
@@ -366,186 +390,37 @@ const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
const dst = db.select().from(servers).where(eq(servers.id, dstId)).limit(1).all()[0]
|
||||
if (!src || !dst) return reply.status(404).send({ error: "Server not found" })
|
||||
try {
|
||||
return await withBtestNodeLocks([srcId, dstId], async () => {
|
||||
const protocol = body.protocol === "udp" ? "udp" : "tcp"
|
||||
const direction = body.direction === "transmit" || body.direction === "receive" ? body.direction : "both"
|
||||
const durationSec = Math.max(3, Number.parseInt(String(body.durationSec ?? ""), 10) || 10)
|
||||
const srcInterface = String(body.srcInterface ?? "").trim()
|
||||
const dstInterface = String(body.dstInterface ?? "").trim()
|
||||
const srcClient = MikrotikClient.fromServer(src)
|
||||
const dstAddressRows = dstInterface
|
||||
? await MikrotikClient.fromServer(dst).getIpAddresses().catch(() => [])
|
||||
: []
|
||||
const srcAddressRows = srcInterface
|
||||
? await srcClient.getIpAddresses().catch(() => [])
|
||||
: []
|
||||
const dstAddress = dstInterface ? firstUsableIpOnInterface(dstAddressRows, dstInterface) : dst.host
|
||||
if (dstInterface && !dstAddress) {
|
||||
return reply.status(400).send({
|
||||
error: `На интерфейсе назначения '${dstInterface}' нет активного IP (проверьте /ip/address — не disabled).`,
|
||||
})
|
||||
}
|
||||
const initialAddress = dstAddress || dst.host
|
||||
const srcAddress = srcInterface ? firstUsableIpOnInterface(srcAddressRows, srcInterface) : src.host
|
||||
const srcInterfaceAddress = srcInterface ? (srcAddress || null) : null
|
||||
const dstInterfaceAddress = dstInterface ? (dstAddress || null) : null
|
||||
|
||||
const rows = await srcClient.bandwidthTest({
|
||||
address: initialAddress,
|
||||
user: dst.username,
|
||||
password: dst.password,
|
||||
protocol,
|
||||
direction,
|
||||
durationSec,
|
||||
})
|
||||
const txSeries = rows
|
||||
.map((r) => parseRateToMbps(r["tx-current"] ?? r["tx-10-second-average"] ?? r["tx-total-average"]))
|
||||
.filter((v) => Number.isFinite(v) && v >= 0)
|
||||
const rxSeries = rows
|
||||
.map((r) => parseRateToMbps(r["rx-current"] ?? r["rx-10-second-average"] ?? r["rx-total-average"]))
|
||||
.filter((v) => Number.isFinite(v) && v >= 0)
|
||||
const last = rows[rows.length - 1] ?? {}
|
||||
const txAvg = parseRateToMbps(last["tx-total-average"] ?? last["tx-10-second-average"])
|
||||
const rxAvg = parseRateToMbps(last["rx-total-average"] ?? last["rx-10-second-average"])
|
||||
const txFinal = txAvg || (txSeries.length ? txSeries.reduce((a, b) => a + b, 0) / txSeries.length : 0)
|
||||
const rxFinal = rxAvg || (rxSeries.length ? rxSeries.reduce((a, b) => a + b, 0) / rxSeries.length : 0)
|
||||
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 3000))
|
||||
|
||||
const pingNow = new Date().toISOString()
|
||||
let afterBtPing: {
|
||||
rttMs: number | null
|
||||
lossPct: number | null
|
||||
error: string | null
|
||||
} = { rttMs: null, lossPct: null, error: null }
|
||||
try {
|
||||
const pingRows = await srcClient.ping(initialAddress, 4, srcInterface || undefined, { interval: "1s" })
|
||||
const pr = parsePing(pingRows as Parameters<typeof parsePing>[0])
|
||||
afterBtPing = {
|
||||
rttMs: pr.avgRtt,
|
||||
lossPct: pr.loss,
|
||||
error: null,
|
||||
}
|
||||
} catch (pe) {
|
||||
afterBtPing = {
|
||||
rttMs: null,
|
||||
lossPct: null,
|
||||
error: pe instanceof Error ? pe.message : String(pe),
|
||||
}
|
||||
}
|
||||
|
||||
const probeId = String(body.probeId ?? "").trim()
|
||||
const runId = String(body.runId ?? "").trim() || `sr-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||||
if (probeId) {
|
||||
db.update(uptimeSpeedProbes).set({
|
||||
lastRunAt: pingNow,
|
||||
lastTxAvgMbps: txFinal,
|
||||
lastRxAvgMbps: rxFinal,
|
||||
lastStatus: "done",
|
||||
lastError: "",
|
||||
lastPingRttMs: afterBtPing.rttMs,
|
||||
lastPingLossPct: afterBtPing.lossPct,
|
||||
lastPingAt: pingNow,
|
||||
lastPingError: afterBtPing.error ?? "",
|
||||
updatedAt: pingNow,
|
||||
}).where(eq(uptimeSpeedProbes.id, probeId)).run()
|
||||
}
|
||||
db.insert(uptimeSpeedTestRuns).values({
|
||||
id: runId,
|
||||
probeId: probeId || null,
|
||||
srcServerId: srcId,
|
||||
dstServerId: dstId,
|
||||
srcInterface: srcInterface || "",
|
||||
dstInterface: dstInterface || "",
|
||||
srcAddress: srcAddress || src.host,
|
||||
dstAddress: initialAddress,
|
||||
srcInterfaceAddress,
|
||||
dstInterfaceAddress,
|
||||
protocol,
|
||||
direction,
|
||||
durationSec,
|
||||
txAvgMbps: txFinal,
|
||||
rxAvgMbps: rxFinal,
|
||||
pingRttMs: afterBtPing.rttMs,
|
||||
pingLossPct: afterBtPing.lossPct,
|
||||
pingError: afterBtPing.error ?? null,
|
||||
status: "done",
|
||||
error: null,
|
||||
createdAt: pingNow,
|
||||
}).run()
|
||||
return reply.send({
|
||||
ok: true,
|
||||
result: {
|
||||
runId,
|
||||
srcInterface: srcInterface || null,
|
||||
dstInterface: dstInterface || null,
|
||||
address: initialAddress,
|
||||
srcAddress: srcAddress || src.host,
|
||||
dstAddress: initialAddress,
|
||||
srcInterfaceAddress,
|
||||
dstInterfaceAddress,
|
||||
protocol,
|
||||
direction,
|
||||
durationSec,
|
||||
txAvgMbps: txFinal,
|
||||
rxAvgMbps: rxFinal,
|
||||
txSeries,
|
||||
rxSeries,
|
||||
raw: rows,
|
||||
afterBtPing,
|
||||
},
|
||||
})
|
||||
const result = await runBandwidthSpeedTest({
|
||||
runId: String(body.runId ?? "").trim() || undefined,
|
||||
probeId: String(body.probeId ?? "").trim() || undefined,
|
||||
srcServerId: srcId,
|
||||
dstServerId: dstId,
|
||||
srcInterface: String(body.srcInterface ?? "").trim() || undefined,
|
||||
dstInterface: String(body.dstInterface ?? "").trim() || undefined,
|
||||
protocol: body.protocol === "udp" ? "udp" : "tcp",
|
||||
direction: body.direction === "transmit" || body.direction === "receive" ? body.direction : "both",
|
||||
durationSec: Math.max(3, Number.parseInt(String(body.durationSec ?? ""), 10) || 10),
|
||||
})
|
||||
return reply.send({ ok: true, result })
|
||||
} catch (e) {
|
||||
const raw = e instanceof Error ? e.message : "Bandwidth test failed"
|
||||
recordSpeedTestFailure({
|
||||
probeId: String(body.probeId ?? "").trim() || undefined,
|
||||
runId: String(body.runId ?? "").trim() || undefined,
|
||||
srcServerId: srcId,
|
||||
dstServerId: dstId,
|
||||
srcInterface: String(body.srcInterface ?? "").trim(),
|
||||
dstInterface: String(body.dstInterface ?? "").trim(),
|
||||
protocol: body.protocol === "udp" ? "udp" : "tcp",
|
||||
direction: body.direction === "transmit" || body.direction === "receive" ? body.direction : "both",
|
||||
durationSec: Math.max(3, Number.parseInt(String(body.durationSec ?? ""), 10) || 10),
|
||||
message: raw,
|
||||
})
|
||||
const message = /timed out|socket hang up|ECONNRESET/i.test(raw)
|
||||
? "BTTest timeout/socket hang up: проверь доступность destination, /tool bandwidth-server, firewall и корректность интерфейсов"
|
||||
: raw
|
||||
const probeId = String((req.body as { probeId?: string }).probeId ?? "").trim()
|
||||
if (probeId) {
|
||||
db.update(uptimeSpeedProbes).set({
|
||||
lastRunAt: new Date().toISOString(),
|
||||
lastStatus: "error",
|
||||
lastError: message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}).where(eq(uptimeSpeedProbes.id, probeId)).run()
|
||||
}
|
||||
const body = req.body as {
|
||||
runId?: string
|
||||
srcServerId?: string | number
|
||||
dstServerId?: string | number
|
||||
srcInterface?: string
|
||||
dstInterface?: string
|
||||
protocol?: "tcp" | "udp"
|
||||
direction?: "transmit" | "receive" | "both"
|
||||
durationSec?: string | number
|
||||
}
|
||||
const srcId = Number.parseInt(String(body.srcServerId ?? ""), 10)
|
||||
const dstId = Number.parseInt(String(body.dstServerId ?? ""), 10)
|
||||
if (Number.isFinite(srcId) && Number.isFinite(dstId)) {
|
||||
db.insert(uptimeSpeedTestRuns).values({
|
||||
id: String(body.runId ?? "").trim() || `sr-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
probeId: probeId || null,
|
||||
srcServerId: srcId,
|
||||
dstServerId: dstId,
|
||||
srcInterface: String(body.srcInterface ?? "").trim(),
|
||||
dstInterface: String(body.dstInterface ?? "").trim(),
|
||||
srcAddress: null,
|
||||
dstAddress: null,
|
||||
srcInterfaceAddress: null,
|
||||
dstInterfaceAddress: null,
|
||||
protocol: body.protocol === "udp" ? "udp" : "tcp",
|
||||
direction: body.direction === "transmit" || body.direction === "receive" ? body.direction : "both",
|
||||
durationSec: Math.max(3, Number.parseInt(String(body.durationSec ?? ""), 10) || 10),
|
||||
txAvgMbps: null,
|
||||
rxAvgMbps: null,
|
||||
pingRttMs: null,
|
||||
pingLossPct: null,
|
||||
pingError: null,
|
||||
status: "error",
|
||||
error: message,
|
||||
createdAt: new Date().toISOString(),
|
||||
}).run()
|
||||
if (/активного IP|Server not found/i.test(raw)) {
|
||||
return reply.status(400).send({ error: message })
|
||||
}
|
||||
return reply.status(502).send({ error: message })
|
||||
}
|
||||
@@ -601,6 +476,10 @@ const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
const allServers = db.select().from(servers).all()
|
||||
const probeRows = readProbeRows()
|
||||
const probeSamples = readProbeSamplesSince(sinceIso)
|
||||
const uptimeCfg = readUptimeSettings()
|
||||
const intervalSec = Math.max(15, uptimeCfg.intervalSec ?? 300)
|
||||
/** Допустимый разрыв между последней попыткой и последним «хорошим» сэмплом (не показывать данные часовой давности). */
|
||||
const resourceFallbackMaxGapMs = Math.min(2 * 3600 * 1000, Math.max(10 * 60 * 1000, intervalSec * 6 * 1000))
|
||||
|
||||
const probes = probeRows.map((p) => {
|
||||
const rows = probeSamples.filter((r) => r.probeId === p.id)
|
||||
@@ -624,18 +503,19 @@ const uptimeRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
|
||||
const resources = allServers.map((s) => {
|
||||
const rows = readResourceSamplesSince(sinceIso, s.id)
|
||||
const last = rows[rows.length - 1]
|
||||
const { row: pick, hasData } = pickResourceDisplayRow(rows, resourceFallbackMaxGapMs)
|
||||
const cpuHistory = toSeries(rows.map((r) => r.cpuLoad), 40)
|
||||
return {
|
||||
serverId: String(s.id),
|
||||
cpu: last?.cpuLoad ?? 0,
|
||||
hasData,
|
||||
cpu: pick?.cpuLoad ?? 0,
|
||||
cpuHistory,
|
||||
ramUsed: Math.max(0, ((last?.totalMemory ?? 0) - (last?.freeMemory ?? 0)) / (1024 * 1024)),
|
||||
ramTotal: Math.max(0, (last?.totalMemory ?? 0) / (1024 * 1024)),
|
||||
hddUsed: Math.max(0, ((last?.totalHddSpace ?? 0) - (last?.freeHddSpace ?? 0)) / (1024 * 1024)),
|
||||
hddTotal: Math.max(0, (last?.totalHddSpace ?? 0) / (1024 * 1024)),
|
||||
uptimeSeconds: last?.uptimeSeconds ?? 0,
|
||||
boardName: last?.boardName || "RouterBOARD",
|
||||
ramUsed: Math.max(0, ((pick?.totalMemory ?? 0) - (pick?.freeMemory ?? 0)) / (1024 * 1024)),
|
||||
ramTotal: Math.max(0, (pick?.totalMemory ?? 0) / (1024 * 1024)),
|
||||
hddUsed: Math.max(0, ((pick?.totalHddSpace ?? 0) - (pick?.freeHddSpace ?? 0)) / (1024 * 1024)),
|
||||
hddTotal: Math.max(0, (pick?.totalHddSpace ?? 0) / (1024 * 1024)),
|
||||
uptimeSeconds: pick?.uptimeSeconds ?? 0,
|
||||
boardName: pick?.boardName || "RouterBOARD",
|
||||
temp: undefined as number | undefined,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* Внутренние хуки: после записи сигналов в SQLite (коллекторы) — отложенный прогон `alert_engine`.
|
||||
* Не импортирует `scheduler.ts` (избегаем цикла); runner задаётся через `wireAlertEngineRunner`.
|
||||
*
|
||||
* Любой код, пишущий таблицы для `buildSignalSnapshot`, после успешного коммита должен вызывать
|
||||
* `scheduleAlertEngineAfterDataCollectors()`.
|
||||
*/
|
||||
import { isSchedulerJobRunning } from "./scheduler-running.js"
|
||||
|
||||
const ALERT_AFTER_COLLECT_DEBOUNCE_MS = 250
|
||||
const ALERT_AFTER_COLLECT_WAIT_MS = 100
|
||||
|
||||
let alertAfterCollectTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let alertAfterCollectChain: Promise<void> = Promise.resolve()
|
||||
|
||||
let alertEngineRunner: (() => Promise<void>) | null = null
|
||||
|
||||
/** Задать runner один раз при старте приложения (обычно `() => executeSchedulerJob("alert_engine")`). */
|
||||
export function wireAlertEngineRunner(run: () => Promise<void>): void {
|
||||
alertEngineRunner = run
|
||||
}
|
||||
|
||||
export function clearAlertCollectorHooksTimer(): void {
|
||||
if (alertAfterCollectTimer != null) {
|
||||
clearTimeout(alertAfterCollectTimer)
|
||||
alertAfterCollectTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
async function runAlertEngineWhenIdle(): Promise<void> {
|
||||
const run = alertEngineRunner
|
||||
if (!run) return
|
||||
while (isSchedulerJobRunning("alert_engine")) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, ALERT_AFTER_COLLECT_WAIT_MS))
|
||||
}
|
||||
await run()
|
||||
}
|
||||
|
||||
/** После успешной записи сигналов в БД коллекторами (не из `alert_engine`). */
|
||||
export function scheduleAlertEngineAfterDataCollectors(): void {
|
||||
if (alertAfterCollectTimer != null) clearTimeout(alertAfterCollectTimer)
|
||||
alertAfterCollectTimer = setTimeout(() => {
|
||||
alertAfterCollectTimer = null
|
||||
alertAfterCollectChain = alertAfterCollectChain
|
||||
.then(() => runAlertEngineWhenIdle())
|
||||
.catch(() => {})
|
||||
}, ALERT_AFTER_COLLECT_DEBOUNCE_MS)
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEngineConfirmPending } from "../../db/schema.js"
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import { computeStabilityReadyMap, deleteConfirmPending } from "./confirm-stability.js"
|
||||
|
||||
const ruleId = "test-confirm-stability-recovery-bypass"
|
||||
|
||||
deleteConfirmPending(ruleId)
|
||||
|
||||
const baseRule: ApiAlertRule = {
|
||||
id: ruleId,
|
||||
name: "Server transition",
|
||||
type: "server",
|
||||
target: "core-1",
|
||||
targets: ["core-1"],
|
||||
groupId: null,
|
||||
condition: "перешёл в offline · восстановился",
|
||||
conditions: ["перешёл в offline", "восстановился"],
|
||||
severity: "critical",
|
||||
enabled: true,
|
||||
cooldown: "5м",
|
||||
confirmStabilitySec: 120,
|
||||
recoveryMode: "always",
|
||||
recoveryStabilitySec: null,
|
||||
chatId: "",
|
||||
lastFiredAt: null,
|
||||
}
|
||||
|
||||
const problemHit = {
|
||||
ruleId,
|
||||
ruleName: baseRule.name,
|
||||
severity: baseRule.severity,
|
||||
message: "Сервер: core-1 — перешёл в offline",
|
||||
transition: "problem" as const,
|
||||
payloadHash: "problem-1",
|
||||
}
|
||||
|
||||
const recoveryHit = {
|
||||
ruleId,
|
||||
ruleName: baseRule.name,
|
||||
severity: baseRule.severity,
|
||||
message: "Сервер: core-1 — восстановился",
|
||||
transition: "recovery" as const,
|
||||
payloadHash: "recovery-1",
|
||||
}
|
||||
|
||||
{
|
||||
const m = computeStabilityReadyMap([baseRule], new Map([[ruleId, problemHit]]))
|
||||
assert.equal(m.get(ruleId), false, "проблемный переход должен ждать confirmStabilitySec")
|
||||
}
|
||||
|
||||
{
|
||||
const m = computeStabilityReadyMap([baseRule], new Map([[ruleId, recoveryHit]]))
|
||||
assert.equal(m.get(ruleId), true, "восстановление должно отправляться без задержки")
|
||||
}
|
||||
|
||||
{
|
||||
const m = computeStabilityReadyMap([baseRule], new Map([[ruleId, null]]))
|
||||
assert.equal(m.get(ruleId), false, "без hit отправка не готова")
|
||||
const left = db
|
||||
.select()
|
||||
.from(alertEngineConfirmPending)
|
||||
.where(eq(alertEngineConfirmPending.ruleId, ruleId))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
assert.equal(left, undefined, "pending должен очищаться после отсутствия hit")
|
||||
}
|
||||
|
||||
console.log("alert-engine confirm-stability tests ok")
|
||||
@@ -0,0 +1,82 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEngineConfirmPending } from "../../db/schema.js"
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import type { RuleEvalHit } from "./types.js"
|
||||
|
||||
export function deleteConfirmPending(ruleId: string) {
|
||||
db.delete(alertEngineConfirmPending).where(eq(alertEngineConfirmPending.ruleId, ruleId)).run()
|
||||
}
|
||||
|
||||
/**
|
||||
* Обновляет состояние ожидания стабильности и возвращает, можно ли отправлять уведомление в этом тике.
|
||||
* При отсутствии срабатывания (`hit == null`) ожидание сбрасывается.
|
||||
*/
|
||||
export function stabilityReadyForSend(
|
||||
ruleId: string,
|
||||
hit: RuleEvalHit | null,
|
||||
delaySec: number | null | undefined,
|
||||
): boolean {
|
||||
if (!hit) {
|
||||
deleteConfirmPending(ruleId)
|
||||
return false
|
||||
}
|
||||
const sec = delaySec != null && delaySec > 0 ? Math.floor(delaySec) : 0
|
||||
if (sec <= 0) {
|
||||
deleteConfirmPending(ruleId)
|
||||
return true
|
||||
}
|
||||
|
||||
const now = Date.now()
|
||||
const nowIso = new Date().toISOString()
|
||||
const existing = db
|
||||
.select()
|
||||
.from(alertEngineConfirmPending)
|
||||
.where(eq(alertEngineConfirmPending.ruleId, ruleId))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
|
||||
if (!existing || existing.payloadHash !== hit.payloadHash) {
|
||||
if (existing) {
|
||||
db.update(alertEngineConfirmPending)
|
||||
.set({ payloadHash: hit.payloadHash, sinceAt: nowIso })
|
||||
.where(eq(alertEngineConfirmPending.ruleId, ruleId))
|
||||
.run()
|
||||
} else {
|
||||
db.insert(alertEngineConfirmPending).values({ ruleId, payloadHash: hit.payloadHash, sinceAt: nowIso }).run()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const t = Date.parse(existing.sinceAt)
|
||||
if (!Number.isFinite(t)) return false
|
||||
return now - t >= sec * 1000
|
||||
}
|
||||
|
||||
/** После успешной отправки — сбросить таймер стабильности для следующего цикла. */
|
||||
export function clearConfirmPendingAfterSend(ruleIds: string[]) {
|
||||
for (const id of ruleIds) deleteConfirmPending(id)
|
||||
}
|
||||
|
||||
export function computeStabilityReadyMap(
|
||||
rules: ApiAlertRule[],
|
||||
hitByRule: Map<string, RuleEvalHit | null>,
|
||||
): Map<string, boolean> {
|
||||
const m = new Map<string, boolean>()
|
||||
for (const rule of rules) {
|
||||
if (!rule.enabled) {
|
||||
deleteConfirmPending(rule.id)
|
||||
m.set(rule.id, false)
|
||||
continue
|
||||
}
|
||||
const hit = hitByRule.get(rule.id) ?? null
|
||||
const delaySec =
|
||||
hit?.transition === "recovery"
|
||||
? rule.recoveryMode === "conditional"
|
||||
? rule.recoveryStabilitySec
|
||||
: 0
|
||||
: rule.confirmStabilitySec
|
||||
m.set(rule.id, stabilityReadyForSend(rule.id, hit, delaySec))
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
|
||||
const COOLDOWN_MS: Record<ApiAlertRule["cooldown"], number> = {
|
||||
"1м": 60_000,
|
||||
"5м": 5 * 60_000,
|
||||
"15м": 15 * 60_000,
|
||||
"1ч": 60 * 60_000,
|
||||
"4ч": 4 * 60 * 60_000,
|
||||
"24ч": 24 * 60 * 60_000,
|
||||
}
|
||||
|
||||
export function cooldownToMs(cd: ApiAlertRule["cooldown"]): number {
|
||||
return COOLDOWN_MS[cd] ?? COOLDOWN_MS["5м"]
|
||||
}
|
||||
|
||||
/** Можно ли отправить снова после lastIso (ISO) */
|
||||
export function isCooldownElapsed(lastFiredIso: string | null | undefined, cooldown: ApiAlertRule["cooldown"]): boolean {
|
||||
if (!lastFiredIso?.trim()) return true
|
||||
const t = Date.parse(lastFiredIso)
|
||||
if (!Number.isFinite(t)) return true
|
||||
return Date.now() - t >= cooldownToMs(cooldown)
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import type { ApiAlertGroup, ApiAlertRule } from "../alerts-service.js"
|
||||
import { computeStabilityReadyMap } from "./confirm-stability.js"
|
||||
import { isCooldownElapsed } from "./cooldown.js"
|
||||
import { groupShouldFire } from "./groups.js"
|
||||
import { isPositiveRecoveryTelegramText } from "./telegram-emoji.js"
|
||||
import { shouldAllowRecoveryByPolicy } from "./state-machine.js"
|
||||
import type { AlertEngineRuleDiag, RuleEvalHit } from "./types.js"
|
||||
|
||||
export type EngineStateMap = Map<string, { lastFiredAt: string; lastPayloadHash: string | null }>
|
||||
|
||||
export type StandaloneDecision = {
|
||||
kind: "rule"
|
||||
rule: ApiAlertRule
|
||||
hit: RuleEvalHit
|
||||
}
|
||||
|
||||
export type GroupDecision = {
|
||||
kind: "group"
|
||||
group: ApiAlertGroup
|
||||
members: ApiAlertRule[]
|
||||
hits: RuleEvalHit[]
|
||||
}
|
||||
|
||||
function ruleScopeKey(ruleId: string, transition?: RuleEvalHit["transition"]): string {
|
||||
const suffix = transition ?? "neutral"
|
||||
return `rule:${ruleId}:${suffix}`
|
||||
}
|
||||
|
||||
function textAfterGroupAlertHeader(fullBody: string): string {
|
||||
const segs = fullBody.split(/\n\n/)
|
||||
if (segs.length >= 2 && /^Группа\s*«/i.test(segs[0] ?? "")) return segs.slice(1).join("\n\n").trim()
|
||||
return fullBody
|
||||
}
|
||||
|
||||
export function buildRuleDiagnostics(
|
||||
rules: ApiAlertRule[],
|
||||
hitByRule: Map<string, RuleEvalHit | null>,
|
||||
stabilityReady: Map<string, boolean>,
|
||||
state: EngineStateMap,
|
||||
canSend: boolean,
|
||||
): AlertEngineRuleDiag[] {
|
||||
const out: AlertEngineRuleDiag[] = []
|
||||
for (const rule of rules) {
|
||||
if (!rule.enabled) continue
|
||||
const hit = hitByRule.get(rule.id) ?? null
|
||||
const inGroup = Boolean(rule.groupId)
|
||||
const evalHit = Boolean(hit)
|
||||
const stab = stabilityReady.get(rule.id) ?? false
|
||||
const key = ruleScopeKey(rule.id, hit?.transition)
|
||||
const prev = state.get(key)
|
||||
const cooldownOk = isCooldownElapsed(prev?.lastFiredAt, rule.cooldown)
|
||||
let blocked: AlertEngineRuleDiag["blocked"]
|
||||
if (!evalHit) blocked = "no_hit"
|
||||
else if (!shouldAllowRecoveryByPolicy(hit, rule.recoveryMode)) blocked = "stability"
|
||||
else if (inGroup) blocked = "in_group"
|
||||
else if (!stab) blocked = "stability"
|
||||
else if (!cooldownOk) blocked = "cooldown"
|
||||
else if (!canSend) blocked = "no_telegram"
|
||||
else if (hit && isPositiveRecoveryTelegramText(hit.message) && prev?.lastPayloadHash === hit.payloadHash) blocked = "dedupe_positive"
|
||||
else blocked = undefined
|
||||
out.push({
|
||||
ruleId: rule.id,
|
||||
inGroup,
|
||||
evalHit,
|
||||
hitTransition: hit?.transition,
|
||||
hitMessage: hit?.message,
|
||||
stabilityOk: stab,
|
||||
cooldownOk,
|
||||
telegramOk: canSend,
|
||||
blocked,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export function computeDecisions(args: {
|
||||
rules: ApiAlertRule[]
|
||||
groups: ApiAlertGroup[]
|
||||
hitByRule: Map<string, RuleEvalHit | null>
|
||||
state: EngineStateMap
|
||||
canSend: boolean
|
||||
}): {
|
||||
stabilityReady: Map<string, boolean>
|
||||
standalone: StandaloneDecision[]
|
||||
grouped: GroupDecision[]
|
||||
ruleDiag: AlertEngineRuleDiag[]
|
||||
} {
|
||||
const { rules, groups, hitByRule, state, canSend } = args
|
||||
const stabilityReady = computeStabilityReadyMap(rules, hitByRule)
|
||||
const ruleDiag = buildRuleDiagnostics(rules, hitByRule, stabilityReady, state, canSend)
|
||||
const standalone: StandaloneDecision[] = []
|
||||
const grouped: GroupDecision[] = []
|
||||
|
||||
for (const rule of rules) {
|
||||
if (!rule.enabled || rule.groupId) continue
|
||||
const hit = hitByRule.get(rule.id)
|
||||
if (!hit) continue
|
||||
if (!shouldAllowRecoveryByPolicy(hit, rule.recoveryMode)) continue
|
||||
if (!stabilityReady.get(rule.id)) continue
|
||||
const key = ruleScopeKey(rule.id, hit.transition)
|
||||
const prev = state.get(key)
|
||||
if (!isCooldownElapsed(prev?.lastFiredAt, rule.cooldown)) continue
|
||||
if (isPositiveRecoveryTelegramText(hit.message) && prev?.lastPayloadHash === hit.payloadHash) continue
|
||||
standalone.push({ kind: "rule", rule, hit })
|
||||
}
|
||||
|
||||
for (const g of groups) {
|
||||
if (!g.enabled) continue
|
||||
const members = rules.filter((r) => r.groupId === g.id && r.enabled)
|
||||
if (members.length === 0) continue
|
||||
const firing = members.map((r) => {
|
||||
const h = hitByRule.get(r.id)
|
||||
if (!shouldAllowRecoveryByPolicy(h ?? null, r.recoveryMode)) return false
|
||||
return Boolean(h) && Boolean(stabilityReady.get(r.id))
|
||||
})
|
||||
if (!groupShouldFire(g.combineMode, firing)) continue
|
||||
const hits: RuleEvalHit[] = []
|
||||
for (const member of members) {
|
||||
const hit = hitByRule.get(member.id)
|
||||
if (!hit) continue
|
||||
if (!shouldAllowRecoveryByPolicy(hit, member.recoveryMode)) continue
|
||||
hits.push(hit)
|
||||
}
|
||||
if (hits.length === 0) continue
|
||||
const key = `group:${g.id}`
|
||||
const prev = state.get(key)
|
||||
const cd = g.cooldownOverride ?? "5м"
|
||||
if (!isCooldownElapsed(prev?.lastFiredAt, cd)) continue
|
||||
const body = `Группа «${g.name}» (${g.combineMode === "any" ? "ANY" : "ALL"})\n\n${hits.map((h) => h.message).join("\n")}`
|
||||
const hash = hits.map((h) => h.payloadHash).sort().join("|")
|
||||
if (isPositiveRecoveryTelegramText(textAfterGroupAlertHeader(body)) && prev?.lastPayloadHash === hash) continue
|
||||
grouped.push({ kind: "group", group: g, members, hits })
|
||||
}
|
||||
|
||||
return { stabilityReady, standalone, grouped, ruleDiag }
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import assert from "node:assert/strict"
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import { evaluateRule } from "./evaluate-rule.js"
|
||||
import type { SignalSnapshot } from "./types.js"
|
||||
|
||||
function emptySnap(over: Partial<SignalSnapshot> = {}): SignalSnapshot {
|
||||
return {
|
||||
sampledAt: "t",
|
||||
servers: [],
|
||||
probes: [],
|
||||
trafficByServer: [],
|
||||
greTunnels: [],
|
||||
bgpSessions: [],
|
||||
...over,
|
||||
}
|
||||
}
|
||||
|
||||
function rule(p: Partial<ApiAlertRule> & Pick<ApiAlertRule, "id" | "name" | "type" | "targets">): ApiAlertRule {
|
||||
return {
|
||||
target: p.targets[0] ?? "",
|
||||
groupId: null,
|
||||
condition: p.conditions?.[0] ?? p.condition ?? "",
|
||||
conditions: p.conditions ?? (p.condition ? [p.condition] : []),
|
||||
severity: p.severity ?? "warning",
|
||||
enabled: p.enabled ?? true,
|
||||
cooldown: p.cooldown ?? "5м",
|
||||
confirmStabilitySec: null,
|
||||
recoveryMode: "always",
|
||||
recoveryStabilitySec: null,
|
||||
chatId: "",
|
||||
lastFiredAt: null,
|
||||
...p,
|
||||
} as ApiAlertRule
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "g1",
|
||||
name: "GRE",
|
||||
type: "gre-tunnel",
|
||||
targets: ["t1 / srv"],
|
||||
conditions: ["перешёл в offline"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
greTunnels: [{ targetLabel: "t1 / srv", status: "down", prevStatus: "up" }],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "g2",
|
||||
name: "GRE rec",
|
||||
type: "gre-tunnel",
|
||||
targets: ["t1 / srv"],
|
||||
conditions: ["восстановился"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
greTunnels: [{ targetLabel: "t1 / srv", status: "up", prevStatus: "down" }],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "b1",
|
||||
name: "BGP",
|
||||
type: "bgp-peer",
|
||||
targets: ["srv: peer1"],
|
||||
conditions: ["разорвал сессию"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
bgpSessions: [{ key: "srv: peer1", state: "Idle", prevState: "Established" }],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "b2",
|
||||
name: "BGP up",
|
||||
type: "bgp-peer",
|
||||
targets: ["srv: peer1"],
|
||||
conditions: ["восстановил сессию"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
bgpSessions: [{ key: "srv: peer1", state: "Established", prevState: "Active" }],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "pfx",
|
||||
name: "pfx",
|
||||
type: "bgp-prefix",
|
||||
targets: ["любой префикс"],
|
||||
conditions: ["был отозван"],
|
||||
})
|
||||
assert.equal(evaluateRule(r, emptySnap()), null)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "srv-rec",
|
||||
name: "rec",
|
||||
type: "server",
|
||||
targets: ["core"],
|
||||
conditions: ["восстановился"],
|
||||
})
|
||||
assert.ok(
|
||||
evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "online",
|
||||
prevStatus: "offline",
|
||||
prev2Status: "offline",
|
||||
sampledAt: "t1",
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
"один online после offline — тоже «восстановился» (ручной REST / ресурс уже online)",
|
||||
)
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: "offline",
|
||||
sampledAt: "t2",
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit, "два подряд online после offline — одно срабатывание")
|
||||
assert.equal(hit?.transition, "recovery")
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "srv-rec-offline-phrase",
|
||||
name: "rec-offline-phrase",
|
||||
type: "server",
|
||||
targets: ["core"],
|
||||
conditions: ["восстановился после offline"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: "offline",
|
||||
sampledAt: "t2",
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit, "в тексте условия есть «offline» — всё равно восстановление по сэмплам")
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "srv-off",
|
||||
name: "off",
|
||||
type: "server",
|
||||
targets: ["core"],
|
||||
conditions: ["перешёл в offline"],
|
||||
})
|
||||
const edgeHit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "offline",
|
||||
prevStatus: "online",
|
||||
prev2Status: "online",
|
||||
sampledAt: "t",
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
assert.ok(edgeHit)
|
||||
assert.equal(edgeHit?.transition, "problem")
|
||||
assert.equal(
|
||||
evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "offline",
|
||||
prevStatus: "offline",
|
||||
prev2Status: "online",
|
||||
sampledAt: "t",
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
null,
|
||||
"устойчивый offline — без повторного «перешёл» на каждом тике",
|
||||
)
|
||||
}
|
||||
|
||||
console.log("alert-engine evaluate-rule tests ok")
|
||||
@@ -0,0 +1,323 @@
|
||||
import { createHash } from "node:crypto"
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import type { RuleEvalHit, SignalSnapshot } from "./types.js"
|
||||
|
||||
function hashPayload(parts: string[]): string {
|
||||
return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 24)
|
||||
}
|
||||
|
||||
function firstNumberInString(s: string): number | null {
|
||||
const m = s.replace(",", ".").match(/(\d+(?:\.\d+)?)/)
|
||||
if (!m) return null
|
||||
const n = Number.parseFloat(m[1] ?? "")
|
||||
return Number.isFinite(n) ? n : null
|
||||
}
|
||||
|
||||
function maxSeverity(
|
||||
a: ApiAlertRule["severity"],
|
||||
b: ApiAlertRule["severity"],
|
||||
): ApiAlertRule["severity"] {
|
||||
const o = { critical: 3, warning: 2, info: 1 } as const
|
||||
return o[a] >= o[b] ? a : b
|
||||
}
|
||||
|
||||
function findServer(snap: SignalSnapshot, name: string) {
|
||||
const n = name.trim().toLowerCase()
|
||||
return snap.servers.find((s) => s.name.trim().toLowerCase() === n)
|
||||
}
|
||||
|
||||
function findProbe(snap: SignalSnapshot, targetKey: string) {
|
||||
const k = targetKey.trim()
|
||||
return snap.probes.find((p) => p.key.trim() === k)
|
||||
}
|
||||
|
||||
function findTraffic(snap: SignalSnapshot, serverName: string) {
|
||||
const n = serverName.trim().toLowerCase()
|
||||
return snap.trafficByServer.find((t) => t.serverName.trim().toLowerCase() === n)
|
||||
}
|
||||
|
||||
function findGreTunnel(snap: SignalSnapshot, targetLabel: string) {
|
||||
const n = targetLabel.trim()
|
||||
return snap.greTunnels.find((g) => g.targetLabel.trim() === n)
|
||||
}
|
||||
|
||||
function findBgpPeer(snap: SignalSnapshot, key: string) {
|
||||
const n = key.trim()
|
||||
return snap.bgpSessions.find((b) => b.key.trim() === n)
|
||||
}
|
||||
|
||||
function isBgpEstablished(state: string): boolean {
|
||||
return state.trim().toLowerCase() === "established"
|
||||
}
|
||||
|
||||
/** Заголовок для Telegram: не «…недоступен: … — восстановился». */
|
||||
function formatServerAlertTitle(ruleName: string, wantRecovery: boolean): string {
|
||||
if (!wantRecovery) return ruleName
|
||||
if (!/недоступен/i.test(ruleName)) return ruleName
|
||||
const trimmed = ruleName.replace(/\s*,\s*недоступен\s*$/i, "").replace(/\s+недоступен\s*$/i, "").trim()
|
||||
if (!trimmed || trimmed === ruleName) return ruleName
|
||||
return `${trimmed} — снова в сети`
|
||||
}
|
||||
|
||||
function evalServerLike(
|
||||
rule: ApiAlertRule,
|
||||
snap: SignalSnapshot,
|
||||
cond: string,
|
||||
): RuleEvalHit | null {
|
||||
const c = cond.toLowerCase()
|
||||
const isOtkluchilsya = /отключился|отключилась|отключилось/.test(c)
|
||||
/** «Перешёл в …» — только на границе, без спама пока сэмплы остаются offline. */
|
||||
const impliesOfflineEdge = /переш(ё|е)л/.test(c) && (/offline|офлайн/.test(c) || /недоступен/.test(c))
|
||||
const impliesDegradedEdge = /переш(ё|е)л/.test(c) && c.includes("degraded")
|
||||
const wantOffline =
|
||||
c.includes("offline") || c.includes("офлайн") || c.includes("отключ")
|
||||
const wantDegraded = c.includes("degraded")
|
||||
/** «Восстановился» / «подключился»: двойной online в сэмплах (см. тесты). Не отсекаем по слову «offline» в фразе вроде «после offline». */
|
||||
const wantRecovery =
|
||||
(c.includes("восстанов") || c.includes("подключ")) && !isOtkluchilsya
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const s = findServer(snap, tgt)
|
||||
if (!s) continue
|
||||
if (wantRecovery) {
|
||||
const p2 = s.prev2Status
|
||||
if (
|
||||
s.status === "online" &&
|
||||
s.prevStatus === "online" &&
|
||||
p2 != null &&
|
||||
(p2 === "offline" || p2 === "degraded")
|
||||
) {
|
||||
hits.push(`${tgt}`)
|
||||
} else if (
|
||||
s.status === "online" &&
|
||||
(s.prevStatus === "offline" || s.prevStatus === "degraded")
|
||||
) {
|
||||
/** Один свежий online после offline (например ресурс уже online, REST только подтвердил). */
|
||||
hits.push(`${tgt}`)
|
||||
}
|
||||
} else if ((impliesOfflineEdge || isOtkluchilsya) && s.status === "offline") {
|
||||
if (s.prevStatus === null || s.prevStatus === "online") hits.push(tgt)
|
||||
} else if (impliesDegradedEdge && s.status === "degraded") {
|
||||
if (s.prevStatus === null || s.prevStatus === "online") hits.push(tgt)
|
||||
} else if (wantDegraded && s.status === "degraded" && !impliesDegradedEdge) {
|
||||
hits.push(tgt)
|
||||
} else if (
|
||||
wantOffline &&
|
||||
s.status === "offline" &&
|
||||
!impliesOfflineEdge &&
|
||||
!isOtkluchilsya
|
||||
) {
|
||||
hits.push(tgt)
|
||||
}
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const title = formatServerAlertTitle(rule.name, wantRecovery)
|
||||
const msg = `${title}: ${hits.join(", ")} — ${cond}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: wantRecovery ? "recovery" : "problem",
|
||||
payloadHash: hashPayload([rule.id, "server", hits.sort().join(","), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalRtt(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const threshold = firstNumberInString(cond)
|
||||
const wantLow = /<|ниже|меньше/i.test(cond) && !/>|выше|больше/i.test(cond)
|
||||
const wantHigh = />|выше|больше/i.test(cond)
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const p = findProbe(snap, tgt)
|
||||
if (!p) continue
|
||||
if (p.status === "down" || p.status === "warn") {
|
||||
hits.push(`${tgt} (${p.status}, RTT ${p.rttMs ?? "—"} мс)`)
|
||||
continue
|
||||
}
|
||||
if (p.rttMs == null) continue
|
||||
if (wantHigh && threshold != null && p.rttMs > threshold) hits.push(`${tgt} (${p.rttMs} мс)`)
|
||||
if (wantLow && threshold != null && p.rttMs < threshold) hits.push(`${tgt} (${p.rttMs} мс)`)
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: RTT ${cond} — ${hits.join("; ")}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: "neutral",
|
||||
payloadHash: hashPayload([rule.id, "rtt", hits.sort().join(";"), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalLoss(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const threshold = firstNumberInString(cond)
|
||||
if (threshold == null) return null
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const p = findProbe(snap, tgt)
|
||||
if (!p || p.lossPct == null) continue
|
||||
if (p.lossPct > threshold) hits.push(`${tgt} (${p.lossPct}%)`)
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: потери ${cond} — ${hits.join("; ")}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: "neutral",
|
||||
payloadHash: hashPayload([rule.id, "loss", hits.sort().join(";"), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalGreTunnel(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const lc = cond.toLowerCase()
|
||||
const wantOffline = lc.includes("offline") && !lc.includes("восстанов")
|
||||
const wantRecovery =
|
||||
lc.includes("восстанов") && !/offline|отключ/i.test(lc)
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const g = findGreTunnel(snap, tgt)
|
||||
if (!g) continue
|
||||
if (wantRecovery) {
|
||||
const prev = g.prevStatus
|
||||
if (
|
||||
g.status === "up" &&
|
||||
prev != null &&
|
||||
(prev === "down" || prev === "degraded")
|
||||
) {
|
||||
hits.push(tgt)
|
||||
}
|
||||
} else if (wantOffline && g.status === "down") {
|
||||
hits.push(tgt)
|
||||
}
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: ${hits.join(", ")} — ${cond}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: wantRecovery ? "recovery" : "problem",
|
||||
payloadHash: hashPayload([rule.id, "gre-tunnel", hits.sort().join(","), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalBgpPeer(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const lc = cond.toLowerCase()
|
||||
const wantDown = lc.includes("разорвал")
|
||||
const wantUp = lc.includes("восстановил")
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const b = findBgpPeer(snap, tgt)
|
||||
if (!b) continue
|
||||
if (wantDown && !isBgpEstablished(b.state)) hits.push(tgt)
|
||||
if (
|
||||
wantUp &&
|
||||
isBgpEstablished(b.state) &&
|
||||
b.prevState != null &&
|
||||
String(b.prevState).trim() !== "" &&
|
||||
!isBgpEstablished(b.prevState)
|
||||
) {
|
||||
hits.push(tgt)
|
||||
}
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: ${hits.join(", ")} — ${cond}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: wantUp ? "recovery" : "problem",
|
||||
payloadHash: hashPayload([rule.id, "bgp-peer", hits.sort().join(","), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalTraffic(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const threshold = firstNumberInString(cond)
|
||||
if (threshold == null) return null
|
||||
const rxLow = /RX\s*</i.test(cond)
|
||||
const txLow = /TX\s*</i.test(cond)
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const t = findTraffic(snap, tgt)
|
||||
if (!t) continue
|
||||
if (rxLow && t.rxMbps < threshold) hits.push(`${tgt} RX ${t.rxMbps.toFixed(2)} Мбит/с`)
|
||||
if (txLow && t.txMbps < threshold) hits.push(`${tgt} TX ${t.txMbps.toFixed(2)} Мбит/с`)
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: трафик ${cond} — ${hits.join("; ")}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: "neutral",
|
||||
payloadHash: hashPayload([rule.id, "traffic", hits.sort().join(";"), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evaluateRuleWithCondition(
|
||||
rule: ApiAlertRule,
|
||||
snap: SignalSnapshot,
|
||||
cond: string,
|
||||
): RuleEvalHit | null {
|
||||
const c = cond.trim()
|
||||
if (!c) return null
|
||||
switch (rule.type) {
|
||||
case "server":
|
||||
case "gre-client":
|
||||
return evalServerLike(rule, snap, c)
|
||||
case "rtt":
|
||||
return evalRtt(rule, snap, c)
|
||||
case "loss":
|
||||
return evalLoss(rule, snap, c)
|
||||
case "traffic":
|
||||
return evalTraffic(rule, snap, c)
|
||||
case "gre-tunnel":
|
||||
return evalGreTunnel(rule, snap, c)
|
||||
case "bgp-peer":
|
||||
return evalBgpPeer(rule, snap, c)
|
||||
case "bgp-prefix":
|
||||
return null
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** Одно правило: OR по targets; OR по conditions (любое из выбранных). */
|
||||
export function evaluateRule(rule: ApiAlertRule, snap: SignalSnapshot): RuleEvalHit | null {
|
||||
if (!rule.enabled || rule.targets.length === 0) return null
|
||||
const conds =
|
||||
rule.conditions?.length > 0
|
||||
? rule.conditions.map((c) => c.trim()).filter(Boolean)
|
||||
: [rule.condition.trim()].filter(Boolean)
|
||||
if (conds.length === 0) return null
|
||||
|
||||
const hits: RuleEvalHit[] = []
|
||||
for (const cond of conds) {
|
||||
const h = evaluateRuleWithCondition(rule, snap, cond)
|
||||
if (h) hits.push(h)
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
if (hits.length === 1) return hits[0]!
|
||||
const message = hits.map((h) => h.message).join("\n")
|
||||
const severity = hits.reduce((a, h) => maxSeverity(a, h.severity), hits[0]!.severity)
|
||||
const payloadHash = hashPayload([rule.id, "or-conds", ...hits.map((h) => h.payloadHash).sort()])
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity,
|
||||
message,
|
||||
transition: hits.some((h) => h.transition === "problem")
|
||||
? "problem"
|
||||
: hits.some((h) => h.transition === "recovery")
|
||||
? "recovery"
|
||||
: "neutral",
|
||||
payloadHash,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { groupShouldFire } from "./groups.js"
|
||||
import { cooldownToMs, isCooldownElapsed } from "./cooldown.js"
|
||||
|
||||
assert.equal(groupShouldFire("any", []), false)
|
||||
assert.equal(groupShouldFire("any", [false, false]), false)
|
||||
assert.equal(groupShouldFire("any", [false, true]), true)
|
||||
assert.equal(groupShouldFire("all", [true, true]), true)
|
||||
assert.equal(groupShouldFire("all", [true, false]), false)
|
||||
assert.equal(groupShouldFire("all", [false, false]), false)
|
||||
|
||||
assert.equal(cooldownToMs("1м"), 60_000)
|
||||
assert.equal(isCooldownElapsed(null, "5м"), true)
|
||||
assert.equal(isCooldownElapsed(new Date(Date.now() - 6 * 60_000).toISOString(), "5м"), true)
|
||||
assert.equal(isCooldownElapsed(new Date(Date.now() - 2 * 60_000).toISOString(), "5м"), false)
|
||||
|
||||
console.log("alert-engine groups/cooldown tests ok")
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* Логика группы: ANY = хотя бы один член «в огне»; ALL = все члены «в огне».
|
||||
* `memberFiring` — только включённые правила группы, по порядку.
|
||||
*/
|
||||
export function groupShouldFire(combineMode: "any" | "all", memberFiring: boolean[]): boolean {
|
||||
if (memberFiring.length === 0) return false
|
||||
if (combineMode === "any") return memberFiring.some(Boolean)
|
||||
return memberFiring.every(Boolean)
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { and, asc, eq, lte } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertOutbox } from "../../db/schema.js"
|
||||
import { appendAlertHistory, sendTelegramAlertMessage } from "../alerts-service.js"
|
||||
|
||||
export type AlertOutboxPayload = {
|
||||
text: string
|
||||
chatId?: string
|
||||
history: {
|
||||
id: string
|
||||
ruleId?: string | null
|
||||
groupId?: string | null
|
||||
ruleName: string
|
||||
severity: "critical" | "warning" | "info"
|
||||
message: string
|
||||
firedAt: string
|
||||
}
|
||||
}
|
||||
|
||||
function safeParsePayload(raw: string): AlertOutboxPayload | null {
|
||||
try {
|
||||
return JSON.parse(raw) as AlertOutboxPayload
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function enqueueTelegramOutbox(input: {
|
||||
id: string
|
||||
dedupeKey: string
|
||||
payload: AlertOutboxPayload
|
||||
maxRetries?: number
|
||||
}): boolean {
|
||||
const existing = db
|
||||
.select()
|
||||
.from(alertOutbox)
|
||||
.where(eq(alertOutbox.dedupeKey, input.dedupeKey))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
if (existing && (existing.status === "pending" || existing.status === "sent")) return false
|
||||
const nowIso = new Date().toISOString()
|
||||
db.insert(alertOutbox)
|
||||
.values({
|
||||
id: input.id,
|
||||
dedupeKey: input.dedupeKey,
|
||||
channel: "telegram",
|
||||
status: "pending",
|
||||
retryCount: 0,
|
||||
maxRetries: Math.max(1, Math.floor(input.maxRetries ?? 3)),
|
||||
nextAttemptAt: nowIso,
|
||||
payloadJson: JSON.stringify(input.payload),
|
||||
})
|
||||
.run()
|
||||
return true
|
||||
}
|
||||
|
||||
export function dispatchPendingOutbox(limit = 25): Promise<{ sent: number; failed: number; errors: string[] }> {
|
||||
return (async () => {
|
||||
const nowIso = new Date().toISOString()
|
||||
const rows = db
|
||||
.select()
|
||||
.from(alertOutbox)
|
||||
.where(and(eq(alertOutbox.status, "pending"), lte(alertOutbox.nextAttemptAt, nowIso)))
|
||||
.orderBy(asc(alertOutbox.createdAt))
|
||||
.limit(Math.max(1, limit))
|
||||
.all()
|
||||
|
||||
let sent = 0
|
||||
let failed = 0
|
||||
const errors: string[] = []
|
||||
|
||||
for (const row of rows) {
|
||||
const payload = safeParsePayload(row.payloadJson)
|
||||
if (!payload) {
|
||||
db.update(alertOutbox)
|
||||
.set({
|
||||
status: "failed",
|
||||
lastError: "invalid payload_json",
|
||||
})
|
||||
.where(eq(alertOutbox.id, row.id))
|
||||
.run()
|
||||
failed += 1
|
||||
continue
|
||||
}
|
||||
const send = await sendTelegramAlertMessage({ text: payload.text, chatId: payload.chatId })
|
||||
if (send.ok) {
|
||||
db.update(alertOutbox)
|
||||
.set({
|
||||
status: "sent",
|
||||
sentAt: new Date().toISOString(),
|
||||
lastError: null,
|
||||
})
|
||||
.where(eq(alertOutbox.id, row.id))
|
||||
.run()
|
||||
appendAlertHistory({
|
||||
id: payload.history.id,
|
||||
ruleId: payload.history.ruleId ?? null,
|
||||
groupId: payload.history.groupId ?? null,
|
||||
ruleName: payload.history.ruleName,
|
||||
severity: payload.history.severity,
|
||||
message: payload.history.message,
|
||||
sentOk: true,
|
||||
firedAt: payload.history.firedAt,
|
||||
})
|
||||
sent += 1
|
||||
continue
|
||||
}
|
||||
const nextRetry = row.retryCount + 1
|
||||
const exhausted = nextRetry >= row.maxRetries
|
||||
db.update(alertOutbox)
|
||||
.set({
|
||||
retryCount: nextRetry,
|
||||
status: exhausted ? "failed" : "pending",
|
||||
nextAttemptAt: new Date(Date.now() + Math.min(300_000, nextRetry * 15_000)).toISOString(),
|
||||
lastError: send.error,
|
||||
})
|
||||
.where(eq(alertOutbox.id, row.id))
|
||||
.run()
|
||||
if (exhausted) {
|
||||
appendAlertHistory({
|
||||
id: payload.history.id,
|
||||
ruleId: payload.history.ruleId ?? null,
|
||||
groupId: payload.history.groupId ?? null,
|
||||
ruleName: payload.history.ruleName,
|
||||
severity: payload.history.severity,
|
||||
message: payload.history.message,
|
||||
sentOk: false,
|
||||
firedAt: payload.history.firedAt,
|
||||
})
|
||||
failed += 1
|
||||
} else {
|
||||
errors.push(`outbox ${row.id}: ${send.error}`)
|
||||
}
|
||||
}
|
||||
|
||||
return { sent, failed, errors }
|
||||
})()
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEnginePrevLive } from "../../db/schema.js"
|
||||
|
||||
export type PrevLiveKind = "gre" | "bgp"
|
||||
|
||||
/** Предыдущие строковые состояния (GRE: up|down|degraded; BGP: state как с роутера). */
|
||||
export type PrevLiveStringMap = Record<string, string>
|
||||
|
||||
function safeParseMap(json: string | null | undefined): PrevLiveStringMap {
|
||||
if (!json || json.trim() === "") return {}
|
||||
try {
|
||||
const v = JSON.parse(json) as unknown
|
||||
if (v == null || typeof v !== "object" || Array.isArray(v)) return {}
|
||||
return v as PrevLiveStringMap
|
||||
} catch {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
export function loadPrevLiveMap(kind: PrevLiveKind): PrevLiveStringMap {
|
||||
const row = db
|
||||
.select()
|
||||
.from(alertEnginePrevLive)
|
||||
.where(eq(alertEnginePrevLive.kind, kind))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
return safeParseMap(row?.payloadJson)
|
||||
}
|
||||
|
||||
export function savePrevLiveMap(kind: PrevLiveKind, map: PrevLiveStringMap) {
|
||||
const payloadJson = JSON.stringify(map)
|
||||
const existing = db
|
||||
.select()
|
||||
.from(alertEnginePrevLive)
|
||||
.where(eq(alertEnginePrevLive.kind, kind))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
if (existing) {
|
||||
db.update(alertEnginePrevLive)
|
||||
.set({ payloadJson, updatedAt: new Date().toISOString() })
|
||||
.where(eq(alertEnginePrevLive.kind, kind))
|
||||
.run()
|
||||
} else {
|
||||
db.insert(alertEnginePrevLive).values({ kind, payloadJson, updatedAt: new Date().toISOString() }).run()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import { evaluateRule } from "./evaluate-rule.js"
|
||||
import type { RuleEvalHit, SignalSnapshot } from "./types.js"
|
||||
|
||||
export function matchRules(rules: ApiAlertRule[], snap: SignalSnapshot): {
|
||||
hitByRule: Map<string, RuleEvalHit | null>
|
||||
errors: string[]
|
||||
} {
|
||||
const errors: string[] = []
|
||||
const hitByRule = new Map<string, RuleEvalHit | null>()
|
||||
for (const rule of rules) {
|
||||
try {
|
||||
hitByRule.set(rule.id, evaluateRule(rule, snap))
|
||||
} catch (e) {
|
||||
errors.push(`rule ${rule.id}: ${e instanceof Error ? e.message : String(e)}`)
|
||||
hitByRule.set(rule.id, null)
|
||||
}
|
||||
}
|
||||
return { hitByRule, errors }
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEngineState } from "../../db/schema.js"
|
||||
import { eq } from "drizzle-orm"
|
||||
import {
|
||||
getTelegramPublic,
|
||||
listAlertGroups,
|
||||
listAlertRules,
|
||||
type ApiAlertRule,
|
||||
} from "../alerts-service.js"
|
||||
import { getGreBgpSnapshotCollectorState } from "../gre-bgp-snapshot-collector.js"
|
||||
import { isAnyAlertSnapshotSourceJobRunning } from "../scheduler-running.js"
|
||||
import { getServersRestPingCollectorState } from "../servers-rest-ping-collector.js"
|
||||
import { isTrafficCollecting } from "../traffic-collector.js"
|
||||
import { getUptimeCollectorsState } from "../uptime-collector.js"
|
||||
import { computeDecisions } from "./decision-engine.js"
|
||||
import { clearConfirmPendingAfterSend } from "./confirm-stability.js"
|
||||
import { dispatchPendingOutbox, enqueueTelegramOutbox } from "./outbox.js"
|
||||
import { matchRules } from "./rule-matcher.js"
|
||||
import { ingestAlertSignals } from "./signal-ingestor.js"
|
||||
import { getLatestSourceFinishedAt, getSourceWatermark, updateSourceWatermark } from "./source-watermark.js"
|
||||
import { pickTelegramAlertEmoji } from "./telegram-emoji.js"
|
||||
import type { AlertEngineRunResult, RuleEvalHit } from "./types.js"
|
||||
|
||||
const SNAPSHOT_SOURCE_WAIT_MS = 30_000
|
||||
const SNAPSHOT_SOURCE_POLL_MS = 20
|
||||
|
||||
/**
|
||||
* Не строить снимок, пока коллекторы пишут в SQLite — иначе гонка с `alert_engine` по таймеру
|
||||
* (в т.ч. слот планировщика занят до первого `await` в коллекторе, когда внутренний `collecting` ещё false).
|
||||
*/
|
||||
async function awaitSnapshotSourcesIdle(): Promise<void> {
|
||||
const t0 = Date.now()
|
||||
while (Date.now() - t0 < SNAPSHOT_SOURCE_WAIT_MS) {
|
||||
const busy =
|
||||
isAnyAlertSnapshotSourceJobRunning() ||
|
||||
getServersRestPingCollectorState().running ||
|
||||
getGreBgpSnapshotCollectorState().running ||
|
||||
getUptimeCollectorsState().resources ||
|
||||
getUptimeCollectorsState().ping ||
|
||||
isTrafficCollecting()
|
||||
if (!busy) return
|
||||
await new Promise<void>((r) => setTimeout(r, SNAPSHOT_SOURCE_POLL_MS))
|
||||
}
|
||||
}
|
||||
|
||||
function newHistoryId(): string {
|
||||
return `ah-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
|
||||
}
|
||||
|
||||
function newOutboxId(): string {
|
||||
return `ao-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
|
||||
}
|
||||
|
||||
function loadEngineStateMap(): Map<string, { lastFiredAt: string; lastPayloadHash: string | null }> {
|
||||
const rows = db.select().from(alertEngineState).all()
|
||||
const m = new Map<string, { lastFiredAt: string; lastPayloadHash: string | null }>()
|
||||
for (const r of rows) {
|
||||
m.set(r.scopeKey, { lastFiredAt: r.lastFiredAt, lastPayloadHash: r.lastPayloadHash ?? null })
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
function upsertEngineState(scopeKey: string, lastFiredAt: string, lastPayloadHash: string | null) {
|
||||
const existing = db.select().from(alertEngineState).where(eq(alertEngineState.scopeKey, scopeKey)).limit(1).all()[0]
|
||||
if (existing) {
|
||||
db.update(alertEngineState)
|
||||
.set({ lastFiredAt, lastPayloadHash })
|
||||
.where(eq(alertEngineState.scopeKey, scopeKey))
|
||||
.run()
|
||||
} else {
|
||||
db.insert(alertEngineState).values({ scopeKey, lastFiredAt, lastPayloadHash }).run()
|
||||
}
|
||||
}
|
||||
|
||||
function severityRank(s: ApiAlertRule["severity"]): number {
|
||||
if (s === "critical") return 3
|
||||
if (s === "warning") return 2
|
||||
return 1
|
||||
}
|
||||
|
||||
function maxSeverity(a: ApiAlertRule["severity"], b: ApiAlertRule["severity"]): ApiAlertRule["severity"] {
|
||||
return severityRank(a) >= severityRank(b) ? a : b
|
||||
}
|
||||
function ruleScopeKey(ruleId: string, transition?: RuleEvalHit["transition"]): string {
|
||||
return `rule:${ruleId}:${transition ?? "neutral"}`
|
||||
}
|
||||
|
||||
/** Один проход движка: оценка правил, группы, Telegram, история, состояние cooldown. */
|
||||
export async function runAlertEngineOnce(): Promise<AlertEngineRunResult> {
|
||||
const errors: string[] = []
|
||||
const sampledAt = new Date().toISOString()
|
||||
const latestSourceFinishedAt = getLatestSourceFinishedAt()
|
||||
const prevSourceFinishedAt = getSourceWatermark()
|
||||
const hasNewSources =
|
||||
latestSourceFinishedAt != null &&
|
||||
(prevSourceFinishedAt == null || latestSourceFinishedAt > prevSourceFinishedAt)
|
||||
|
||||
const rules = listAlertRules()
|
||||
const groups = listAlertGroups()
|
||||
const state = loadEngineStateMap()
|
||||
const pub = getTelegramPublic()
|
||||
const canSend = pub.tokenConfigured && Boolean(pub.chatId?.trim())
|
||||
let standaloneFires = 0
|
||||
let groupFires = 0
|
||||
|
||||
let ruleDiag: AlertEngineRunResult["ruleDiag"] = []
|
||||
if (hasNewSources) {
|
||||
await awaitSnapshotSourcesIdle()
|
||||
const snap = ingestAlertSignals()
|
||||
const matched = matchRules(rules, snap)
|
||||
errors.push(...matched.errors)
|
||||
const { standalone, grouped, ruleDiag: diag } = computeDecisions({
|
||||
rules,
|
||||
groups,
|
||||
hitByRule: matched.hitByRule,
|
||||
state,
|
||||
canSend,
|
||||
})
|
||||
ruleDiag = diag
|
||||
|
||||
for (const d of standalone) {
|
||||
if (!canSend) break
|
||||
const firedAt = new Date().toISOString()
|
||||
const key = ruleScopeKey(d.rule.id, d.hit.transition)
|
||||
const queued = enqueueTelegramOutbox({
|
||||
id: newOutboxId(),
|
||||
dedupeKey: `${key}:${d.hit.payloadHash}`,
|
||||
payload: {
|
||||
text: `${pickTelegramAlertEmoji(d.hit.message)} ${d.hit.message}`,
|
||||
chatId: d.rule.chatId?.trim() || undefined,
|
||||
history: {
|
||||
id: newHistoryId(),
|
||||
ruleId: d.rule.id,
|
||||
groupId: null,
|
||||
ruleName: d.hit.ruleName,
|
||||
severity: d.hit.severity,
|
||||
message: d.hit.message,
|
||||
firedAt,
|
||||
},
|
||||
},
|
||||
})
|
||||
if (!queued) continue
|
||||
standaloneFires += 1
|
||||
clearConfirmPendingAfterSend([d.rule.id])
|
||||
upsertEngineState(key, firedAt, d.hit.payloadHash)
|
||||
state.set(key, { lastFiredAt: firedAt, lastPayloadHash: d.hit.payloadHash })
|
||||
}
|
||||
|
||||
for (const d of grouped) {
|
||||
if (!canSend) break
|
||||
const firedAt = new Date().toISOString()
|
||||
const lines = d.hits.map((h) => h.message)
|
||||
const sev = d.hits.reduce((a, h) => maxSeverity(a, h.severity), d.hits[0]!.severity)
|
||||
const body = `Группа «${d.group.name}» (${d.group.combineMode === "any" ? "ANY" : "ALL"})\n\n${lines.join("\n")}`
|
||||
const hash = d.hits.map((h) => h.payloadHash).sort().join("|")
|
||||
const key = `group:${d.group.id}`
|
||||
const queued = enqueueTelegramOutbox({
|
||||
id: newOutboxId(),
|
||||
dedupeKey: `${key}:${hash}`,
|
||||
payload: {
|
||||
text: `${pickTelegramAlertEmoji(body)} ${body}`,
|
||||
history: {
|
||||
id: newHistoryId(),
|
||||
ruleId: null,
|
||||
groupId: d.group.id,
|
||||
ruleName: `Группа: ${d.group.name}`,
|
||||
severity: sev,
|
||||
message: body,
|
||||
firedAt,
|
||||
},
|
||||
},
|
||||
})
|
||||
if (!queued) continue
|
||||
groupFires += 1
|
||||
upsertEngineState(key, firedAt, hash)
|
||||
state.set(key, { lastFiredAt: firedAt, lastPayloadHash: hash })
|
||||
clearConfirmPendingAfterSend(d.members.map((r) => r.id))
|
||||
}
|
||||
updateSourceWatermark(latestSourceFinishedAt)
|
||||
}
|
||||
|
||||
const outbox = await dispatchPendingOutbox()
|
||||
errors.push(...outbox.errors)
|
||||
|
||||
return {
|
||||
sampledAt,
|
||||
rulesChecked: hasNewSources ? rules.length : 0,
|
||||
standaloneFires,
|
||||
groupFires,
|
||||
skippedNoTelegram: !canSend,
|
||||
errors,
|
||||
ruleDiag: ruleDiag ?? [],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { buildSignalSnapshotFromCollectors } from "./signals-from-collectors.js"
|
||||
|
||||
/** Единая точка входа для подготовки сигналов к rule matching. */
|
||||
export function ingestAlertSignals() {
|
||||
return buildSignalSnapshotFromCollectors()
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
import { sqliteDatabase } from "../../db/index.js"
|
||||
import type {
|
||||
GreBgpSnapshotRunSnapshot,
|
||||
PingRunSnapshot,
|
||||
ResourcesRunSnapshot,
|
||||
SchedulerRunSnapshot,
|
||||
ServersRestPingRunSnapshot,
|
||||
TrafficRunSnapshot,
|
||||
} from "../../types/scheduler-run-snapshot.js"
|
||||
import type {
|
||||
BgpPeerSignal,
|
||||
GreTunnelSignal,
|
||||
ProbeSignal,
|
||||
ServerSignal,
|
||||
SignalSnapshot,
|
||||
TrafficServerSignal,
|
||||
} from "./types.js"
|
||||
|
||||
function normalizeNameKey(v: string): string {
|
||||
return v.trim().toLowerCase()
|
||||
}
|
||||
|
||||
function safeParseRunSnapshot(raw: string | null): SchedulerRunSnapshot | null {
|
||||
if (!raw) return null
|
||||
try {
|
||||
return JSON.parse(raw) as SchedulerRunSnapshot
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function readLatestOkRunSnapshots(jobKey: string, limit = 32): SchedulerRunSnapshot[] {
|
||||
const rows = sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT result_json AS resultJson
|
||||
FROM scheduler_runs
|
||||
WHERE job_key = ? AND status = 'ok' AND result_json IS NOT NULL
|
||||
ORDER BY finished_at DESC
|
||||
LIMIT ?`,
|
||||
)
|
||||
.all(jobKey, limit) as { resultJson: string | null }[]
|
||||
const out: SchedulerRunSnapshot[] = []
|
||||
for (const row of rows) {
|
||||
const parsed = safeParseRunSnapshot(row.resultJson)
|
||||
if (!parsed || parsed.job !== jobKey) continue
|
||||
out.push(parsed)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type TriState = "online" | "offline" | "degraded"
|
||||
|
||||
function timelineToServerSignal(name: string, sampledAt: string, statuses: TriState[]): ServerSignal {
|
||||
return {
|
||||
name,
|
||||
status: statuses[0] ?? "offline",
|
||||
prevStatus: statuses[1] ?? null,
|
||||
prev2Status: statuses[2] ?? null,
|
||||
sampledAt,
|
||||
}
|
||||
}
|
||||
|
||||
function collectServerSignalsFromResourceRuns(runs: ResourcesRunSnapshot[]): Map<string, ServerSignal> {
|
||||
const byKey = new Map<
|
||||
string,
|
||||
{ name: string; sampledAt: string; statuses: TriState[] }
|
||||
>()
|
||||
for (const run of runs) {
|
||||
for (const s of run.servers ?? []) {
|
||||
const name = String(s.name ?? "").trim()
|
||||
if (!name) continue
|
||||
const key = normalizeNameKey(name)
|
||||
const rec = byKey.get(key) ?? { name, sampledAt: run.sampledAt, statuses: [] }
|
||||
if (rec.statuses.length < 3) rec.statuses.push(s.status === "online" ? "online" : "offline")
|
||||
if (!byKey.has(key)) rec.sampledAt = run.sampledAt
|
||||
byKey.set(key, rec)
|
||||
}
|
||||
}
|
||||
const out = new Map<string, ServerSignal>()
|
||||
for (const [key, rec] of byKey) out.set(key, timelineToServerSignal(rec.name, rec.sampledAt, rec.statuses))
|
||||
return out
|
||||
}
|
||||
|
||||
function collectServerSignalsFromRestRuns(runs: ServersRestPingRunSnapshot[]): Map<string, ServerSignal> {
|
||||
const byKey = new Map<
|
||||
string,
|
||||
{ name: string; sampledAt: string; statuses: TriState[] }
|
||||
>()
|
||||
for (const run of runs) {
|
||||
for (const s of run.servers ?? []) {
|
||||
const name = String(s.name ?? "").trim()
|
||||
if (!name) continue
|
||||
const key = normalizeNameKey(name)
|
||||
const rec = byKey.get(key) ?? { name, sampledAt: run.sampledAt, statuses: [] }
|
||||
if (rec.statuses.length < 3) rec.statuses.push(s.ok ? "online" : "offline")
|
||||
if (!byKey.has(key)) rec.sampledAt = run.sampledAt
|
||||
byKey.set(key, rec)
|
||||
}
|
||||
}
|
||||
const out = new Map<string, ServerSignal>()
|
||||
for (const [key, rec] of byKey) out.set(key, timelineToServerSignal(rec.name, rec.sampledAt, rec.statuses))
|
||||
return out
|
||||
}
|
||||
|
||||
function mergeResourceAndRestServer(resource: ServerSignal, rest: ServerSignal): ServerSignal {
|
||||
const rUp = resource.status === "online"
|
||||
const tUp = rest.status === "online"
|
||||
if (tUp && !rUp) {
|
||||
const rBad = resource.status === "offline" || resource.status === "degraded"
|
||||
if (rBad) {
|
||||
return {
|
||||
name: rest.name,
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: resource.status,
|
||||
sampledAt: rest.sampledAt,
|
||||
}
|
||||
}
|
||||
return rest
|
||||
}
|
||||
if (rUp && !tUp) return resource
|
||||
if (tUp && rUp) {
|
||||
if (rest.sampledAt >= resource.sampledAt) {
|
||||
if (resource.prevStatus === "offline" || resource.prevStatus === "degraded") {
|
||||
return {
|
||||
name: rest.name,
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: resource.prevStatus === "degraded" ? "degraded" : "offline",
|
||||
sampledAt: rest.sampledAt,
|
||||
}
|
||||
}
|
||||
if (
|
||||
resource.prevStatus === "online" &&
|
||||
resource.prev2Status != null &&
|
||||
(resource.prev2Status === "offline" || resource.prev2Status === "degraded")
|
||||
) {
|
||||
return {
|
||||
name: rest.name,
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: resource.prev2Status,
|
||||
sampledAt: rest.sampledAt,
|
||||
}
|
||||
}
|
||||
}
|
||||
return rest.sampledAt >= resource.sampledAt ? rest : resource
|
||||
}
|
||||
return rest.sampledAt >= resource.sampledAt ? rest : resource
|
||||
}
|
||||
|
||||
function collectProbeSignalsFromRuns(runs: PingRunSnapshot[]): ProbeSignal[] {
|
||||
const out = new Map<string, ProbeSignal>()
|
||||
for (const run of runs) {
|
||||
for (const p of run.probes ?? []) {
|
||||
const key = `${String(p.name ?? "").trim()} → ${String(p.target ?? "").trim()}`
|
||||
if (!key.trim() || out.has(key)) continue
|
||||
out.set(key, {
|
||||
key,
|
||||
status: p.status,
|
||||
rttMs: p.rttMs ?? null,
|
||||
lossPct: p.lossPct ?? null,
|
||||
sampledAt: run.sampledAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
return [...out.values()]
|
||||
}
|
||||
|
||||
function collectTrafficSignalsFromRuns(runs: TrafficRunSnapshot[]): TrafficServerSignal[] {
|
||||
const out = new Map<string, TrafficServerSignal>()
|
||||
for (const run of runs) {
|
||||
for (const s of run.servers ?? []) {
|
||||
const name = String(s.name ?? "").trim()
|
||||
if (!name) continue
|
||||
const key = normalizeNameKey(name)
|
||||
if (out.has(key)) continue
|
||||
out.set(key, {
|
||||
serverName: name,
|
||||
rxMbps: Number(s.sumRxMbps ?? 0),
|
||||
txMbps: Number(s.sumTxMbps ?? 0),
|
||||
sampledAt: run.sampledAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
return [...out.values()]
|
||||
}
|
||||
|
||||
function collectGreSignalsFromRuns(runs: GreBgpSnapshotRunSnapshot[]): GreTunnelSignal[] {
|
||||
const byTarget = new Map<string, GreTunnelSignal>()
|
||||
for (const run of runs) {
|
||||
const greRows = run.greTunnels ?? []
|
||||
for (const row of greRows) {
|
||||
const label = String(row.targetLabel ?? "").trim()
|
||||
if (!label) continue
|
||||
const prev = byTarget.get(label)
|
||||
if (!prev) {
|
||||
byTarget.set(label, {
|
||||
targetLabel: label,
|
||||
status: row.status,
|
||||
prevStatus: null,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (prev.prevStatus == null) prev.prevStatus = row.status
|
||||
}
|
||||
}
|
||||
return [...byTarget.values()]
|
||||
}
|
||||
|
||||
function collectBgpSignalsFromRuns(runs: GreBgpSnapshotRunSnapshot[]): BgpPeerSignal[] {
|
||||
const byKey = new Map<string, BgpPeerSignal>()
|
||||
for (const run of runs) {
|
||||
const peers = run.bgpPeers ?? []
|
||||
for (const row of peers) {
|
||||
const key = String(row.key ?? "").trim()
|
||||
if (!key) continue
|
||||
const prev = byKey.get(key)
|
||||
if (!prev) {
|
||||
byKey.set(key, { key, state: String(row.state ?? "").trim(), prevState: null })
|
||||
continue
|
||||
}
|
||||
if (prev.prevState == null) prev.prevState = String(row.state ?? "").trim()
|
||||
}
|
||||
}
|
||||
return [...byKey.values()]
|
||||
}
|
||||
|
||||
/** Собирает сигналы alert_engine напрямую из результатов collector jobs (`scheduler_runs.result_json`). */
|
||||
export function buildSignalSnapshotFromCollectors(): SignalSnapshot {
|
||||
const sampledAt = new Date().toISOString()
|
||||
const resourceRuns = readLatestOkRunSnapshots("uptime_resources", 16) as ResourcesRunSnapshot[]
|
||||
const restRuns = readLatestOkRunSnapshots("servers_rest_ping", 16) as ServersRestPingRunSnapshot[]
|
||||
const pingRuns = readLatestOkRunSnapshots("uptime_ping", 24) as PingRunSnapshot[]
|
||||
const trafficRuns = readLatestOkRunSnapshots("traffic", 8) as TrafficRunSnapshot[]
|
||||
const greBgpRuns = readLatestOkRunSnapshots("gre_bgp", 8) as GreBgpSnapshotRunSnapshot[]
|
||||
|
||||
const resourceByKey = collectServerSignalsFromResourceRuns(resourceRuns)
|
||||
const restByKey = collectServerSignalsFromRestRuns(restRuns)
|
||||
const serverKeys = new Set([...resourceByKey.keys(), ...restByKey.keys()])
|
||||
const servers: ServerSignal[] = []
|
||||
for (const key of serverKeys) {
|
||||
const resource = resourceByKey.get(key)
|
||||
const rest = restByKey.get(key)
|
||||
if (!rest) {
|
||||
if (resource) servers.push(resource)
|
||||
continue
|
||||
}
|
||||
if (!resource) {
|
||||
servers.push(rest)
|
||||
continue
|
||||
}
|
||||
servers.push(mergeResourceAndRestServer(resource, rest))
|
||||
}
|
||||
servers.sort((a, b) => a.name.localeCompare(b.name, undefined, { sensitivity: "base" }))
|
||||
|
||||
return {
|
||||
sampledAt,
|
||||
servers,
|
||||
probes: collectProbeSignalsFromRuns(pingRuns),
|
||||
trafficByServer: collectTrafficSignalsFromRuns(trafficRuns),
|
||||
greTunnels: collectGreSignalsFromRuns(greBgpRuns),
|
||||
bgpSessions: collectBgpSignalsFromRuns(greBgpRuns),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { buildSignalSnapshotFromCollectors } from "./signals-from-collectors.js"
|
||||
import type { SignalSnapshot } from "./types.js"
|
||||
|
||||
/** Legacy-совместимость: единый источник сигналов — collector snapshots из scheduler_runs. */
|
||||
export function buildSignalSnapshot(): SignalSnapshot {
|
||||
return buildSignalSnapshotFromCollectors()
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { desc, eq, inArray } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEngineCursor, schedulerRuns } from "../../db/schema.js"
|
||||
|
||||
const SOURCE_JOBS = [
|
||||
"traffic",
|
||||
"servers_rest_ping",
|
||||
"uptime_resources",
|
||||
"uptime_ping",
|
||||
"uptime_speed",
|
||||
"gre_bgp",
|
||||
] as const
|
||||
|
||||
export function getLatestSourceFinishedAt(): string | null {
|
||||
const row = db
|
||||
.select({ finishedAt: schedulerRuns.finishedAt })
|
||||
.from(schedulerRuns)
|
||||
.where(inArray(schedulerRuns.jobKey, [...SOURCE_JOBS]))
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
return row?.finishedAt ?? null
|
||||
}
|
||||
|
||||
export function getSourceWatermark(): string | null {
|
||||
const row = db.select().from(alertEngineCursor).where(eq(alertEngineCursor.id, 1)).limit(1).all()[0]
|
||||
return row?.lastSourceFinishedAt ?? null
|
||||
}
|
||||
|
||||
export function updateSourceWatermark(lastSourceFinishedAt: string | null): void {
|
||||
const existing = db
|
||||
.select()
|
||||
.from(alertEngineCursor)
|
||||
.where(eq(alertEngineCursor.id, 1))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
if (existing) {
|
||||
db.update(alertEngineCursor)
|
||||
.set({
|
||||
lastSourceFinishedAt,
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
.where(eq(alertEngineCursor.id, 1))
|
||||
.run()
|
||||
return
|
||||
}
|
||||
db.insert(alertEngineCursor)
|
||||
.values({
|
||||
id: 1,
|
||||
lastSourceFinishedAt,
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
.run()
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { shouldAllowRecoveryByPolicy } from "./state-machine.js"
|
||||
|
||||
const recoveryHit = {
|
||||
ruleId: "r",
|
||||
ruleName: "r",
|
||||
severity: "warning" as const,
|
||||
message: "восстановился",
|
||||
transition: "recovery" as const,
|
||||
payloadHash: "h1",
|
||||
}
|
||||
|
||||
const problemHit = {
|
||||
...recoveryHit,
|
||||
transition: "problem" as const,
|
||||
}
|
||||
|
||||
assert.equal(shouldAllowRecoveryByPolicy(recoveryHit, "always"), true)
|
||||
assert.equal(shouldAllowRecoveryByPolicy(recoveryHit, "conditional"), true)
|
||||
assert.equal(shouldAllowRecoveryByPolicy(recoveryHit, "never"), false)
|
||||
assert.equal(shouldAllowRecoveryByPolicy(problemHit, "never"), true)
|
||||
|
||||
console.log("alert-engine state-machine tests ok")
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { RuleEvalHit } from "./types.js"
|
||||
|
||||
export type RecoveryMode = "always" | "never" | "conditional"
|
||||
|
||||
export function isRecoveryTransition(hit: RuleEvalHit | null): boolean {
|
||||
return hit?.transition === "recovery"
|
||||
}
|
||||
|
||||
export function shouldAllowRecoveryByPolicy(
|
||||
hit: RuleEvalHit | null,
|
||||
policy: RecoveryMode,
|
||||
): boolean {
|
||||
if (!isRecoveryTransition(hit)) return true
|
||||
if (policy === "never") return false
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { isPositiveRecoveryTelegramText, pickTelegramAlertEmoji } from "./telegram-emoji.js"
|
||||
|
||||
assert.equal(pickTelegramAlertEmoji("Сервер: x — снова в сети: x — восстановился"), "✅")
|
||||
assert.equal(pickTelegramAlertEmoji("Сервер: x недоступен: x — перешёл в offline"), "🚨")
|
||||
assert.equal(isPositiveRecoveryTelegramText("A — восстановился"), true)
|
||||
assert.equal(isPositiveRecoveryTelegramText("A — перешёл в offline"), false)
|
||||
assert.equal(isPositiveRecoveryTelegramText("BGP: p — восстановил сессию"), true)
|
||||
|
||||
console.log("telegram-emoji tests ok")
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Эмодзи в Telegram: восстановление / «подключился» / BGP session up — не сирена.
|
||||
* Текст совпадает с форматом `evaluate-rule` (`… — условие`, несколько строк через \\n).
|
||||
*/
|
||||
function isRecoveryLine(line: string): boolean {
|
||||
const t = line.toLowerCase()
|
||||
return (
|
||||
/[—\-]\s*(восстанов|подключ)/i.test(t) ||
|
||||
/восстановил[аи]?\s+сессию/i.test(t) ||
|
||||
/снова\s+в\s+сети/i.test(t)
|
||||
)
|
||||
}
|
||||
|
||||
/** «Хорошие» переходы: не слать дубль при том же payloadHash (см. `run-once`). */
|
||||
export function isPositiveRecoveryTelegramText(fullText: string): boolean {
|
||||
const lines = fullText
|
||||
.split(/\n+/)
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean)
|
||||
if (lines.length === 0) return false
|
||||
if (lines.length === 1) return isRecoveryLine(lines[0]!)
|
||||
return lines.every(isRecoveryLine)
|
||||
}
|
||||
|
||||
export function pickTelegramAlertEmoji(fullText: string): string {
|
||||
return isPositiveRecoveryTelegramText(fullText) ? "✅" : "🚨"
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
|
||||
export type ServerSignal = {
|
||||
name: string
|
||||
status: "online" | "offline" | "degraded"
|
||||
/** Предпоследний сэмпл (для условий «восстановился» / «подключился») */
|
||||
prevStatus: "online" | "offline" | "degraded" | null
|
||||
/** Третий с конца сэмпл — чтобы «восстановился» не висел истинным до второго online-сэмпла в БД. */
|
||||
prev2Status: "online" | "offline" | "degraded" | null
|
||||
sampledAt: string
|
||||
}
|
||||
|
||||
export type ProbeSignal = {
|
||||
/** Как в UI: `имя пробы → target` */
|
||||
key: string
|
||||
status: "up" | "warn" | "down"
|
||||
rttMs: number | null
|
||||
lossPct: number | null
|
||||
sampledAt: string
|
||||
}
|
||||
|
||||
export type TrafficServerSignal = {
|
||||
serverName: string
|
||||
rxMbps: number
|
||||
txMbps: number
|
||||
sampledAt: string
|
||||
}
|
||||
|
||||
export type GreTunnelSignal = {
|
||||
/** Как в UI: `tunnelName / serverName` */
|
||||
targetLabel: string
|
||||
status: "up" | "down" | "degraded"
|
||||
/** Состояние на прошлом тике движка (из БД), для «восстановился». */
|
||||
prevStatus: "up" | "down" | "degraded" | null
|
||||
}
|
||||
|
||||
export type BgpPeerSignal = {
|
||||
/** Как в UI: `serverName: peerName` */
|
||||
key: string
|
||||
state: string
|
||||
/** Состояние на прошлом тике (строка state с роутера). */
|
||||
prevState: string | null
|
||||
}
|
||||
|
||||
export type SignalSnapshot = {
|
||||
sampledAt: string
|
||||
servers: ServerSignal[]
|
||||
probes: ProbeSignal[]
|
||||
trafficByServer: TrafficServerSignal[]
|
||||
greTunnels: GreTunnelSignal[]
|
||||
bgpSessions: BgpPeerSignal[]
|
||||
}
|
||||
|
||||
export type RuleEvalHit = {
|
||||
ruleId: string
|
||||
ruleName: string
|
||||
severity: ApiAlertRule["severity"]
|
||||
message: string
|
||||
/** Тип перехода для диагностики и логики задержки отправки. */
|
||||
transition: "problem" | "recovery" | "neutral"
|
||||
/** Для дедупа */
|
||||
payloadHash: string
|
||||
}
|
||||
|
||||
import type { AlertEngineRuleDiagSnapshot } from "../../types/scheduler-run-snapshot.js"
|
||||
|
||||
export type AlertEngineRuleDiag = AlertEngineRuleDiagSnapshot
|
||||
|
||||
export type AlertEngineRunResult = {
|
||||
sampledAt: string
|
||||
rulesChecked: number
|
||||
standaloneFires: number
|
||||
groupFires: number
|
||||
skippedNoTelegram: boolean
|
||||
errors: string[]
|
||||
ruleDiag: AlertEngineRuleDiag[]
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
import { asc, desc, eq, sql } from "drizzle-orm"
|
||||
import { db, sqliteDatabase } from "../db/index.js"
|
||||
import {
|
||||
alertGroups,
|
||||
alertHistory,
|
||||
alertRuleConditions,
|
||||
alertRuleTargets,
|
||||
alertRules,
|
||||
alertTelegramSettings,
|
||||
servers,
|
||||
uptimeProbes,
|
||||
} from "../db/schema.js"
|
||||
|
||||
export type ApiAlertGroup = {
|
||||
id: string
|
||||
name: string
|
||||
combineMode: "any" | "all"
|
||||
enabled: boolean
|
||||
cooldownOverride: (typeof alertRules.$inferSelect)["cooldown"] | null
|
||||
}
|
||||
|
||||
export type ApiAlertRule = {
|
||||
id: string
|
||||
name: string
|
||||
type: (typeof alertRules.$inferSelect)["type"]
|
||||
/** Сводная строка для списка / поиска */
|
||||
target: string
|
||||
/** Все объекты правила (OR) */
|
||||
targets: string[]
|
||||
groupId: string | null
|
||||
/** Сводная строка условий для списка */
|
||||
condition: string
|
||||
/** Все условия (OR); если пусто в БД — используется `condition` */
|
||||
conditions: string[]
|
||||
severity: (typeof alertRules.$inferSelect)["severity"]
|
||||
enabled: boolean
|
||||
cooldown: (typeof alertRules.$inferSelect)["cooldown"]
|
||||
/** Секунды стабильности перед отправкой; null — выключено */
|
||||
confirmStabilitySec: number | null
|
||||
/** Политика отправки recovery для этого правила. */
|
||||
recoveryMode: "always" | "never" | "conditional"
|
||||
/** Доп. задержка подтверждения recovery при `conditional`. */
|
||||
recoveryStabilitySec: number | null
|
||||
chatId: string
|
||||
lastFiredAt: string | null
|
||||
}
|
||||
|
||||
export type ApiAlertHistoryEntry = {
|
||||
id: string
|
||||
ruleName: string
|
||||
severity: (typeof alertHistory.$inferSelect)["severity"]
|
||||
message: string
|
||||
sent: boolean
|
||||
firedAt: string
|
||||
source: "db" | "live"
|
||||
}
|
||||
|
||||
function ensureTelegramRow() {
|
||||
let row = db.select().from(alertTelegramSettings).where(eq(alertTelegramSettings.id, 1)).limit(1).all()[0]
|
||||
if (!row) {
|
||||
db.insert(alertTelegramSettings).values({ id: 1 }).run()
|
||||
row = db.select().from(alertTelegramSettings).where(eq(alertTelegramSettings.id, 1)).limit(1).all()[0]
|
||||
}
|
||||
return row!
|
||||
}
|
||||
|
||||
function lastFiredByRuleId(): Map<string, string> {
|
||||
const rows = sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT rule_id AS ruleId, MAX(fired_at) AS mx
|
||||
FROM alert_history
|
||||
WHERE rule_id IS NOT NULL AND rule_id != ''
|
||||
GROUP BY rule_id`,
|
||||
)
|
||||
.all() as { ruleId: string; mx: string }[]
|
||||
const m = new Map<string, string>()
|
||||
for (const r of rows) {
|
||||
if (r.ruleId && r.mx) m.set(r.ruleId, r.mx)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
function targetsByRuleId(): Map<string, string[]> {
|
||||
const rows = db
|
||||
.select()
|
||||
.from(alertRuleTargets)
|
||||
.orderBy(asc(alertRuleTargets.sortIndex), asc(alertRuleTargets.id))
|
||||
.all()
|
||||
const m = new Map<string, string[]>()
|
||||
for (const t of rows) {
|
||||
const rid = t.ruleId
|
||||
const arr = m.get(rid) ?? []
|
||||
arr.push(t.target)
|
||||
m.set(rid, arr)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
function conditionsByRuleId(): Map<string, string[]> {
|
||||
const rows = db
|
||||
.select()
|
||||
.from(alertRuleConditions)
|
||||
.orderBy(asc(alertRuleConditions.sortIndex), asc(alertRuleConditions.id))
|
||||
.all()
|
||||
const m = new Map<string, string[]>()
|
||||
for (const t of rows) {
|
||||
const rid = t.ruleId
|
||||
const arr = m.get(rid) ?? []
|
||||
arr.push(t.conditionLine)
|
||||
m.set(rid, arr)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
export function listAlertGroups(): ApiAlertGroup[] {
|
||||
return db
|
||||
.select()
|
||||
.from(alertGroups)
|
||||
.orderBy(asc(alertGroups.name))
|
||||
.all()
|
||||
.map((g) => ({
|
||||
id: g.id,
|
||||
name: g.name,
|
||||
combineMode: g.combineMode,
|
||||
enabled: g.enabled,
|
||||
cooldownOverride: g.cooldownOverride ?? null,
|
||||
}))
|
||||
}
|
||||
|
||||
export function listAlertRules(): ApiAlertRule[] {
|
||||
const lf = lastFiredByRuleId()
|
||||
const tgtMap = targetsByRuleId()
|
||||
const condMap = conditionsByRuleId()
|
||||
const rows = db.select().from(alertRules).all()
|
||||
return rows.map((r) => {
|
||||
const targets = tgtMap.get(r.id)
|
||||
const ts = targets?.length ? targets : [r.target]
|
||||
const condLines = condMap.get(r.id)
|
||||
const cs = condLines?.length ? condLines : [r.condition]
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
type: r.type,
|
||||
target: r.target,
|
||||
targets: ts,
|
||||
groupId: r.groupId ?? null,
|
||||
condition: r.condition,
|
||||
conditions: cs,
|
||||
severity: r.severity,
|
||||
enabled: r.enabled,
|
||||
cooldown: r.cooldown,
|
||||
confirmStabilitySec:
|
||||
r.confirmStabilitySec != null && r.confirmStabilitySec > 0 ? r.confirmStabilitySec : null,
|
||||
recoveryMode:
|
||||
r.recoveryMode === "never" || r.recoveryMode === "conditional" ? r.recoveryMode : "always",
|
||||
recoveryStabilitySec:
|
||||
r.recoveryStabilitySec != null && r.recoveryStabilitySec > 0 ? r.recoveryStabilitySec : null,
|
||||
chatId: r.chatId ?? "",
|
||||
lastFiredAt: lf.get(r.id) ?? null,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function summarizeTargets(targets: string[]): string {
|
||||
const u = targets.map((t) => t.trim()).filter(Boolean)
|
||||
if (u.length === 0) return ""
|
||||
if (u.length <= 2) return u.join(", ")
|
||||
return `${u[0]}, ${u[1]} +${u.length - 2}`
|
||||
}
|
||||
|
||||
function summarizeConditionLines(lines: string[]): string {
|
||||
const u = lines.map((t) => t.trim()).filter(Boolean)
|
||||
if (u.length === 0) return "—"
|
||||
if (u.length === 1) return u[0]!
|
||||
if (u.length <= 3) return u.join(" · ")
|
||||
return `${u[0]} · ${u[1]} · +${u.length - 2}`
|
||||
}
|
||||
|
||||
export type ReplaceAlertRuleInput = {
|
||||
id: string
|
||||
name: string
|
||||
type: ApiAlertRule["type"]
|
||||
/** @deprecated используйте targets; если нет targets — берётся target */
|
||||
target?: string
|
||||
targets?: string[]
|
||||
/** Сводная строка; если не передана — из conditions */
|
||||
condition?: string
|
||||
/** Несколько условий (OR); если пусто — используется `condition` */
|
||||
conditions?: string[]
|
||||
severity: ApiAlertRule["severity"]
|
||||
enabled: boolean
|
||||
cooldown: ApiAlertRule["cooldown"]
|
||||
/** Секунды; 0 / undefined / null — без задержки подтверждения */
|
||||
confirmStabilitySec?: number | null
|
||||
recoveryMode?: "always" | "never" | "conditional"
|
||||
recoveryStabilitySec?: number | null
|
||||
chatId: string
|
||||
groupId?: string | null
|
||||
}
|
||||
|
||||
export type ReplaceAlertGroupInput = {
|
||||
id: string
|
||||
name: string
|
||||
combineMode: "any" | "all"
|
||||
enabled: boolean
|
||||
cooldownOverride: ApiAlertRule["cooldown"] | null
|
||||
}
|
||||
|
||||
/** Атомарно заменяет группы, правила и строки targets. */
|
||||
export function replaceAlertsConfig(payload: { groups: ReplaceAlertGroupInput[]; rules: ReplaceAlertRuleInput[] }) {
|
||||
const now = sql`(datetime('now'))`
|
||||
const { groups, rules } = payload
|
||||
db.transaction((tx) => {
|
||||
tx.delete(alertRuleConditions).run()
|
||||
tx.delete(alertRuleTargets).run()
|
||||
tx.delete(alertRules).run()
|
||||
tx.delete(alertGroups).run()
|
||||
for (const g of groups) {
|
||||
tx.insert(alertGroups).values({
|
||||
id: g.id,
|
||||
name: g.name.trim() || g.id,
|
||||
combineMode: g.combineMode,
|
||||
enabled: g.enabled,
|
||||
cooldownOverride: g.cooldownOverride ?? null,
|
||||
updatedAt: now,
|
||||
}).run()
|
||||
}
|
||||
for (const r of rules) {
|
||||
const rawTargets =
|
||||
r.targets && r.targets.length > 0
|
||||
? r.targets.map((t) => t.trim()).filter(Boolean)
|
||||
: [String(r.target ?? "").trim()].filter(Boolean)
|
||||
const summary = summarizeTargets(rawTargets.length ? rawTargets : ["—"])
|
||||
const rawConds =
|
||||
r.conditions && r.conditions.length > 0
|
||||
? r.conditions.map((c) => c.trim()).filter(Boolean)
|
||||
: [String(r.condition ?? "").trim()].filter(Boolean)
|
||||
const condSummary = summarizeConditionLines(rawConds.length ? rawConds : ["—"])
|
||||
const stab =
|
||||
r.confirmStabilitySec != null &&
|
||||
Number.isFinite(r.confirmStabilitySec) &&
|
||||
r.confirmStabilitySec > 0
|
||||
? Math.min(86400, Math.max(1, Math.floor(r.confirmStabilitySec)))
|
||||
: null
|
||||
const recoveryMode =
|
||||
r.recoveryMode === "never" || r.recoveryMode === "conditional" ? r.recoveryMode : "always"
|
||||
const recoveryStabilitySec =
|
||||
r.recoveryStabilitySec != null &&
|
||||
Number.isFinite(r.recoveryStabilitySec) &&
|
||||
r.recoveryStabilitySec > 0
|
||||
? Math.min(86400, Math.max(1, Math.floor(r.recoveryStabilitySec)))
|
||||
: null
|
||||
tx.insert(alertRules).values({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
type: r.type,
|
||||
target: summary || "—",
|
||||
condition: condSummary,
|
||||
severity: r.severity,
|
||||
enabled: r.enabled,
|
||||
cooldown: r.cooldown,
|
||||
confirmStabilitySec: stab,
|
||||
recoveryMode,
|
||||
recoveryStabilitySec,
|
||||
chatId: r.chatId ?? "",
|
||||
groupId: r.groupId && r.groupId.trim() ? r.groupId.trim() : null,
|
||||
updatedAt: now,
|
||||
}).run()
|
||||
rawTargets.forEach((t, i) => {
|
||||
tx.insert(alertRuleTargets).values({
|
||||
id: `rt-${r.id}-${i}`,
|
||||
ruleId: r.id,
|
||||
target: t,
|
||||
sortIndex: i,
|
||||
}).run()
|
||||
})
|
||||
rawConds.forEach((c, i) => {
|
||||
tx.insert(alertRuleConditions).values({
|
||||
id: `rc-${r.id}-${i}`,
|
||||
ruleId: r.id,
|
||||
conditionLine: c,
|
||||
sortIndex: i,
|
||||
}).run()
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** Совместимость: только правила, группы не трогаем. */
|
||||
export function replaceAlertRules(rules: ReplaceAlertRuleInput[]) {
|
||||
const curGroups = listAlertGroups()
|
||||
replaceAlertsConfig({ groups: curGroups.map((g) => ({ ...g })), rules })
|
||||
}
|
||||
|
||||
export function getTelegramSettingsRow() {
|
||||
return ensureTelegramRow()
|
||||
}
|
||||
|
||||
export function getTelegramPublic() {
|
||||
const row = ensureTelegramRow()
|
||||
const tid = row.messageThreadId
|
||||
const messageThreadId =
|
||||
typeof tid === "number" && Number.isFinite(tid) && Math.floor(tid) >= 1 ? Math.floor(tid) : null
|
||||
return {
|
||||
chatId: row.chatId ?? "",
|
||||
tokenConfigured: Boolean(row.botToken?.trim()),
|
||||
messageThreadId,
|
||||
}
|
||||
}
|
||||
|
||||
export function updateTelegramSettings(patch: {
|
||||
token?: string | null
|
||||
chatId?: string | undefined
|
||||
/** undefined — не менять; null — сбросить тему */
|
||||
messageThreadId?: number | null
|
||||
}) {
|
||||
const cur = ensureTelegramRow()
|
||||
let nextToken = cur.botToken
|
||||
let nextChat = cur.chatId ?? ""
|
||||
let nextThread: number | null | undefined = undefined
|
||||
|
||||
if (patch.token !== undefined) {
|
||||
if (patch.token === null || patch.token === "") nextToken = ""
|
||||
else nextToken = patch.token.trim()
|
||||
}
|
||||
if (patch.chatId !== undefined) nextChat = patch.chatId.trim()
|
||||
if (patch.messageThreadId !== undefined) {
|
||||
if (patch.messageThreadId === null) nextThread = null
|
||||
else {
|
||||
const n = Math.floor(patch.messageThreadId)
|
||||
nextThread = Number.isFinite(n) && n >= 1 ? n : null
|
||||
}
|
||||
}
|
||||
|
||||
const base = {
|
||||
botToken: nextToken,
|
||||
chatId: nextChat,
|
||||
updatedAt: sql`(datetime('now'))`,
|
||||
}
|
||||
if (nextThread !== undefined) {
|
||||
db.update(alertTelegramSettings)
|
||||
.set({ ...base, messageThreadId: nextThread })
|
||||
.where(eq(alertTelegramSettings.id, 1))
|
||||
.run()
|
||||
} else {
|
||||
db.update(alertTelegramSettings).set(base).where(eq(alertTelegramSettings.id, 1)).run()
|
||||
}
|
||||
return getTelegramPublic()
|
||||
}
|
||||
|
||||
/** Токен только для внутреннего вызова Telegram API (не отдаётся клиенту). */
|
||||
export function getTelegramBotToken(): string {
|
||||
return ensureTelegramRow().botToken?.trim() ?? ""
|
||||
}
|
||||
|
||||
/** Для `sendMessage`: только если в БД задана валидная тема. */
|
||||
export function getTelegramMessageThreadIdForApi(): number | undefined {
|
||||
const { messageThreadId } = getTelegramPublic()
|
||||
return messageThreadId ?? undefined
|
||||
}
|
||||
|
||||
/** Текст тестового сообщения по полям правила (как в UI предпросмотра, без записи в историю). */
|
||||
export function formatAlertRuleTestTelegramText(opts: {
|
||||
name: string
|
||||
targets: string[]
|
||||
conditionLine: string
|
||||
severity: "critical" | "warning" | "info"
|
||||
cooldown: string
|
||||
}): string {
|
||||
const emoji =
|
||||
opts.severity === "critical" ? "🔴" : opts.severity === "warning" ? "🟡" : "🔵"
|
||||
const obj = opts.targets.length ? opts.targets.join(", ") : "—"
|
||||
return [
|
||||
"🧪 MikroTik Manager — тестовая отправка по полям формы правила (в историю срабатываний не записывается).",
|
||||
"",
|
||||
`${emoji} ${opts.name}`,
|
||||
`Объект: ${obj}`,
|
||||
`Событие: ${opts.conditionLine}`,
|
||||
`Cooldown: ${opts.cooldown}`,
|
||||
].join("\n")
|
||||
}
|
||||
|
||||
/** Отправка текста в Telegram (движок алертов и тест). */
|
||||
export async function sendTelegramAlertMessage(opts: {
|
||||
text: string
|
||||
/** Если задан — вместо токена из БД (ручной тест) */
|
||||
token?: string
|
||||
/** Пусто — глобальный chat из БД */
|
||||
chatId?: string
|
||||
/** Переопределение темы; undefined — как в настройках */
|
||||
messageThreadId?: number | null
|
||||
}): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
const token = (opts.token?.trim() || getTelegramBotToken()).trim()
|
||||
if (!token) return { ok: false, error: "Не задан Bot Token" }
|
||||
const pub = getTelegramPublic()
|
||||
const chatId = (opts.chatId?.trim() || pub.chatId || "").trim()
|
||||
if (!chatId) return { ok: false, error: "Не задан Chat ID" }
|
||||
const threadId =
|
||||
opts.messageThreadId !== undefined && opts.messageThreadId !== null
|
||||
? opts.messageThreadId >= 1
|
||||
? Math.floor(opts.messageThreadId)
|
||||
: undefined
|
||||
: getTelegramMessageThreadIdForApi()
|
||||
const url = `https://api.telegram.org/bot${encodeURIComponent(token)}/sendMessage`
|
||||
try {
|
||||
const payload: { chat_id: string; text: string; message_thread_id?: number } = {
|
||||
chat_id: chatId,
|
||||
text: opts.text,
|
||||
}
|
||||
if (threadId != null) payload.message_thread_id = threadId
|
||||
const res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
const data = (await res.json().catch(() => ({}))) as { ok?: boolean; description?: string }
|
||||
if (!res.ok || data.ok === false) {
|
||||
return { ok: false, error: data.description ?? res.statusText ?? "Telegram API error" }
|
||||
}
|
||||
return { ok: true }
|
||||
} catch (e) {
|
||||
return { ok: false, error: e instanceof Error ? e.message : "Ошибка сети" }
|
||||
}
|
||||
}
|
||||
|
||||
export function appendAlertHistory(entry: {
|
||||
id: string
|
||||
ruleId?: string | null
|
||||
groupId?: string | null
|
||||
ruleName: string
|
||||
severity: (typeof alertHistory.$inferSelect)["severity"]
|
||||
message: string
|
||||
sentOk: boolean
|
||||
firedAt?: string
|
||||
}) {
|
||||
const firedAt = entry.firedAt ?? new Date().toISOString()
|
||||
db.insert(alertHistory)
|
||||
.values({
|
||||
id: entry.id,
|
||||
ruleId: entry.ruleId ?? null,
|
||||
groupId: entry.groupId ?? null,
|
||||
ruleName: entry.ruleName,
|
||||
severity: entry.severity,
|
||||
message: entry.message,
|
||||
sentOk: entry.sentOk,
|
||||
firedAt,
|
||||
})
|
||||
.run()
|
||||
}
|
||||
|
||||
interface LiveResRow {
|
||||
serverName: string
|
||||
sampledAt: string
|
||||
}
|
||||
|
||||
interface LiveProbeRow {
|
||||
probeName: string
|
||||
target: string
|
||||
sampledAt: string
|
||||
status: string
|
||||
rttMs: number | null
|
||||
lossPct: number | null
|
||||
}
|
||||
|
||||
function loadLiveResourceIssues(): LiveResRow[] {
|
||||
return sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT s.name AS serverName, urs.sampled_at AS sampledAt
|
||||
FROM uptime_resource_samples urs
|
||||
INNER JOIN (
|
||||
SELECT server_id, MAX(id) AS mid FROM uptime_resource_samples GROUP BY server_id
|
||||
) latest ON latest.mid = urs.id
|
||||
INNER JOIN servers s ON s.id = urs.server_id
|
||||
WHERE urs.status = 'offline'`,
|
||||
)
|
||||
.all() as LiveResRow[]
|
||||
}
|
||||
|
||||
function loadLiveProbeIssues(): LiveProbeRow[] {
|
||||
return sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT p.name AS probeName, p.target AS target, ups.sampled_at AS sampledAt,
|
||||
ups.status AS status, ups.rtt_ms AS rttMs, ups.loss_pct AS lossPct
|
||||
FROM uptime_probe_samples ups
|
||||
INNER JOIN (
|
||||
SELECT probe_id, MAX(id) AS mid FROM uptime_probe_samples GROUP BY probe_id
|
||||
) latest ON latest.mid = ups.id
|
||||
INNER JOIN uptime_probes p ON p.id = ups.probe_id
|
||||
WHERE ups.status IN ('down', 'warn')`,
|
||||
)
|
||||
.all() as LiveProbeRow[]
|
||||
}
|
||||
|
||||
function toLiveHistoryEntries(): ApiAlertHistoryEntry[] {
|
||||
const out: ApiAlertHistoryEntry[] = []
|
||||
for (const r of loadLiveResourceIssues()) {
|
||||
const safeId = `live-res-${r.serverName}-${r.sampledAt}`.replace(/[^a-zA-Z0-9._-]+/g, "_")
|
||||
out.push({
|
||||
id: safeId,
|
||||
ruleName: "Сервер (ресурсы)",
|
||||
severity: "critical",
|
||||
message: `${r.serverName}: статус offline по данным мониторинга`,
|
||||
sent: true,
|
||||
firedAt: r.sampledAt,
|
||||
source: "live",
|
||||
})
|
||||
}
|
||||
for (const p of loadLiveProbeIssues()) {
|
||||
const sev = p.status === "down" ? "critical" : "warning"
|
||||
const loss = p.lossPct != null ? `${p.lossPct}%` : "—"
|
||||
const rtt = p.rttMs != null ? `${p.rttMs} мс` : "—"
|
||||
const safeId = `live-prb-${p.probeName}-${p.target}-${p.sampledAt}`.replace(/[^a-zA-Z0-9._-]+/g, "_")
|
||||
out.push({
|
||||
id: safeId,
|
||||
ruleName: "Ping-проба",
|
||||
severity: sev,
|
||||
message: `${p.probeName} → ${p.target}: ${p.status} (RTT ${rtt}, потери ${loss})`,
|
||||
sent: true,
|
||||
firedAt: p.sampledAt,
|
||||
source: "live",
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export function listPersistedHistory(limit: number): ApiAlertHistoryEntry[] {
|
||||
const rows = db
|
||||
.select()
|
||||
.from(alertHistory)
|
||||
.orderBy(desc(alertHistory.firedAt))
|
||||
.limit(limit)
|
||||
.all()
|
||||
return rows.map((h) => ({
|
||||
id: h.id,
|
||||
ruleName: h.ruleName,
|
||||
severity: h.severity,
|
||||
message: h.message,
|
||||
sent: h.sentOk,
|
||||
firedAt: h.firedAt,
|
||||
source: "db" as const,
|
||||
}))
|
||||
}
|
||||
|
||||
export function listMergedHistory(persistLimit = 40, liveCap = 25): ApiAlertHistoryEntry[] {
|
||||
const persisted = listPersistedHistory(persistLimit)
|
||||
const live = toLiveHistoryEntries().slice(0, liveCap)
|
||||
const merged = [...persisted, ...live].sort((a, b) => (a.firedAt < b.firedAt ? 1 : a.firedAt > b.firedAt ? -1 : 0))
|
||||
const seen = new Set<string>()
|
||||
const dedup: ApiAlertHistoryEntry[] = []
|
||||
for (const e of merged) {
|
||||
if (seen.has(e.id)) continue
|
||||
seen.add(e.id)
|
||||
dedup.push(e)
|
||||
if (dedup.length >= 50) break
|
||||
}
|
||||
return dedup
|
||||
}
|
||||
|
||||
export function getAlertsMeta() {
|
||||
const srvRows = db
|
||||
.select({
|
||||
name: servers.name,
|
||||
host: servers.host,
|
||||
site: servers.site,
|
||||
country: servers.country,
|
||||
type: servers.type,
|
||||
})
|
||||
.from(servers)
|
||||
.all()
|
||||
const labels = srvRows.map((s) => (s.name?.trim() ? s.name.trim() : s.host))
|
||||
const serversDetail = srvRows.map((r) => ({
|
||||
name: r.name?.trim() || "",
|
||||
host: r.host?.trim() || "",
|
||||
site: r.site?.trim() || "",
|
||||
country: r.country?.trim() || "",
|
||||
type: (r.type ?? "home-router") as string,
|
||||
}))
|
||||
const probes = db.select({ name: uptimeProbes.name, target: uptimeProbes.target }).from(uptimeProbes).all()
|
||||
const probeTargets = probes.map((p) => `${p.name} → ${p.target}`)
|
||||
const defaults = ["8.8.8.8", "1.1.1.1"]
|
||||
const rttLossTargets: string[] = []
|
||||
for (const L of labels.slice(0, 12)) {
|
||||
for (const d of defaults) rttLossTargets.push(`${L} → ${d}`)
|
||||
}
|
||||
return {
|
||||
servers: labels,
|
||||
greClients: labels,
|
||||
probeTargets: probeTargets.length ? probeTargets : rttLossTargets,
|
||||
rttLossTargets: rttLossTargets.length ? rttLossTargets : labels.map((L) => `${L} → 8.8.8.8`),
|
||||
trafficServers: labels,
|
||||
serversDetail,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { servers } from "../db/schema.js"
|
||||
import type { BgpSessionRead, RosBgpSession } from "../types/server.js"
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect
|
||||
|
||||
/** Parse RouterOS duration like "1m30s", "1d5h13m23s580ms", "30s" → seconds */
|
||||
function parseDuration(s: string | undefined): number {
|
||||
if (!s) return 0
|
||||
let total = 0
|
||||
const matches = s.matchAll(/(\d+)(w|d|h|m(?!s)|s|ms)/g)
|
||||
for (const m of matches) {
|
||||
const n = parseInt(m[1])
|
||||
switch (m[2]) {
|
||||
case "w":
|
||||
total += n * 604800
|
||||
break
|
||||
case "d":
|
||||
total += n * 86400
|
||||
break
|
||||
case "h":
|
||||
total += n * 3600
|
||||
break
|
||||
case "m":
|
||||
total += n * 60
|
||||
break
|
||||
case "s":
|
||||
total += n
|
||||
break
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
function parseCapabilities(capStr: string | undefined): string[] {
|
||||
if (!capStr) return []
|
||||
const codeMap: Record<string, string> = {
|
||||
mp: "MP-BGP",
|
||||
rr: "Route Refresh",
|
||||
gr: "Graceful Restart",
|
||||
as4: "4-byte-AS",
|
||||
enhe: "Extended Next-Hop",
|
||||
role: "BGP Role",
|
||||
err: "Extended Route Refresh",
|
||||
llgr: "Long-Lived GR",
|
||||
"add-path": "ADD-PATH",
|
||||
}
|
||||
return capStr
|
||||
.split(",")
|
||||
.map((c) => c.trim().toLowerCase())
|
||||
.filter(Boolean)
|
||||
.map((c) => codeMap[c] ?? c)
|
||||
.filter((v, i, a) => a.indexOf(v) === i)
|
||||
}
|
||||
|
||||
function detectState(s: RosBgpSession): string {
|
||||
if (s["established"] === "true") return "Established"
|
||||
|
||||
if (s["state"]) {
|
||||
const map: Record<string, string> = {
|
||||
established: "Established",
|
||||
active: "Active",
|
||||
idle: "Idle",
|
||||
connect: "Connect",
|
||||
opensent: "OpenSent",
|
||||
openconfirm: "OpenConfirm",
|
||||
"open sent": "OpenSent",
|
||||
"open confirm": "OpenConfirm",
|
||||
}
|
||||
const key = s["state"].trim().toLowerCase()
|
||||
if (map[key]) return map[key]
|
||||
for (const [k, v] of Object.entries(map)) {
|
||||
if (key.startsWith(k)) return v
|
||||
}
|
||||
return key.charAt(0).toUpperCase() + key.slice(1)
|
||||
}
|
||||
|
||||
if (s["uptime"]) return "Active"
|
||||
|
||||
return "Idle"
|
||||
}
|
||||
|
||||
/** Парсинг BGP-сессий с роутера — общая логика для HTTP-маршрута и alert-engine. */
|
||||
export function parseBgpSessions(server: ServerRow, raw: RosBgpSession[]): BgpSessionRead[] {
|
||||
return raw.map((s, idx) => {
|
||||
const remoteAs = parseInt(s["remote.as"] ?? "0") || 0
|
||||
const localAs = parseInt(s["local.as"] ?? "0") || 0
|
||||
const type: "eBGP" | "iBGP" = localAs > 0 && localAs === remoteAs ? "iBGP" : "eBGP"
|
||||
|
||||
const localCaps = parseCapabilities(s["local.capabilities"])
|
||||
const remoteCaps = parseCapabilities(s["remote.capabilities"])
|
||||
const legacyCaps: string[] = []
|
||||
if (s["4-octet-as-capability"] === "true") legacyCaps.push("4-byte-AS")
|
||||
if (s["as4-capability"] === "true") legacyCaps.push("4-byte-AS")
|
||||
if (s["refresh-capability"] === "true") legacyCaps.push("Route Refresh")
|
||||
if (s["add-path-capability"] === "true") legacyCaps.push("ADD-PATH")
|
||||
if (s["graceful-restart-capability"] === "true") legacyCaps.push("Graceful Restart")
|
||||
if (s["extended-message-capability"] === "true") legacyCaps.push("Extended Messages")
|
||||
const caps = [...new Set([...localCaps, ...remoteCaps, ...legacyCaps])]
|
||||
|
||||
const holdTime = parseDuration(s["hold-time"]) || parseInt(s["active-holdtime"] ?? "0") || 90
|
||||
|
||||
const keepalive = parseDuration(s["keepalive-time"]) || 30
|
||||
|
||||
const inputMessages =
|
||||
parseInt(s["remote.messages"] ?? s["total-messages-received"] ?? "0") || 0
|
||||
const outputMessages = parseInt(s["local.messages"] ?? s["total-messages-sent"] ?? "0") || 0
|
||||
|
||||
const peerIp = (s["remote.address"] ?? "").replace(/\/\d+$/, "")
|
||||
|
||||
return {
|
||||
id: s[".id"] ?? String(idx),
|
||||
serverId: server.id,
|
||||
serverName: server.name || server.host,
|
||||
serverSite: server.site,
|
||||
serverCountry: server.country,
|
||||
name: s["name"] ?? peerIp,
|
||||
peerIp,
|
||||
remoteAs,
|
||||
localAs,
|
||||
localId: s["local.id"] ?? "",
|
||||
remoteId: s["remote.id"] ?? "",
|
||||
state: detectState(s),
|
||||
type,
|
||||
uptime: s["uptime"] || null,
|
||||
holdTime,
|
||||
keepalive,
|
||||
prefixesRx: parseInt(s["prefix-count"] ?? s["total-updates-received"] ?? "0") || 0,
|
||||
prefixesTx: parseInt(s["total-updates-sent"] ?? "0") || 0,
|
||||
inputMessages,
|
||||
outputMessages,
|
||||
capabilities: caps,
|
||||
lastError: s["last-notification"] || null,
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers } from "../db/schema.js"
|
||||
import type { BgpSessionRead } from "../types/server.js"
|
||||
import { parseBgpSessions } from "./bgp-parse-sessions.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
/** Как в UI `/alerts`: `serverName: peerName`. */
|
||||
export function bgpPeerAlertKey(s: BgpSessionRead): string {
|
||||
const sn = String(s.serverName ?? "").trim()
|
||||
const nm = String(s.name ?? "").trim()
|
||||
return `${sn}: ${nm}`.trim()
|
||||
}
|
||||
|
||||
/** Агрегат BGP-сессий с включённых серверов — как `/api/bgp/sessions`. */
|
||||
export async function fetchBgpSessionsForAlerts(): Promise<BgpSessionRead[]> {
|
||||
const allServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
const results = await Promise.all(
|
||||
allServers.map(async (server) => {
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(server)
|
||||
const raw = await client.getBgpSessions()
|
||||
return parseBgpSessions(server, raw)
|
||||
} catch {
|
||||
return [] as BgpSessionRead[]
|
||||
}
|
||||
}),
|
||||
)
|
||||
return results.flat()
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/** Серийный доступ к bandwidth-test по node id (пара узлов не пересекаются). */
|
||||
const btestNodeLocks = new Map<number, Promise<void>>()
|
||||
|
||||
export async function withBtestNodeLocks<T>(nodeIds: number[], fn: () => Promise<T>): Promise<T> {
|
||||
const ids = [...new Set(nodeIds.filter((id) => Number.isFinite(id)))].sort((a, b) => a - b)
|
||||
const releases: Array<() => void> = []
|
||||
try {
|
||||
for (const id of ids) {
|
||||
const prev = btestNodeLocks.get(id)
|
||||
let resolveCurrent!: () => void
|
||||
const current = new Promise<void>((resolve) => {
|
||||
resolveCurrent = resolve
|
||||
})
|
||||
btestNodeLocks.set(id, current)
|
||||
if (prev) await prev
|
||||
releases.push(() => {
|
||||
if (btestNodeLocks.get(id) === current) btestNodeLocks.delete(id)
|
||||
resolveCurrent()
|
||||
})
|
||||
}
|
||||
return await fn()
|
||||
} finally {
|
||||
for (let i = releases.length - 1; i >= 0; i -= 1) releases[i]()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { db, sqliteDatabase } from "../db/index.js"
|
||||
import { alertBgpPeerSamples, alertGreTunnelSamples } from "../db/schema.js"
|
||||
import { bgpPeerAlertKey, fetchBgpSessionsForAlerts } from "./bgp-peers-live.js"
|
||||
import { fetchGreTunnelLiveRows } from "./gre-tunnels-live.js"
|
||||
import type { GreBgpSnapshotRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
|
||||
let collecting = false
|
||||
|
||||
export function getGreBgpSnapshotCollectorState(): { running: boolean } {
|
||||
return { running: collecting }
|
||||
}
|
||||
|
||||
/**
|
||||
* Один опрос GRE + BGP по включённым серверам и запись строк в SQLite для `buildSignalSnapshot`.
|
||||
* Движок оповещений больше не дублирует эти REST-запросы.
|
||||
*/
|
||||
export async function collectGreBgpSnapshotOnce(): Promise<GreBgpSnapshotRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collecting) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "gre_bgp",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
greWritten: 0,
|
||||
bgpWritten: 0,
|
||||
errors: [],
|
||||
}
|
||||
}
|
||||
collecting = true
|
||||
const errors: string[] = []
|
||||
const snapshot: GreBgpSnapshotRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "gre_bgp",
|
||||
sampledAt,
|
||||
greWritten: 0,
|
||||
bgpWritten: 0,
|
||||
greTunnels: [],
|
||||
bgpPeers: [],
|
||||
errors: [],
|
||||
}
|
||||
|
||||
try {
|
||||
const [greSettled, bgpSettled] = await Promise.allSettled([
|
||||
fetchGreTunnelLiveRows(),
|
||||
fetchBgpSessionsForAlerts(),
|
||||
])
|
||||
|
||||
if (greSettled.status === "rejected") {
|
||||
const msg =
|
||||
greSettled.reason instanceof Error ? greSettled.reason.message : String(greSettled.reason)
|
||||
errors.push(`GRE: ${msg}`)
|
||||
}
|
||||
if (bgpSettled.status === "rejected") {
|
||||
const msg =
|
||||
bgpSettled.reason instanceof Error ? bgpSettled.reason.message : String(bgpSettled.reason)
|
||||
errors.push(`BGP: ${msg}`)
|
||||
}
|
||||
|
||||
const greRows = greSettled.status === "fulfilled" ? greSettled.value : []
|
||||
const bgpRows = bgpSettled.status === "fulfilled" ? bgpSettled.value : []
|
||||
snapshot.greTunnels = greRows.map((r) => ({ targetLabel: r.targetLabel, status: r.status }))
|
||||
snapshot.bgpPeers = bgpRows.map((s) => ({ key: bgpPeerAlertKey(s), state: s.state }))
|
||||
|
||||
db.transaction((tx) => {
|
||||
for (const r of greRows) {
|
||||
tx.insert(alertGreTunnelSamples).values({
|
||||
sampledAt,
|
||||
targetLabel: r.targetLabel,
|
||||
status: r.status,
|
||||
}).run()
|
||||
snapshot.greWritten += 1
|
||||
}
|
||||
for (const s of bgpRows) {
|
||||
tx.insert(alertBgpPeerSamples).values({
|
||||
sampledAt,
|
||||
peerKey: bgpPeerAlertKey(s),
|
||||
state: s.state,
|
||||
}).run()
|
||||
snapshot.bgpWritten += 1
|
||||
}
|
||||
})
|
||||
|
||||
sqliteDatabase
|
||||
.prepare(`DELETE FROM alert_gre_tunnel_samples WHERE sampled_at < datetime('now', '-30 days')`)
|
||||
.run()
|
||||
sqliteDatabase
|
||||
.prepare(`DELETE FROM alert_bgp_peer_samples WHERE sampled_at < datetime('now', '-30 days')`)
|
||||
.run()
|
||||
|
||||
if (errors.length) snapshot.errors = errors
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
snapshot.fatalError = msg
|
||||
} finally {
|
||||
collecting = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers } from "../db/schema.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect
|
||||
|
||||
interface RosGre {
|
||||
".id"?: string
|
||||
name?: string
|
||||
"local-address"?: string
|
||||
"remote-address"?: string
|
||||
"allow-fast-path"?: string
|
||||
"clamp-tcp-mss"?: string
|
||||
mtu?: string
|
||||
"keepalive"?: string
|
||||
dscp?: string
|
||||
running?: string
|
||||
disabled?: string
|
||||
comment?: string
|
||||
}
|
||||
|
||||
export type GreTunnelLiveStatus = "up" | "down" | "degraded"
|
||||
|
||||
/** Как в UI `/alerts`: `${tunnelName} / ${serverName}`. */
|
||||
export type GreTunnelLiveRow = {
|
||||
targetLabel: string
|
||||
status: GreTunnelLiveStatus
|
||||
}
|
||||
|
||||
function mapGreRow(server: ServerRow, g: RosGre, idx: number): GreTunnelLiveRow {
|
||||
const running = g.running === "true"
|
||||
const disabled = g.disabled === "true"
|
||||
const status: GreTunnelLiveStatus = disabled ? "down" : running ? "up" : "degraded"
|
||||
const sn = String(server.name ?? "").trim() || String(server.host ?? "").trim() || String(server.id)
|
||||
const tn = (String(g.name ?? "").trim() || String(g[".id"] ?? "").trim() || `gre-${idx + 1}`).trim()
|
||||
const label = `${tn} / ${sn}`.trim()
|
||||
return { targetLabel: label, status }
|
||||
}
|
||||
|
||||
async function fetchGreForServer(server: ServerRow): Promise<GreTunnelLiveRow[]> {
|
||||
const client = MikrotikClient.fromServer(server)
|
||||
const greRaw = await client.get<RosGre[]>("/interface/gre")
|
||||
return greRaw.map((g, idx) => mapGreRow(server, g, idx))
|
||||
}
|
||||
|
||||
/** GRE с включённых серверов — тот же смысл, что `/api/filters/gre-tunnels` (без правил фильтров). */
|
||||
export async function fetchGreTunnelLiveRows(): Promise<GreTunnelLiveRow[]> {
|
||||
const enabledServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
const results = await Promise.all(
|
||||
enabledServers.map(async (server) => {
|
||||
try {
|
||||
return await fetchGreForServer(server)
|
||||
} catch {
|
||||
return [] as GreTunnelLiveRow[]
|
||||
}
|
||||
}),
|
||||
)
|
||||
return results.flat()
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import http from "node:http"
|
||||
import https from "node:https"
|
||||
import type { Server } from "../db/schema.js"
|
||||
import { parseRosDataSizeBytes } from "./ros-metric-parse.js"
|
||||
import type {
|
||||
RosIdentity, RosInterface, RosIpAddress, RosResource,
|
||||
RosBgpSession,
|
||||
@@ -199,6 +200,76 @@ function rosDelete(
|
||||
})
|
||||
}
|
||||
|
||||
function rosPatch(
|
||||
params: MikrotikConnectParams,
|
||||
path: string,
|
||||
body: Record<string, string>,
|
||||
timeoutMs: number,
|
||||
): Promise<unknown> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const basePath = params.apiPath ?? "/rest"
|
||||
const authHeader = "Basic " + Buffer.from(`${params.username}:${params.password}`).toString("base64")
|
||||
const payload = JSON.stringify(body)
|
||||
|
||||
const options: https.RequestOptions = {
|
||||
hostname: params.host,
|
||||
port: params.port,
|
||||
path: basePath + path,
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
Authorization: authHeader,
|
||||
"Content-Type": "application/json",
|
||||
"Content-Length": Buffer.byteLength(payload),
|
||||
},
|
||||
rejectUnauthorized: params.useSsl ? params.verifySsl : undefined,
|
||||
}
|
||||
|
||||
const lib = params.useSsl ? https : http
|
||||
const timer = setTimeout(() => {
|
||||
req.destroy(new Error(`Connection to ${params.host}:${params.port} timed out after ${timeoutMs / 1000}s`))
|
||||
}, timeoutMs)
|
||||
|
||||
const req = lib.request(options, (res) => {
|
||||
let buf = ""
|
||||
res.setEncoding("utf8")
|
||||
res.on("data", (chunk: string) => { buf += chunk })
|
||||
res.on("end", () => {
|
||||
clearTimeout(timer)
|
||||
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
|
||||
reject(new MikrotikError(res.statusCode ?? 0, path, buf)); return
|
||||
}
|
||||
if (buf.trim() === "") {
|
||||
resolve({})
|
||||
return
|
||||
}
|
||||
try { resolve(JSON.parse(buf)) } catch { resolve({}) }
|
||||
})
|
||||
})
|
||||
|
||||
req.on("error", (err) => {
|
||||
clearTimeout(timer)
|
||||
reject(err)
|
||||
})
|
||||
req.write(payload)
|
||||
req.end()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* RouterOS REST для одиночных меню (`/system/resource`, `/system/identity`) отдаёт JSON-массив из одного объекта `[{...}]`,
|
||||
* а не сам объект. Без распаковки поля читаются с массива → всегда undefined и в БД уходят нули при «успехе».
|
||||
* @see https://help.mikrotik.com/docs/spaces/ROS/pages/47579162/REST+API
|
||||
*/
|
||||
function unwrapRestSingle<T extends object>(raw: unknown): T {
|
||||
if (Array.isArray(raw)) {
|
||||
const first = raw[0]
|
||||
if (first && typeof first === "object") return first as T
|
||||
throw new Error("RouterOS REST: пустой массив вместо записи")
|
||||
}
|
||||
if (raw && typeof raw === "object") return raw as T
|
||||
throw new Error("RouterOS REST: неожиданное тело ответа")
|
||||
}
|
||||
|
||||
// ── MikrotikClient ─────────────────────────────────────────────────────────────
|
||||
|
||||
export class MikrotikClient {
|
||||
@@ -228,14 +299,43 @@ export class MikrotikClient {
|
||||
return rosDelete(this.params, path, timeoutMs)
|
||||
}
|
||||
|
||||
async patch<T>(path: string, body: Record<string, string>, timeoutMs = 15_000): Promise<T> {
|
||||
return rosPatch(this.params, path, body, timeoutMs) as Promise<T>
|
||||
}
|
||||
|
||||
// ── typed helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
async getIdentity(): Promise<RosIdentity> {
|
||||
return this.get<RosIdentity>("/system/identity")
|
||||
const raw = await this.get<unknown>("/system/identity")
|
||||
return unwrapRestSingle<RosIdentity>(raw)
|
||||
}
|
||||
|
||||
async getResource(): Promise<RosResource> {
|
||||
return this.get<RosResource>("/system/resource")
|
||||
const raw = await this.get<unknown>("/system/resource")
|
||||
if (Array.isArray(raw)) {
|
||||
if (raw.length === 0) throw new Error("RouterOS REST: пустой массив вместо записи /system/resource")
|
||||
if (raw.length === 1) return unwrapRestSingle<RosResource>(raw[0])
|
||||
/** Несколько объектов: взять строку с total-memory (агрегат), а не дочернюю запись с «чужим» uptime. */
|
||||
const objs = raw.filter((x): x is Record<string, unknown> => x != null && typeof x === "object" && !Array.isArray(x))
|
||||
/** На части железа несколько строк; «первая с непустым total-memory» может парситься в 0 — берём запись с максимальным RAM. */
|
||||
let best: Record<string, unknown> | undefined
|
||||
let bestBytes = -1
|
||||
for (const o of objs) {
|
||||
const tm = parseRosDataSizeBytes(o["total-memory"])
|
||||
const fm = parseRosDataSizeBytes(o["free-memory"])
|
||||
const score = tm > 0 ? tm : fm
|
||||
if (score > bestBytes) {
|
||||
bestBytes = score
|
||||
best = o
|
||||
}
|
||||
}
|
||||
const main = objs.find((o) => {
|
||||
const tm = o["total-memory"]
|
||||
return tm != null && String(tm).trim() !== ""
|
||||
})
|
||||
return ((bestBytes > 0 ? best : undefined) ?? main ?? objs[0]) as unknown as RosResource
|
||||
}
|
||||
return unwrapRestSingle<RosResource>(raw)
|
||||
}
|
||||
|
||||
async getInterfaces(): Promise<RosInterface[]> {
|
||||
@@ -277,6 +377,11 @@ export class MikrotikClient {
|
||||
return this.get<RosBfdSession[]>("/routing/bfd/session")
|
||||
}
|
||||
|
||||
async setOspfInterfaceTemplateCost(templateId: string, cost: number): Promise<void> {
|
||||
const safeId = templateId.startsWith("*") ? templateId : `*${templateId.replace(/^\*/, "")}`
|
||||
await this.patch(`/routing/ospf/interface-template/${encodeURIComponent(safeId)}`, { cost: String(cost) })
|
||||
}
|
||||
|
||||
// ── extra endpoints for exec route ────────────────────────────────────────
|
||||
|
||||
async getIpRoutes(): Promise<RosIpRoute[]> {
|
||||
|
||||
@@ -4,6 +4,7 @@ import { servers, serverSnapshots } from "../db/schema.js"
|
||||
import type { SnapshotInsert } from "../db/schema.js"
|
||||
import type { SnapshotRead } from "../types/server.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
import { parseRosCpuLoadPercent, parseRosDataSizeBytes } from "./ros-metric-parse.js"
|
||||
|
||||
// ── pollServer ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -45,10 +46,9 @@ export async function pollServer(serverId: number): Promise<SnapshotRead> {
|
||||
|
||||
const latencyMs = performance.now() - t0
|
||||
|
||||
// Parse RouterOS string values (RouterOS REST API returns everything as strings)
|
||||
const cpuLoad = parseInt(resource["cpu-load"], 10)
|
||||
const freeMem = parseInt(resource["free-memory"], 10)
|
||||
const totalMem = parseInt(resource["total-memory"], 10)
|
||||
const cpuLoad = parseRosCpuLoadPercent(resource["cpu-load"])
|
||||
const freeMem = parseRosDataSizeBytes(resource["free-memory"])
|
||||
const totalMem = parseRosDataSizeBytes(resource["total-memory"])
|
||||
|
||||
Object.assign(partialSnap, {
|
||||
status: "online",
|
||||
@@ -57,9 +57,9 @@ export async function pollServer(serverId: number): Promise<SnapshotRead> {
|
||||
rosVersion: resource["version"],
|
||||
boardName: resource["board-name"],
|
||||
uptime: resource["uptime"],
|
||||
cpuLoad: isNaN(cpuLoad) ? null : cpuLoad,
|
||||
freeMemory: isNaN(freeMem) ? null : freeMem,
|
||||
totalMemory: isNaN(totalMem) ? null : totalMem,
|
||||
cpuLoad,
|
||||
freeMemory: freeMem,
|
||||
totalMemory: totalMem,
|
||||
rawInterfaces: JSON.stringify(ifaces),
|
||||
rawIpAddresses: JSON.stringify(addresses),
|
||||
} satisfies Partial<SnapshotInsert>)
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* Разбор полей /system/resource из RouterOS REST: не всегда «сырые» байты в виде десятичной строки.
|
||||
* На железе часто встречаются значения вида "224.0MiB", "94.2MiB"; parseInt от этого даёт мусор или 0.
|
||||
*/
|
||||
|
||||
export function parseRosDataSizeBytes(raw: unknown): number {
|
||||
if (raw == null) return 0
|
||||
if (typeof raw === "number" && Number.isFinite(raw)) return Math.max(0, Math.floor(raw))
|
||||
let s = String(raw).trim().replace(/\s+/g, "")
|
||||
if (!s) return 0
|
||||
if (/^\d+$/.test(s)) {
|
||||
const n = Number.parseInt(s, 10)
|
||||
return Number.isFinite(n) ? Math.max(0, n) : 0
|
||||
}
|
||||
const m = s.match(/^(\d+(?:[.,]\d+)?)(KiB|MiB|GiB|TiB|Ki|Mi|Gi|Ti|KB|MB|GB|TB)$/i)
|
||||
if (m) {
|
||||
const v = Number.parseFloat(m[1].replace(",", "."))
|
||||
if (!Number.isFinite(v) || v < 0) return 0
|
||||
const u = m[2].toLowerCase()
|
||||
const mult =
|
||||
u === "ki" || u === "kib" || u === "kb" ? 1024
|
||||
: u === "mi" || u === "mib" || u === "mb" ? 1024 ** 2
|
||||
: u === "gi" || u === "gib" || u === "gb" ? 1024 ** 3
|
||||
: u === "ti" || u === "tib" || u === "tb" ? 1024 ** 4
|
||||
: 1
|
||||
return Math.max(0, Math.floor(v * mult))
|
||||
}
|
||||
const fallback = Number.parseInt(s, 10)
|
||||
return Number.isFinite(fallback) ? Math.max(0, fallback) : 0
|
||||
}
|
||||
|
||||
/** cpu-load: "7%", "7", 7 */
|
||||
export function parseRosCpuLoadPercent(raw: unknown): number {
|
||||
if (raw == null) return 0
|
||||
if (typeof raw === "number" && Number.isFinite(raw)) {
|
||||
return Math.min(100, Math.max(0, Math.round(raw)))
|
||||
}
|
||||
const s = String(raw).trim().replace("%", "").replace(",", ".").trim()
|
||||
if (!s) return 0
|
||||
const n = Number.parseFloat(s)
|
||||
if (!Number.isFinite(n)) return 0
|
||||
return Math.min(100, Math.max(0, Math.round(n)))
|
||||
}
|
||||
|
||||
/**
|
||||
* `uptime` из /system/resource (REST): обычно "2d20h12m20s", "39m", "29s".
|
||||
* Встречаются варианты: только секунды строкой, `d.hh:mm:ss`, `hh:mm:ss` без суффиксов.
|
||||
* @see https://help.mikrotik.com/docs/display/ROS/Resource
|
||||
*/
|
||||
export function parseRosUptimeToSeconds(raw: unknown): number {
|
||||
if (raw == null) return 0
|
||||
if (typeof raw === "number" && Number.isFinite(raw)) {
|
||||
return Math.max(0, Math.floor(raw))
|
||||
}
|
||||
const s0 = String(raw).trim().replace(/\u00a0/g, " ").trim()
|
||||
if (!s0) return 0
|
||||
|
||||
// Только десятичные цифры — секунды (некоторые клиенты/прокси)
|
||||
if (/^\d+$/.test(s0)) {
|
||||
const n = Number.parseInt(s0, 10)
|
||||
return Number.isFinite(n) ? Math.max(0, n) : 0
|
||||
}
|
||||
|
||||
const s = s0.toLowerCase().replace(/\s+/g, "")
|
||||
|
||||
// 1d18:27:15
|
||||
const dClock = s.match(/^(\d+)d(\d{1,3}):(\d{2}):(\d{2})$/i)
|
||||
if (dClock) {
|
||||
const d = Number.parseInt(dClock[1], 10)
|
||||
const hh = Number.parseInt(dClock[2], 10)
|
||||
const mm = Number.parseInt(dClock[3], 10)
|
||||
const ss = Number.parseInt(dClock[4], 10)
|
||||
if ([d, hh, mm, ss].every((x) => Number.isFinite(x) && x >= 0)) {
|
||||
return d * 86400 + hh * 3600 + mm * 60 + ss
|
||||
}
|
||||
}
|
||||
|
||||
// 18:27:15 — только если нет суффиксов w/d/h/m/s (иначе это часть "2d20h12m20s")
|
||||
if (!/[wdhms]/i.test(s0) && /^\d{1,7}:\d{2}:\d{2}$/.test(s)) {
|
||||
const p = s.split(":")
|
||||
if (p.length === 3) {
|
||||
const hh = Number.parseInt(p[0], 10)
|
||||
const mm = Number.parseInt(p[1], 10)
|
||||
const sec = Number.parseInt(p[2], 10)
|
||||
if ([hh, mm, sec].every((x) => Number.isFinite(x) && x >= 0)) {
|
||||
return hh * 3600 + mm * 60 + sec
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total = 0
|
||||
for (const m of s.matchAll(/(\d+)(w|d|h|m(?!s)|s)/gi)) {
|
||||
const n = Number.parseInt(m[1], 10)
|
||||
if (!Number.isFinite(n) || n < 0) continue
|
||||
const u = m[2].toLowerCase()
|
||||
switch (u) {
|
||||
case "w": total += n * 604800; break
|
||||
case "d": total += n * 86400; break
|
||||
case "h": total += n * 3600; break
|
||||
case "m": total += n * 60; break
|
||||
case "s": total += n; break
|
||||
default: break
|
||||
}
|
||||
}
|
||||
return Math.max(0, total)
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Реестр активных джоб планировщика (без импорта `scheduler.ts` → нет цикла с `run-once`).
|
||||
* Слот берётся до `await` в теле джобы — так `alert_engine` может дождаться REST и др. до чтения БД.
|
||||
*/
|
||||
const runningSchedulerJobs = new Set<string>()
|
||||
|
||||
const ALERT_SNAPSHOT_SOURCE_JOB_KEYS = new Set<string>([
|
||||
"traffic",
|
||||
"servers_rest_ping",
|
||||
"uptime_resources",
|
||||
"uptime_ping",
|
||||
"uptime_speed",
|
||||
"gre_bgp",
|
||||
])
|
||||
|
||||
/** true, если слот занят (джоба уже выполняется). */
|
||||
export function tryBeginSchedulerJob(jobKey: string): boolean {
|
||||
if (runningSchedulerJobs.has(jobKey)) return false
|
||||
runningSchedulerJobs.add(jobKey)
|
||||
return true
|
||||
}
|
||||
|
||||
export function endSchedulerJob(jobKey: string): void {
|
||||
runningSchedulerJobs.delete(jobKey)
|
||||
}
|
||||
|
||||
export function isSchedulerJobRunning(jobKey: string): boolean {
|
||||
return runningSchedulerJobs.has(jobKey)
|
||||
}
|
||||
|
||||
/** Есть ли коллектор, который пишет таблицы снимка для `buildSignalSnapshot`. */
|
||||
export function isAnyAlertSnapshotSourceJobRunning(): boolean {
|
||||
for (const k of ALERT_SNAPSHOT_SOURCE_JOB_KEYS) {
|
||||
if (runningSchedulerJobs.has(k)) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
/**
|
||||
* In-process планировщик: отдельный интервал на job (`refreshScheduler`).
|
||||
*
|
||||
* **Приоритеты при конкуренции** (для дальнейшего per-server mutex; сейчас зафиксировано в дизайне):
|
||||
* 1. `uptime_ping` — выше (короткий критичный сигнал).
|
||||
* 2. `traffic`, `uptime_resources`, `servers_rest_ping` — средний (REST `/system/identity` по каталогу).
|
||||
* 3. `uptime_speed` — ниже (BW-test, тяжёлый); локи узлов — `withBtestNodeLocks` в speed-сервисе.
|
||||
*
|
||||
* **Оповещения (`alert_engine`):** читают SQLite после джоб сбора (см. `buildSignalSnapshot`), в т.ч.
|
||||
* `gre_bgp` → `alert_gre_tunnel_samples` / `alert_bgp_peer_samples`. После успешного завершения джоб
|
||||
* `traffic`, `uptime_*`, `servers_rest_ping`, `gre_bgp` планируется **дополнительный** прогон движка
|
||||
* (debounce), см. [`alert-collector-hooks.ts`](./alert-collector-hooks.ts); любой другой писатель сэмплов
|
||||
* для снимка оповещений тоже должен вызывать `scheduleAlertEngineAfterDataCollectors()` после коммита.
|
||||
*
|
||||
* **Кастомные job без произвольного кода (MVP):** пер-тайминг ping в `uptime_probes.interval_sec`;
|
||||
* таблица `scheduler_job_instances` (kind + config JSON) — при необходимости следующий этап.
|
||||
*/
|
||||
import { desc, eq, lt } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { schedulerRuns } from "../db/schema.js"
|
||||
import type { SchedulerRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import {
|
||||
collectServersRestPingOnce,
|
||||
getServersApiPingSettings,
|
||||
} from "./servers-rest-ping-collector.js"
|
||||
import { collectTrafficOnce, getTrafficSettings } from "./traffic-collector.js"
|
||||
import {
|
||||
collectPingProbesOnce,
|
||||
collectResourceSamplesOnce,
|
||||
getSettings as getUptimeSettings,
|
||||
} from "./uptime-collector.js"
|
||||
import { runScheduledSpeedProbesOnce } from "./uptime-speed-service.js"
|
||||
import { collectGreBgpSnapshotOnce } from "./gre-bgp-snapshot-collector.js"
|
||||
import { runAlertEngineOnce } from "./alert-engine/run-once.js"
|
||||
import {
|
||||
clearAlertCollectorHooksTimer,
|
||||
scheduleAlertEngineAfterDataCollectors,
|
||||
wireAlertEngineRunner,
|
||||
} from "./alert-collector-hooks.js"
|
||||
import {
|
||||
endSchedulerJob,
|
||||
isSchedulerJobRunning,
|
||||
tryBeginSchedulerJob,
|
||||
} from "./scheduler-running.js"
|
||||
|
||||
export const JOB_KEYS = [
|
||||
"traffic",
|
||||
"servers_rest_ping",
|
||||
"uptime_resources",
|
||||
"uptime_ping",
|
||||
"uptime_speed",
|
||||
"gre_bgp",
|
||||
"alert_engine",
|
||||
] as const
|
||||
export type SchedulerJobKey = (typeof JOB_KEYS)[number]
|
||||
|
||||
const timers = new Map<string, ReturnType<typeof setInterval>>()
|
||||
|
||||
const RUN_LOG_RETENTION_MS = 30 * 24 * 60 * 60 * 1000
|
||||
|
||||
function newRunId(): string {
|
||||
return `sch-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
|
||||
}
|
||||
|
||||
function appendSchedulerRun(row: {
|
||||
jobKey: string
|
||||
startedAt: string
|
||||
finishedAt: string
|
||||
status: "ok" | "error"
|
||||
error: string | null
|
||||
durationMs: number
|
||||
result?: SchedulerRunSnapshot | null
|
||||
}) {
|
||||
db.insert(schedulerRuns).values({
|
||||
id: newRunId(),
|
||||
jobKey: row.jobKey,
|
||||
startedAt: row.startedAt,
|
||||
finishedAt: row.finishedAt,
|
||||
status: row.status,
|
||||
error: row.error,
|
||||
durationMs: row.durationMs,
|
||||
resultJson: row.result ? JSON.stringify(row.result) : null,
|
||||
}).run()
|
||||
const cutoff = new Date(Date.now() - RUN_LOG_RETENTION_MS).toISOString()
|
||||
db.delete(schedulerRuns).where(lt(schedulerRuns.finishedAt, cutoff)).run()
|
||||
}
|
||||
|
||||
async function runSchedulerJobBody(jobKey: SchedulerJobKey): Promise<void> {
|
||||
const startedAt = Date.now()
|
||||
const startedIso = new Date().toISOString()
|
||||
let snapshot: SchedulerRunSnapshot | undefined
|
||||
try {
|
||||
switch (jobKey) {
|
||||
case "traffic":
|
||||
snapshot = await collectTrafficOnce()
|
||||
break
|
||||
case "uptime_resources":
|
||||
snapshot = await collectResourceSamplesOnce()
|
||||
break
|
||||
case "uptime_ping":
|
||||
snapshot = await collectPingProbesOnce()
|
||||
break
|
||||
case "uptime_speed":
|
||||
snapshot = await runScheduledSpeedProbesOnce()
|
||||
break
|
||||
case "servers_rest_ping":
|
||||
snapshot = await collectServersRestPingOnce()
|
||||
break
|
||||
case "gre_bgp":
|
||||
snapshot = await collectGreBgpSnapshotOnce()
|
||||
break
|
||||
case "alert_engine": {
|
||||
const r = await runAlertEngineOnce()
|
||||
snapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "alert_engine",
|
||||
sampledAt: r.sampledAt,
|
||||
rulesChecked: r.rulesChecked,
|
||||
standaloneFires: r.standaloneFires,
|
||||
groupFires: r.groupFires,
|
||||
skippedNoTelegram: r.skippedNoTelegram,
|
||||
...(r.ruleDiag.length ? { ruleDiag: r.ruleDiag } : {}),
|
||||
...(r.errors.length ? { errors: r.errors } : {}),
|
||||
}
|
||||
break
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unknown job: ${jobKey}`)
|
||||
}
|
||||
const finishedIso = new Date().toISOString()
|
||||
appendSchedulerRun({
|
||||
jobKey,
|
||||
startedAt: startedIso,
|
||||
finishedAt: finishedIso,
|
||||
status: "ok",
|
||||
error: null,
|
||||
durationMs: Date.now() - startedAt,
|
||||
result: snapshot ?? null,
|
||||
})
|
||||
if (
|
||||
jobKey === "traffic" ||
|
||||
jobKey === "servers_rest_ping" ||
|
||||
jobKey === "uptime_resources" ||
|
||||
jobKey === "uptime_ping" ||
|
||||
jobKey === "uptime_speed" ||
|
||||
jobKey === "gre_bgp"
|
||||
) {
|
||||
scheduleAlertEngineAfterDataCollectors()
|
||||
}
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
const finishedIso = new Date().toISOString()
|
||||
appendSchedulerRun({
|
||||
jobKey,
|
||||
startedAt: startedIso,
|
||||
finishedAt: finishedIso,
|
||||
status: "error",
|
||||
error: msg,
|
||||
durationMs: Date.now() - startedAt,
|
||||
result: snapshot ?? null,
|
||||
})
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
/** Фоновый тик: пропуск, если предыдущий прогон ещё идёт. */
|
||||
export async function executeSchedulerJob(jobKey: SchedulerJobKey): Promise<void> {
|
||||
if (!tryBeginSchedulerJob(jobKey)) return
|
||||
try {
|
||||
await runSchedulerJobBody(jobKey)
|
||||
} catch {
|
||||
/* залогировано в runSchedulerJobBody */
|
||||
} finally {
|
||||
endSchedulerJob(jobKey)
|
||||
}
|
||||
}
|
||||
|
||||
/** Ручной запуск: 409, если job уже выполняется. */
|
||||
export async function runSchedulerJobNow(jobKey: string): Promise<void> {
|
||||
if (!JOB_KEYS.includes(jobKey as SchedulerJobKey)) {
|
||||
throw new Error(`Unknown job key: ${jobKey}`)
|
||||
}
|
||||
if (!tryBeginSchedulerJob(jobKey)) {
|
||||
const err = new Error("Job already running")
|
||||
;(err as Error & { statusCode?: number }).statusCode = 409
|
||||
throw err
|
||||
}
|
||||
try {
|
||||
await runSchedulerJobBody(jobKey as SchedulerJobKey)
|
||||
} finally {
|
||||
endSchedulerJob(jobKey)
|
||||
}
|
||||
}
|
||||
|
||||
function clearAllTimers() {
|
||||
clearAlertCollectorHooksTimer()
|
||||
for (const t of timers.values()) clearInterval(t)
|
||||
timers.clear()
|
||||
}
|
||||
|
||||
/** Пересоздать интервалы после смены настроек. */
|
||||
export function refreshScheduler(): void {
|
||||
clearAllTimers()
|
||||
|
||||
const traffic = getTrafficSettings()
|
||||
if (traffic.enabled) {
|
||||
const ms = Math.max(5_000, traffic.intervalSec * 1000)
|
||||
void executeSchedulerJob("traffic").catch(() => {})
|
||||
timers.set(
|
||||
"traffic",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("traffic").catch(() => {})
|
||||
}, ms),
|
||||
)
|
||||
}
|
||||
|
||||
const apiPing = getServersApiPingSettings()
|
||||
if (apiPing.enabled) {
|
||||
const apiMs = Math.max(10_000, apiPing.intervalSec * 1000)
|
||||
void executeSchedulerJob("servers_rest_ping").catch(() => {})
|
||||
timers.set(
|
||||
"servers_rest_ping",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("servers_rest_ping").catch(() => {})
|
||||
}, apiMs),
|
||||
)
|
||||
}
|
||||
|
||||
const uptime = getUptimeSettings()
|
||||
const resOn = uptime.resourcesEnabled ?? uptime.enabled
|
||||
const pingOn = uptime.pingEnabled ?? uptime.enabled
|
||||
const spdOn = uptime.speedEnabled ?? uptime.enabled
|
||||
|
||||
if (resOn) {
|
||||
const resMs = Math.max(5_000, uptime.intervalSec * 1000)
|
||||
void executeSchedulerJob("uptime_resources").catch(() => {})
|
||||
timers.set(
|
||||
"uptime_resources",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("uptime_resources").catch(() => {})
|
||||
}, resMs),
|
||||
)
|
||||
}
|
||||
|
||||
if (pingOn) {
|
||||
const pingMs = Math.max(1_000, uptime.probeIntervalSec * 1000)
|
||||
void executeSchedulerJob("uptime_ping").catch(() => {})
|
||||
timers.set(
|
||||
"uptime_ping",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("uptime_ping").catch(() => {})
|
||||
}, pingMs),
|
||||
)
|
||||
}
|
||||
|
||||
if (spdOn) {
|
||||
const spdMs = Math.max(10_000, uptime.speedIntervalSec * 1000)
|
||||
void executeSchedulerJob("uptime_speed").catch(() => {})
|
||||
timers.set(
|
||||
"uptime_speed",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("uptime_speed").catch(() => {})
|
||||
}, spdMs),
|
||||
)
|
||||
}
|
||||
|
||||
const greBgpMs = 30_000
|
||||
void executeSchedulerJob("gre_bgp").catch(() => {})
|
||||
timers.set(
|
||||
"gre_bgp",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("gre_bgp").catch(() => {})
|
||||
}, greBgpMs),
|
||||
)
|
||||
|
||||
const alertMs = 20_000
|
||||
void executeSchedulerJob("alert_engine").catch(() => {})
|
||||
timers.set(
|
||||
"alert_engine",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("alert_engine").catch(() => {})
|
||||
}, alertMs),
|
||||
)
|
||||
}
|
||||
|
||||
export function stopScheduler(): void {
|
||||
clearAllTimers()
|
||||
}
|
||||
|
||||
export function isJobRunning(jobKey: string): boolean {
|
||||
return isSchedulerJobRunning(jobKey)
|
||||
}
|
||||
|
||||
function lastRunForJob(jobKey: string) {
|
||||
return db.select().from(schedulerRuns)
|
||||
.where(eq(schedulerRuns.jobKey, jobKey))
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
}
|
||||
|
||||
export function getSchedulerStatus() {
|
||||
const traffic = getTrafficSettings()
|
||||
const uptime = getUptimeSettings()
|
||||
const apiPing = getServersApiPingSettings()
|
||||
|
||||
const resOn = uptime.resourcesEnabled ?? uptime.enabled
|
||||
const pingOn = uptime.pingEnabled ?? uptime.enabled
|
||||
const spdOn = uptime.speedEnabled ?? uptime.enabled
|
||||
|
||||
const jobMeta: Record<SchedulerJobKey, { enabled: boolean; intervalSec: number }> = {
|
||||
traffic: { enabled: traffic.enabled, intervalSec: traffic.intervalSec },
|
||||
servers_rest_ping: { enabled: apiPing.enabled, intervalSec: apiPing.intervalSec },
|
||||
uptime_resources: { enabled: resOn, intervalSec: uptime.intervalSec },
|
||||
uptime_ping: { enabled: pingOn, intervalSec: uptime.probeIntervalSec },
|
||||
uptime_speed: { enabled: spdOn, intervalSec: uptime.speedIntervalSec },
|
||||
gre_bgp: { enabled: true, intervalSec: 30 },
|
||||
alert_engine: { enabled: true, intervalSec: 20 },
|
||||
}
|
||||
|
||||
return {
|
||||
jobs: JOB_KEYS.map((jobKey) => {
|
||||
const last = lastRunForJob(jobKey)
|
||||
const m = jobMeta[jobKey]
|
||||
return {
|
||||
jobKey,
|
||||
enabled: m.enabled,
|
||||
intervalSec: m.intervalSec,
|
||||
running: isSchedulerJobRunning(jobKey),
|
||||
lastFinishedAt: last?.finishedAt ?? null,
|
||||
lastStatus: last?.status ?? null,
|
||||
lastDurationMs: last?.durationMs ?? null,
|
||||
lastError: last?.error ?? null,
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export function listSchedulerRuns(opts: { jobKey?: string; limit: number; offset: number }) {
|
||||
const limit = Math.min(200, Math.max(1, opts.limit))
|
||||
const offset = Math.max(0, opts.offset)
|
||||
if (opts.jobKey) {
|
||||
return {
|
||||
runs: db.select().from(schedulerRuns)
|
||||
.where(eq(schedulerRuns.jobKey, opts.jobKey))
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
.all(),
|
||||
}
|
||||
}
|
||||
return {
|
||||
runs: db.select().from(schedulerRuns)
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
.all(),
|
||||
}
|
||||
}
|
||||
|
||||
wireAlertEngineRunner(() => executeSchedulerJob("alert_engine"))
|
||||
@@ -0,0 +1,202 @@
|
||||
import { eq, or } from "drizzle-orm"
|
||||
import { db, sqliteDatabase } from "../db/index.js"
|
||||
import { servers, serversApiPingSettings, serversRestPingSamples } from "../db/schema.js"
|
||||
import type { ServersRestPingRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
let collecting = false
|
||||
|
||||
function getSettingsRow() {
|
||||
const row = db.select().from(serversApiPingSettings).where(eq(serversApiPingSettings.id, 1)).limit(1).all()[0]
|
||||
if (row) return row
|
||||
const now = new Date().toISOString()
|
||||
db.insert(serversApiPingSettings).values({
|
||||
id: 1,
|
||||
enabled: false,
|
||||
intervalSec: 120,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
}).run()
|
||||
return db.select().from(serversApiPingSettings).where(eq(serversApiPingSettings.id, 1)).limit(1).all()[0]
|
||||
}
|
||||
|
||||
/** Как uptime: включённые в каталоге или Home Router (часто «выкл.» при доступе только из LAN). */
|
||||
function serversForRestProbe() {
|
||||
return db
|
||||
.select()
|
||||
.from(servers)
|
||||
.where(or(eq(servers.enabled, true), eq(servers.type, "home-router")))
|
||||
.all()
|
||||
}
|
||||
|
||||
export function getServersApiPingSettings() {
|
||||
return getSettingsRow()
|
||||
}
|
||||
|
||||
export function updateServersApiPingSettings(patch: { enabled?: boolean; intervalSec?: number }) {
|
||||
const prev = getSettingsRow()
|
||||
const next = {
|
||||
enabled: patch.enabled ?? prev.enabled,
|
||||
intervalSec: patch.intervalSec ?? prev.intervalSec,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
db.update(serversApiPingSettings).set(next).where(eq(serversApiPingSettings.id, 1)).run()
|
||||
return getSettingsRow()
|
||||
}
|
||||
|
||||
export function getServersRestPingCollectorState(): { running: boolean } {
|
||||
return { running: collecting }
|
||||
}
|
||||
|
||||
function loadLastRestSampleByServer(): Map<number, { ok: boolean }> {
|
||||
const rows = sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT s.server_id AS serverId, s.ok
|
||||
FROM servers_rest_ping_samples s
|
||||
INNER JOIN (
|
||||
SELECT server_id, MAX(id) AS mid FROM servers_rest_ping_samples GROUP BY server_id
|
||||
) t ON t.mid = s.id`,
|
||||
)
|
||||
.all() as { serverId: number; ok: number }[]
|
||||
const m = new Map<number, { ok: boolean }>()
|
||||
for (const r of rows) m.set(r.serverId, { ok: Boolean(r.ok) })
|
||||
return m
|
||||
}
|
||||
|
||||
/** Последний сэмпл ресурсов — для согласования «восстановился» с REST без второго тика uptime. */
|
||||
function loadLatestResourceDownServerIds(): Set<number> {
|
||||
const rows = sqliteDatabase
|
||||
.prepare(
|
||||
`WITH r AS (
|
||||
SELECT urs.server_id AS serverId, urs.status,
|
||||
ROW_NUMBER() OVER (PARTITION BY urs.server_id ORDER BY urs.id DESC) AS rn
|
||||
FROM uptime_resource_samples urs
|
||||
)
|
||||
SELECT serverId FROM r WHERE rn = 1 AND status IN ('offline', 'degraded')`,
|
||||
)
|
||||
.all() as { serverId: number }[]
|
||||
return new Set(rows.map((x) => x.serverId))
|
||||
}
|
||||
|
||||
/**
|
||||
* Лёгкая проверка: GET `/system/identity` (REST) на узел, замер RTT.
|
||||
* Пишет строки в `servers_rest_ping_samples` для алерт-движка (см. `buildSignalSnapshot`).
|
||||
*/
|
||||
export async function collectServersRestPingOnce(): Promise<ServersRestPingRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collecting) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "servers_rest_ping",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
servers: [],
|
||||
}
|
||||
}
|
||||
collecting = true
|
||||
const startedAt = Date.now()
|
||||
const snapshot: ServersRestPingRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "servers_rest_ping",
|
||||
sampledAt,
|
||||
servers: [],
|
||||
}
|
||||
|
||||
try {
|
||||
const targets = serversForRestProbe()
|
||||
const rows = await Promise.all(
|
||||
targets.map(async (srv) => {
|
||||
const name = srv.name || srv.host
|
||||
const t0 = performance.now()
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(srv)
|
||||
await client.getIdentity()
|
||||
const latencyMs = Math.round(performance.now() - t0)
|
||||
return {
|
||||
serverId: srv.id,
|
||||
name,
|
||||
host: srv.host,
|
||||
ok: true as const,
|
||||
latencyMs,
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
serverId: srv.id,
|
||||
name,
|
||||
host: srv.host,
|
||||
ok: false as const,
|
||||
latencyMs: null as number | null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
snapshot.servers = rows.sort((a, b) => a.serverId - b.serverId)
|
||||
|
||||
const lastByServer = loadLastRestSampleByServer()
|
||||
const resourceDown = loadLatestResourceDownServerIds()
|
||||
|
||||
db.transaction((tx) => {
|
||||
for (const row of snapshot.servers) {
|
||||
const sid = row.serverId
|
||||
const last = lastByServer.get(sid)
|
||||
const errText = row.ok ? null : (row.error ?? null)
|
||||
|
||||
const insertOne = (ok: boolean, latencyMs: number | null, error: string | null) => {
|
||||
tx.insert(serversRestPingSamples).values({
|
||||
serverId: sid,
|
||||
sampledAt,
|
||||
ok,
|
||||
latencyMs,
|
||||
error,
|
||||
}).run()
|
||||
}
|
||||
|
||||
const lat = row.latencyMs ?? null
|
||||
|
||||
if (row.ok) {
|
||||
const downRes = resourceDown.has(sid)
|
||||
if (last === undefined && downRes) {
|
||||
insertOne(false, null, null)
|
||||
}
|
||||
insertOne(true, lat, null)
|
||||
if (last?.ok === false || (last === undefined && downRes)) {
|
||||
insertOne(true, lat, null)
|
||||
}
|
||||
} else {
|
||||
insertOne(false, null, errText)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
sqliteDatabase
|
||||
.prepare(`DELETE FROM servers_rest_ping_samples WHERE sampled_at < datetime('now', '-30 days')`)
|
||||
.run()
|
||||
|
||||
db.update(serversApiPingSettings)
|
||||
.set({
|
||||
lastCollectedAt: sampledAt,
|
||||
lastDurationMs: Date.now() - startedAt,
|
||||
lastError: "",
|
||||
updatedAt: sampledAt,
|
||||
})
|
||||
.where(eq(serversApiPingSettings.id, 1))
|
||||
.run()
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
snapshot.fatalError = msg
|
||||
db.update(serversApiPingSettings)
|
||||
.set({
|
||||
lastCollectedAt: sampledAt,
|
||||
lastDurationMs: Date.now() - startedAt,
|
||||
lastError: msg,
|
||||
updatedAt: sampledAt,
|
||||
})
|
||||
.where(eq(serversApiPingSettings.id, 1))
|
||||
.run()
|
||||
} finally {
|
||||
collecting = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import { and, asc, eq, gte, lt } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers, trafficSamples, trafficSettings } from "../db/schema.js"
|
||||
import type { TrafficRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
interface RosIfaceTraffic {
|
||||
@@ -19,13 +21,7 @@ export interface TrafficCollectorState {
|
||||
lastError: string | null
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null
|
||||
let collecting = false
|
||||
const state: TrafficCollectorState = {
|
||||
running: false,
|
||||
lastRunAt: null,
|
||||
lastError: null,
|
||||
}
|
||||
|
||||
function toNum(raw: unknown): number {
|
||||
const n = Number.parseFloat(String(raw ?? "0"))
|
||||
@@ -54,36 +50,74 @@ function cleanupOldSamples(retentionDays: number) {
|
||||
.run()
|
||||
}
|
||||
|
||||
export async function collectTrafficOnce(): Promise<void> {
|
||||
if (collecting) return
|
||||
export async function collectTrafficOnce(): Promise<TrafficRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collecting) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "traffic",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
servers: [],
|
||||
}
|
||||
}
|
||||
collecting = true
|
||||
const startedAt = Date.now()
|
||||
const now = new Date().toISOString()
|
||||
const now = sampledAt
|
||||
const settings = getSettingsRow()
|
||||
const snapshot: TrafficRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "traffic",
|
||||
sampledAt: now,
|
||||
servers: [],
|
||||
}
|
||||
|
||||
try {
|
||||
const enabledServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
|
||||
for (const srv of enabledServers) {
|
||||
const name = srv.name || srv.host
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(srv)
|
||||
const ifaces = await client.get<RosIfaceTraffic[]>("/interface")
|
||||
if (ifaces.length === 0) continue
|
||||
db.insert(trafficSamples).values(
|
||||
ifaces.map((i) => ({
|
||||
serverId: srv.id,
|
||||
interfaceName: i.name ?? "unknown",
|
||||
sampledAt: now,
|
||||
rxBytes: toNum(i["rx-byte"]),
|
||||
txBytes: toNum(i["tx-byte"]),
|
||||
rxBps: toNum(i["rx-bits-per-second"]),
|
||||
txBps: toNum(i["tx-bits-per-second"]),
|
||||
running: (i.running ?? "false") === "true",
|
||||
disabled: (i.disabled ?? "false") === "true",
|
||||
})),
|
||||
).run()
|
||||
} catch {
|
||||
// Continue collecting from remaining servers
|
||||
let sumRx = 0
|
||||
let sumTx = 0
|
||||
for (const i of ifaces) {
|
||||
sumRx += toNum(i["rx-bits-per-second"]) / 1_000_000
|
||||
sumTx += toNum(i["tx-bits-per-second"]) / 1_000_000
|
||||
}
|
||||
if (ifaces.length > 0) {
|
||||
db.insert(trafficSamples).values(
|
||||
ifaces.map((i) => ({
|
||||
serverId: srv.id,
|
||||
interfaceName: i.name ?? "unknown",
|
||||
sampledAt: now,
|
||||
rxBytes: toNum(i["rx-byte"]),
|
||||
txBytes: toNum(i["tx-byte"]),
|
||||
rxBps: toNum(i["rx-bits-per-second"]),
|
||||
txBps: toNum(i["tx-bits-per-second"]),
|
||||
running: (i.running ?? "false") === "true",
|
||||
disabled: (i.disabled ?? "false") === "true",
|
||||
})),
|
||||
).run()
|
||||
}
|
||||
snapshot.servers.push({
|
||||
serverId: srv.id,
|
||||
name,
|
||||
host: srv.host,
|
||||
ok: true,
|
||||
interfaces: ifaces.length,
|
||||
sumRxMbps: Math.round(sumRx),
|
||||
sumTxMbps: Math.round(sumTx),
|
||||
})
|
||||
} catch (err) {
|
||||
snapshot.servers.push({
|
||||
serverId: srv.id,
|
||||
name,
|
||||
host: srv.host,
|
||||
ok: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,45 +128,38 @@ export async function collectTrafficOnce(): Promise<void> {
|
||||
lastError: "",
|
||||
updatedAt: now,
|
||||
}).where(eq(trafficSettings.id, 1)).run()
|
||||
|
||||
state.lastRunAt = now
|
||||
state.lastError = null
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
snapshot.fatalError = msg
|
||||
db.update(trafficSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - startedAt,
|
||||
lastError: msg,
|
||||
updatedAt: now,
|
||||
}).where(eq(trafficSettings.id, 1)).run()
|
||||
state.lastRunAt = now
|
||||
state.lastError = msg
|
||||
} finally {
|
||||
collecting = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
export function stopTrafficCollector() {
|
||||
if (timer) {
|
||||
clearInterval(timer)
|
||||
timer = null
|
||||
}
|
||||
state.running = false
|
||||
}
|
||||
/** Таймеры трафика ведёт планировщик. */
|
||||
export function stopTrafficCollector() {}
|
||||
|
||||
export function restartTrafficCollector() {
|
||||
stopTrafficCollector()
|
||||
const settings = getSettingsRow()
|
||||
if (!settings.enabled) return
|
||||
const intervalMs = Math.max(5, settings.intervalSec) * 1000
|
||||
timer = setInterval(() => {
|
||||
void collectTrafficOnce()
|
||||
}, intervalMs)
|
||||
state.running = true
|
||||
}
|
||||
export function restartTrafficCollector() {}
|
||||
|
||||
export function getTrafficCollectorState(): TrafficCollectorState {
|
||||
return { ...state }
|
||||
const s = getSettingsRow()
|
||||
return {
|
||||
running: Boolean(s.enabled),
|
||||
lastRunAt: s.lastCollectedAt ?? null,
|
||||
lastError: s.lastError || null,
|
||||
}
|
||||
}
|
||||
|
||||
/** Идёт активный прогон `collectTrafficOnce` (запись в SQLite ещё не завершена). */
|
||||
export function isTrafficCollecting(): boolean {
|
||||
return collecting
|
||||
}
|
||||
|
||||
export function getTrafficSettings() {
|
||||
@@ -152,7 +179,6 @@ export function updateTrafficSettings(patch: {
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
db.update(trafficSettings).set(next).where(eq(trafficSettings.id, 1)).run()
|
||||
restartTrafficCollector()
|
||||
return getSettingsRow()
|
||||
}
|
||||
|
||||
@@ -166,4 +192,3 @@ export function readServerSamplesInRange(serverId: number, sinceIso: string) {
|
||||
.orderBy(asc(trafficSamples.sampledAt))
|
||||
.all()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, asc, eq, gte, lt } from "drizzle-orm"
|
||||
import { and, asc, eq, gte, inArray, lt, max, or } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import {
|
||||
servers,
|
||||
@@ -7,31 +7,18 @@ import {
|
||||
uptimeResourceSamples,
|
||||
uptimeSettings,
|
||||
} from "../db/schema.js"
|
||||
import type { PingRunSnapshot, ResourcesRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import { PING_PROBE_WARN_RTT_MS } from "../constants/ping-probe.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
import { parseRosCpuLoadPercent, parseRosDataSizeBytes, parseRosUptimeToSeconds } from "./ros-metric-parse.js"
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null
|
||||
let collecting = false
|
||||
let collectingResources = false
|
||||
let collectingPing = false
|
||||
|
||||
function parseDurationToSeconds(raw: string | undefined): number {
|
||||
if (!raw) return 0
|
||||
let total = 0
|
||||
for (const m of raw.matchAll(/(\d+)(w|d|h|m(?!s)|s)/g)) {
|
||||
const n = Number.parseInt(m[1], 10)
|
||||
if (!Number.isFinite(n)) continue
|
||||
switch (m[2]) {
|
||||
case "w": total += n * 604800; break
|
||||
case "d": total += n * 86400; break
|
||||
case "h": total += n * 3600; break
|
||||
case "m": total += n * 60; break
|
||||
case "s": total += n; break
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
function toNum(raw: unknown): number {
|
||||
const n = Number.parseInt(String(raw ?? "0"), 10)
|
||||
return Number.isFinite(n) ? Math.max(0, n) : 0
|
||||
/** Параллельный прогон сборщиков uptime — чтобы `alert_engine` не читал БД до коммита сэмплов. */
|
||||
export function getUptimeCollectorsState(): { resources: boolean; ping: boolean } {
|
||||
return { resources: collectingResources, ping: collectingPing }
|
||||
}
|
||||
|
||||
function parsePingTimeMs(raw: string | undefined): number | null {
|
||||
@@ -44,7 +31,6 @@ function parsePingTimeMs(raw: string | undefined): number | null {
|
||||
const sec = s.match(/^(\d+(?:\.\d+)?)\s*s$/)
|
||||
if (sec) return Number.parseFloat(sec[1]) * 1000
|
||||
|
||||
// HH:MM:SS(.sss) from some RouterOS outputs
|
||||
const clock = s.match(/^(\d+):(\d+):(\d+(?:\.\d+)?)$/)
|
||||
if (clock) {
|
||||
const h = Number.parseFloat(clock[1])
|
||||
@@ -57,16 +43,19 @@ function parsePingTimeMs(raw: string | undefined): number | null {
|
||||
return Number.isFinite(n) ? n : null
|
||||
}
|
||||
|
||||
function getSettings() {
|
||||
export function getSettings() {
|
||||
const row = db.select().from(uptimeSettings).where(eq(uptimeSettings.id, 1)).limit(1).all()[0]
|
||||
if (row) return row
|
||||
const now = new Date().toISOString()
|
||||
db.insert(uptimeSettings).values({
|
||||
id: 1,
|
||||
enabled: true,
|
||||
intervalSec: 15,
|
||||
resourcesEnabled: true,
|
||||
pingEnabled: true,
|
||||
speedEnabled: true,
|
||||
intervalSec: 300,
|
||||
probeIntervalSec: 15,
|
||||
speedIntervalSec: 60,
|
||||
speedIntervalSec: 3600,
|
||||
retentionDays: 14,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
@@ -105,38 +94,101 @@ export function parsePing(results: Array<{ time?: string; status?: string; sent?
|
||||
typeof lossSource === "number" && Number.isFinite(lossSource)
|
||||
? Math.max(0, Math.min(100, lossSource))
|
||||
: 100
|
||||
const status: "up" | "warn" | "down" = loss >= 100 ? "down" : (loss > 1 || (avgRtt ?? 0) > 60 ? "warn" : "up")
|
||||
const status: "up" | "warn" | "down" =
|
||||
loss >= 100 ? "down" : (loss > 1 || (avgRtt ?? 0) > PING_PROBE_WARN_RTT_MS ? "warn" : "up")
|
||||
return { avgRtt, loss, status }
|
||||
}
|
||||
|
||||
export async function collectUptimeOnce(): Promise<void> {
|
||||
if (collecting) return
|
||||
collecting = true
|
||||
/** Узлы с REST-опросом для uptime: включённые в каталоге или любой Home Router (см. /servers — HR часто «выкл.» при доступе только из LAN). */
|
||||
function serversForUptimeRosPoll() {
|
||||
return db
|
||||
.select()
|
||||
.from(servers)
|
||||
.where(or(eq(servers.enabled, true), eq(servers.type, "home-router")))
|
||||
.all()
|
||||
}
|
||||
|
||||
function lastProbeSampleTimes(): Map<string, string> {
|
||||
const rows = db.select({
|
||||
probeId: uptimeProbeSamples.probeId,
|
||||
lastAt: max(uptimeProbeSamples.sampledAt),
|
||||
}).from(uptimeProbeSamples).groupBy(uptimeProbeSamples.probeId).all()
|
||||
const m = new Map<string, string>()
|
||||
for (const r of rows) {
|
||||
if (r.lastAt) m.set(r.probeId, String(r.lastAt))
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
/** Сбор метрик ресурсов (CPU/RAM/…) по всем включённым серверам. */
|
||||
export async function collectResourceSamplesOnce(): Promise<ResourcesRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collectingResources) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_resources",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
servers: [],
|
||||
}
|
||||
}
|
||||
collectingResources = true
|
||||
const started = Date.now()
|
||||
const now = new Date().toISOString()
|
||||
const now = sampledAt
|
||||
const settings = getSettings()
|
||||
const snapshot: ResourcesRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_resources",
|
||||
sampledAt: now,
|
||||
servers: [],
|
||||
}
|
||||
|
||||
try {
|
||||
const enabledServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
const resourceTargets = serversForUptimeRosPoll()
|
||||
|
||||
for (const s of enabledServers) {
|
||||
for (const s of resourceTargets) {
|
||||
const name = s.name || s.host
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(s)
|
||||
const resource = await client.getResource()
|
||||
const cpuLoad = parseRosCpuLoadPercent(resource["cpu-load"])
|
||||
const freeMem = parseRosDataSizeBytes(resource["free-memory"])
|
||||
const totalMem = parseRosDataSizeBytes(resource["total-memory"])
|
||||
const freeHdd = parseRosDataSizeBytes(resource["free-hdd-space"])
|
||||
const totalHdd = parseRosDataSizeBytes(resource["total-hdd-space"])
|
||||
const uptimeSeconds = parseRosUptimeToSeconds(resource["uptime"])
|
||||
db.insert(uptimeResourceSamples).values({
|
||||
serverId: s.id,
|
||||
sampledAt: now,
|
||||
status: "online",
|
||||
cpuLoad: toNum(resource["cpu-load"]),
|
||||
freeMemory: toNum(resource["free-memory"]),
|
||||
totalMemory: toNum(resource["total-memory"]),
|
||||
freeHddSpace: toNum(resource["free-hdd-space"]),
|
||||
totalHddSpace: toNum(resource["total-hdd-space"]),
|
||||
uptimeSeconds: parseDurationToSeconds(resource["uptime"]),
|
||||
cpuLoad,
|
||||
freeMemory: freeMem,
|
||||
totalMemory: totalMem,
|
||||
freeHddSpace: freeHdd,
|
||||
totalHddSpace: totalHdd,
|
||||
uptimeSeconds,
|
||||
boardName: String(resource["board-name"] ?? ""),
|
||||
rosVersion: String(resource["version"] ?? ""),
|
||||
}).run()
|
||||
} catch {
|
||||
const memUsedMb = totalMem > 0 ? Math.round((totalMem - freeMem) / (1024 * 1024)) : 0
|
||||
const memTotalMb = totalMem > 0 ? Math.round(totalMem / (1024 * 1024)) : 0
|
||||
snapshot.servers.push({
|
||||
serverId: s.id,
|
||||
name,
|
||||
host: s.host,
|
||||
status: "online",
|
||||
cpuLoadPct: cpuLoad,
|
||||
memUsedMb,
|
||||
memTotalMb,
|
||||
memUsedPct: totalMem > 0 ? Math.round(((totalMem - freeMem) / totalMem) * 100) : undefined,
|
||||
diskFreeMb: freeHdd > 0 ? Math.round(freeHdd / (1024 * 1024)) : 0,
|
||||
diskTotalMb: totalHdd > 0 ? Math.round(totalHdd / (1024 * 1024)) : 0,
|
||||
uptimeSeconds,
|
||||
boardName: String(resource["board-name"] ?? "") || undefined,
|
||||
rosVersion: String(resource["version"] ?? "") || undefined,
|
||||
})
|
||||
} catch (err) {
|
||||
console.warn(`[uptime-collector] resource serverId=${s.id} (${s.host}):`, err instanceof Error ? err.message : err)
|
||||
db.insert(uptimeResourceSamples).values({
|
||||
serverId: s.id,
|
||||
sampledAt: now,
|
||||
@@ -150,12 +202,180 @@ export async function collectUptimeOnce(): Promise<void> {
|
||||
boardName: "",
|
||||
rosVersion: "",
|
||||
}).run()
|
||||
snapshot.servers.push({
|
||||
serverId: s.id,
|
||||
name,
|
||||
host: s.host,
|
||||
status: "offline",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
cleanup(Math.max(1, settings.retentionDays))
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: "",
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} catch (e) {
|
||||
snapshot.fatalError = e instanceof Error ? e.message : String(e)
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: e instanceof Error ? e.message : String(e),
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} finally {
|
||||
collectingResources = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
/** Ping-пробы: учитывается interval_sec у пробы или глобальный probe_interval_sec. */
|
||||
export async function collectPingProbesOnce(): Promise<PingRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collectingPing) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_ping",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
probes: [],
|
||||
}
|
||||
}
|
||||
collectingPing = true
|
||||
const started = Date.now()
|
||||
const now = sampledAt
|
||||
const settings = getSettings()
|
||||
const defaultProbeSec = Math.max(5, settings.probeIntervalSec ?? settings.intervalSec)
|
||||
const lastByProbe = lastProbeSampleTimes()
|
||||
const nowMs = Date.now()
|
||||
const snapshot: PingRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_ping",
|
||||
sampledAt: now,
|
||||
probes: [],
|
||||
skippedByInterval: 0,
|
||||
}
|
||||
|
||||
try {
|
||||
const rosPollServers = serversForUptimeRosPoll()
|
||||
const probes = db.select().from(uptimeProbes).where(eq(uptimeProbes.enabled, true)).orderBy(asc(uptimeProbes.sortOrder)).all()
|
||||
|
||||
for (const p of probes) {
|
||||
const src = enabledServers.find((s) => s.id === p.srcServerId)
|
||||
const effectiveSec = (p.intervalSec != null && p.intervalSec > 0) ? p.intervalSec : defaultProbeSec
|
||||
const lastIso = lastByProbe.get(p.id)
|
||||
if (lastIso) {
|
||||
const elapsed = nowMs - Date.parse(lastIso)
|
||||
if (elapsed < effectiveSec * 1000) {
|
||||
snapshot.skippedByInterval = (snapshot.skippedByInterval ?? 0) + 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
const src = rosPollServers.find((s) => s.id === p.srcServerId)
|
||||
if (!src) continue
|
||||
const srcServerName = src.name || src.host
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(src)
|
||||
const results = await client.ping(p.target, 4, p.srcInterface || undefined)
|
||||
const parsed = parsePing(results)
|
||||
db.insert(uptimeProbeSamples).values({
|
||||
probeId: p.id,
|
||||
sampledAt: now,
|
||||
rttMs: parsed.avgRtt,
|
||||
lossPct: parsed.loss,
|
||||
status: parsed.status,
|
||||
}).run()
|
||||
snapshot.probes.push({
|
||||
probeId: p.id,
|
||||
name: p.name,
|
||||
target: p.target,
|
||||
srcServerId: p.srcServerId,
|
||||
srcServerName,
|
||||
srcInterface: p.srcInterface || "",
|
||||
ok: true,
|
||||
rttMs: parsed.avgRtt,
|
||||
lossPct: parsed.loss,
|
||||
status: parsed.status,
|
||||
})
|
||||
} catch (err) {
|
||||
db.insert(uptimeProbeSamples).values({
|
||||
probeId: p.id,
|
||||
sampledAt: now,
|
||||
rttMs: null,
|
||||
lossPct: 100,
|
||||
status: "down",
|
||||
}).run()
|
||||
snapshot.probes.push({
|
||||
probeId: p.id,
|
||||
name: p.name,
|
||||
target: p.target,
|
||||
srcServerId: p.srcServerId,
|
||||
srcServerName,
|
||||
srcInterface: p.srcInterface || "",
|
||||
ok: false,
|
||||
rttMs: null,
|
||||
lossPct: 100,
|
||||
status: "down",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
cleanup(Math.max(1, settings.retentionDays))
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: "",
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} catch (e) {
|
||||
snapshot.fatalError = e instanceof Error ? e.message : String(e)
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: e instanceof Error ? e.message : String(e),
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} finally {
|
||||
collectingPing = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
const MAX_PROBE_IDS_PER_GROUP_COLLECT = 64
|
||||
|
||||
/**
|
||||
* Немедленный ping по указанным пробам (без учёта interval_sec) — запись в uptime_probe_samples.
|
||||
* Используется кнопкой «Обновить» у группы «сервер + назначение» на вкладке проб.
|
||||
*/
|
||||
export async function collectPingForProbeIds(probeIds: string[]): Promise<{ polled: number }> {
|
||||
const uniqueIds = [...new Set(probeIds.filter((id) => typeof id === "string" && id.trim() !== ""))].slice(
|
||||
0,
|
||||
MAX_PROBE_IDS_PER_GROUP_COLLECT,
|
||||
)
|
||||
if (uniqueIds.length === 0) return { polled: 0 }
|
||||
|
||||
if (collectingPing) {
|
||||
const e = new Error("Сбор ping уже выполняется, подождите")
|
||||
;(e as Error & { statusCode?: number }).statusCode = 409
|
||||
throw e
|
||||
}
|
||||
|
||||
collectingPing = true
|
||||
const now = new Date().toISOString()
|
||||
const settings = getSettings()
|
||||
let polled = 0
|
||||
|
||||
try {
|
||||
const rosPollServers = serversForUptimeRosPoll()
|
||||
const rows = db.select().from(uptimeProbes).where(inArray(uptimeProbes.id, uniqueIds)).all()
|
||||
|
||||
for (const p of rows) {
|
||||
const src = rosPollServers.find((s) => s.id === p.srcServerId)
|
||||
if (!src) continue
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(src)
|
||||
@@ -177,64 +397,74 @@ export async function collectUptimeOnce(): Promise<void> {
|
||||
status: "down",
|
||||
}).run()
|
||||
}
|
||||
polled += 1
|
||||
}
|
||||
|
||||
cleanup(Math.max(1, settings.retentionDays))
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: "",
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} catch (e) {
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: e instanceof Error ? e.message : String(e),
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
return { polled }
|
||||
} finally {
|
||||
collecting = false
|
||||
collectingPing = false
|
||||
}
|
||||
}
|
||||
|
||||
export function restartUptimeCollector() {
|
||||
const s = getSettings()
|
||||
if (timer) clearInterval(timer)
|
||||
timer = null
|
||||
if (!s.enabled) return
|
||||
const intervalMs = Math.max(5, s.probeIntervalSec || s.intervalSec) * 1000
|
||||
timer = setInterval(() => { void collectUptimeOnce() }, intervalMs)
|
||||
/** Совместимость: один проход ресурсы + ping (для ручного вызова без планировщика). */
|
||||
export async function collectUptimeOnce(): Promise<void> {
|
||||
await collectResourceSamplesOnce()
|
||||
await collectPingProbesOnce()
|
||||
}
|
||||
|
||||
export function stopUptimeCollector() {
|
||||
if (timer) clearInterval(timer)
|
||||
timer = null
|
||||
}
|
||||
/** no-op: таймеры управляются планировщиком */
|
||||
export function restartUptimeCollector() {}
|
||||
|
||||
export function stopUptimeCollector() {}
|
||||
|
||||
export function readUptimeSettings() {
|
||||
return getSettings()
|
||||
}
|
||||
|
||||
export function updateUptimeSettings(patch: {
|
||||
/** Устаревший: если передан без per-job флагов — выставляет все три job-флага. */
|
||||
enabled?: boolean
|
||||
resourcesEnabled?: boolean
|
||||
pingEnabled?: boolean
|
||||
speedEnabled?: boolean
|
||||
/** Интервал сбора метрик ресурсов (сек). */
|
||||
intervalSec?: number
|
||||
/** Интервал тика ping и значение по умолчанию для проб без своего interval_sec. */
|
||||
probeIntervalSec?: number
|
||||
speedIntervalSec?: number
|
||||
retentionDays?: number
|
||||
}) {
|
||||
const prev = getSettings()
|
||||
const nextProbeInterval = patch.probeIntervalSec ?? patch.intervalSec ?? prev.probeIntervalSec ?? prev.intervalSec
|
||||
const anyJobFlag =
|
||||
patch.resourcesEnabled !== undefined
|
||||
|| patch.pingEnabled !== undefined
|
||||
|| patch.speedEnabled !== undefined
|
||||
|
||||
let resourcesEnabled = patch.resourcesEnabled ?? prev.resourcesEnabled ?? prev.enabled
|
||||
let pingEnabled = patch.pingEnabled ?? prev.pingEnabled ?? prev.enabled
|
||||
let speedEnabled = patch.speedEnabled ?? prev.speedEnabled ?? prev.enabled
|
||||
|
||||
if (patch.enabled !== undefined && !anyJobFlag) {
|
||||
resourcesEnabled = patch.enabled
|
||||
pingEnabled = patch.enabled
|
||||
speedEnabled = patch.enabled
|
||||
}
|
||||
|
||||
const masterEnabled = Boolean(resourcesEnabled || pingEnabled || speedEnabled)
|
||||
|
||||
const next = {
|
||||
enabled: patch.enabled ?? prev.enabled,
|
||||
intervalSec: nextProbeInterval,
|
||||
probeIntervalSec: nextProbeInterval,
|
||||
enabled: masterEnabled,
|
||||
resourcesEnabled,
|
||||
pingEnabled,
|
||||
speedEnabled,
|
||||
intervalSec: patch.intervalSec ?? prev.intervalSec,
|
||||
probeIntervalSec: patch.probeIntervalSec ?? prev.probeIntervalSec,
|
||||
speedIntervalSec: patch.speedIntervalSec ?? prev.speedIntervalSec ?? 60,
|
||||
retentionDays: patch.retentionDays ?? prev.retentionDays,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
db.update(uptimeSettings).set(next).where(eq(uptimeSettings.id, 1)).run()
|
||||
restartUptimeCollector()
|
||||
return readUptimeSettings()
|
||||
}
|
||||
|
||||
@@ -260,20 +490,20 @@ export function replaceProbes(rows: Array<{
|
||||
probeFilter: string
|
||||
enabled: boolean
|
||||
showOnDashboard?: boolean
|
||||
intervalSec?: number
|
||||
}>) {
|
||||
const now = new Date().toISOString()
|
||||
const existing = db.select({ id: uptimeProbes.id }).from(uptimeProbes).all()
|
||||
const nextIds = new Set(rows.map((r) => r.id))
|
||||
|
||||
// Remove only probes that user actually deleted (their samples are removed by FK cascade).
|
||||
for (const p of existing) {
|
||||
if (nextIds.has(p.id)) continue
|
||||
db.delete(uptimeProbes).where(eq(uptimeProbes.id, p.id)).run()
|
||||
}
|
||||
|
||||
// Upsert rows in-place to preserve sample history for unchanged probe ids.
|
||||
for (let i = 0; i < rows.length; i += 1) {
|
||||
const r = rows[i]
|
||||
const intervalSec = r.intervalSec != null && r.intervalSec > 0 ? Math.max(5, Math.floor(r.intervalSec)) : 0
|
||||
const patch = {
|
||||
srcServerId: r.srcServerId,
|
||||
srcInterface: r.srcInterface || "",
|
||||
@@ -282,6 +512,7 @@ export function replaceProbes(rows: Array<{
|
||||
probeFilter: r.probeFilter || "—",
|
||||
enabled: r.enabled,
|
||||
showOnDashboard: r.showOnDashboard ?? false,
|
||||
intervalSec,
|
||||
sortOrder: i,
|
||||
updatedAt: now,
|
||||
}
|
||||
@@ -311,4 +542,3 @@ export function readResourceSamplesSince(sinceIso: string, serverId: number) {
|
||||
.orderBy(asc(uptimeResourceSamples.sampledAt))
|
||||
.all()
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
import { asc, eq } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers, uptimeSpeedProbes, uptimeSpeedTestRuns } from "../db/schema.js"
|
||||
import type { SpeedRunSnapshot, SpeedScheduledRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import type { RosIpAddress } from "../types/server.js"
|
||||
import { withBtestNodeLocks } from "./btest-node-locks.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
import { parsePing } from "./uptime-collector.js"
|
||||
|
||||
function firstUsableIpOnInterface(rows: RosIpAddress[], iface: string): string {
|
||||
const name = iface.trim()
|
||||
const candidates = rows.filter((a) => {
|
||||
if (String(a.interface ?? "").trim() !== name) return false
|
||||
if (String(a.disabled ?? "false").toLowerCase() === "true") return false
|
||||
if (String(a.invalid ?? "false").toLowerCase() === "true") return false
|
||||
const addr = String(a.address ?? "").trim()
|
||||
return addr.includes("/")
|
||||
})
|
||||
candidates.sort((a, b) => {
|
||||
const ad = String(a.dynamic ?? "false").toLowerCase() === "true" ? 1 : 0
|
||||
const bd = String(b.dynamic ?? "false").toLowerCase() === "true" ? 1 : 0
|
||||
return ad - bd
|
||||
})
|
||||
const raw = candidates[0]?.address
|
||||
return raw ? String(raw).split("/")[0]?.trim() ?? "" : ""
|
||||
}
|
||||
|
||||
function parseRateToMbps(raw: unknown): number {
|
||||
const txt = String(raw ?? "").trim().toLowerCase()
|
||||
if (!txt) return 0
|
||||
const m = txt.match(/^(\d+(?:\.\d+)?)\s*(g|m|k)?(?:bps|bit\/s|bits\/s)?$/)
|
||||
if (m) {
|
||||
const n = Number.parseFloat(m[1])
|
||||
const u = m[2] ?? ""
|
||||
if (!Number.isFinite(n)) return 0
|
||||
if (u === "g") return n * 1000
|
||||
if (u === "m") return n
|
||||
if (u === "k") return n / 1000
|
||||
return n / 1_000_000
|
||||
}
|
||||
const n = Number.parseFloat(txt.replace(/[^\d.]/g, ""))
|
||||
if (!Number.isFinite(n)) return 0
|
||||
return n > 10000 ? n / 1_000_000 : n
|
||||
}
|
||||
|
||||
export interface SpeedTestRunParams {
|
||||
runId?: string
|
||||
probeId?: string
|
||||
srcServerId: number
|
||||
dstServerId: number
|
||||
srcInterface?: string
|
||||
dstInterface?: string
|
||||
protocol?: "tcp" | "udp"
|
||||
direction?: "transmit" | "receive" | "both"
|
||||
durationSec?: number
|
||||
}
|
||||
|
||||
export interface SpeedTestRunResult {
|
||||
runId: string
|
||||
srcInterface: string | null
|
||||
dstInterface: string | null
|
||||
address: string
|
||||
srcAddress: string
|
||||
dstAddress: string
|
||||
srcInterfaceAddress: string | null
|
||||
dstInterfaceAddress: string | null
|
||||
protocol: "tcp" | "udp"
|
||||
direction: "transmit" | "receive" | "both"
|
||||
durationSec: number
|
||||
txAvgMbps: number
|
||||
rxAvgMbps: number
|
||||
txSeries: number[]
|
||||
rxSeries: number[]
|
||||
raw: Array<Record<string, string>>
|
||||
afterBtPing: {
|
||||
rttMs: number | null
|
||||
lossPct: number | null
|
||||
error: string | null
|
||||
}
|
||||
}
|
||||
|
||||
function humanizeSpeedError(raw: string): string {
|
||||
return /timed out|socket hang up|ECONNRESET/i.test(raw)
|
||||
? "BTTest timeout/socket hang up: проверь доступность destination, /tool bandwidth-server, firewall и корректность интерфейсов"
|
||||
: raw
|
||||
}
|
||||
|
||||
/** Полный цикл bandwidth-test + after-ping + запись в БД. Вызывать внутри withBtestNodeLocks при необходимости — здесь lock уже снаружи. */
|
||||
export async function executeBandwidthSpeedTestInner(params: SpeedTestRunParams): Promise<SpeedTestRunResult> {
|
||||
const srcId = params.srcServerId
|
||||
const dstId = params.dstServerId
|
||||
const src = db.select().from(servers).where(eq(servers.id, srcId)).limit(1).all()[0]
|
||||
const dst = db.select().from(servers).where(eq(servers.id, dstId)).limit(1).all()[0]
|
||||
if (!src || !dst) throw new Error("Server not found")
|
||||
|
||||
const protocol = params.protocol === "udp" ? "udp" : "tcp"
|
||||
const direction = params.direction === "transmit" || params.direction === "receive" ? params.direction : "both"
|
||||
const durationSec = Math.max(3, Number.parseInt(String(params.durationSec ?? ""), 10) || 10)
|
||||
const srcInterface = String(params.srcInterface ?? "").trim()
|
||||
const dstInterface = String(params.dstInterface ?? "").trim()
|
||||
const srcClient = MikrotikClient.fromServer(src)
|
||||
const dstAddressRows = dstInterface
|
||||
? await MikrotikClient.fromServer(dst).getIpAddresses().catch(() => [])
|
||||
: []
|
||||
const srcAddressRows = srcInterface
|
||||
? await srcClient.getIpAddresses().catch(() => [])
|
||||
: []
|
||||
const dstAddress = dstInterface ? firstUsableIpOnInterface(dstAddressRows, dstInterface) : dst.host
|
||||
if (dstInterface && !dstAddress) {
|
||||
throw new Error(`На интерфейсе назначения '${dstInterface}' нет активного IP (проверьте /ip/address — не disabled).`)
|
||||
}
|
||||
const initialAddress = dstAddress || dst.host
|
||||
const srcAddress = srcInterface ? firstUsableIpOnInterface(srcAddressRows, srcInterface) : src.host
|
||||
const srcInterfaceAddress = srcInterface ? (srcAddress || null) : null
|
||||
const dstInterfaceAddress = dstInterface ? (dstAddress || null) : null
|
||||
|
||||
const rows = await srcClient.bandwidthTest({
|
||||
address: initialAddress,
|
||||
user: dst.username,
|
||||
password: dst.password,
|
||||
protocol,
|
||||
direction,
|
||||
durationSec,
|
||||
})
|
||||
const txSeries = rows
|
||||
.map((r) => parseRateToMbps(r["tx-current"] ?? r["tx-10-second-average"] ?? r["tx-total-average"]))
|
||||
.filter((v) => Number.isFinite(v) && v >= 0)
|
||||
const rxSeries = rows
|
||||
.map((r) => parseRateToMbps(r["rx-current"] ?? r["rx-10-second-average"] ?? r["rx-total-average"]))
|
||||
.filter((v) => Number.isFinite(v) && v >= 0)
|
||||
const last = rows[rows.length - 1] ?? {}
|
||||
const txAvg = parseRateToMbps(last["tx-total-average"] ?? last["tx-10-second-average"])
|
||||
const rxAvg = parseRateToMbps(last["rx-total-average"] ?? last["rx-10-second-average"])
|
||||
const txFinal = txAvg || (txSeries.length ? txSeries.reduce((a, b) => a + b, 0) / txSeries.length : 0)
|
||||
const rxFinal = rxAvg || (rxSeries.length ? rxSeries.reduce((a, b) => a + b, 0) / rxSeries.length : 0)
|
||||
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 3000))
|
||||
|
||||
const pingNow = new Date().toISOString()
|
||||
let afterBtPing: SpeedTestRunResult["afterBtPing"] = { rttMs: null, lossPct: null, error: null }
|
||||
try {
|
||||
const pingRows = await srcClient.ping(initialAddress, 4, srcInterface || undefined, { interval: "1s" })
|
||||
const pr = parsePing(pingRows as Parameters<typeof parsePing>[0])
|
||||
afterBtPing = { rttMs: pr.avgRtt, lossPct: pr.loss, error: null }
|
||||
} catch (pe) {
|
||||
afterBtPing = {
|
||||
rttMs: null,
|
||||
lossPct: null,
|
||||
error: pe instanceof Error ? pe.message : String(pe),
|
||||
}
|
||||
}
|
||||
|
||||
const probeId = String(params.probeId ?? "").trim()
|
||||
const runId = String(params.runId ?? "").trim() || `sr-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||||
if (probeId) {
|
||||
db.update(uptimeSpeedProbes).set({
|
||||
lastRunAt: pingNow,
|
||||
lastTxAvgMbps: txFinal,
|
||||
lastRxAvgMbps: rxFinal,
|
||||
lastStatus: "done",
|
||||
lastError: "",
|
||||
lastPingRttMs: afterBtPing.rttMs,
|
||||
lastPingLossPct: afterBtPing.lossPct,
|
||||
lastPingAt: pingNow,
|
||||
lastPingError: afterBtPing.error ?? "",
|
||||
updatedAt: pingNow,
|
||||
}).where(eq(uptimeSpeedProbes.id, probeId)).run()
|
||||
}
|
||||
db.insert(uptimeSpeedTestRuns).values({
|
||||
id: runId,
|
||||
probeId: probeId || null,
|
||||
srcServerId: srcId,
|
||||
dstServerId: dstId,
|
||||
srcInterface: srcInterface || "",
|
||||
dstInterface: dstInterface || "",
|
||||
srcAddress: srcAddress || src.host,
|
||||
dstAddress: initialAddress,
|
||||
srcInterfaceAddress,
|
||||
dstInterfaceAddress,
|
||||
protocol,
|
||||
direction,
|
||||
durationSec,
|
||||
txAvgMbps: txFinal,
|
||||
rxAvgMbps: rxFinal,
|
||||
pingRttMs: afterBtPing.rttMs,
|
||||
pingLossPct: afterBtPing.lossPct,
|
||||
pingError: afterBtPing.error ?? null,
|
||||
status: "done",
|
||||
error: null,
|
||||
createdAt: pingNow,
|
||||
}).run()
|
||||
|
||||
return {
|
||||
runId,
|
||||
srcInterface: srcInterface || null,
|
||||
dstInterface: dstInterface || null,
|
||||
address: initialAddress,
|
||||
srcAddress: srcAddress || src.host,
|
||||
dstAddress: initialAddress,
|
||||
srcInterfaceAddress,
|
||||
dstInterfaceAddress,
|
||||
protocol,
|
||||
direction,
|
||||
durationSec,
|
||||
txAvgMbps: txFinal,
|
||||
rxAvgMbps: rxFinal,
|
||||
txSeries,
|
||||
rxSeries,
|
||||
raw: rows,
|
||||
afterBtPing,
|
||||
}
|
||||
}
|
||||
|
||||
export async function runBandwidthSpeedTest(params: SpeedTestRunParams): Promise<SpeedTestRunResult> {
|
||||
const srcId = params.srcServerId
|
||||
const dstId = params.dstServerId
|
||||
if (srcId === dstId) throw new Error("Source and destination must be different")
|
||||
return withBtestNodeLocks([srcId, dstId], () => executeBandwidthSpeedTestInner(params))
|
||||
}
|
||||
|
||||
export function recordSpeedTestFailure(params: {
|
||||
probeId?: string
|
||||
runId?: string
|
||||
srcServerId: number
|
||||
dstServerId: number
|
||||
srcInterface?: string
|
||||
dstInterface?: string
|
||||
protocol?: "tcp" | "udp"
|
||||
direction?: "transmit" | "receive" | "both"
|
||||
durationSec?: number
|
||||
message: string
|
||||
}) {
|
||||
const probeId = String(params.probeId ?? "").trim()
|
||||
const message = humanizeSpeedError(params.message)
|
||||
if (probeId) {
|
||||
db.update(uptimeSpeedProbes).set({
|
||||
lastRunAt: new Date().toISOString(),
|
||||
lastStatus: "error",
|
||||
lastError: message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}).where(eq(uptimeSpeedProbes.id, probeId)).run()
|
||||
}
|
||||
db.insert(uptimeSpeedTestRuns).values({
|
||||
id: String(params.runId ?? "").trim() || `sr-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
probeId: probeId || null,
|
||||
srcServerId: params.srcServerId,
|
||||
dstServerId: params.dstServerId,
|
||||
srcInterface: String(params.srcInterface ?? "").trim(),
|
||||
dstInterface: String(params.dstInterface ?? "").trim(),
|
||||
srcAddress: null,
|
||||
dstAddress: null,
|
||||
srcInterfaceAddress: null,
|
||||
dstInterfaceAddress: null,
|
||||
protocol: params.protocol === "udp" ? "udp" : "tcp",
|
||||
direction: params.direction === "transmit" || params.direction === "receive" ? params.direction : "both",
|
||||
durationSec: Math.max(3, Number.parseInt(String(params.durationSec ?? ""), 10) || 10),
|
||||
txAvgMbps: null,
|
||||
rxAvgMbps: null,
|
||||
pingRttMs: null,
|
||||
pingLossPct: null,
|
||||
pingError: null,
|
||||
status: "error",
|
||||
error: message,
|
||||
createdAt: new Date().toISOString(),
|
||||
}).run()
|
||||
}
|
||||
|
||||
function serverDisplayName(id: number): string {
|
||||
const s = db.select().from(servers).where(eq(servers.id, id)).limit(1).all()[0]
|
||||
return s ? (s.name || s.host) : `#${id}`
|
||||
}
|
||||
|
||||
/** Фоновый прогон всех включённых speed-проб (по очереди). */
|
||||
export async function runScheduledSpeedProbesOnce(): Promise<SpeedScheduledRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
const runs: SpeedRunSnapshot[] = []
|
||||
const rows = db.select().from(uptimeSpeedProbes).orderBy(asc(uptimeSpeedProbes.sortOrder)).all()
|
||||
const enabled = rows.filter((r) => r.enabled !== false && r.srcServerId !== r.dstServerId)
|
||||
const protocol = (p: (typeof enabled)[number]) => (p.protocol === "udp" ? "udp" : "tcp")
|
||||
const direction = (p: (typeof enabled)[number]) =>
|
||||
p.direction === "transmit" || p.direction === "receive" ? p.direction : "both"
|
||||
const durationSec = (p: (typeof enabled)[number]) => Math.max(3, p.durationSec || 10)
|
||||
|
||||
for (const p of enabled) {
|
||||
try {
|
||||
const r = await runBandwidthSpeedTest({
|
||||
probeId: p.id,
|
||||
srcServerId: p.srcServerId,
|
||||
dstServerId: p.dstServerId,
|
||||
srcInterface: p.srcInterface || undefined,
|
||||
dstInterface: p.dstInterface || undefined,
|
||||
protocol: protocol(p) as "tcp" | "udp",
|
||||
direction: direction(p) as "transmit" | "receive" | "both",
|
||||
durationSec: durationSec(p),
|
||||
})
|
||||
runs.push({
|
||||
probeId: p.id,
|
||||
srcServerId: p.srcServerId,
|
||||
srcServerName: serverDisplayName(p.srcServerId),
|
||||
dstServerId: p.dstServerId,
|
||||
dstServerName: serverDisplayName(p.dstServerId),
|
||||
srcInterface: r.srcInterface ?? "",
|
||||
dstInterface: r.dstInterface ?? "",
|
||||
protocol: r.protocol,
|
||||
direction: r.direction,
|
||||
durationSec: r.durationSec,
|
||||
ok: true,
|
||||
txAvgMbps: r.txAvgMbps,
|
||||
rxAvgMbps: r.rxAvgMbps,
|
||||
pingRttMs: r.afterBtPing.rttMs,
|
||||
pingLossPct: r.afterBtPing.lossPct,
|
||||
pingError: r.afterBtPing.error,
|
||||
})
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
recordSpeedTestFailure({
|
||||
probeId: p.id,
|
||||
srcServerId: p.srcServerId,
|
||||
dstServerId: p.dstServerId,
|
||||
srcInterface: p.srcInterface,
|
||||
dstInterface: p.dstInterface,
|
||||
protocol: protocol(p),
|
||||
direction: direction(p),
|
||||
durationSec: p.durationSec,
|
||||
message: msg,
|
||||
})
|
||||
runs.push({
|
||||
probeId: p.id,
|
||||
srcServerId: p.srcServerId,
|
||||
srcServerName: serverDisplayName(p.srcServerId),
|
||||
dstServerId: p.dstServerId,
|
||||
dstServerName: serverDisplayName(p.dstServerId),
|
||||
srcInterface: p.srcInterface || "",
|
||||
dstInterface: p.dstInterface || "",
|
||||
protocol: protocol(p),
|
||||
direction: direction(p),
|
||||
durationSec: durationSec(p),
|
||||
ok: false,
|
||||
error: msg,
|
||||
})
|
||||
}
|
||||
}
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_speed",
|
||||
sampledAt,
|
||||
runs,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
/**
|
||||
* Снимок результатов одного прогона планировщика (JSON в `scheduler_runs.result_json`).
|
||||
* Версия `v` для совместимости при расширении схемы.
|
||||
*/
|
||||
|
||||
export const SCHEDULER_RUN_SNAPSHOT_VERSION = 1 as const
|
||||
|
||||
export interface TrafficServerSnapshot {
|
||||
serverId: number
|
||||
name: string
|
||||
host: string
|
||||
ok: boolean
|
||||
error?: string
|
||||
/** Сколько интерфейсов записано в сэмплы */
|
||||
interfaces?: number
|
||||
/** Сумма RX по интерфейсам, Мбит/с (округл.) */
|
||||
sumRxMbps?: number
|
||||
/** Сумма TX по интерфейсам, Мбит/с (округл.) */
|
||||
sumTxMbps?: number
|
||||
}
|
||||
|
||||
export interface TrafficRunSnapshot {
|
||||
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
|
||||
job: "traffic"
|
||||
sampledAt: string
|
||||
/** Прогон пропущен — предыдущий сбор ещё выполняется */
|
||||
skipped?: boolean
|
||||
/** Критическая ошибка до/вне цикла по серверам */
|
||||
fatalError?: string
|
||||
servers: TrafficServerSnapshot[]
|
||||
}
|
||||
|
||||
export interface ResourceServerSnapshot {
|
||||
serverId: number
|
||||
name: string
|
||||
host: string
|
||||
status: "online" | "offline"
|
||||
error?: string
|
||||
cpuLoadPct?: number
|
||||
memUsedMb?: number
|
||||
memTotalMb?: number
|
||||
memUsedPct?: number
|
||||
diskFreeMb?: number
|
||||
diskTotalMb?: number
|
||||
uptimeSeconds?: number
|
||||
boardName?: string
|
||||
rosVersion?: string
|
||||
}
|
||||
|
||||
export interface ResourcesRunSnapshot {
|
||||
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
|
||||
job: "uptime_resources"
|
||||
sampledAt: string
|
||||
skipped?: boolean
|
||||
fatalError?: string
|
||||
servers: ResourceServerSnapshot[]
|
||||
}
|
||||
|
||||
export interface PingProbeSnapshot {
|
||||
probeId: string
|
||||
name: string
|
||||
target: string
|
||||
srcServerId: number
|
||||
srcServerName: string
|
||||
srcInterface: string
|
||||
ok: boolean
|
||||
rttMs: number | null
|
||||
lossPct: number
|
||||
status: "up" | "warn" | "down"
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface PingRunSnapshot {
|
||||
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
|
||||
job: "uptime_ping"
|
||||
sampledAt: string
|
||||
skipped?: boolean
|
||||
fatalError?: string
|
||||
/** Пробы, для которых в этом тике записан сэмпл (не пропущены по interval) */
|
||||
probes: PingProbeSnapshot[]
|
||||
/** Сколько проб пропущено из‑за interval_sec */
|
||||
skippedByInterval?: number
|
||||
}
|
||||
|
||||
export interface SpeedRunSnapshot {
|
||||
probeId: string
|
||||
srcServerId: number
|
||||
srcServerName: string
|
||||
dstServerId: number
|
||||
dstServerName: string
|
||||
srcInterface: string
|
||||
dstInterface: string
|
||||
protocol: string
|
||||
direction: string
|
||||
durationSec: number
|
||||
ok: boolean
|
||||
txAvgMbps?: number | null
|
||||
rxAvgMbps?: number | null
|
||||
pingRttMs?: number | null
|
||||
pingLossPct?: number | null
|
||||
pingError?: string | null
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface SpeedScheduledRunSnapshot {
|
||||
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
|
||||
job: "uptime_speed"
|
||||
sampledAt: string
|
||||
runs: SpeedRunSnapshot[]
|
||||
}
|
||||
|
||||
/** Узел каталога: успешный GET `/system/identity` (REST) и время ответа. */
|
||||
export interface ServerRestPingSnapshot {
|
||||
serverId: number
|
||||
name: string
|
||||
host: string
|
||||
ok: boolean
|
||||
/** RTT успешного REST-запроса, мс */
|
||||
latencyMs?: number | null
|
||||
error?: string
|
||||
}
|
||||
|
||||
export interface ServersRestPingRunSnapshot {
|
||||
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
|
||||
job: "servers_rest_ping"
|
||||
sampledAt: string
|
||||
skipped?: boolean
|
||||
fatalError?: string
|
||||
servers: ServerRestPingSnapshot[]
|
||||
}
|
||||
|
||||
/** Одна строка диагностики прогона движка (для `scheduler_runs.result_json`). */
|
||||
export type AlertEngineRuleDiagSnapshot = {
|
||||
ruleId: string
|
||||
inGroup: boolean
|
||||
evalHit: boolean
|
||||
hitTransition?: "problem" | "recovery" | "neutral"
|
||||
hitMessage?: string
|
||||
stabilityOk: boolean
|
||||
cooldownOk: boolean
|
||||
telegramOk: boolean
|
||||
blocked?:
|
||||
| "no_hit"
|
||||
| "stability"
|
||||
| "cooldown"
|
||||
| "no_telegram"
|
||||
| "dedupe_positive"
|
||||
| "in_group"
|
||||
}
|
||||
|
||||
export interface AlertEngineRunSnapshot {
|
||||
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
|
||||
job: "alert_engine"
|
||||
sampledAt: string
|
||||
rulesChecked: number
|
||||
standaloneFires: number
|
||||
groupFires: number
|
||||
skippedNoTelegram: boolean
|
||||
errors?: string[]
|
||||
/** Почему правило не дало отдельную отправку в этом тике (если массив пуст — движок без правил). */
|
||||
ruleDiag?: AlertEngineRuleDiagSnapshot[]
|
||||
}
|
||||
|
||||
/** Снимок джобы `gre_bgp`: запись GRE/BGP в SQLite для алерт-движка. */
|
||||
export interface GreBgpSnapshotRunSnapshot {
|
||||
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
|
||||
job: "gre_bgp"
|
||||
sampledAt: string
|
||||
skipped?: boolean
|
||||
fatalError?: string
|
||||
greWritten: number
|
||||
bgpWritten: number
|
||||
/** Текущее состояние GRE на момент прогона (для alert_engine). */
|
||||
greTunnels?: Array<{
|
||||
targetLabel: string
|
||||
status: "up" | "down" | "degraded"
|
||||
}>
|
||||
/** Текущее состояние BGP-пиров на момент прогона (для alert_engine). */
|
||||
bgpPeers?: Array<{
|
||||
key: string
|
||||
state: string
|
||||
}>
|
||||
errors?: string[]
|
||||
}
|
||||
|
||||
export type SchedulerRunSnapshot =
|
||||
| TrafficRunSnapshot
|
||||
| ResourcesRunSnapshot
|
||||
| PingRunSnapshot
|
||||
| SpeedScheduledRunSnapshot
|
||||
| ServersRestPingRunSnapshot
|
||||
| GreBgpSnapshotRunSnapshot
|
||||
| AlertEngineRunSnapshot
|
||||
@@ -0,0 +1,10 @@
|
||||
/**
|
||||
* Парсит id сервера из БД (положительное целое).
|
||||
* Не извлекает цифры из строк вроде «srv1» — только целая строка из цифр.
|
||||
*/
|
||||
export function parseDbServerId(raw: unknown): number | null {
|
||||
const s = String(raw ?? "").trim()
|
||||
if (!/^\d+$/.test(s)) return null
|
||||
const n = Number.parseInt(s, 10)
|
||||
return Number.isFinite(n) && n >= 1 ? n : null
|
||||
}
|
||||
Reference in New Issue
Block a user