Compare commits

..
5 Commits
Author SHA1 Message Date
DenozordecandCursor 6332d83a12 feat(traffic): показать поток NetFlow на карте сети
Docker images / prepare-release (push) Successful in 10s
Docker images / backend-image (push) Successful in 2m14s
Docker images / frontend-image (push) Successful in 3m14s
Docker images / notify-webhook (push) Skipped
Docker images / updater-image (push) Successful in 46s
Docker images / publish-release (push) Successful in 11s
Скорость между узлами считается как в Трафике (5 мин, без overlay/mesh), отдельно от ёмкости WAN и BT.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-07 12:37:47 +07:00
DenozordecandCursor 3834c40aa8 fix(traffic): починить сборку tsc для частичных IPFIX-фикстур
Docker images / prepare-release (push) Successful in 6s
Docker images / backend-image (push) Successful in 1m50s
Docker images / frontend-image (push) Successful in 3m15s
Docker images / notify-webhook (push) Skipped
Docker images / updater-image (push) Successful in 47s
Docker images / publish-release (push) Successful in 11s
Тесты не должны попадать в Docker tsc; ingest принимает ParsedFlowInput и нормализует поля.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-07 11:45:22 +07:00
DenozordecandCursor 90c8c393e5 feat(traffic): разделить плоскости IPFIX и показать путь JH→EN
Docker images / prepare-release (push) Successful in 8s
Docker images / backend-image (push) Failing after 1m31s
Docker images / frontend-image (push) Successful in 2m36s
Docker images / notify-webhook (push) Skipped
Docker images / updater-image (push) Successful in 42s
Docker images / publish-release (push) Skipped
Считать payload отдельно от overlay GRE/ESP и mesh; вкладка Пути и KPI Wire из счётчиков iface.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-07 11:39:14 +07:00
DenozordecandCursor cb799da13a fix(netflow): изолировать коллектор IPFIX и срезать раздувание базы
Docker images / prepare-release (push) Successful in 8s
Docker images / backend-image (push) Successful in 2m23s
Docker images / frontend-image (push) Successful in 3m16s
Docker images / notify-webhook (push) Skipped
Docker images / updater-image (push) Successful in 57s
Docker images / publish-release (push) Successful in 12s
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-07 09:50:24 +07:00
DenozordecandCursor e0ddb17539 feat(traffic): классифицировать потоки по брендам и разгрузить ingest
Docker images / prepare-release (push) Successful in 12s
Docker images / backend-image (push) Successful in 1m48s
Docker images / frontend-image (push) Successful in 2m53s
Docker images / notify-webhook (push) Skipped
Docker images / updater-image (push) Successful in 41s
Docker images / publish-release (push) Successful in 8s
Починить ISO-страну Cloudflare, не держать holder как сервис, клик по срезу открывает Сессии. Не вызывать REST /interface с UDP, flush SQLite пачкой — uptime не должен ловить timeout 10s.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-07 03:01:14 +07:00
50 changed files with 4625 additions and 516 deletions
+172 -7
View File
@@ -41,6 +41,15 @@ import {
wanJhEdgeMapKey,
type GreSpeedProbeSnapshot,
} from "@/lib/map-gre-speed-probe"
import {
formatNetflowDir,
formatNetflowRate,
hopHasRate,
matchNetflowForGreEdge,
matchNetflowForWan,
type MatchedNetflowHop,
} from "@/lib/map-netflow-hops"
import type { FlowMapHop, FlowMapHopsDto } from "@mmapp/contracts/traffic-flow"
import { Button } from "@/components/ui/button"
import { StatusBadge } from "@/components/status-badge"
import { StatusDot } from "@/components/status-dot"
@@ -420,6 +429,53 @@ function GreEdgeMetricBadge({
)
}
/** Живой поток NetFlow (не ёмкость канала / не BT). */
function NetflowRateBadge({
mx,
my,
hop,
onOpen,
}: {
mx: number
my: number
hop: MatchedNetflowHop
onOpen?: (e: React.MouseEvent<SVGElement>) => void
}) {
const showDir = hop.bpsFwd > 0 && hop.bpsRev > 0
const bw = showDir ? 86 : 72
const bh = showDir ? 32 : 20
return (
<g
transform={`translate(${mx},${my})`}
style={{ cursor: onOpen ? "pointer" : "default" }}
onPointerDown={(e) => { e.stopPropagation() }}
onClick={(e) => { e.stopPropagation(); onOpen?.(e) }}
>
<title>
Поток NetFlow между узлами (как в «Трафик»: 5 мин, без overlay/mesh). Скорость канала отдельно.
</title>
<rect
x={-bw / 2}
y={-bh / 2}
width={bw}
height={bh}
rx="6"
fill="rgba(6,13,26,0.94)"
stroke="#34d399"
strokeWidth="1.15"
/>
<text textAnchor="middle" y={showDir ? "-4" : "4"} fontFamily="ui-monospace,monospace">
<tspan fill="#6ee7b7" fontSize="8" fontWeight="700">{formatNetflowRate(hop)}</tspan>
</text>
{showDir && (
<text textAnchor="middle" y="10" fontFamily="ui-monospace,monospace">
<tspan fill="#34d399" fontSize="6.5" fontWeight="600">{formatNetflowDir(hop)}</tspan>
</text>
)}
</g>
)
}
function SvgTooltip({ n }: { n: Server & { x: number; y: number } }) {
const ss = STATUS_STYLE[n.status]
const ts = TYPE_STYLE[n.type]
@@ -784,6 +840,7 @@ export default function NetworkMapPage() {
const [mapServers, setMapServers] = useState<Server[]>([])
const [mapGreTunnels, setMapGreTunnels] = useState<GreTunnel[]>([])
const [speedProbes, setSpeedProbes] = useState<GreSpeedProbeSnapshot[]>([])
const [mapHops, setMapHops] = useState<FlowMapHop[]>([])
/** FQDN из GRE outer → IPv4 (ответ POST /api/network/resolve-hosts), для матчинга с WAN. */
const [greResolvedIpv4ByHost, setGreResolvedIpv4ByHost] = useState<Record<string, string>>({})
const [dataError, setDataError] = useState<string | null>(null)
@@ -935,11 +992,31 @@ export default function NetworkMapPage() {
const [filter, setFilter] = useState<FilterKey>("all")
const [search, setSearch] = useState("")
const [showPingBadges, setShowPingBadges] = useState(true)
const [showNetflow, setShowNetflow] = useState(true)
const [showAnimDots, setShowAnimDots] = useState(true)
const [showMinimap, setShowMinimap] = useState(true)
const [showHints, setShowHints] = useState(false)
const [showLayers, setShowLayers] = useState(false)
useEffect(() => {
if (!useLiveData || !showNetflow) {
queueMicrotask(() => setMapHops([]))
return
}
let cancelled = false
const tick = () => {
apiFetch<FlowMapHopsDto>("/api/traffic/flow/map-hops?range=5m")
.then((res) => { if (!cancelled) setMapHops(res.hops ?? []) })
.catch(() => { if (!cancelled) setMapHops([]) })
}
tick()
const id = window.setInterval(tick, 4000)
return () => {
cancelled = true
window.clearInterval(id)
}
}, [useLiveData, showNetflow, apiFetch])
const effectiveSatPos = useMemo(() => {
const out: Record<string, { x: number; y: number }[]> = {}
mapServers
@@ -1100,6 +1177,32 @@ export default function NetworkMapPage() {
return out
}, [greEdges])
const netflowByGreKey = useMemo(() => {
const m = new Map<string, MatchedNetflowHop>()
if (!showNetflow) return m
for (const e of greEdges) {
const hop = matchNetflowForGreEdge(e, mapHops)
if (hopHasRate(hop)) m.set(greEdgeKey(e), hop)
}
return m
}, [greEdges, mapHops, showNetflow])
const netflowByWanKey = useMemo(() => {
const m = new Map<string, MatchedNetflowHop>()
if (!showNetflow) return m
for (const home of homeRouters) {
for (const [wIdx, wan] of (home.wanUplinks ?? []).entries()) {
const hop = matchNetflowForWan(home.id, wan.iface, mapHops)
if (!hopHasRate(hop)) continue
m.set(`${home.id}\t${wIdx}`, hop)
for (const e of wanJhEdges) {
if (e.homeId === home.id && e.wanIdx === wIdx) m.set(wanJhEdgeMapKey(e), hop)
}
}
}
return m
}, [homeRouters, wanJhEdges, mapHops, showNetflow])
const nodes = mapServers
.map((s) => ({ ...s, ...nodePosById[s.id]! }))
// Визуальный приоритет: HR поверх JH, JH поверх EN.
@@ -1461,6 +1564,7 @@ export default function NetworkMapPage() {
onMouseLeave={() => setShowLayers(false)}>
{([
{ key: "showPingBadges", label: "Ping-значки", val: showPingBadges, set: setShowPingBadges, hint: "P" },
{ key: "showNetflow", label: "NetFlow", val: showNetflow, set: setShowNetflow, hint: "" },
{ key: "showAnimDots", label: "Анимация трафика", val: showAnimDots, set: setShowAnimDots, hint: "" },
{ key: "showMinimap", label: "Минимап", val: showMinimap, set: setShowMinimap, hint: "M" },
{ key: "showHints", label: "Горячие клавиши", val: showHints, set: setShowHints, hint: "" },
@@ -1588,6 +1692,15 @@ export default function NetworkMapPage() {
tBadge,
normalPx,
)
const flowHop = netflowByGreKey.get(edgeId)
const flowPos = edgeBadgePosition(
e.from.x,
e.from.y,
e.to.x,
e.to.y,
tBadge,
-normalPx - (normalPx === 0 ? 22 : 0),
)
function openGreDetail(ev: React.MouseEvent<SVGElement>) {
ev.stopPropagation()
setSelectedGreEdge(e)
@@ -1598,7 +1711,7 @@ export default function NetworkMapPage() {
<g key={edgeId} opacity={dimmed ? 0.05 : 1} style={{ transition: "opacity 0.3s" }}>
<line
x1={e.from.x} y1={e.from.y} x2={e.to.x} y2={e.to.y}
stroke={ts.stroke} strokeWidth="1.5"
stroke={ts.stroke} strokeWidth={hopHasRate(flowHop) ? 2.6 : 1.5}
strokeDasharray={e.tunnel.ipsec ? "7 4" : "none"}
opacity={ts.opacity}
/>
@@ -1632,6 +1745,14 @@ export default function NetworkMapPage() {
outerSummary={greOuterSummaryLine(e.tunnel, fromN, toN, greResolvedMap)}
/>
)}
{showNetflow && hopHasRate(flowHop) && (
<NetflowRateBadge
mx={flowPos.mx}
my={flowPos.my}
hop={flowHop}
onOpen={openGreDetail}
/>
)}
</g>
)
})}
@@ -1663,14 +1784,16 @@ export default function NetworkMapPage() {
const color = WAN_COLORS[edge.wanIdx] ?? "#888"
const vis = filter === "all" || filter === "home-router" || filter === "jump-host" || filter === "online"
const { mx, my } = edgeBadgePosition(satPos.x, satPos.y, jh.x, jh.y, 0.62, -17)
const flowPos = edgeBadgePosition(satPos.x, satPos.y, jh.x, jh.y, 0.38, 18)
const isHL = selected?.id === edge.homeId && (selWanIdx === null || selWanIdx === edge.wanIdx)
const wanFlow = netflowByWanKey.get(wanJhEdgeMapKey(edge))
return (
<g key={edgeKey} opacity={vis ? (isHL ? 1 : 0.45) : 0.05}
style={{ transition: "opacity 0.3s" }}>
<line
x1={satPos.x} y1={satPos.y} x2={jh.x} y2={jh.y}
stroke={color}
strokeWidth={edge.active ? 2 : 1.2}
strokeWidth={edge.active ? (hopHasRate(wanFlow) ? 2.8 : 2) : 1.2}
strokeDasharray={edge.active ? "none" : "5 4"}
opacity={edge.active ? 0.7 : 0.4}
filter={isHL ? `url(#glow-wan-${edge.wanIdx})` : undefined}
@@ -1710,6 +1833,14 @@ export default function NetworkMapPage() {
}
return <PingBadge mx={mx} my={my} ping={edge.pingMs} color={pingColor(edge.pingMs)} />
})()}
{showNetflow && hopHasRate(wanFlow) && (
<NetflowRateBadge
mx={flowPos.mx}
my={flowPos.my}
hop={wanFlow}
onOpen={(ev) => openWanJhSpeedDetail(ev, edge)}
/>
)}
</g>
)
})}
@@ -1976,7 +2107,7 @@ export default function NetworkMapPage() {
</div>
<div className="flex items-center justify-between py-2 border-b border-border/50">
<span className="text-xs text-muted-foreground">
{bwMon ? "TX / RX (BT)" : "Скорость (модель)"}
{bwMon ? "TX / RX (BT)" : "Скорость канала (модель)"}
</span>
<span className="text-xs font-mono font-semibold text-sky-400">
{merged.dlMbps != null && merged.ulMbps != null
@@ -1984,10 +2115,23 @@ export default function NetworkMapPage() {
: "—"}
</span>
</div>
<div className="flex items-center justify-between py-2 border-b border-border/50">
<span className="text-xs text-muted-foreground">Поток (NetFlow)</span>
<span className="text-xs font-mono font-semibold text-emerald-400">
{(() => {
const hop = netflowByGreKey.get(selectedEdgeId)
if (!hopHasRate(hop)) return "—"
return hop.bpsFwd > 0 && hop.bpsRev > 0
? formatNetflowDir(hop)
: formatNetflowRate(hop)
})()}
</span>
</div>
<p className="text-[10px] text-muted-foreground pt-2 leading-snug">
{merged.hasSpeedMonitor
? "Ping и/или TX/RX — с последнего прогона speed-пробы; проба сопоставляется с этим GRE по WAN и интерфейсам."
: "«Модель RTT» и «скорость» — демо до появления подходящей speed-пробы в «Мониторинг → скорость»."}
? "Ping и/или TX/RX — с последнего прогона speed-пробы; проба сопоставляется с этим GRE по WAN и интерфейсам. "
: "«Модель RTT» и «скорость канала» — демо до появления подходящей speed-пробы в «Мониторинг → скорость». "}
Поток живой NetFlow за 5 мин (как в «Трафик»: без overlay/mesh), не ёмкость канала.
</p>
</div>
</div>
@@ -2168,12 +2312,26 @@ export default function NetworkMapPage() {
</div>
<div className="flex flex-col gap-1">
{[["ISP", wan.isp], ["Iface", wan.iface], ["IP", wan.ip],
["BW", `${wan.maxDl}${wan.maxUl} Мбит`]].map(([k, v]) => (
["Канал", `${wan.maxDl}${wan.maxUl} Мбит`]].map(([k, v]) => (
<div key={k} className="flex items-center justify-between">
<span className="text-[10px] text-muted-foreground">{k}</span>
<span className="text-[10px] font-mono">{v}</span>
</div>
))}
{(() => {
const hop = netflowByWanKey.get(`${selected.id}\t${wIdx}`)
if (!hopHasRate(hop)) return null
return (
<div className="flex items-center justify-between">
<span className="text-[10px] text-muted-foreground">Поток</span>
<span className="text-[10px] font-mono text-emerald-400">
{hop.bpsFwd > 0 && hop.bpsRev > 0
? formatNetflowDir(hop)
: formatNetflowRate(hop)}
</span>
</div>
)
})()}
</div>
{myEdges.length > 0 && (
<div className="mt-2 pt-2 border-t border-border/40">
@@ -2230,9 +2388,11 @@ export default function NetworkMapPage() {
fromServer && toServer ? speedProbeByTunnelId.get(tunnelPanelKey) : undefined
const merged = mergeGreMetricsWithSpeedProbe(spGre, baseProbe)
const pc = pingColor(merged.pingMs)
const greFlow = netflowByGreKey.get(tunnelPanelKey)
const showMetrics =
merged.pingMs != null ||
(merged.dlMbps != null && merged.ulMbps != null)
(merged.dlMbps != null && merged.ulMbps != null) ||
hopHasRate(greFlow)
return (
<div key={tunnelPanelKey} className="rounded-md border border-border/60 px-3 py-2 bg-muted/20">
<div className="flex items-center justify-between mb-1">
@@ -2272,6 +2432,11 @@ export default function NetworkMapPage() {
{merged.dlMbps} {merged.ulMbps}
</span>
)}
{hopHasRate(greFlow) && (
<span className="text-[9px] ml-1.5 text-emerald-400">
{formatNetflowRate(greFlow)}
</span>
)}
</span>
</div>
)}
+43 -3
View File
@@ -32,6 +32,8 @@ import {
import { cn } from "@/lib/utils"
import { requestJson } from "@/shared/api/http-client"
import { downloadSystemDatabaseBackup, restoreSystemDatabaseBackup } from "@/shared/api/system-database"
import { purgeTrafficFlowData } from "@/shared/api/traffic-flow"
import { formatFlowPurgeResult, NetflowPurgeConfirm } from "@/components/traffic/netflow-purge-dialog"
import { toast } from "sonner"
interface ApiKey { id: string; name: string; prefix: string; created: string; last: string; scopes: string[] }
@@ -148,6 +150,8 @@ export default function SettingsPage() {
const [dbRestoreBusy, setDbRestoreBusy] = useState(false)
const [dbRestoreFile, setDbRestoreFile] = useState<File | null>(null)
const [dbRestoreDialogOpen, setDbRestoreDialogOpen] = useState(false)
const [dbPurgeOpen, setDbPurgeOpen] = useState(false)
const [dbPurgeBusy, setDbPurgeBusy] = useState(false)
// notifications
const [notifEmail, setNotifEmail] = useState(true)
@@ -266,6 +270,20 @@ export default function SettingsPage() {
}
}, [backendUrl, dbRestoreFile, systemDbAvailable])
const handleNetflowPurgeConfirm = useCallback(async () => {
if (!systemDbAvailable) return
setDbPurgeBusy(true)
try {
const result = await purgeTrafficFlowData(backendUrl)
toast.success(formatFlowPurgeResult(result))
setDbPurgeOpen(false)
} catch (e) {
toast.error(e instanceof Error ? e.message : "Не удалось сбросить NetFlow")
} finally {
setDbPurgeBusy(false)
}
}, [backendUrl, systemDbAvailable])
const renderContent = () => {
const ra = DEFAULT_ROUTE_AI_OPTIMIZER_SETTINGS
@@ -417,7 +435,7 @@ export default function SettingsPage() {
<OpsPanel
title="База данных приложения"
description="Резервная копия SQLite бекенда: серверы, мониторинг, оповещения, EvoBGP. На время операции планировщик сбора данных приостанавливается."
description="Резервная копия SQLite бекенда и сброс таблиц NetFlow. На время операции планировщик и коллектор IPFIX приостанавливаются. Preview: https://reui.io/preview/base/settings-16"
contentClassName="divide-y px-5"
>
{!systemDbAvailable && (
@@ -434,7 +452,7 @@ export default function SettingsPage() {
variant="outline"
size="sm"
className="h-8"
disabled={!systemDbAvailable || dbBackupBusy || dbRestoreBusy}
disabled={!systemDbAvailable || dbBackupBusy || dbRestoreBusy || dbPurgeBusy}
onClick={() => { void handleSystemDatabaseBackup() }}
>
{dbBackupBusy ? <LoaderCircleIcon className="size-4 animate-spin" /> : <DownloadIcon className="size-4" />}
@@ -450,7 +468,7 @@ export default function SettingsPage() {
variant="outline"
size="sm"
className="h-8"
disabled={!systemDbAvailable || dbBackupBusy || dbRestoreBusy}
disabled={!systemDbAvailable || dbBackupBusy || dbRestoreBusy || dbPurgeBusy}
onClick={() => setDbRestoreDialogOpen(true)}
>
<UploadIcon className="size-4" />
@@ -461,6 +479,21 @@ export default function SettingsPage() {
)}
</div>
</SettingRow>
<SettingRow
label="Сбросить данные NetFlow"
description="Удалит сессии и агрегаты из SQLite, затем VACUUM. Ключи WG и пиры не трогает"
>
<Button
variant="destructive"
size="sm"
className="h-8"
disabled={!systemDbAvailable || dbBackupBusy || dbRestoreBusy || dbPurgeBusy}
onClick={() => setDbPurgeOpen(true)}
>
{dbPurgeBusy ? <LoaderCircleIcon className="size-4 animate-spin" /> : <TrashIcon className="size-4" />}
{dbPurgeBusy ? "Сброс…" : "Сбросить"}
</Button>
</SettingRow>
</OpsPanel>
<OpsPanel
@@ -893,6 +926,13 @@ export default function SettingsPage() {
/>
)}
<NetflowPurgeConfirm
open={dbPurgeOpen}
busy={dbPurgeBusy}
onConfirm={() => { void handleNetflowPurgeConfirm() }}
onCancel={() => { if (!dbPurgeBusy) setDbPurgeOpen(false) }}
/>
<FileImportDialog
open={dbRestoreDialogOpen}
onOpenChange={setDbRestoreDialogOpen}
+87 -18
View File
@@ -19,11 +19,11 @@ import { useDataSource } from "@/lib/data-source"
import { useTrafficLive } from "@/hooks/use-traffic-live"
import { useFlowLive } from "@/hooks/use-flow-live"
import { requestJson } from "@/shared/api/http-client"
import { getFlowAnalytics, getFlowClients, getFlowExporters, getTrafficFlows } from "@/shared/api/traffic-flow"
import { getFlowAnalytics, getFlowClients, getFlowExporters, getFlowMonthly, getTrafficFlows } from "@/shared/api/traffic-flow"
import { listServers } from "@/shared/api/servers"
import { FlowOverlaySheet } from "@/components/traffic/flow-overlay-sheet"
import { FlowAnalyticsDetail, FlowEntityCardView } from "@/components/traffic/flow-analytics-panel"
import type { FlowAnalyticsDto, FlowEntityCard, FlowStatsDto } from "@mmapp/contracts/traffic-flow"
import type { FlowAnalyticsDto, FlowEntityCard, FlowMonthlyDto, FlowStatsDto } from "@mmapp/contracts/traffic-flow"
import type { ServerRead } from "@mmapp/contracts/servers"
import { Badge } from "@/components/reui/badge"
import {
@@ -63,18 +63,57 @@ function flowIngestLine(stats: FlowStatsDto | null): string | null {
return `Коллектор: ${listener} · пакеты ${stats.packetsReceived ?? 0} · последний ${last}${exporter}${err}`
}
function flowEmptyHint(stats: FlowStatsDto | null): string | undefined {
function flowEmptyHint(stats: FlowStatsDto | null, collectorAlive?: boolean): string | undefined {
if (!stats) return undefined
if (stats.lastError) return stats.lastError
if (stats.packetsReceived) {
return `IPFIX приходит (${stats.lastExporterIp ?? "экспортёр"}), но сессии ещё не записаны.`
}
if (stats.listenerBound === false) {
if (stats.listenerBound === false && !collectorAlive) {
return "Коллектор UDP не слушает. Подключите JH ещё раз — ingest включится автоматически."
}
if (stats.listenerBound || collectorAlive) {
return "Коллектор жив, IPFIX ещё не доходит. На jump-host у target Src должен быть 0.0.0.0 (авто)."
}
return "IPFIX ещё не доходит до коллектора. На jump-host у target Src должен быть 0.0.0.0 (авто). На хосте MM проверьте bind 10.255.254.1:4739 после wg-flow."
}
function monthlyToAnalytics(m: FlowMonthlyDto): FlowAnalyticsDto {
const emptySeries = Array(60).fill(0) as number[]
return {
bpsNow: 0,
bytes: m.bytes,
packets: 0,
conversations: 0,
conversationsRaw: 0,
uniqueSrc: 0,
uniqueDst: 0,
topProto: "—",
topCategory: "—",
rxSeries: emptySeries,
txSeries: emptySeries,
applications: [],
protocols: [],
sources: [],
destinations: [],
interfaces: [],
asns: m.asns,
countries: m.countries,
categories: [],
services: m.services,
mapEdges: [],
conversationsList: [],
paths: [],
ifaces: [],
live: false,
degraded: false,
bytesPayload: m.bytes,
bytesOverlay: 0,
bytesMesh: 0,
bytesWire: 0,
}
}
// ─── data model ───────────────────────────────────────────────────────────────
interface BoundIfaceTraffic {
@@ -298,9 +337,9 @@ const userTraffic: UserTraffic[] = INIT_USERS.map((u) =>
/** Ключи совпадают с `rangeToMinutes` в API (`/api/traffic/...`). */
const TRAFFIC_RANGE_KEYS = ["5m", "15m", "1h", "4h", "24h"] as const
type Range = (typeof TRAFFIC_RANGE_KEYS)[number]
type Range = (typeof TRAFFIC_RANGE_KEYS)[number] | "30d"
const TRAFFIC_RANGE_LABELS: Record<Range, string> = {
const TRAFFIC_RANGE_LABELS: Record<(typeof TRAFFIC_RANGE_KEYS)[number], string> = {
"5m": "5м",
"15m": "15м",
"1h": "1ч",
@@ -772,6 +811,8 @@ export default function TrafficPage() {
const [flowAnalytics, setFlowAnalytics] = useState<FlowAnalyticsDto | null>(null)
const [flowIface, setFlowIface] = useState("__all__")
const [flowDedup, setFlowDedup] = useState(true)
const [flowExcludeMesh, setFlowExcludeMesh] = useState(true)
const [flowExcludeOverlay, setFlowExcludeOverlay] = useState(true)
const [overlayOpen, setOverlayOpen] = useState(false)
const [catalogServers, setCatalogServers] = useState<ServerRead[]>([])
const effectiveMode: GroupMode = groupMode
@@ -781,7 +822,7 @@ export default function TrafficPage() {
serverId: selectedId,
iface: selectedIface,
})
const flowLiveEnabled = isLive && effectiveMode === "flows" && Boolean(selectedId)
const flowLiveEnabled = isLive && effectiveMode === "flows" && Boolean(selectedId) && range === "5m"
const { sample: flowLiveSample, error: flowLiveError } = useFlowLive({
enabled: flowLiveEnabled,
backendUrl,
@@ -790,6 +831,8 @@ export default function TrafficPage() {
userId: flowScope === "users" ? selectedId : undefined,
iface: flowIface,
dedup: flowDedup,
excludeMesh: flowExcludeMesh,
excludeOverlay: flowExcludeOverlay,
})
const toLiveServer = (s: LiveTrafficServer): ServerTraffic => {
@@ -835,7 +878,7 @@ export default function TrafficPage() {
setLiveBusy(true)
setLiveError(null)
try {
const q = encodeURIComponent(targetRange)
const q = encodeURIComponent(targetRange === "30d" ? "24h" : targetRange)
const [srvRes, usersRes, ifacesRes] = await Promise.all([
apiFetch<{ servers: LiveTrafficServer[] }>(`/api/traffic/servers?range=${q}`),
apiFetch<{ users: UserTraffic[] }>(`/api/traffic/users?range=${q}`),
@@ -902,8 +945,6 @@ export default function TrafficPage() {
useEffect(() => {
if (!isLive || effectiveMode !== "flows") return
void loadFlows()
const t = window.setInterval(() => { void loadFlows() }, 5000)
return () => window.clearInterval(t)
}, [isLive, effectiveMode, loadFlows])
useEffect(() => {
@@ -911,14 +952,28 @@ export default function TrafficPage() {
setFlowAnalytics(null)
return
}
if (range === "5m") {
setFlowAnalytics(null)
return
}
if (range === "30d") {
const month = new Date().toISOString().slice(0, 7)
void getFlowMonthly(backendUrl, {
month,
serverId: flowScope === "servers" ? selectedId : undefined,
}).then((m) => setFlowAnalytics(monthlyToAnalytics(m))).catch(() => setFlowAnalytics(null))
return
}
void getFlowAnalytics(backendUrl, {
range,
serverId: flowScope === "servers" ? selectedId : undefined,
userId: flowScope === "users" ? selectedId : undefined,
iface: flowIface,
dedup: flowDedup,
excludeMesh: flowExcludeMesh,
excludeOverlay: flowExcludeOverlay,
}).then(setFlowAnalytics).catch(() => setFlowAnalytics(null))
}, [isLive, effectiveMode, selectedId, range, flowScope, flowIface, flowDedup, backendUrl])
}, [isLive, effectiveMode, selectedId, range, flowScope, flowIface, flowDedup, flowExcludeMesh, flowExcludeOverlay, backendUrl])
useEffect(() => {
setFlowIface("__all__")
@@ -969,12 +1024,19 @@ export default function TrafficPage() {
const handleModeChange = (next: GroupMode) => {
setGroupMode(next)
if (next === "servers") setSelectedId(activeServerTraffic[0]?.id ?? "srv1")
else if (next === "users") setSelectedId((isLive ? liveUsers : userTraffic)[0]?.id ?? "u1")
else if (next === "ifaces") setSelectedId((isLive ? liveBoundIfaces : boundIfaces)[0]?.id ?? "")
else if (next === "flows") {
if (next === "servers") {
setSelectedId(activeServerTraffic[0]?.id ?? "srv1")
if (range === "30d") setRange("1h")
} else if (next === "users") {
setSelectedId((isLive ? liveUsers : userTraffic)[0]?.id ?? "u1")
if (range === "30d") setRange("1h")
} else if (next === "ifaces") {
setSelectedId((isLive ? liveBoundIfaces : boundIfaces)[0]?.id ?? "")
if (range === "30d") setRange("1h")
} else if (next === "flows") {
setFlowScope("servers")
setFlowIface("__all__")
setRange("5m")
setSelectedId(flowExporters[0]?.id ?? "")
}
setSortField("rx")
@@ -1064,7 +1126,10 @@ export default function TrafficPage() {
const visibleSortFields = SORT_FIELDS.filter(s => !s.modesOnly || s.modesOnly.includes(effectiveMode))
const ingestLine = flowIngestLine(flowStats)
const flowError = liveError || flowLiveError
const collectorAlive = Boolean(flowStats?.listenerBound || flowStats?.packetsReceived)
const flowError = liveError
|| (flowLiveError && !(collectorAlive && /live HTTP 500/.test(flowLiveError)) ? flowLiveError : null)
|| (displayedFlow?.degraded ? "Коллектор перегружен: упрощённая аналитика" : null)
const flowKpiItems = [
{
@@ -1258,7 +1323,7 @@ export default function TrafficPage() {
))}
{sortedFlowCards.length === 0 ? (
<p className="text-xs text-muted-foreground">
{flowEmptyHint(flowStats) ?? "Нет экспортёров IPFIX. Подключите jump-host."}
{flowEmptyHint(flowStats, collectorAlive) ?? "Нет экспортёров IPFIX. Подключите jump-host."}
</p>
) : null}
</div>
@@ -1274,8 +1339,12 @@ export default function TrafficPage() {
onIface={setFlowIface}
dedup={flowDedup}
onDedup={setFlowDedup}
excludeMesh={flowExcludeMesh}
onExcludeMesh={setFlowExcludeMesh}
excludeOverlay={flowExcludeOverlay}
onExcludeOverlay={setFlowExcludeOverlay}
liveHint={displayedFlow?.live ? "live" : undefined}
emptyHint={flowEmptyHint(flowStats)}
emptyHint={flowEmptyHint(flowStats, collectorAlive)}
/>
</FramePanel>
</Frame>
+1 -1
View File
@@ -14,7 +14,7 @@
"test:auth": "tsx src/lib/permissions.test.ts && tsx src/plugins/auth.smoke.test.ts",
"test:wireguard": "npx tsx src/services/wireguard-config.test.ts",
"test:traffic-rate": "tsx src/services/traffic-rate.test.ts",
"test:traffic-flow": "tsx src/services/traffic-flow-parse.test.ts && tsx src/services/traffic-flow-map-exporter.test.ts && tsx src/services/traffic-flow-ifaces.test.ts && tsx src/services/traffic-flow-dedup.test.ts && tsx src/services/traffic-flow-classify.test.ts && tsx src/services/traffic-flow-ripe.test.ts && tsx src/services/traffic-flow-analytics.test.ts",
"test:traffic-flow": "tsx src/services/traffic-flow-parse.test.ts && tsx src/services/traffic-flow-map-exporter.test.ts && tsx src/services/traffic-flow-ifaces.test.ts && tsx src/services/traffic-flow-dedup.test.ts && tsx src/services/traffic-flow-planes.test.ts && tsx src/services/traffic-flow-classify.test.ts && tsx src/services/traffic-flow-ripe.test.ts && tsx src/services/traffic-flow-brands.test.ts && tsx src/services/traffic-flow-ingest.test.ts && tsx src/services/traffic-flow-analytics.test.ts && tsx src/services/traffic-flow-map-hops.test.ts && tsx src/services/traffic-flow-hardening.test.ts && tsx src/services/traffic-flow-purge.test.ts",
"test:users": "tsx src/modules/users/iface-type.test.ts && tsx src/modules/users/bindings.test.ts"
},
"dependencies": {
+88 -6
View File
@@ -7,11 +7,23 @@ import { drizzle } from "drizzle-orm/better-sqlite3"
import { env } from "../config.js"
import * as schema from "./schema.js"
const sqlite = new Database(env.DATABASE_PATH)
export const SQLITE_BUSY_TIMEOUT_MS = 5000
export function applySqlitePragmas(handle: SqliteHandle): void {
handle.pragma("journal_mode = WAL")
handle.pragma("foreign_keys = ON")
handle.pragma(`busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`)
handle.pragma("synchronous = NORMAL")
}
function openSqlite(): SqliteHandle {
const handle = new Database(env.DATABASE_PATH)
applySqlitePragmas(handle)
return handle
}
let sqlite = openSqlite()
// WAL mode for better concurrent read performance
sqlite.pragma("journal_mode = WAL")
sqlite.pragma("foreign_keys = ON")
sqlite.exec(`
CREATE TABLE IF NOT EXISTS servers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -155,6 +167,10 @@ CREATE TABLE IF NOT EXISTS flow_buckets (
bytes INTEGER NOT NULL DEFAULT 0,
packets INTEGER NOT NULL DEFAULT 0,
in_iface TEXT NOT NULL DEFAULT '',
out_iface TEXT NOT NULL DEFAULT '',
next_hop TEXT NOT NULL DEFAULT '',
flow_start_ms INTEGER NOT NULL DEFAULT 0,
flow_end_ms INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY (server_id) REFERENCES servers(id) ON DELETE CASCADE
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_flow_buckets_unique
@@ -162,6 +178,39 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_flow_buckets_unique
CREATE INDEX IF NOT EXISTS idx_flow_buckets_server_time
ON flow_buckets(server_id, bucket_at);
CREATE TABLE IF NOT EXISTS flow_minute_stats (
server_id INTEGER NOT NULL,
bucket_at TEXT NOT NULL,
bytes INTEGER NOT NULL DEFAULT 0,
packets INTEGER NOT NULL DEFAULT 0,
unique_src INTEGER NOT NULL DEFAULT 0,
unique_dst INTEGER NOT NULL DEFAULT 0,
conversations INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (server_id, bucket_at)
);
CREATE TABLE IF NOT EXISTS flow_minute_dims (
server_id INTEGER NOT NULL,
bucket_at TEXT NOT NULL,
dim TEXT NOT NULL,
key TEXT NOT NULL,
bytes INTEGER NOT NULL DEFAULT 0,
packets INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (server_id, bucket_at, dim, key)
);
CREATE INDEX IF NOT EXISTS idx_flow_minute_dims_time ON flow_minute_dims(bucket_at, dim);
CREATE TABLE IF NOT EXISTS flow_daily_dims (
server_id INTEGER NOT NULL,
day TEXT NOT NULL,
dim TEXT NOT NULL,
key TEXT NOT NULL,
bytes INTEGER NOT NULL DEFAULT 0,
packets INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (server_id, day, dim, key)
);
CREATE INDEX IF NOT EXISTS idx_flow_daily_dims_day ON flow_daily_dims(day, dim);
CREATE TABLE IF NOT EXISTS flow_ip_meta (
prefix TEXT PRIMARY KEY,
asn INTEGER NOT NULL DEFAULT 0,
@@ -844,6 +893,15 @@ WHERE NOT EXISTS (SELECT 1 FROM acme_settings WHERE id = 1);
}
}
{
const flowCols = sqlite.prepare(`PRAGMA table_info('flow_buckets')`).all() as Array<{ name?: string }>
const names = new Set(flowCols.map((c) => c.name))
if (!names.has("out_iface")) sqlite.exec(`ALTER TABLE flow_buckets ADD COLUMN out_iface TEXT NOT NULL DEFAULT ''`)
if (!names.has("next_hop")) sqlite.exec(`ALTER TABLE flow_buckets ADD COLUMN next_hop TEXT NOT NULL DEFAULT ''`)
if (!names.has("flow_start_ms")) sqlite.exec(`ALTER TABLE flow_buckets ADD COLUMN flow_start_ms INTEGER NOT NULL DEFAULT 0`)
if (!names.has("flow_end_ms")) sqlite.exec(`ALTER TABLE flow_buckets ADD COLUMN flow_end_ms INTEGER NOT NULL DEFAULT 0`)
}
const certIssueJobCols = sqlite.prepare(`PRAGMA table_info('certificate_issue_jobs')`).all() as Array<{ name?: string }>
if (!certIssueJobCols.some((c) => c.name === "source")) {
sqlite.exec(`ALTER TABLE certificate_issue_jobs ADD COLUMN source TEXT NOT NULL DEFAULT 'manual'`)
@@ -902,7 +960,31 @@ if (backupEntryCount.c === 0) {
}
}
export const db = drizzle(sqlite, { schema })
export let db = drizzle(sqlite, { schema })
/** Прямой доступ к better-sqlite3 для сложных read-only запросов (напр. /api/alerts). */
export const sqliteDatabase: SqliteHandle = sqlite
export let sqliteDatabase: SqliteHandle = sqlite
let sqliteExclusiveOp = false
export function beginSqliteExclusiveOp(): void {
if (sqliteExclusiveOp) {
throw new Error("Операция с базой данных уже выполняется")
}
sqliteExclusiveOp = true
}
export function endSqliteExclusiveOp(): void {
sqliteExclusiveOp = false
}
export function reopenSqlite(): void {
try {
sqlite.close()
} catch {
/* already closed */
}
sqlite = openSqlite()
sqliteDatabase = sqlite
db = drizzle(sqlite, { schema })
}
+38
View File
@@ -182,6 +182,40 @@ export const trafficFlowSettings = sqliteTable("traffic_flow_settings", {
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
})
export const flowMinuteStats = sqliteTable("flow_minute_stats", {
serverId: integer("server_id").notNull(),
bucketAt: text("bucket_at").notNull(),
bytes: integer("bytes").notNull().default(0),
packets: integer("packets").notNull().default(0),
uniqueSrc: integer("unique_src").notNull().default(0),
uniqueDst: integer("unique_dst").notNull().default(0),
conversations: integer("conversations").notNull().default(0),
}, (t) => [
uniqueIndex("idx_flow_minute_stats_pk").on(t.serverId, t.bucketAt),
])
export const flowMinuteDims = sqliteTable("flow_minute_dims", {
serverId: integer("server_id").notNull(),
bucketAt: text("bucket_at").notNull(),
dim: text("dim").notNull(),
key: text("key").notNull(),
bytes: integer("bytes").notNull().default(0),
packets: integer("packets").notNull().default(0),
}, (t) => [
uniqueIndex("idx_flow_minute_dims_pk").on(t.serverId, t.bucketAt, t.dim, t.key),
])
export const flowDailyDims = sqliteTable("flow_daily_dims", {
serverId: integer("server_id").notNull(),
day: text("day").notNull(),
dim: text("dim").notNull(),
key: text("key").notNull(),
bytes: integer("bytes").notNull().default(0),
packets: integer("packets").notNull().default(0),
}, (t) => [
uniqueIndex("idx_flow_daily_dims_pk").on(t.serverId, t.day, t.dim, t.key),
])
export const flowBuckets = sqliteTable("flow_buckets", {
id: integer("id").primaryKey({ autoIncrement: true }),
serverId: integer("server_id")
@@ -196,6 +230,10 @@ export const flowBuckets = sqliteTable("flow_buckets", {
bytes: integer("bytes").notNull().default(0),
packets: integer("packets").notNull().default(0),
inIface: text("in_iface").notNull().default(""),
outIface: text("out_iface").notNull().default(""),
nextHop: text("next_hop").notNull().default(""),
flowStartMs: integer("flow_start_ms").notNull().default(0),
flowEndMs: integer("flow_end_ms").notNull().default(0),
}, (t) => [
uniqueIndex("idx_flow_buckets_unique").on(
t.serverId, t.bucketAt, t.src, t.dst, t.proto, t.srcPort, t.dstPort, t.inIface,
+44 -3
View File
@@ -1,6 +1,7 @@
import Fastify, { type FastifyInstance } from "fastify"
import Fastify, { type FastifyError, type FastifyInstance } from "fastify"
import cors from "@fastify/cors"
import { serializerCompiler, validatorCompiler } from "@fastify/type-provider-zod"
import { monitorEventLoopDelay } from "node:perf_hooks"
import { env } from "./config.js"
import authPlugin, { requireAuth } from "./plugins/auth.js"
import serversRoutes from "./routes/servers.js"
@@ -28,7 +29,10 @@ import wireguardRoutes from "./routes/wireguard.js"
import firewallRoutes from "./routes/firewall.js"
import usersRoutes from "./routes/users.js"
import { refreshScheduler, stopScheduler } from "./services/scheduler.js"
import { startTrafficFlowListener, stopTrafficFlowListener } from "./services/traffic-flow-ingest.js"
import { getFlowWorkerHealth, startTrafficFlowListener, stopTrafficFlowListener } from "./services/traffic-flow-ingest.js"
const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 })
eventLoopDelay.enable()
export async function buildApp(opts?: {
logger?: boolean
@@ -37,7 +41,7 @@ export async function buildApp(opts?: {
const usePrettyLogger =
opts?.logger !== false && process.env.NODE_ENV !== "production"
const app = Fastify({
bodyLimit: 512 * 1024 * 1024,
bodyLimit: 2 * 1024 * 1024,
requestTimeout: 10 * 60 * 1000,
logger:
opts?.logger === false
@@ -59,6 +63,18 @@ export async function buildApp(opts?: {
app.setValidatorCompiler(validatorCompiler)
app.setSerializerCompiler(serializerCompiler)
app.setErrorHandler((error: FastifyError, request, reply) => {
const status = typeof error.statusCode === "number" && error.statusCode >= 400
? error.statusCode
: 500
if (status >= 500) {
request.log.error(error)
return reply.status(status).send({ error: "Внутренняя ошибка сервера" })
}
const message = error instanceof Error ? error.message : "Ошибка запроса"
return reply.status(status).send({ error: message })
})
await app.register(cors, {
origin: env.CORS_ORIGIN,
methods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
@@ -70,6 +86,8 @@ export async function buildApp(opts?: {
status: "ok",
timestamp: new Date().toISOString(),
version: process.env.APP_VERSION ?? "dev",
eventLoopDelayMs: Math.round(eventLoopDelay.mean / 1e6),
flowWorker: getFlowWorkerHealth(),
}))
app.get("/api/auth/config", async () => ({
@@ -132,6 +150,29 @@ const isMain =
if (isMain) {
try {
const app = await buildApp()
let shuttingDown = false
const shutdown = async (code: number) => {
if (shuttingDown) return
shuttingDown = true
try {
stopTrafficFlowListener()
await app.close()
} catch (err) {
console.error(err)
} finally {
process.exit(code)
}
}
process.on("SIGTERM", () => { void shutdown(0) })
process.on("SIGINT", () => { void shutdown(0) })
process.on("uncaughtException", (err) => {
console.error(err)
void shutdown(1)
})
process.on("unhandledRejection", (reason) => {
console.error(reason)
void shutdown(1)
})
await app.listen({ port: env.PORT, host: "0.0.0.0" })
console.log(
`\n🚀 MikroTik Manager Backend running at http://localhost:${env.PORT}`,
+4
View File
@@ -21,6 +21,10 @@ assert.equal(
permissionForRequest("GET", "/api/traffic/servers/1/live"),
"mm:traffic:read",
)
assert.equal(
permissionForRequest("POST", "/api/traffic/flow/purge"),
"mm:traffic:write",
)
assert.equal(
permissionForRequest("GET", "/api/unknown-thing"),
"mm:dashboard:read",
+89 -2
View File
@@ -13,18 +13,39 @@ import {
} from "../services/traffic-flow-settings.js"
import {
getFlowListenerState,
purgeTrafficFlowStore,
startTrafficFlowListener,
listFlowTalkers,
} from "../services/traffic-flow-ingest.js"
import {
buildFlowAnalytics,
getFlowMonthly,
listFlowClients,
listFlowExporters,
safeBuildLiveFlowSample,
} from "../services/traffic-flow-analytics.js"
import { buildFlowMapHops } from "../services/traffic-flow-map-hops.js"
import { applyFlowOverlay } from "../services/traffic-flow-overlay.js"
import { listTrafficFlowHostFiles } from "../services/traffic-flow-host-files.js"
import { appendEvent } from "../modules/events/service/events-service.js"
const LIVE_TICK_MS = 2000
export const MAX_FLOW_LIVE_SUBSCRIBERS = 4
let liveSubscribers = 0
export function tryAcquireFlowLiveSlot(): boolean {
if (liveSubscribers >= MAX_FLOW_LIVE_SUBSCRIBERS) return false
liveSubscribers += 1
return true
}
export function releaseFlowLiveSlot(): void {
liveSubscribers = Math.max(0, liveSubscribers - 1)
}
export function resetFlowLiveSlotsForTests(): void {
liveSubscribers = 0
}
function rangeToMinutes(range: string | undefined): number {
switch ((range ?? "5m").toLowerCase()) {
@@ -33,6 +54,7 @@ function rangeToMinutes(range: string | undefined): number {
case "1h": return 60
case "4h": return 240
case "24h": return 1440
case "30d": return 1440
default: return 5
}
}
@@ -50,13 +72,23 @@ function parseDedup(raw: unknown): boolean {
}
function analyticsQuery(req: FastifyRequest) {
const q = req.query as { range?: string; serverId?: string; userId?: string; iface?: string; dedup?: string }
const q = req.query as {
range?: string
serverId?: string
userId?: string
iface?: string
dedup?: string
excludeMesh?: string
excludeOverlay?: string
}
return {
minutes: rangeToMinutes(q.range),
serverId: parseId(q.serverId),
userId: q.userId?.trim() || undefined,
iface: q.iface?.trim() || undefined,
dedup: parseDedup(q.dedup),
excludeMesh: parseDedup(q.excludeMesh),
excludeOverlay: parseDedup(q.excludeOverlay),
}
}
@@ -142,6 +174,35 @@ const trafficFlowRoutes: FastifyPluginAsyncZod = async (app) => {
return reply.send({ files: listTrafficFlowHostFiles() })
})
app.post("/traffic/flow/purge", async (_req, reply) => {
try {
const result = await purgeTrafficFlowStore()
appendEvent({
level: "warning",
eventType: "traffic.flow.purge",
sourceModule: "traffic",
title: "Сброшены данные NetFlow",
message: `Удалены сессии ${result.deleted.buckets}, minute ${result.deleted.minuteStats}, daily ${result.deleted.dailyDims}`,
entityType: "traffic_flow",
entityId: "purge",
payload: {
buckets: result.deleted.buckets,
minuteStats: result.deleted.minuteStats,
minuteDims: result.deleted.minuteDims,
dailyDims: result.deleted.dailyDims,
fileBytesBefore: result.fileBytesBefore,
fileBytesAfter: result.fileBytesAfter,
vacuumed: result.vacuumed,
},
})
return reply.send(result)
} catch (err) {
const message = err instanceof Error ? err.message : String(err)
const status = message.includes("уже выполняется") ? 409 : 500
return reply.status(status).send({ error: message })
}
})
app.post("/traffic/flow/overlay", applyOverlayHandler)
app.post("/traffic/flow-overlay", applyOverlayHandler)
@@ -162,8 +223,32 @@ const trafficFlowRoutes: FastifyPluginAsyncZod = async (app) => {
return reply.send(buildFlowAnalytics(analyticsQuery(req)))
})
app.get("/traffic/flow/map-hops", async (req, reply) => {
return reply.send(buildFlowMapHops(analyticsQuery(req)))
})
app.get("/traffic/flow/monthly", async (req, reply) => {
const q = req.query as { month?: string; serverId?: string }
const now = new Date()
const month = /^\d{4}-\d{2}$/.test(q.month ?? "")
? (q.month as string)
: `${now.getUTCFullYear()}-${String(now.getUTCMonth() + 1).padStart(2, "0")}`
return reply.send(getFlowMonthly(month, parseId(q.serverId)))
})
app.get("/traffic/flow/live", async (req, reply) => {
if (!tryAcquireFlowLiveSlot()) {
return reply.status(429).send({ error: "Слишком много live-подписок" })
}
const query = analyticsQuery(req)
const liveQuery = {
serverId: query.serverId,
userId: query.userId,
iface: query.iface,
dedup: query.dedup,
excludeMesh: query.excludeMesh,
excludeOverlay: query.excludeOverlay,
}
const abort = new AbortController()
const onClose = () => abort.abort()
req.raw.on("close", onClose)
@@ -190,12 +275,14 @@ const trafficFlowRoutes: FastifyPluginAsyncZod = async (app) => {
try {
while (!abort.signal.aborted) {
writeSse(reply.raw, "sample", buildFlowAnalytics(query))
const payload = safeBuildLiveFlowSample(liveQuery)
writeSse(reply.raw, payload.event, payload.data)
await sleep(LIVE_TICK_MS, abort.signal)
}
} catch {
/* abort / disconnect */
} finally {
releaseFlowLiveSlot()
req.raw.off("close", onClose)
try {
reply.raw.end()
+15 -5
View File
@@ -11,6 +11,16 @@ import type {
FirewallFamily, FirewallTable,
} from "../types/server.js"
const MAX_ROS_BODY_BYTES = 8 * 1024 * 1024
function appendRosBody(body: string, chunk: string, req?: http.ClientRequest): string {
if (body.length + chunk.length > MAX_ROS_BODY_BYTES) {
req?.destroy(new Error("RouterOS: ответ больше 8 МиБ"))
return body
}
return body + chunk
}
// ── connection params ─────────────────────────────────────────────────────────
export interface MikrotikConnectParams {
@@ -52,7 +62,7 @@ function rosRequest(
const req = lib.request(options, (res) => {
let body = ""
res.setEncoding("utf8")
res.on("data", (chunk: string) => { body += chunk })
res.on("data", (chunk: string) => { body = appendRosBody(body, chunk, req) })
res.on("end", () => {
clearTimeout(timer)
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
@@ -135,7 +145,7 @@ function rosPost(
req = lib.request(options, (res) => {
let buf = ""
res.setEncoding("utf8")
res.on("data", (chunk: string) => { buf += chunk })
res.on("data", (chunk: string) => { buf = appendRosBody(buf, chunk, req) })
res.on("end", () => {
settle(() => {
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
@@ -189,7 +199,7 @@ function rosPut(
const req = lib.request(options, (res) => {
let buf = ""
res.setEncoding("utf8")
res.on("data", (chunk: string) => { buf += chunk })
res.on("data", (chunk: string) => { buf = appendRosBody(buf, chunk, req) })
res.on("end", () => {
clearTimeout(timer)
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
@@ -239,7 +249,7 @@ function rosDelete(
const req = lib.request(options, (res) => {
let body = ""
res.setEncoding("utf8")
res.on("data", (chunk: string) => { body += chunk })
res.on("data", (chunk: string) => { body = appendRosBody(body, chunk, req) })
res.on("end", () => {
clearTimeout(timer)
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
@@ -289,7 +299,7 @@ function rosPatch(
const req = lib.request(options, (res) => {
let buf = ""
res.setEncoding("utf8")
res.on("data", (chunk: string) => { buf += chunk })
res.on("data", (chunk: string) => { buf = appendRosBody(buf, chunk, req) })
res.on("end", () => {
clearTimeout(timer)
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
+12 -8
View File
@@ -4,16 +4,19 @@ import os from "node:os"
import path from "node:path"
import Database from "better-sqlite3"
import { env } from "../config.js"
import { sqliteDatabase } from "../db/index.js"
import { beginSqliteExclusiveOp, endSqliteExclusiveOp, reopenSqlite, sqliteDatabase } from "../db/index.js"
import { refreshScheduler, stopScheduler } from "./scheduler.js"
import {
reattachFlowSqlite,
startTrafficFlowListener,
stopTrafficFlowListener,
} from "./traffic-flow-ingest.js"
const SQLITE_MAGIC = Buffer.from("SQLite format 3\0")
const MAX_RESTORE_BYTES = 512 * 1024 * 1024
type SqliteHandle = InstanceType<typeof Database>
let operationInFlight = false
function fmtTimestamp(date = new Date()): string {
const pad = (n: number) => String(n).padStart(2, "0")
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}_${pad(date.getHours())}-${pad(date.getMinutes())}-${pad(date.getSeconds())}`
@@ -33,16 +36,15 @@ function assertSqliteFile(buffer: Buffer): void {
}
async function withDatabaseOperation<T>(fn: () => Promise<T> | T): Promise<T> {
if (operationInFlight) {
throw new Error("Операция с базой данных уже выполняется")
}
operationInFlight = true
beginSqliteExclusiveOp()
stopTrafficFlowListener()
stopScheduler()
try {
return await fn()
} finally {
startTrafficFlowListener()
refreshScheduler()
operationInFlight = false
endSqliteExclusiveOp()
}
}
@@ -80,6 +82,8 @@ export async function restoreSystemDatabaseBackup(buffer: Buffer): Promise<void>
await writeFile(tempPath, buffer)
source = new Database(tempPath, { readonly: true, fileMustExist: true })
await source.backup(resolveDatabasePath())
reopenSqlite()
reattachFlowSqlite()
sqliteDatabase.pragma("wal_checkpoint(TRUNCATE)")
} finally {
source?.close()
@@ -4,7 +4,9 @@ import {
ingestParsedFlowsForServerForTests,
resetFlowRingsForTests,
} from "./traffic-flow-ingest.js"
import { buildFlowAnalytics } from "./traffic-flow-analytics.js"
import { buildFlowAnalytics, formatLiveSseFromBuilder, getFlowMonthly, listFlowClients, listFlowExporters } from "./traffic-flow-analytics.js"
import { sqliteDatabase } from "../db/index.js"
import { seedFlowTopologyForTests, type FlowTopology } from "./traffic-flow-topology.js"
import { disableCatalogFetchForTests, resetFlowCatalogForTests, seedFlowCatalogForTests } from "./traffic-flow-classify.js"
import {
disableRipeEnqueueForTests,
@@ -20,8 +22,18 @@ resetRipeCacheForTests()
disableRipeEnqueueForTests()
resetIfaceCacheForTests()
resetFlowRingsForTests()
seedFlowTopologyForTests({
clientIfaces: new Map(),
clientByIface: new Map(),
enNodes: [],
enHosts: new Set(),
jhHosts: new Set(),
wanIfaces: new Map(),
plane: { clientIfaceNames: new Set(), enHosts: new Set(), jhHosts: new Set() },
})
rememberServerIfaces(7, [
{ ".id": "*2", name: "ether1" },
{ ".id": "*B", name: "ether2" },
{ ".id": "*A", name: "wg-flow" },
])
@@ -35,7 +47,7 @@ ingestParsedFlowsForServerForTests(7, [
bytes: 12_000,
packets: 10,
inIface: "2",
outIface: "10",
outIface: "11",
},
{
src: "10.1.1.8",
@@ -81,6 +93,7 @@ resetFlowRingsForTests()
resetIfaceCacheForTests()
rememberServerIfaces(7, [
{ ".id": "*2", name: "ether1" },
{ ".id": "*B", name: "ether2" },
{ ".id": "*A", name: "wg-flow" },
])
ingestParsedFlowsForServerForTests(7, [
@@ -93,7 +106,7 @@ ingestParsedFlowsForServerForTests(7, [
bytes: 12_000,
packets: 10,
inIface: "2",
outIface: "10",
outIface: "11",
},
{
src: "10.1.1.8",
@@ -103,7 +116,7 @@ ingestParsedFlowsForServerForTests(7, [
dstPort: 443,
bytes: 9_000,
packets: 9,
inIface: "10",
inIface: "11",
outIface: "",
},
])
@@ -157,7 +170,9 @@ try {
assert.ok(geo.asns?.some((r) => r.label.includes("AS15169")))
assert.equal(geo.countries?.[0]?.id, "US")
assert.equal(geo.mapEdges?.[0]?.toCountry, "US")
assert.ok(geo.mapEdges?.every((e) => e.toCountry !== "?"))
assert.equal(geo.conversationsList[0]?.dstCountry, "US")
assert.equal(geo.asns?.[0]?.id, "15169")
} finally {
resetFlowRingsForTests()
resetIfaceCacheForTests()
@@ -165,4 +180,242 @@ try {
resetFlowCatalogForTests()
}
resetFlowRingsForTests()
resetIfaceCacheForTests()
disableRipeEnqueueForTests()
resetRipeCacheForTests()
resetFlowCatalogForTests()
seedRipeCacheForTests({
prefix: "1.1.1.0/24",
asn: 13335,
country: "?",
lat: null,
lng: null,
holder: "CLOUDFLARENET, US",
ok: true,
fetchedAt: Date.now(),
})
rememberServerIfaces(7, [{ ".id": "*2", name: "ether1" }])
ingestParsedFlowsForServerForTests(7, [
{
src: "10.1.1.8",
dst: "1.1.1.1",
proto: 6,
srcPort: 51234,
dstPort: 443,
bytes: 5000,
packets: 5,
inIface: "2",
outIface: "",
},
])
try {
const cf = buildFlowAnalytics({ minutes: 5, serverId: 7 })
assert.equal(cf.countries?.[0]?.id, "US")
assert.ok(cf.mapEdges?.every((e) => e.toCountry !== "?"))
assert.equal(cf.services?.[0]?.label, "Cloudflare")
assert.equal(cf.categories?.[0]?.label, "CDN")
assert.equal(cf.conversationsList[0]?.dstCountry, "US")
assert.equal(cf.asns?.[0]?.id, "13335")
} finally {
resetFlowRingsForTests()
resetIfaceCacheForTests()
resetRipeCacheForTests()
resetFlowCatalogForTests()
}
{
ingestParsedFlowsForServerForTests(7, [
{
src: "10.1.1.8",
dst: "8.8.8.8",
proto: 6,
srcPort: 51234,
dstPort: 443,
bytes: 12_000,
packets: 10,
inIface: "2",
outIface: "",
},
])
const degraded = buildFlowAnalytics({ minutes: 5, serverId: 7, skipHeavy: true })
assert.equal(degraded.degraded, true)
assert.equal(degraded.conversationsList.length, 0)
assert.ok((degraded.bytes ?? 0) >= 12_000)
const liveErr = formatLiveSseFromBuilder(() => {
throw new Error("SQLITE_BUSY")
})
assert.equal(liveErr.event, "error")
assert.equal((liveErr.data as { error: string }).error, "SQLITE_BUSY")
const liveOk = formatLiveSseFromBuilder(() => ({ ok: true }))
assert.equal(liveOk.event, "sample")
const exporters = listFlowExporters(5)
const clients = listFlowClients(5)
assert.ok(Array.isArray(exporters.exporters))
assert.ok(Array.isArray(clients.clients))
resetFlowRingsForTests()
}
{
sqliteDatabase.prepare(`DELETE FROM flow_daily_dims WHERE server_id = 7 AND day LIKE '2026-09-%'`).run()
sqliteDatabase.exec(`
INSERT INTO flow_daily_dims (server_id, day, dim, key, bytes, packets)
VALUES
(7, '2026-09-01', 'country', 'US', 1000, 10),
(7, '2026-09-02', 'country', 'US', 500, 5),
(7, '2026-09-01', 'service', 'steam', 800, 8),
(7, '2026-09-01', 'asn', '15169', 900, 9),
(7, '2026-09-01', 'asn', 'other', 100, 1)
`)
const monthly = getFlowMonthly("2026-09", 7)
assert.equal(monthly.bytes, 1500)
assert.equal(monthly.countries[0]?.id, "US")
assert.equal(monthly.countries[0]?.bytes, 1500)
assert.ok(monthly.asns.some((row) => row.id === "other"))
sqliteDatabase.prepare(`DELETE FROM flow_daily_dims WHERE server_id = 7 AND day LIKE '2026-09-%'`).run()
}
{
resetFlowRingsForTests()
resetIfaceCacheForTests()
resetRipeCacheForTests()
resetFlowCatalogForTests()
disableRipeEnqueueForTests()
rememberServerIfaces(7, [
{ ".id": "*2", name: "gre-client" },
{ ".id": "*3", name: "NSK-SERVHOST-RTK" },
{ ".id": "*4", name: "gre-en-nsk" },
])
const topo: FlowTopology = {
clientIfaces: new Map([[7, new Set(["gre-client"])]]),
clientByIface: new Map([["7|gre-client", {
userId: "u1",
login: "alice",
name: "Alice",
serverId: 7,
interfaceName: "gre-client",
}]]),
enNodes: [{ id: 9, name: "NSK-SERVHOST-RTK", hosts: ["198.51.100.1"] }],
enHosts: new Set(["198.51.100.1"]),
jhHosts: new Set(["203.0.113.10"]),
wanIfaces: new Map(),
plane: {
clientIfaceNames: new Set(["gre-client"]),
enHosts: new Set(["198.51.100.1"]),
jhHosts: new Set(["203.0.113.10"]),
},
}
seedFlowTopologyForTests(topo)
seedRipeCacheForTests({
prefix: "173.194.0.0/16",
asn: 15169,
country: "US",
lat: null,
lng: null,
holder: "GOOGLE",
ok: true,
fetchedAt: Date.now(),
})
ingestParsedFlowsForServerForTests(7, [
{
src: "10.100.1.17",
dst: "173.194.160.163",
proto: 6,
srcPort: 51234,
dstPort: 443,
bytes: 12_000,
packets: 10,
inIface: "2",
outIface: "3",
nextHop: "198.51.100.1",
},
{
src: "203.0.113.10",
dst: "198.51.100.1",
proto: 47,
srcPort: 0,
dstPort: 0,
bytes: 5_000_000,
packets: 4000,
inIface: "4",
outIface: "4",
},
{
src: "10.100.1.17",
dst: "10.100.1.18",
proto: 6,
srcPort: 50000,
dstPort: 443,
bytes: 8000,
packets: 8,
inIface: "2",
outIface: "2",
},
])
try {
const def = buildFlowAnalytics({ minutes: 5, serverId: 7 })
assert.equal(def.bytes, 12_000)
assert.equal(def.bytesPayload, 12_000)
assert.equal(def.bytesOverlay, 5_000_000)
assert.equal(def.bytesMesh, 8000)
assert.equal(def.excludeOverlayApplied, true)
assert.equal(def.excludeMeshApplied, true)
assert.ok(!def.conversationsList.some((r) => r.proto === 47))
assert.equal(def.conversationsList[0]?.service, "Google")
assert.equal(def.conversationsList[0]?.category, "Веб")
assert.equal(def.conversationsList[0]?.clientName, "Alice")
assert.equal(def.conversationsList[0]?.enName, "NSK-SERVHOST-RTK")
assert.equal(def.conversationsList[0]?.plane, "payload")
const path = def.paths?.[0]
assert.ok(path)
assert.equal(path.clientName, "Alice")
assert.equal(path.enName, "NSK-SERVHOST-RTK")
assert.equal(path.dst, "173.194.160.163")
const withAll = buildFlowAnalytics({ minutes: 5, serverId: 7, excludeOverlay: false, excludeMesh: false })
assert.equal(withAll.bytes, 12_000 + 5_000_000 + 8000)
assert.ok(withAll.conversationsList.some((r) => r.plane === "overlay"))
assert.ok(withAll.conversationsList.some((r) => r.plane === "client_mesh"))
} finally {
seedFlowTopologyForTests(null)
resetFlowRingsForTests()
resetIfaceCacheForTests()
resetRipeCacheForTests()
resetFlowCatalogForTests()
}
}
{
resetFlowRingsForTests()
resetIfaceCacheForTests()
const sidRow = sqliteDatabase.prepare(`SELECT id FROM servers LIMIT 1`).get() as { id?: number } | undefined
if (sidRow?.id) {
const sid = sidRow.id
rememberServerIfaces(sid, [{ ".id": "*4", name: "gre-en-nsk" }])
ingestParsedFlowsForServerForTests(sid, [{
src: "10.100.1.17",
dst: "8.8.8.8",
proto: 6,
srcPort: 1,
dstPort: 443,
bytes: 100,
packets: 1,
inIface: "4",
outIface: "4",
}])
sqliteDatabase.prepare(`
INSERT INTO traffic_samples (server_id, interface_name, sampled_at, rx_bytes, tx_bytes, rx_bps, tx_bps)
VALUES (?, 'gre-en-nsk', datetime('now'), 9000000, 1000000, 40000000, 2000000)
`).run(sid)
try {
const wire = buildFlowAnalytics({ minutes: 5, serverId: sid })
assert.ok((wire.bpsWire ?? 0) >= 40_000_000)
assert.notEqual(wire.bpsWire, (wire.bytes * 8) / 300)
} finally {
sqliteDatabase.prepare(`DELETE FROM traffic_samples WHERE server_id = ? AND interface_name = 'gre-en-nsk'`).run(sid)
}
}
resetFlowRingsForTests()
resetIfaceCacheForTests()
}
console.log("traffic-flow-analytics.test.ts: ok")
+359 -106
View File
@@ -1,5 +1,5 @@
import { eq } from "drizzle-orm"
import { db } from "../db/index.js"
import { db, sqliteDatabase } from "../db/index.js"
import { appUsers, servers, userInterfaceBindings } from "../db/schema.js"
import type {
FlowAnalyticsDto,
@@ -8,21 +8,38 @@ import type {
FlowEntityCard,
FlowExportersDto,
FlowMapEdge,
FlowMonthlyDto,
FlowPathRow,
FlowTalkerDto,
} from "@mmapp/contracts/traffic-flow"
import { protoName } from "./traffic-flow-parse.js"
import {
getFlowListenerState,
getFlowRuntimeCounters,
getFlowWorkerHealth,
getRingMbps,
listStoredFlowRows,
listFlowRowsForWindow,
type PendingFlowRow,
} from "./traffic-flow-ingest.js"
import { MAX_PENDING, RING_OVERLAY } from "./traffic-flow-engine.js"
import { resolveIfaceName } from "./traffic-flow-ifaces.js"
import { getTrafficFlowSettingsRow, listHostPeers } from "./traffic-flow-settings.js"
import { applicationName, flowRowMatchesFilter } from "./traffic-flow-apps.js"
import { dedupFlowRowsMaxBytes, flowTupleKey } from "./traffic-flow-dedup.js"
import { enqueueRipeMisses, lookupRipeCached } from "./traffic-flow-ripe.js"
import { classifyFlowDst, refreshFlowCatalogInBackground } from "./traffic-flow-classify.js"
import { isIsoCountry } from "./traffic-flow-brands.js"
import { classifyFlowPlane, flowBps, shouldKeepPlane } from "./traffic-flow-planes.js"
import {
enGreIfaceNames,
latestWireBps,
loadFlowTopology,
resolveClient,
resolveEn,
} from "./traffic-flow-topology.js"
export const LIVE_ANALYTICS_MINUTES = 5
const LIVE_DEGRADED_PENDING = Math.floor(MAX_PENDING * 0.8)
export interface FlowAnalyticsQuery {
minutes: number
@@ -31,20 +48,29 @@ export interface FlowAnalyticsQuery {
iface?: string
/** Default true: один 5-tuple = max байт по ifaces. */
dedup?: boolean
/** Default true: скрыть GRE/WG между клиентами JH. */
excludeMesh?: boolean
/** Default true: скрыть overlay GRE/ESP JH↔EN из payload KPI. */
excludeOverlay?: boolean
skipHeavy?: boolean
}
function bpsToMbps(bps: number): number {
return bps / 1_000_000
}
function topN(map: Map<string, { bytes: number; packets: number }>, windowSec: number, n: number): FlowBreakdownRow[] {
function topN(
map: Map<string, { bytes: number; packets: number; label?: string }>,
windowSec: number,
n: number,
): FlowBreakdownRow[] {
const total = [...map.values()].reduce((a, v) => a + v.bytes, 0) || 1
return [...map.entries()]
.sort((a, b) => b[1].bytes - a[1].bytes)
.slice(0, n)
.map(([id, v]) => ({
id,
label: id,
label: v.label || id,
bytes: v.bytes,
packets: v.packets,
bps: (v.bytes * 8) / windowSec,
@@ -52,10 +78,17 @@ function topN(map: Map<string, { bytes: number; packets: number }>, windowSec: n
}))
}
function bump(map: Map<string, { bytes: number; packets: number }>, id: string, bytes: number, packets: number) {
const prev = map.get(id) ?? { bytes: 0, packets: 0 }
function bump(
map: Map<string, { bytes: number; packets: number; label?: string }>,
id: string,
bytes: number,
packets: number,
label?: string,
) {
const prev = map.get(id) ?? { bytes: 0, packets: 0, label }
prev.bytes += bytes
prev.packets += packets
if (label) prev.label = label
map.set(id, prev)
}
@@ -95,13 +128,13 @@ function snapshotStatus(serverId: number): FlowEntityCard["status"] {
return "online"
}
function topLabel(map: Map<string, { bytes: number; packets: number }>, fallback = "—"): string {
function topLabel(map: Map<string, { bytes: number; packets: number; label?: string }>, fallback = "—"): string {
let best = fallback
let bestBytes = 0
for (const [label, v] of map) {
for (const [id, v] of map) {
if (v.bytes > bestBytes) {
bestBytes = v.bytes
best = label
best = v.label || id
}
}
return best
@@ -111,35 +144,59 @@ export function buildFlowAnalytics(q: FlowAnalyticsQuery): FlowAnalyticsDto {
const settings = getTrafficFlowSettingsRow()
const top = Math.min(50, Math.max(10, settings.topN))
const windowSec = Math.max(60, q.minutes * 60)
const sinceIso = new Date(Date.now() - q.minutes * 60_000).toISOString()
const raw = listStoredFlowRows(sinceIso)
const raw = listFlowRowsForWindow(q.minutes)
const allow = q.userId ? userIfaceAllow(q.userId) : null
const serverRows = db.select().from(servers).all()
const nameById = new Map(serverRows.map((s) => [s.id, s.name || s.host]))
const countryById = new Map(serverRows.map((s) => [s.id, (s.country || "").toUpperCase() || "UN"]))
const ifaceFilter = q.iface && q.iface !== "__all__" ? q.iface : ""
const wantDedup = q.dedup !== false && !ifaceFilter
const excludeMesh = q.excludeMesh !== false
const excludeOverlay = q.excludeOverlay !== false
const topo = loadFlowTopology()
refreshFlowCatalogInBackground()
const applications = new Map<string, { bytes: number; packets: number }>()
const protocols = new Map<string, { bytes: number; packets: number }>()
const sources = new Map<string, { bytes: number; packets: number }>()
const destinations = new Map<string, { bytes: number; packets: number }>()
const applications = new Map<string, { bytes: number; packets: number; label?: string }>()
const protocols = new Map<string, { bytes: number; packets: number; label?: string }>()
const sources = new Map<string, { bytes: number; packets: number; label?: string }>()
const destinations = new Map<string, { bytes: number; packets: number; label?: string }>()
const ifacesMap = new Map<string, { bytes: number; packets: number; index: string }>()
const asns = new Map<string, { bytes: number; packets: number }>()
const countries = new Map<string, { bytes: number; packets: number }>()
const categories = new Map<string, { bytes: number; packets: number }>()
const services = new Map<string, { bytes: number; packets: number }>()
const conv = new Map<string, FlowTalkerDto & { rawBytes: number }>()
const asns = new Map<string, { bytes: number; packets: number; label?: string }>()
const countries = new Map<string, { bytes: number; packets: number; label?: string }>()
const categories = new Map<string, { bytes: number; packets: number; label?: string }>()
const services = new Map<string, { bytes: number; packets: number; label?: string }>()
const conv = new Map<string, FlowTalkerDto & { rawBytes: number; flowStartMs: number; flowEndMs: number }>()
const edgeAcc = new Map<string, FlowMapEdge & { catBytes: Map<string, number> }>()
const pathAcc = new Map<string, FlowPathRow>()
const srcs = new Set<string>()
const dsts = new Set<string>()
const matched: PendingFlowRow[] = []
const skipHeavy = Boolean(q.skipHeavy)
let bytesPayload = 0
let bytesOverlay = 0
let bytesMesh = 0
const ifacesForWire = new Set<string>()
for (const r of raw) {
const resolved = resolveIfaceName(r.serverId, r.inIface)
const outResolved = resolveIfaceName(r.serverId, r.outIface)
if (!flowRowMatchesFilter(r, resolved.name, q, allow)) continue
ifacesForWire.add(resolved.name)
if (outResolved.name && outResolved.name !== "—") ifacesForWire.add(outResolved.name)
const plane = classifyFlowPlane({
src: r.src,
dst: r.dst,
proto: r.proto,
srcPort: r.srcPort,
dstPort: r.dstPort,
inIface: resolved.name,
outIface: outResolved.name,
}, topo.plane)
if (plane === "payload") bytesPayload += r.bytes
else if (plane === "overlay") bytesOverlay += r.bytes
else if (plane === "client_mesh") bytesMesh += r.bytes
if (!shouldKeepPlane(plane, { excludeMesh, excludeOverlay })) continue
matched.push(r)
const ifaceKey = resolved.name
@@ -170,77 +227,138 @@ export function buildFlowAnalytics(q: FlowAnalyticsQuery): FlowAnalyticsDto {
bump(categories, classified.category, r.bytes, r.packets)
bump(services, classified.service, r.bytes, r.packets)
if (ripe?.ok && ripe.asn) {
const asnId = String(ripe.asn)
const asnLabel = ripe.holder ? `AS${ripe.asn} ${ripe.holder}` : `AS${ripe.asn}`
bump(asns, asnLabel, r.bytes, r.packets)
bump(asns, asnId, r.bytes, r.packets, asnLabel)
}
if (ripe?.ok && ripe.country && ripe.country !== "") {
bump(countries, ripe.country, r.bytes, r.packets)
const dstCountry = ripe?.ok && isIsoCountry(ripe.country) ? ripe.country : ""
if (dstCountry) {
bump(countries, dstCountry, r.bytes, r.packets)
}
const ckey = wantDedup
? flowTupleKey(r)
: `${flowTupleKey(r)}|${r.inIface}`
const prev = conv.get(ckey)
if (prev) {
prev.rawBytes += r.bytes
prev.bytes += r.bytes
prev.packets += r.packets
} else {
conv.set(ckey, {
serverId: String(r.serverId),
serverName: nameById.get(r.serverId) ?? String(r.serverId),
if (!skipHeavy) {
const outResolved = resolveIfaceName(r.serverId, r.outIface)
const plane = classifyFlowPlane({
src: r.src,
dst: r.dst,
proto: r.proto,
protoName: protoName(r.proto),
srcPort: r.srcPort,
dstPort: r.dstPort,
bytes: r.bytes,
packets: r.packets,
bps: 0,
inIface: resolved.name,
inIfaceIndex: resolved.index,
application: app,
category: classified.category,
service: classified.service,
dstCountry: ripe?.country && ripe.country !== "—" ? ripe.country : undefined,
dstAsn: ripe?.asn || undefined,
rawBytes: r.bytes,
})
}
const toCountry = ripe?.ok && ripe.country && ripe.country !== "—" ? ripe.country : ""
if (toCountry) {
const fromCountry = countryById.get(r.serverId) || "UN"
const ekey = `${r.serverId}|${toCountry}`
let edge = edgeAcc.get(ekey)
if (!edge) {
edge = {
fromId: String(r.serverId),
fromLabel: nameById.get(r.serverId) ?? String(r.serverId),
fromCountry,
toCountry,
toAsn: ripe?.asn ?? 0,
category: classified.category,
bytes: 0,
outIface: outResolved.name,
}, topo.plane)
const client = resolveClient(topo, r.serverId, resolved.name)
const en = resolveEn(topo, r.nextHop, outResolved.name)
const ckey = wantDedup
? flowTupleKey(r)
: `${flowTupleKey(r)}|${r.inIface}`
const prev = conv.get(ckey)
if (prev) {
prev.rawBytes += r.bytes
prev.bytes += r.bytes
prev.packets += r.packets
if (r.flowStartMs && (!prev.flowStartMs || r.flowStartMs < prev.flowStartMs)) prev.flowStartMs = r.flowStartMs
if (r.flowEndMs > prev.flowEndMs) prev.flowEndMs = r.flowEndMs
} else {
conv.set(ckey, {
serverId: String(r.serverId),
serverName: nameById.get(r.serverId) ?? String(r.serverId),
src: r.src,
dst: r.dst,
proto: r.proto,
protoName: protoName(r.proto),
srcPort: r.srcPort,
dstPort: r.dstPort,
bytes: r.bytes,
packets: r.packets,
bps: 0,
catBytes: new Map(),
inIface: resolved.name,
inIfaceIndex: resolved.index,
outIface: outResolved.name !== "—" ? outResolved.name : undefined,
nextHop: r.nextHop || undefined,
application: app,
category: classified.category,
service: classified.service,
dstCountry: dstCountry || undefined,
dstAsn: ripe?.asn || undefined,
clientId: client?.userId,
clientName: client?.name,
enId: en ? String(en.id) : undefined,
enName: en?.name,
plane,
rawBytes: r.bytes,
flowStartMs: r.flowStartMs ?? 0,
flowEndMs: r.flowEndMs ?? 0,
})
}
const pathKey = `${client?.userId || "unknown"}|${r.serverId}|${en?.id || ""}|${r.dst}|${resolved.name}`
const pathPrev = pathAcc.get(pathKey)
if (pathPrev) {
pathPrev.bytes += r.bytes
pathPrev.packets += r.packets
} else {
pathAcc.set(pathKey, {
id: pathKey,
clientId: client?.userId || "unknown",
clientName: client?.name || "Неизвестный клиент",
ifaces: client ? [...(topo.clientIfaces.get(r.serverId) ?? [resolved.name])].join(", ") : resolved.name,
serverId: String(r.serverId),
serverName: nameById.get(r.serverId) ?? String(r.serverId),
inIface: resolved.name,
outIface: outResolved.name !== "—" ? outResolved.name : "",
enId: en ? String(en.id) : "",
enName: en?.name || "",
dst: r.dst,
service: classified.service,
category: classified.category,
plane,
bytes: r.bytes,
packets: r.packets,
bps: 0,
})
}
const toCountry = dstCountry
if (toCountry) {
const fromCountry = countryById.get(r.serverId) || "UN"
const ekey = `${r.serverId}|${toCountry}`
let edge = edgeAcc.get(ekey)
if (!edge) {
edge = {
fromId: String(r.serverId),
fromLabel: nameById.get(r.serverId) ?? String(r.serverId),
fromCountry,
toCountry,
toAsn: ripe?.asn ?? 0,
category: classified.category,
bytes: 0,
bps: 0,
catBytes: new Map(),
}
edgeAcc.set(ekey, edge)
}
edgeAcc.set(ekey, edge)
edge.bytes += r.bytes
if (ripe?.asn) edge.toAsn = ripe.asn
edge.catBytes.set(classified.category, (edge.catBytes.get(classified.category) ?? 0) + r.bytes)
}
edge.bytes += r.bytes
if (ripe?.asn) edge.toAsn = ripe.asn
edge.catBytes.set(classified.category, (edge.catBytes.get(classified.category) ?? 0) + r.bytes)
}
}
enqueueRipeMisses(dsts)
const conversationsList = [...conv.values()]
.map((t) => ({ ...t, bps: (t.rawBytes * 8) / windowSec }))
.map((t) => {
const { rawBytes, flowStartMs, flowEndMs, ...rest } = t
return { ...rest, bps: flowBps(rawBytes, flowStartMs, flowEndMs, windowSec) }
})
.sort((a, b) => b.bytes - a.bytes)
.slice(0, top)
const paths: FlowPathRow[] = [...pathAcc.values()]
.map((p) => ({ ...p, bps: (p.bytes * 8) / windowSec }))
.sort((a, b) => b.bytes - a.bytes)
.slice(0, top)
.map(({ rawBytes: _raw, ...rest }) => rest)
const topProto = topLabel(protocols)
const topCategory = topLabel(categories)
@@ -288,6 +406,12 @@ export function buildFlowAnalytics(q: FlowAnalyticsQuery): FlowAnalyticsDto {
.sort((a, b) => b.bytes - a.bytes)
.slice(0, top)
const overlayRing = ringServer
? getRingMbps(ringServer, RING_OVERLAY)
: { rxNow: 0, txNow: 0 }
const greNames = ringServer ? enGreIfaceNames(topo, ringServer, [...ifacesForWire]) : []
const wire = ringServer ? latestWireBps(ringServer, greNames) : { bps: 0, bytes: 0 }
return {
bpsNow: (ring.rxNow + ring.txNow) * 1_000_000 || (totalBytes * 8) / windowSec,
bytes: totalBytes,
@@ -318,53 +442,68 @@ export function buildFlowAnalytics(q: FlowAnalyticsQuery): FlowAnalyticsDto {
services: topN(services, windowSec, top),
mapEdges,
conversationsList,
paths,
ifaces: ifaceRows,
live: listener.bound,
dedupApplied: wantDedup,
degraded: skipHeavy,
bytesPayload,
bytesOverlay,
bytesMesh,
bytesWire: wire.bytes,
bpsOverlay: (overlayRing.rxNow + overlayRing.txNow) * 1_000_000 || (bytesOverlay * 8) / windowSec,
bpsWire: wire.bps,
excludeMeshApplied: excludeMesh,
excludeOverlayApplied: excludeOverlay,
}
}
function cardFromServer(
s: typeof servers.$inferSelect,
minutes: number,
): FlowEntityCard {
const analytics = buildFlowAnalytics({ minutes, serverId: s.id })
const ring = getRingMbps(s.id, "__all__")
return {
id: String(s.id),
name: s.name || s.host,
subtitle: s.host,
site: s.site || "—",
country: s.country || "UN",
status: snapshotStatus(s.id),
rxNow: ring.rxNow || bpsToMbps(analytics.bpsNow),
txNow: ring.txNow,
sessions: analytics.conversations,
rxSeries: ring.rx.some((v) => v > 0) ? ring.rx : analytics.rxSeries,
txSeries: ring.tx,
bytes: analytics.bytes,
function summarizeByServer(rows: PendingFlowRow[]) {
const bytes = new Map<number, number>()
const sessions = new Map<number, number>()
for (const r of rows) {
bytes.set(r.serverId, (bytes.get(r.serverId) ?? 0) + r.bytes)
sessions.set(r.serverId, (sessions.get(r.serverId) ?? 0) + 1)
}
return { bytes, sessions }
}
export function listFlowExporters(minutes: number): FlowExportersDto {
const settings = getTrafficFlowSettingsRow()
const sinceIso = new Date(Date.now() - minutes * 60_000).toISOString()
const rows = listStoredFlowRows(sinceIso)
const ids = new Set<number>()
for (const r of rows) ids.add(r.serverId)
const runtime = getFlowRuntimeCounters()
const rows = listFlowRowsForWindow(minutes)
const { bytes, sessions } = summarizeByServer(rows)
const ids = new Set<number>([...bytes.keys()])
for (const p of listHostPeers()) ids.add(p.serverId)
const serverRows = db.select().from(servers).all()
const emptySeries = Array(60).fill(0) as number[]
const exporters = serverRows
.filter((s) => ids.has(s.id))
.map((s) => cardFromServer(s, minutes))
.map((s) => {
const ring = getRingMbps(s.id, "__all__")
const total = bytes.get(s.id) ?? 0
return {
id: String(s.id),
name: s.name || s.host,
subtitle: s.host,
site: s.site || "—",
country: s.country || "UN",
status: snapshotStatus(s.id),
rxNow: ring.rxNow || (total * 8) / Math.max(60, minutes * 60) / 1_000_000,
txNow: ring.txNow,
sessions: sessions.get(s.id) ?? 0,
rxSeries: ring.rx.some((v) => v > 0) ? ring.rx : emptySeries,
txSeries: ring.tx,
bytes: total,
} satisfies FlowEntityCard
})
.sort((a, b) => b.rxNow - a.rxNow)
const listener = getFlowListenerState()
return {
exporters,
lastExporterIp: settings.lastExporterIp ?? null,
lastError: settings.lastError || null,
packetsReceived: settings.packetsReceived,
lastDatagramAt: settings.lastDatagramAt ?? null,
lastExporterIp: runtime.lastExporterIp,
lastError: runtime.lastError,
packetsReceived: runtime.packetsReceived,
lastDatagramAt: runtime.lastDatagramAt,
listenerBound: listener.bound,
listenerAddress: listener.address,
}
@@ -379,13 +518,31 @@ export function listFlowClients(minutes: number): FlowClientsDto {
list.push(b)
byUser.set(b.userId, list)
}
const rows = listFlowRowsForWindow(minutes)
const emptySeries = Array(60).fill(0) as number[]
const windowSec = Math.max(60, minutes * 60)
const clients: FlowEntityCard[] = []
for (const u of users) {
const userBinds = byUser.get(u.id) ?? []
if (userBinds.length === 0) continue
const analytics = buildFlowAnalytics({ minutes, userId: u.id })
const allow = new Map<number, Set<string>>()
for (const b of userBinds) {
const set = allow.get(b.serverId) ?? new Set<string>()
set.add(b.interfaceName)
allow.set(b.serverId, set)
}
let total = 0
let sessions = 0
for (const r of rows) {
const resolved = resolveIfaceName(r.serverId, r.inIface)
const names = allow.get(r.serverId)
if (!names) continue
if (!names.has(resolved.name) && !names.has(r.inIface)) continue
total += r.bytes
sessions += 1
}
const firstServer = userBinds[0]?.serverId
const ring = firstServer ? getRingMbps(firstServer, "__all__") : { rx: Array(60).fill(0) as number[], tx: Array(60).fill(0) as number[], rxNow: 0, txNow: 0 }
const ring = firstServer ? getRingMbps(firstServer, "__all__") : { rx: emptySeries, tx: emptySeries, rxNow: 0, txNow: 0 }
clients.push({
id: u.id,
name: u.login,
@@ -393,14 +550,110 @@ export function listFlowClients(minutes: number): FlowClientsDto {
site: `${userBinds.length} ifaces`,
country: "UN",
status: u.active ? "online" : "offline",
rxNow: bpsToMbps(analytics.bpsNow) || ring.rxNow,
rxNow: (total * 8) / windowSec / 1_000_000 || ring.rxNow,
txNow: ring.txNow,
sessions: analytics.conversations,
rxSeries: analytics.rxSeries,
txSeries: analytics.txSeries,
bytes: analytics.bytes,
sessions,
rxSeries: ring.rx.some((v) => v > 0) ? ring.rx : emptySeries,
txSeries: ring.tx,
bytes: total,
})
}
clients.sort((a, b) => b.rxNow - a.rxNow)
return { clients }
}
export function formatLiveSseFromBuilder(build: () => unknown): { event: "sample" | "error"; data: unknown } {
try {
return { event: "sample", data: build() }
} catch (err) {
const message = err instanceof Error ? err.message : String(err)
return { event: "error", data: { error: message } }
}
}
export function isFlowAnalyticsDegraded(): boolean {
const health = getFlowWorkerHealth()
return health.pendingSize >= LIVE_DEGRADED_PENDING
}
export function safeBuildLiveFlowSample(q: Omit<FlowAnalyticsQuery, "minutes" | "skipHeavy">): {
event: "sample" | "error"
data: unknown
} {
return formatLiveSseFromBuilder(() => {
const skipHeavy = isFlowAnalyticsDegraded()
return buildFlowAnalytics({ ...q, minutes: LIVE_ANALYTICS_MINUTES, skipHeavy })
})
}
function monthBounds(month: string): { start: string; end: string } | null {
if (!/^\d{4}-\d{2}$/.test(month)) return null
const [yearRaw, monthRaw] = month.split("-")
const year = Number(yearRaw)
const monthIdx = Number(monthRaw)
if (!Number.isFinite(year) || monthIdx < 1 || monthIdx > 12) return null
const start = `${month}-01`
const endDate = new Date(Date.UTC(year, monthIdx, 1))
const end = endDate.toISOString().slice(0, 10)
return { start, end }
}
function toBreakdown(
rows: Array<{ key: string; bytes: number; packets: number }>,
totalBytes: number,
windowSec: number,
): FlowBreakdownRow[] {
const denom = totalBytes || 1
return rows
.sort((a, b) => b.bytes - a.bytes)
.map((r) => ({
id: r.key,
label: r.key,
bytes: r.bytes,
packets: r.packets,
bps: (r.bytes * 8) / windowSec,
percent: (r.bytes / denom) * 100,
}))
}
export function getFlowMonthly(month: string, serverId?: number): FlowMonthlyDto {
const bounds = monthBounds(month)
if (!bounds) {
return { month, bytes: 0, countries: [], services: [], asns: [] }
}
const params: Array<string | number> = [bounds.start, bounds.end]
let where = "day >= ? AND day < ? AND dim IN ('country', 'service', 'asn')"
if (serverId != null) {
where += " AND server_id = ?"
params.push(serverId)
}
const rows = sqliteDatabase.prepare(`
SELECT dim AS dim, key AS key, SUM(bytes) AS bytes, SUM(packets) AS packets
FROM flow_daily_dims
WHERE ${where}
GROUP BY dim, key
`).all(...params) as Array<{ dim: string; key: string; bytes: number; packets: number }>
const countries: Array<{ key: string; bytes: number; packets: number }> = []
const services: Array<{ key: string; bytes: number; packets: number }> = []
const asns: Array<{ key: string; bytes: number; packets: number }> = []
let bytes = 0
for (const row of rows) {
const rec = { key: row.key, bytes: Number(row.bytes) || 0, packets: Number(row.packets) || 0 }
if (row.dim === "country") {
countries.push(rec)
bytes += rec.bytes
} else if (row.dim === "service") services.push(rec)
else if (row.dim === "asn") asns.push(rec)
}
const daysInMonth = Math.max(1, Math.round((Date.parse(`${bounds.end}T00:00:00Z`) - Date.parse(`${bounds.start}T00:00:00Z`)) / 86_400_000))
const windowSec = daysInMonth * 86_400
const countryTotal = countries.reduce((a, r) => a + r.bytes, 0) || bytes || 1
return {
month,
bytes,
countries: toBreakdown(countries, countryTotal, windowSec),
services: toBreakdown(services, services.reduce((a, r) => a + r.bytes, 0) || 1, windowSec),
asns: toBreakdown(asns, asns.reduce((a, r) => a + r.bytes, 0) || 1, windowSec),
}
}
@@ -32,7 +32,12 @@ const WELL_KNOWN: Record<string, string> = {
"17:500": "IKE",
"17:4500": "NAT-T",
"17:1194": "OpenVPN",
"17:443": "QUIC",
"17:853": "DNS",
"6:853": "DNS",
"17:51820": "WireGuard",
"17:13232": "WireGuard",
"17:51821": "WireGuard",
"17:4789": "VXLAN",
"17:4739": "IPFIX",
"17:2055": "NetFlow",
@@ -51,6 +56,7 @@ export function applicationName(proto: number, dstPort: number, srcPort = 0): st
if (proto === 47) return "GRE"
if (proto === 50) return "ESP"
if (proto === 89) return "OSPF"
if (proto === 17 && (dstPort === 443 || srcPort === 443)) return "QUIC"
const dstKey = `${proto}:${dstPort}`
const srcKey = `${proto}:${srcPort}`
return WELL_KNOWN[dstKey] ?? WELL_KNOWN[srcKey] ?? `${protoName(proto)}/${dstPort || srcPort || "—"}`
@@ -0,0 +1,29 @@
import assert from "node:assert/strict"
import {
brandByAsn,
countryFromHolder,
lookupBrand,
OTHER_SERVICE,
resolveRipeCountry,
} from "./traffic-flow-brands.js"
assert.equal(resolveRipeCountry("?", 13335, "CLOUDFLARENET, US"), "US")
assert.equal(resolveRipeCountry("EU", 13335, ""), "US")
assert.equal(resolveRipeCountry("?", 0, "CLOUDFLARENET, US"), "US")
assert.equal(countryFromHolder("CLOUDFLARENET, US"), "US")
assert.equal(resolveRipeCountry("NL", 0, ""), "NL")
assert.equal(resolveRipeCountry("?", 0, ""), "")
assert.equal(brandByAsn(13335)?.service, "Cloudflare")
assert.equal(brandByAsn(13335)?.category, "CDN")
assert.equal(brandByAsn(15169)?.service, "Google")
assert.equal(brandByAsn(15169)?.category, "Веб")
assert.equal(lookupBrand("208.65.153.1", 0)?.service, "YouTube")
assert.equal(brandByAsn(32590)?.service, "Steam")
assert.equal(brandByAsn(32590)?.category, "Игры")
assert.equal(brandByAsn(401115)?.service, "ChatGPT")
assert.equal(lookupBrand("1.1.1.1", 13335)?.service, "Cloudflare")
assert.equal(lookupBrand("203.0.113.9", 64500), null)
assert.equal(OTHER_SERVICE, "Прочее")
console.log("traffic-flow-brands.test.ts: ok")
+107
View File
@@ -0,0 +1,107 @@
import { ipInCidrV4, parseCidrV4 } from "./traffic-flow-ip.js"
export const OTHER_SERVICE = "Прочее"
export interface BrandHit {
service: string
category: string
}
const ASN_BRANDS = new Map<number, BrandHit>([
[13335, { service: "Cloudflare", category: "CDN" }],
[209242, { service: "Cloudflare", category: "CDN" }],
[54113, { service: "Fastly", category: "CDN" }],
[20940, { service: "Akamai", category: "CDN" }],
[16509, { service: "Amazon", category: "CDN" }],
[14618, { service: "Amazon", category: "CDN" }],
[8075, { service: "Microsoft", category: "CDN" }],
[13238, { service: "Yandex", category: "CDN" }],
[32590, { service: "Steam", category: "Игры" }],
[2906, { service: "Netflix", category: "Видео / стриминг" }],
[40027, { service: "Netflix", category: "Видео / стриминг" }],
[15169, { service: "Google", category: "Веб" }],
[36040, { service: "YouTube", category: "Видео / стриминг" }],
[46489, { service: "Twitch", category: "Видео / стриминг" }],
[401115, { service: "ChatGPT", category: "ИИ" }],
[49544, { service: "Discord", category: "Голос" }],
[62041, { service: "Telegram", category: "Голос" }],
[59930, { service: "Telegram", category: "Голос" }],
[211157, { service: "Telegram", category: "Голос" }],
[32934, { service: "Meta", category: "CDN" }],
[396986, { service: "TikTok", category: "Видео / стриминг" }],
])
const ASN_HQ_COUNTRY = new Map<number, string>([
[13335, "US"],
[209242, "US"],
[54113, "US"],
[20940, "US"],
[16509, "US"],
[14618, "US"],
[8075, "US"],
[15169, "US"],
[32590, "US"],
[2906, "US"],
[40027, "US"],
[36040, "US"],
[46489, "US"],
[401115, "US"],
[49544, "US"],
[32934, "US"],
[13238, "RU"],
[62041, "NL"],
[59930, "NL"],
[211157, "NL"],
])
const CIDR_BRANDS: Array<{ cidr: string; prefixLen: number; hit: BrandHit }> = [
{ cidr: "104.16.0.0/13", prefixLen: 13, hit: { service: "Cloudflare", category: "CDN" } },
{ cidr: "104.24.0.0/14", prefixLen: 14, hit: { service: "Cloudflare", category: "CDN" } },
{ cidr: "172.64.0.0/13", prefixLen: 13, hit: { service: "Cloudflare", category: "CDN" } },
{ cidr: "162.158.0.0/15", prefixLen: 15, hit: { service: "Cloudflare", category: "CDN" } },
{ cidr: "208.65.152.0/22", prefixLen: 22, hit: { service: "YouTube", category: "Видео / стриминг" } },
{ cidr: "208.117.224.0/19", prefixLen: 19, hit: { service: "YouTube", category: "Видео / стриминг" } },
].sort((a, b) => b.prefixLen - a.prefixLen)
const NON_ISO = new Set(["EU", "AP", "ZZ", "XX", "A1", "A2", "O1"])
export function isIsoCountry(code: string): boolean {
const c = String(code ?? "").trim().toUpperCase()
return /^[A-Z]{2}$/.test(c) && !NON_ISO.has(c)
}
export function normalizeIsoCountry(code: string): string {
const c = String(code ?? "").trim().toUpperCase()
return isIsoCountry(c) ? c : ""
}
/** `CLOUDFLARENET, US` → `US`. */
export function countryFromHolder(holder: string): string {
const m = String(holder ?? "").trim().match(/,\s*([A-Za-z]{2})\s*$/)
return m?.[1] ? normalizeIsoCountry(m[1]) : ""
}
export function countryForAsn(asn: number): string {
if (!asn) return ""
return ASN_HQ_COUNTRY.get(asn) ?? ""
}
export function resolveRipeCountry(country: string, asn: number, holder: string): string {
return normalizeIsoCountry(country) || countryFromHolder(holder) || countryForAsn(asn)
}
export function brandByAsn(asn: number): BrandHit | null {
if (!asn) return null
return ASN_BRANDS.get(asn) ?? null
}
export function brandByCidr(ip: string): BrandHit | null {
for (const row of CIDR_BRANDS) {
if (parseCidrV4(row.cidr) && ipInCidrV4(ip, row.cidr)) return row.hit
}
return null
}
export function lookupBrand(ip: string, asn: number): BrandHit | null {
return brandByCidr(ip) || brandByAsn(asn)
}
@@ -1,4 +1,5 @@
import assert from "node:assert/strict"
import { applicationName } from "./traffic-flow-apps.js"
import { classifyFlowDst, disableCatalogFetchForTests, resetFlowCatalogForTests, seedFlowCatalogForTests } from "./traffic-flow-classify.js"
disableCatalogFetchForTests()
@@ -16,5 +17,44 @@ assert.equal(miss.category, "DNS")
const cdn = classifyFlowDst("203.0.113.9", 6, 443, 1, { prefix: "203.0.113.0/24", asn: 13335, country: "US", lat: null, lng: null, holder: "CLOUDFLARENET", ok: true, fetchedAt: Date.now() })
assert.equal(cdn.category, "CDN")
assert.equal(cdn.service, "Cloudflare")
const amazonHolder = classifyFlowDst("203.0.113.50", 6, 443, 1, { prefix: "203.0.113.0/24", asn: 64500, country: "RU", lat: null, lng: null, holder: "AMAZON-AES - Amazon.com, Inc.", ok: true, fetchedAt: Date.now() })
assert.equal(amazonHolder.service, "Прочее")
assert.notEqual(amazonHolder.service, "AMAZON-AES - Amazon.com, Inc.")
const google = classifyFlowDst("173.194.160.163", 6, 443, 1, {
prefix: "173.194.0.0/16",
asn: 15169,
country: "US",
lat: null,
lng: null,
holder: "GOOGLE",
ok: true,
fetchedAt: Date.now(),
})
assert.equal(google.service, "Google")
assert.equal(google.category, "Веб")
const youtube = classifyFlowDst("173.194.160.163", 6, 443, 1, {
prefix: "173.194.0.0/16",
asn: 15169,
country: "US",
lat: null,
lng: null,
holder: "YouTube LLC",
ok: true,
fetchedAt: Date.now(),
})
assert.equal(youtube.service, "YouTube")
assert.equal(youtube.category, "Видео / стриминг")
const gre = classifyFlowDst("198.51.100.1", 47, 0, 0, null)
assert.equal(gre.service, "GRE")
assert.equal(gre.category, "Туннель")
const esp = classifyFlowDst("198.51.100.1", 50, 0, 0, null)
assert.equal(esp.category, "Туннель")
assert.equal(applicationName(17, 443, 50000), "QUIC")
assert.equal(applicationName(17, 853, 50000), "DNS")
console.log("traffic-flow-classify.test.ts: ok")
+19 -7
View File
@@ -1,3 +1,4 @@
import { lookupBrand, OTHER_SERVICE } from "./traffic-flow-brands.js"
import { db } from "../db/index.js"
import { evobgpSettings } from "../db/schema.js"
import { ipInCidrV4, parseCidrV4 } from "./traffic-flow-ip.js"
@@ -50,9 +51,12 @@ export function categoryFromPurpose(purpose: string, proto: number, dstPort: num
if (/streaming|youtube|netflix|twitch|video/.test(p)) return "Видео / стриминг"
if (/cdn|cloudflare|akamai|fastly/.test(p)) return "CDN"
if (/voip|discord|zoom/.test(p)) return "Голос"
if (/openai|chatgpt|\bai\b/.test(p)) return "ИИ"
if (/веб|web|google/.test(p)) return "Веб"
const app = applicationName(proto, dstPort, srcPort)
if (app === "DNS" || app === "SSH" || app === "BGP") return app
return "Проче"
if (app === "GRE" || app === "ESP" || app === "WireGuard") return "Туннель"
return OTHER_SERVICE
}
function matchCidr(ip: string): CatalogCidr | null {
@@ -69,14 +73,22 @@ export function classifyFlowDst(
srcPort: number,
ripe: FlowIpMeta | null,
): FlowClassification {
if (proto === 47) return { service: "GRE", category: "Туннель" }
if (proto === 50) return { service: "ESP", category: "Туннель" }
const app = applicationName(proto, dstPort, srcPort)
if (app === "WireGuard") return { service: "WireGuard", category: "Туннель" }
const hit = matchCidr(dst)
const holder = ripe?.holder ?? ""
const youtubeHolder = /youtube/i.test(holder)
const brand = youtubeHolder
? { service: "YouTube", category: "Видео / стриминг" }
: lookupBrand(dst, ripe?.asn ?? 0)
const asnName = ripe?.asn ? asnPurpose.get(ripe.asn) : undefined
const purpose = hit?.purpose || asnName || ripe?.holder || ""
const service = (hit?.purpose || asnName || ripe?.holder || "Проче").trim() || "Проче"
return {
service,
category: categoryFromPurpose(purpose, proto, dstPort, srcPort),
}
const service = (hit?.purpose || brand?.service || asnName || OTHER_SERVICE).trim() || OTHER_SERVICE
const category = hit
? categoryFromPurpose(hit.purpose, proto, dstPort, srcPort)
: (brand?.category || categoryFromPurpose(asnName || "", proto, dstPort, srcPort))
return { service, category }
}
async function fetchCatalog(): Promise<void> {
@@ -0,0 +1,41 @@
import type { OverlayPeerRef } from "./traffic-flow-map-exporter.js"
export interface ExporterMapPayload {
overlayPrefix: string
byTunnelIp: Array<[string, number]>
peers: OverlayPeerRef[]
hostIps: Array<[string, number]>
}
export interface CollectorStartPayload {
dbPath: string
listenHost: string
listenPort: number
topN: number
retentionHours: number
exporterMap: ExporterMapPayload
}
export interface CollectorHeartbeat {
bound: boolean
address: string | null
packetsReceived: number
lastExporterIp: string | null
lastError: string
lastDatagramAt: string | null
pendingSize: number
dropped: number
rowsStored: number
workerAlive: boolean
rings: Array<{ key: string; inBps: number[]; outBps: number[] }>
}
export type MainToWorker =
| { type: "start"; payload: CollectorStartPayload }
| { type: "stop" }
| { type: "updateExporterMap"; payload: ExporterMapPayload }
| { type: "updateSettings"; payload: { topN: number; retentionHours: number } }
export type WorkerToMain =
| { type: "heartbeat"; payload: CollectorHeartbeat }
| { type: "error"; payload: { message: string } }
@@ -0,0 +1,141 @@
import { createSocket, type Socket } from "node:dgram"
import { parentPort } from "node:worker_threads"
import { sqliteDatabase } from "../db/index.js"
import type {
CollectorStartPayload,
ExporterMapPayload,
MainToWorker,
WorkerToMain,
} from "./traffic-flow-collector-ipc.js"
import {
TICK_MS,
attachEngineSqlite,
configureEngine,
flushPending,
getEngineStats,
ingestDatagram,
setEngineError,
setExporterResolveCtx,
snapshotRings,
} from "./traffic-flow-engine.js"
let socket: Socket | null = null
let flushTimer: ReturnType<typeof setInterval> | null = null
let bound = false
let address: string | null = null
let attached = false
function send(msg: WorkerToMain): void {
parentPort?.postMessage(msg)
}
function heartbeat(): void {
const stats = getEngineStats()
send({
type: "heartbeat",
payload: {
bound,
address,
packetsReceived: stats.packetsReceived,
lastExporterIp: stats.lastExporterIp,
lastError: stats.lastError,
lastDatagramAt: stats.lastDatagramAt,
pendingSize: stats.pendingSize,
dropped: stats.dropped,
rowsStored: stats.rowsStored,
workerAlive: true,
rings: snapshotRings(),
},
})
}
function applyExporterMap(payload: ExporterMapPayload): void {
setExporterResolveCtx({
overlayPrefix: payload.overlayPrefix,
byTunnelIp: new Map(payload.byTunnelIp),
peers: payload.peers,
hostIps: new Map(payload.hostIps),
})
}
function ensureSqlite(): void {
if (attached) return
attachEngineSqlite(sqliteDatabase)
attached = true
}
function stopListener(): void {
if (flushTimer) {
clearInterval(flushTimer)
flushTimer = null
}
try {
flushPending()
} catch (e) {
setEngineError(e instanceof Error ? e.message : String(e))
}
if (socket) {
try { socket.close() } catch { /* ignore */ }
socket = null
}
bound = false
address = null
}
function startListener(payload: CollectorStartPayload): void {
stopListener()
ensureSqlite()
configureEngine({ topN: payload.topN, retentionHours: payload.retentionHours })
applyExporterMap(payload.exporterMap)
const sock = createSocket("udp4")
sock.on("error", (err) => {
setEngineError(err.message)
bound = false
address = null
send({ type: "error", payload: { message: err.message } })
heartbeat()
})
sock.on("message", (msg, rinfo) => {
try {
ingestDatagram(msg, rinfo.address)
} catch (e) {
setEngineError(e instanceof Error ? e.message : String(e))
}
})
try {
sock.setRecvBufferSize(8 * 1024 * 1024)
} catch {
/* platform may ignore */
}
sock.bind(payload.listenPort, payload.listenHost, () => {
bound = true
address = `${payload.listenHost}:${payload.listenPort}`
setEngineError("")
heartbeat()
})
socket = sock
flushTimer = setInterval(() => {
try {
flushPending()
} catch (e) {
setEngineError(e instanceof Error ? e.message : String(e))
}
heartbeat()
}, TICK_MS)
}
parentPort?.on("message", (msg: MainToWorker) => {
try {
if (msg.type === "start") startListener(msg.payload)
else if (msg.type === "stop") {
stopListener()
heartbeat()
} else if (msg.type === "updateExporterMap") applyExporterMap(msg.payload)
else if (msg.type === "updateSettings") configureEngine(msg.payload)
} catch (e) {
const message = e instanceof Error ? e.message : String(e)
setEngineError(message)
send({ type: "error", payload: { message } })
}
})
+746
View File
@@ -0,0 +1,746 @@
import type Database from "better-sqlite3"
import { normalizeParsedFlow, parseFlowPacket, protoName, type ParsedFlow, type ParsedFlowInput } from "./traffic-flow-parse.js"
import { classifyFlowPlaneLite } from "./traffic-flow-planes.js"
import { pickServerIdForExporter, type OverlayPeerRef } from "./traffic-flow-map-exporter.js"
import { applicationName } from "./traffic-flow-apps.js"
import { classifyFlowDst } from "./traffic-flow-classify.js"
import { enqueueRipeMisses, lookupRipeCached } from "./traffic-flow-ripe.js"
import { isIsoCountry } from "./traffic-flow-brands.js"
import { maybeRefreshIfaces } from "./traffic-flow-ifaces.js"
type SqliteHandle = InstanceType<typeof Database>
export const TICK_MS = 2_000
export const RING_LEN = 60
export const MAX_PENDING = 50_000
export const DAILY_ASN_TOP = 500
export const DAILY_RETENTION_DAYS = 396
export const MINUTE_RETENTION_HOURS = 48
let pendingCap = MAX_PENDING
export interface PendingFlowRow {
serverId: number
bucketAt: string
src: string
dst: string
proto: number
srcPort: number
dstPort: number
bytes: number
packets: number
inIface: string
outIface: string
nextHop: string
flowStartMs: number
flowEndMs: number
}
export interface EngineStats {
packetsReceived: number
lastExporterIp: string | null
lastError: string
lastDatagramAt: string | null
dropped: number
rowsStored: number
pendingSize: number
}
interface PendingEntry {
serverId: number
bucketAt: string
flow: ParsedFlow
bytes: number
packets: number
}
interface MinuteRollup {
bytes: number
packets: number
srcs: Set<string>
dsts: Set<string>
conversations: number
}
interface DimAcc {
bytes: number
packets: number
}
export interface ExporterResolveCtx {
overlayPrefix: string
byTunnelIp: Map<string, number>
peers: OverlayPeerRef[]
hostIps: Map<string, number>
}
let sqliteRef: SqliteHandle | null = null
let topN = 200
let retentionHours = 24
const pending = new Map<string, PendingEntry>()
const recent = new Map<string, PendingFlowRow>()
const tickAccum = new Map<string, { inBytes: number; outBytes: number }>()
const rings = new Map<string, { inBps: number[]; outBps: number[] }>()
const minuteRollup = new Map<string, MinuteRollup>()
const minuteDims = new Map<string, DimAcc>()
let packetsReceived = 0
let lastExporterIp: string | null = null
let lastError = ""
let lastDatagramAt: string | null = null
let dropped = 0
let rowsStored = 0
let lastFlushUsedTransaction = false
let lastPruneAt = 0
let exporterCtx: ExporterResolveCtx | null = null
const PRUNE_MS = 5 * 60_000
const LIVE_WINDOW_MS = 15 * 60_000
function nowIso(): string {
return new Date().toISOString()
}
export function minuteBucketIso(at = Date.now()): string {
const d = new Date(at)
d.setSeconds(0, 0)
return d.toISOString()
}
function dayKey(bucketAt: string): string {
return bucketAt.slice(0, 10)
}
export const RING_PAYLOAD = "__all__"
export const RING_OVERLAY = "__overlay__"
export const RING_MESH = "__mesh__"
function ringKey(serverId: number, iface: string): string {
return `${serverId}\0${iface || RING_PAYLOAD}`
}
function pendingKey(serverId: number, bucketAt: string, flow: ParsedFlow): string {
return `${serverId}\0${bucketAt}\0${flow.src}\0${flow.dst}\0${flow.proto}\0${flow.srcPort}\0${flow.dstPort}\0${flow.inIface}`
}
function rowKey(row: PendingFlowRow): string {
return `${row.serverId}|${row.bucketAt}|${row.src}|${row.dst}|${row.proto}|${row.srcPort}|${row.dstPort}|${row.inIface}`
}
function rollupKey(serverId: number, bucketAt: string): string {
return `${serverId}\0${bucketAt}`
}
function dimKey(serverId: number, bucketAt: string, dim: string, key: string): string {
return `${serverId}\0${bucketAt}\0${dim}\0${key}`
}
function bumpTick(key: string, inBytes: number, outBytes: number): void {
const prev = tickAccum.get(key) ?? { inBytes: 0, outBytes: 0 }
prev.inBytes += inBytes
prev.outBytes += outBytes
tickAccum.set(key, prev)
}
function addToTick(serverId: number, flow: ParsedFlow, bytes: number): void {
const plane = classifyFlowPlaneLite(flow)
if (plane === "mgmt") return
const bucket = plane === "overlay" ? RING_OVERLAY : plane === "client_mesh" ? RING_MESH : RING_PAYLOAD
bumpTick(ringKey(serverId, bucket), bytes, 0)
if (flow.inIface) bumpTick(ringKey(serverId, flow.inIface), bytes, 0)
if (flow.outIface && flow.outIface !== flow.inIface) bumpTick(ringKey(serverId, flow.outIface), 0, bytes)
}
function emptyRing(): { inBps: number[]; outBps: number[] } {
return { inBps: Array(RING_LEN).fill(0), outBps: Array(RING_LEN).fill(0) }
}
function bumpDim(serverId: number, bucketAt: string, dim: string, key: string, bytes: number, packets: number): void {
if (!key) return
const k = dimKey(serverId, bucketAt, dim, key)
const prev = minuteDims.get(k)
if (prev) {
prev.bytes += bytes
prev.packets += packets
return
}
minuteDims.set(k, { bytes, packets })
}
function bumpRollup(serverId: number, bucketAt: string, flow: ParsedFlow, bytes: number, packets: number): void {
const k = rollupKey(serverId, bucketAt)
let acc = minuteRollup.get(k)
if (!acc) {
acc = { bytes: 0, packets: 0, srcs: new Set(), dsts: new Set(), conversations: 0 }
minuteRollup.set(k, acc)
}
acc.bytes += bytes
acc.packets += packets
if (flow.src) acc.srcs.add(flow.src)
if (flow.dst) acc.dsts.add(flow.dst)
acc.conversations += 1
}
export function attachEngineSqlite(handle: SqliteHandle): void {
sqliteRef = handle
}
export function setPendingCapForTests(n: number | null): void {
pendingCap = n == null ? MAX_PENDING : Math.max(1, n)
}
export function configureEngine(opts: { topN?: number; retentionHours?: number }): void {
if (opts.topN != null) topN = Math.max(20, opts.topN)
if (opts.retentionHours != null) retentionHours = Math.max(1, opts.retentionHours)
}
export function setExporterResolveCtx(ctx: ExporterResolveCtx | null): void {
exporterCtx = ctx
}
export function resolveServerId(exporterIp: string): number | null {
if (!exporterCtx) return null
return pickServerIdForExporter({
exporterIp,
overlayPrefix: exporterCtx.overlayPrefix,
byTunnelIp: exporterCtx.byTunnelIp,
peers: exporterCtx.peers,
hostIps: exporterCtx.hostIps,
})
}
export function bumpPacketMeta(exporterIp: string): void {
packetsReceived += 1
lastExporterIp = exporterIp
lastDatagramAt = nowIso()
}
export function setEngineError(message: string): void {
lastError = message
}
export function getEngineStats(): EngineStats {
return {
packetsReceived,
lastExporterIp,
lastError,
lastDatagramAt,
dropped,
rowsStored,
pendingSize: pending.size,
}
}
export function queueParsedFlows(serverId: number, flows: ParsedFlowInput[]): void {
const bucketAt = minuteBucketIso()
const ripeMisses: string[] = []
for (const raw of flows) {
const flow = normalizeParsedFlow(raw)
addToTick(serverId, flow, flow.bytes)
bumpRollup(serverId, bucketAt, flow, flow.bytes, flow.packets)
const ripe = lookupRipeCached(flow.dst)
if (flow.dst && !ripe) ripeMisses.push(flow.dst)
const classified = classifyFlowDst(flow.dst, flow.proto, flow.dstPort, flow.srcPort, ripe)
const app = applicationName(flow.proto, flow.dstPort, flow.srcPort)
const country = ripe?.ok && isIsoCountry(ripe.country)
? ripe.country
: (ripe?.ok ? "" : "unknown")
const asnKey = ripe?.ok && ripe.asn ? String(ripe.asn) : "unknown"
bumpDim(serverId, bucketAt, "proto", protoName(flow.proto), flow.bytes, flow.packets)
bumpDim(serverId, bucketAt, "app", app, flow.bytes, flow.packets)
bumpDim(serverId, bucketAt, "iface", flow.inIface || "__unknown__", flow.bytes, flow.packets)
bumpDim(serverId, bucketAt, "category", classified.category, flow.bytes, flow.packets)
bumpDim(serverId, bucketAt, "service", classified.service, flow.bytes, flow.packets)
if (country) bumpDim(serverId, bucketAt, "country", country, flow.bytes, flow.packets)
bumpDim(serverId, bucketAt, "asn", asnKey, flow.bytes, flow.packets)
const key = pendingKey(serverId, bucketAt, flow)
const prev = pending.get(key)
if (prev) {
prev.bytes += flow.bytes
prev.packets += flow.packets
if (flow.outIface && !prev.flow.outIface) prev.flow.outIface = flow.outIface
if (flow.nextHop && !prev.flow.nextHop) prev.flow.nextHop = flow.nextHop
if (flow.flowStartMs && (!prev.flow.flowStartMs || flow.flowStartMs < prev.flow.flowStartMs)) {
prev.flow.flowStartMs = flow.flowStartMs
}
if (flow.flowEndMs > (prev.flow.flowEndMs ?? 0)) prev.flow.flowEndMs = flow.flowEndMs
continue
}
if (pending.size >= pendingCap) {
dropped += 1
continue
}
pending.set(key, {
serverId,
bucketAt,
flow: { ...flow },
bytes: flow.bytes,
packets: flow.packets,
})
}
if (ripeMisses.length) enqueueRipeMisses(ripeMisses)
}
export function ingestDatagram(msg: Buffer, exporterIp: string): boolean {
bumpPacketMeta(exporterIp)
const flows = parseFlowPacket(msg, exporterIp)
if (!flows.length) return true
const serverId = resolveServerId(exporterIp)
if (serverId == null) {
setEngineError(
`IPFIX от ${exporterIp}: нет jump-host с адресом wg-flow. Docker SNAT (172.x) при нескольких JH не различим.`,
)
return false
}
setEngineError("")
maybeRefreshIfaces(serverId)
queueParsedFlows(serverId, flows)
return true
}
function toPendingRow(row: PendingEntry): PendingFlowRow {
const flow = normalizeParsedFlow(row.flow)
return {
serverId: row.serverId,
bucketAt: row.bucketAt,
src: flow.src || "0.0.0.0",
dst: flow.dst || "0.0.0.0",
proto: flow.proto,
srcPort: flow.srcPort,
dstPort: flow.dstPort,
bytes: row.bytes,
packets: row.packets,
inIface: flow.inIface,
outIface: flow.outIface,
nextHop: flow.nextHop,
flowStartMs: flow.flowStartMs,
flowEndMs: flow.flowEndMs,
}
}
function mergeInto(map: Map<string, PendingFlowRow>, row: PendingFlowRow): void {
const key = rowKey(row)
const prev = map.get(key)
if (prev) {
prev.bytes += row.bytes
prev.packets += row.packets
if (row.outIface && !prev.outIface) prev.outIface = row.outIface
if (row.nextHop && !prev.nextHop) prev.nextHop = row.nextHop
if (row.flowStartMs && (!prev.flowStartMs || row.flowStartMs < prev.flowStartMs)) prev.flowStartMs = row.flowStartMs
if (row.flowEndMs > (prev.flowEndMs ?? 0)) prev.flowEndMs = row.flowEndMs
return
}
map.set(key, { ...row })
}
function pruneRecent(sinceMs = Date.now() - LIVE_WINDOW_MS): void {
const cutoff = new Date(sinceMs).toISOString()
for (const [key, row] of recent) {
if (row.bucketAt < cutoff) recent.delete(key)
}
while (recent.size > MAX_PENDING) {
const first = recent.keys().next().value
if (first == null) break
recent.delete(first)
}
}
export function peekPendingFlows(): PendingFlowRow[] {
return [...pending.values()].map(toPendingRow)
}
export function listLiveFlowRows(sinceIso: string): PendingFlowRow[] {
const merged = new Map<string, PendingFlowRow>()
for (const row of recent.values()) {
if (row.bucketAt < sinceIso) continue
mergeInto(merged, row)
}
for (const row of peekPendingFlows()) {
if (row.bucketAt < sinceIso) continue
mergeInto(merged, row)
}
return [...merged.values()]
}
export function rollFlowRings(): void {
const keys = new Set([...tickAccum.keys(), ...rings.keys()])
const sec = TICK_MS / 1000
for (const key of keys) {
const acc = tickAccum.get(key) ?? { inBytes: 0, outBytes: 0 }
tickAccum.delete(key)
const inBps = (acc.inBytes * 8) / sec
const outBps = (acc.outBytes * 8) / sec
let ring = rings.get(key)
if (!ring) {
ring = emptyRing()
rings.set(key, ring)
}
ring.inBps.push(inBps)
ring.inBps.shift()
ring.outBps.push(outBps)
ring.outBps.shift()
const silent = ring.inBps.every((v) => v === 0) && ring.outBps.every((v) => v === 0)
if (silent && !tickAccum.has(key)) rings.delete(key)
}
}
export function getRingMbps(serverId: number, iface = RING_PAYLOAD): {
rx: number[]
tx: number[]
rxNow: number
txNow: number
} {
const ring = rings.get(ringKey(serverId, iface))
const scale = 1_000_000
if (!ring) {
return { rx: Array(RING_LEN).fill(0), tx: Array(RING_LEN).fill(0), rxNow: 0, txNow: 0 }
}
return {
rx: ring.inBps.map((b) => b / scale),
tx: ring.outBps.map((b) => b / scale),
rxNow: (ring.inBps[RING_LEN - 1] ?? 0) / scale,
txNow: (ring.outBps[RING_LEN - 1] ?? 0) / scale,
}
}
export function snapshotRings(): Array<{ key: string; inBps: number[]; outBps: number[] }> {
return [...rings.entries()].map(([key, ring]) => ({
key,
inBps: [...ring.inBps],
outBps: [...ring.outBps],
}))
}
export function applyRingSnapshot(rows: Array<{ key: string; inBps: number[]; outBps: number[] }>): void {
rings.clear()
for (const row of rows) {
rings.set(row.key, { inBps: row.inBps, outBps: row.outBps })
}
}
function persistListenerStats(handle: SqliteHandle): void {
handle.prepare(`
UPDATE traffic_flow_settings
SET packets_received = @packetsReceived,
last_datagram_at = @lastDatagramAt,
last_exporter_ip = @lastExporterIp,
last_error = @lastError,
updated_at = @updatedAt
WHERE id = 1
`).run({
packetsReceived,
lastDatagramAt,
lastExporterIp,
lastError,
updatedAt: nowIso(),
})
}
function upsertMinuteAndDaily(handle: SqliteHandle): void {
const upsertMinute = handle.prepare(`
INSERT INTO flow_minute_stats (
server_id, bucket_at, bytes, packets, unique_src, unique_dst, conversations
) VALUES (
@serverId, @bucketAt, @bytes, @packets, @uniqueSrc, @uniqueDst, @conversations
)
ON CONFLICT(server_id, bucket_at) DO UPDATE SET
bytes = bytes + excluded.bytes,
packets = packets + excluded.packets,
unique_src = MAX(unique_src, excluded.unique_src),
unique_dst = MAX(unique_dst, excluded.unique_dst),
conversations = conversations + excluded.conversations
`)
const upsertDim = handle.prepare(`
INSERT INTO flow_minute_dims (server_id, bucket_at, dim, key, bytes, packets)
VALUES (@serverId, @bucketAt, @dim, @key, @bytes, @packets)
ON CONFLICT(server_id, bucket_at, dim, key) DO UPDATE SET
bytes = bytes + excluded.bytes,
packets = packets + excluded.packets
`)
const upsertDaily = handle.prepare(`
INSERT INTO flow_daily_dims (server_id, day, dim, key, bytes, packets)
VALUES (@serverId, @day, @dim, @key, @bytes, @packets)
ON CONFLICT(server_id, day, dim, key) DO UPDATE SET
bytes = bytes + excluded.bytes,
packets = packets + excluded.packets
`)
const tx = handle.transaction(() => {
for (const [k, acc] of minuteRollup) {
const [serverIdRaw, bucketAt] = k.split("\0")
upsertMinute.run({
serverId: Number(serverIdRaw),
bucketAt,
bytes: acc.bytes,
packets: acc.packets,
uniqueSrc: acc.srcs.size,
uniqueDst: acc.dsts.size,
conversations: acc.conversations,
})
}
for (const [k, acc] of minuteDims) {
const [serverIdRaw, bucketAt, dim, key] = k.split("\0")
upsertDim.run({
serverId: Number(serverIdRaw),
bucketAt,
dim,
key,
bytes: acc.bytes,
packets: acc.packets,
})
if (dim === "country" || dim === "service" || dim === "asn") {
upsertDaily.run({
serverId: Number(serverIdRaw),
day: dayKey(bucketAt ?? ""),
dim,
key,
bytes: acc.bytes,
packets: acc.packets,
})
}
}
})
tx()
minuteRollup.clear()
minuteDims.clear()
}
function capDailyAsn(handle: SqliteHandle): void {
const today = nowIso().slice(0, 10)
const rows = handle.prepare(`
SELECT server_id AS serverId, key, bytes, packets
FROM flow_daily_dims
WHERE day = ? AND dim = 'asn'
ORDER BY server_id, bytes DESC
`).all(today) as Array<{ serverId: number; key: string; bytes: number; packets: number }>
const byServer = new Map<number, typeof rows>()
for (const row of rows) {
const list = byServer.get(row.serverId) ?? []
list.push(row)
byServer.set(row.serverId, list)
}
const del = handle.prepare(`
DELETE FROM flow_daily_dims WHERE server_id = ? AND day = ? AND dim = 'asn' AND key = ?
`)
const upsertOther = handle.prepare(`
INSERT INTO flow_daily_dims (server_id, day, dim, key, bytes, packets)
VALUES (?, ?, 'asn', 'other', ?, ?)
ON CONFLICT(server_id, day, dim, key) DO UPDATE SET
bytes = bytes + excluded.bytes,
packets = packets + excluded.packets
`)
for (const [serverId, list] of byServer) {
if (list.length <= DAILY_ASN_TOP) continue
let otherBytes = 0
let otherPackets = 0
for (const row of list.slice(DAILY_ASN_TOP)) {
if (row.key === "other") continue
otherBytes += row.bytes
otherPackets += row.packets
del.run(serverId, today, row.key)
}
if (otherBytes > 0) upsertOther.run(serverId, today, otherBytes, otherPackets)
}
}
function pruneStored(handle: SqliteHandle): void {
const now = Date.now()
if (now - lastPruneAt < PRUNE_MS) return
lastPruneAt = now
const flowCutoff = new Date(now - retentionHours * 3600_000).toISOString()
const minuteCutoff = new Date(now - MINUTE_RETENTION_HOURS * 3600_000).toISOString()
const dailyCutoff = new Date(now - DAILY_RETENTION_DAYS * 86400_000).toISOString().slice(0, 10)
handle.prepare(`DELETE FROM flow_buckets WHERE bucket_at < ?`).run(flowCutoff)
handle.prepare(`DELETE FROM flow_minute_stats WHERE bucket_at < ?`).run(minuteCutoff)
handle.prepare(`DELETE FROM flow_minute_dims WHERE bucket_at < ?`).run(minuteCutoff)
handle.prepare(`DELETE FROM flow_daily_dims WHERE day < ?`).run(dailyCutoff)
const keep = Math.max(20, topN)
try {
handle.prepare(`
DELETE FROM flow_buckets WHERE id IN (
SELECT id FROM (
SELECT id, ROW_NUMBER() OVER (
PARTITION BY server_id, bucket_at ORDER BY bytes DESC
) AS rn
FROM flow_buckets
) ranked WHERE rn > ?
)
`).run(keep)
} catch {
const buckets = handle.prepare(`
SELECT DISTINCT server_id AS serverId, bucket_at AS bucketAt FROM flow_buckets
`).all() as Array<{ serverId: number; bucketAt: string }>
for (const b of buckets) {
const rows = handle.prepare(`
SELECT id, bytes FROM flow_buckets
WHERE server_id = ? AND bucket_at = ?
ORDER BY bytes DESC
`).all(b.serverId, b.bucketAt) as Array<{ id: number; bytes: number }>
for (const extra of rows.slice(keep)) {
handle.prepare(`DELETE FROM flow_buckets WHERE id = ?`).run(extra.id)
}
}
}
}
function topNPending(rows: PendingFlowRow[]): PendingFlowRow[] {
const keep = Math.max(20, topN)
const groups = new Map<string, PendingFlowRow[]>()
for (const row of rows) {
const k = `${row.serverId}\0${row.bucketAt}`
const list = groups.get(k) ?? []
list.push(row)
groups.set(k, list)
}
const out: PendingFlowRow[] = []
for (const list of groups.values()) {
list.sort((a, b) => b.bytes - a.bytes)
out.push(...list.slice(0, keep))
}
return out
}
export function flushPending(): void {
pruneRecent()
rollFlowRings()
const handle = sqliteRef
if (!handle) {
lastFlushUsedTransaction = false
return
}
persistListenerStats(handle)
if (pending.size === 0 && minuteRollup.size === 0 && minuteDims.size === 0) {
pruneStored(handle)
lastFlushUsedTransaction = false
return
}
const rows = topNPending([...pending.values()].map(toPendingRow))
pending.clear()
for (const row of rows) mergeInto(recent, row)
const upsertFlow = handle.prepare(`
INSERT INTO flow_buckets (
server_id, bucket_at, src, dst, proto, src_port, dst_port, bytes, packets, in_iface, out_iface, next_hop, flow_start_ms, flow_end_ms
) VALUES (
@serverId, @bucketAt, @src, @dst, @proto, @srcPort, @dstPort, @bytes, @packets, @inIface, @outIface, @nextHop, @flowStartMs, @flowEndMs
)
ON CONFLICT(server_id, bucket_at, src, dst, proto, src_port, dst_port, in_iface)
DO UPDATE SET
bytes = bytes + excluded.bytes,
packets = packets + excluded.packets,
out_iface = CASE WHEN excluded.out_iface != '' THEN excluded.out_iface ELSE out_iface END,
next_hop = CASE WHEN excluded.next_hop != '' THEN excluded.next_hop ELSE next_hop END,
flow_start_ms = CASE
WHEN excluded.flow_start_ms > 0 AND (flow_start_ms = 0 OR excluded.flow_start_ms < flow_start_ms)
THEN excluded.flow_start_ms ELSE flow_start_ms END,
flow_end_ms = MAX(flow_end_ms, excluded.flow_end_ms)
`)
lastFlushUsedTransaction = false
try {
const tx = handle.transaction((batch: PendingFlowRow[]) => {
for (const r of batch) {
upsertFlow.run({
serverId: r.serverId,
bucketAt: r.bucketAt,
src: r.src,
dst: r.dst,
proto: r.proto,
srcPort: r.srcPort,
dstPort: r.dstPort,
bytes: r.bytes,
packets: r.packets,
inIface: r.inIface,
outIface: r.outIface,
nextHop: r.nextHop,
flowStartMs: r.flowStartMs,
flowEndMs: r.flowEndMs,
})
}
})
tx(rows)
lastFlushUsedTransaction = true
rowsStored += rows.length
} catch {
for (const r of rows) {
try {
upsertFlow.run({
serverId: r.serverId,
bucketAt: r.bucketAt,
src: r.src,
dst: r.dst,
proto: r.proto,
srcPort: r.srcPort,
dstPort: r.dstPort,
bytes: r.bytes,
packets: r.packets,
inIface: r.inIface,
outIface: r.outIface,
nextHop: r.nextHop,
flowStartMs: r.flowStartMs,
flowEndMs: r.flowEndMs,
})
rowsStored += 1
} catch {
/* ignore single-row failures */
}
}
}
try {
upsertMinuteAndDaily(handle)
capDailyAsn(handle)
} catch {
/* rollup best-effort */
}
pruneStored(handle)
try {
handle.pragma("wal_checkpoint(TRUNCATE)")
} catch {
/* ignore */
}
}
export function lastFlushUsedTransactionForTests(): boolean {
return lastFlushUsedTransaction
}
export function flushPendingForTests(): void {
flushPending()
}
export function onEngineTick(): void {
flushPending()
}
export function ingestParsedFlowsForServerForTests(serverId: number, flows: ParsedFlowInput[]): void {
queueParsedFlows(serverId, flows)
rollFlowRings()
}
export function resetEngineForTests(): void {
pending.clear()
recent.clear()
tickAccum.clear()
rings.clear()
minuteRollup.clear()
minuteDims.clear()
packetsReceived = 0
lastExporterIp = null
lastError = ""
lastDatagramAt = null
dropped = 0
rowsStored = 0
lastFlushUsedTransaction = false
lastPruneAt = 0
pendingCap = MAX_PENDING
}
export function pendingSizeForTests(): number {
return pending.size
}
export function droppedForTests(): number {
return dropped
}
@@ -0,0 +1,23 @@
import assert from "node:assert/strict"
import { SQLITE_BUSY_TIMEOUT_MS, sqliteDatabase } from "../db/index.js"
import {
MAX_FLOW_LIVE_SUBSCRIBERS,
resetFlowLiveSlotsForTests,
tryAcquireFlowLiveSlot,
releaseFlowLiveSlot,
} from "../routes/traffic-flow.js"
const busy = sqliteDatabase.pragma("busy_timeout") as Array<{ busy_timeout: number }>
const busyValue = Array.isArray(busy) ? Number(Object.values(busy[0] ?? {})[0]) : Number(busy)
assert.equal(busyValue, SQLITE_BUSY_TIMEOUT_MS)
resetFlowLiveSlotsForTests()
for (let i = 0; i < MAX_FLOW_LIVE_SUBSCRIBERS; i++) {
assert.equal(tryAcquireFlowLiveSlot(), true)
}
assert.equal(tryAcquireFlowLiveSlot(), false)
releaseFlowLiveSlot()
assert.equal(tryAcquireFlowLiveSlot(), true)
resetFlowLiveSlotsForTests()
console.log("traffic-flow-hardening.test.ts: ok")
@@ -4,6 +4,8 @@ import {
resetIfaceCacheForTests,
resolveIfaceName,
rosIdToIfIndex,
shouldRefreshIfaces,
markIfaceRefreshAttempt,
} from "./traffic-flow-ifindex.js"
import { applicationName, flowRowMatchesFilter } from "./traffic-flow-apps.js"
@@ -35,6 +37,14 @@ assert.equal(rosIdToIfIndex("*12"), 18)
assert.equal(resolveIfaceName(8, "10").name, "gre1")
assert.equal(resolveIfaceName(8, "18").name, "gre1")
resetIfaceCacheForTests()
assert.equal(shouldRefreshIfaces(9), true)
rememberServerIfaces(9, [{ ".id": "*2", name: "ether1" }])
assert.equal(shouldRefreshIfaces(9), false)
resetIfaceCacheForTests()
markIfaceRefreshAttempt(9)
assert.equal(shouldRefreshIfaces(9), false)
assert.equal(applicationName(6, 443), "HTTPS")
assert.equal(applicationName(17, 53), "DNS")
assert.equal(applicationName(6, 22), "SSH")
+22 -3
View File
@@ -3,8 +3,9 @@ import { db } from "../db/index.js"
import { servers } from "../db/schema.js"
import { MikrotikClient } from "./mikrotik.js"
import {
ifaceCacheFresh,
rememberServerIfaces,
shouldRefreshIfaces,
markIfaceRefreshAttempt,
type RosIfaceIndexRow,
} from "./traffic-flow-ifindex.js"
@@ -15,13 +16,16 @@ export {
resetIfaceCacheForTests,
resolveIfaceName,
rosIdToIfIndex,
shouldRefreshIfaces,
markIfaceRefreshAttempt,
} from "./traffic-flow-ifindex.js"
const inflight = new Set<number>()
let refreshIfacesImpl: (serverId: number, force?: boolean) => Promise<void> = refreshServerIfacesInner
export async function refreshServerIfaces(serverId: number, force = false): Promise<void> {
async function refreshServerIfacesInner(serverId: number, force = false): Promise<void> {
if (inflight.has(serverId)) return
if (!force && ifaceCacheFresh(serverId)) return
if (!force && !shouldRefreshIfaces(serverId)) return
inflight.add(serverId)
try {
const row = db.select().from(servers).where(eq(servers.id, serverId)).limit(1).all()[0]
@@ -32,6 +36,21 @@ export async function refreshServerIfaces(serverId: number, force = false): Prom
} catch {
/* keep previous cache */
} finally {
markIfaceRefreshAttempt(serverId)
inflight.delete(serverId)
}
}
export async function refreshServerIfaces(serverId: number, force = false): Promise<void> {
return refreshIfacesImpl(serverId, force)
}
export function maybeRefreshIfaces(serverId: number): boolean {
if (!shouldRefreshIfaces(serverId)) return false
void refreshIfacesImpl(serverId)
return true
}
export function setRefreshIfacesForTests(fn: typeof refreshServerIfacesInner | null): void {
refreshIfacesImpl = fn ?? refreshServerIfacesInner
}
@@ -6,6 +6,7 @@ export interface RosIfaceIndexRow {
const cache = new Map<number, Map<number, string>>()
const fetchedAt = new Map<number, number>()
const lastAttempt = new Map<number, number>()
export const IFACE_CACHE_TTL_MS = 60_000
@@ -52,7 +53,19 @@ export function ifaceCacheFresh(serverId: number, ttlMs = IFACE_CACHE_TTL_MS): b
return Boolean(prev && Date.now() - prev < ttlMs && cache.has(serverId))
}
/** Не ходить в REST, пока кэш жив или с момента последней попытки не прошёл TTL. */
export function shouldRefreshIfaces(serverId: number, ttlMs = IFACE_CACHE_TTL_MS): boolean {
if (ifaceCacheFresh(serverId, ttlMs)) return false
const attempted = lastAttempt.get(serverId) ?? 0
return !(attempted && Date.now() - attempted < ttlMs)
}
export function markIfaceRefreshAttempt(serverId: number, at = Date.now()): void {
lastAttempt.set(serverId, at)
}
export function resetIfaceCacheForTests(): void {
cache.clear()
fetchedAt.clear()
lastAttempt.clear()
}
@@ -0,0 +1,120 @@
import assert from "node:assert/strict"
import {
markIfaceRefreshAttempt,
rememberServerIfaces,
resetIfaceCacheForTests,
shouldRefreshIfaces,
} from "./traffic-flow-ifindex.js"
import {
applyHeartbeatForTests,
flushPendingForTests,
getFlowListenerState,
getFlowRuntimeCounters,
getFlowWorkerHealth,
ingestParsedFlowsForServerForTests,
lastFlushUsedTransactionForTests,
maybeRefreshIfaces,
peekPendingFlows,
resetFlowRingsForTests,
setPendingCapForTests,
setRefreshIfacesForTests,
setWantListenForTests,
simulateWorkerExitForTests,
} from "./traffic-flow-ingest.js"
import { configureEngine, droppedForTests, pendingSizeForTests } from "./traffic-flow-engine.js"
import { sqliteDatabase } from "../db/index.js"
resetIfaceCacheForTests()
resetFlowRingsForTests()
let refreshCalls = 0
setRefreshIfacesForTests(async () => {
refreshCalls += 1
})
rememberServerIfaces(1, [{ ".id": "*A", name: "wg-flow" }])
assert.equal(shouldRefreshIfaces(1), false)
assert.equal(maybeRefreshIfaces(1), false)
assert.equal(refreshCalls, 0)
resetIfaceCacheForTests()
assert.equal(shouldRefreshIfaces(2), true)
assert.equal(maybeRefreshIfaces(2), true)
assert.equal(refreshCalls, 1)
markIfaceRefreshAttempt(2)
assert.equal(shouldRefreshIfaces(2), false)
assert.equal(maybeRefreshIfaces(2), false)
assert.equal(refreshCalls, 1)
assert.equal(lastFlushUsedTransactionForTests(), false)
resetFlowRingsForTests()
setPendingCapForTests(3)
const many = Array.from({ length: 6 }, (_, i) => ({
src: `10.1.1.${i + 1}`,
dst: "8.8.8.8",
proto: 6,
srcPort: 50000 + i,
dstPort: 443,
bytes: 1000,
packets: 1,
inIface: "2",
outIface: "",
}))
ingestParsedFlowsForServerForTests(9, many)
assert.equal(pendingSizeForTests(), 3)
assert.equal(droppedForTests(), 3)
assert.equal(peekPendingFlows().length, 3)
setPendingCapForTests(null)
resetFlowRingsForTests()
configureEngine({ topN: 20 })
const talkers = Array.from({ length: 25 }, (_, i) => ({
src: `10.2.1.${i + 1}`,
dst: "1.1.1.1",
proto: 6,
srcPort: 40000 + i,
dstPort: 443,
bytes: 1000 + i,
packets: 1,
inIface: "2",
outIface: "",
}))
ingestParsedFlowsForServerForTests(9, talkers)
flushPendingForTests()
const stored = sqliteDatabase.prepare(`
SELECT COUNT(*) AS n FROM flow_buckets WHERE server_id = 9
`).get() as { n: number }
assert.ok(stored.n <= 20, `expected topN cap, got ${stored.n}`)
sqliteDatabase.prepare(`DELETE FROM flow_buckets WHERE server_id = 9`).run()
sqliteDatabase.prepare(`DELETE FROM flow_minute_stats WHERE server_id = 9`).run()
sqliteDatabase.prepare(`DELETE FROM flow_minute_dims WHERE server_id = 9`).run()
sqliteDatabase.prepare(`DELETE FROM flow_daily_dims WHERE server_id = 9`).run()
applyHeartbeatForTests({
bound: true,
address: "127.0.0.1:4739",
packetsReceived: 42,
lastExporterIp: "10.255.254.3",
lastError: "",
lastDatagramAt: new Date().toISOString(),
pendingSize: 1,
dropped: 0,
rowsStored: 1,
workerAlive: true,
rings: [],
})
assert.equal(getFlowListenerState().bound, true)
assert.equal(getFlowRuntimeCounters().packetsReceived, 42)
assert.equal(getFlowWorkerHealth().alive, false)
setWantListenForTests(true)
assert.equal(simulateWorkerExitForTests(), 1)
assert.equal(getFlowListenerState().bound, false)
setWantListenForTests(false)
resetFlowRingsForTests()
resetIfaceCacheForTests()
setRefreshIfacesForTests(null)
console.log("traffic-flow-ingest.test.ts: ok")
+359 -292
View File
@@ -1,317 +1,302 @@
import { createSocket, type Socket } from "node:dgram"
import { desc, eq, gte, sql } from "drizzle-orm"
import { db } from "../db/index.js"
import { Worker } from "node:worker_threads"
import { existsSync, statSync } from "node:fs"
import path from "node:path"
import { gte, sql } from "drizzle-orm"
import { beginSqliteExclusiveOp, db, endSqliteExclusiveOp, sqliteDatabase } from "../db/index.js"
import { env } from "../config.js"
import { flowBuckets, servers } from "../db/schema.js"
import type { FlowStatsDto, FlowTalkerDto } from "@mmapp/contracts/traffic-flow"
import { parseFlowPacket, protoName, type ParsedFlow } from "./traffic-flow-parse.js"
import { pickServerIdForExporter } from "./traffic-flow-map-exporter.js"
import type { FlowPurgeDto, FlowStatsDto, FlowTalkerDto } from "@mmapp/contracts/traffic-flow"
import { protoName, type ParsedFlowInput } from "./traffic-flow-parse.js"
import type { CollectorHeartbeat, ExporterMapPayload, MainToWorker, WorkerToMain } from "./traffic-flow-collector-ipc.js"
import {
attachEngineSqlite,
applyRingSnapshot,
configureEngine,
flushPending,
getEngineStats,
getRingMbps as engineGetRingMbps,
ingestParsedFlowsForServerForTests as engineIngestForServer,
lastFlushUsedTransactionForTests as engineLastFlushTx,
listLiveFlowRows as engineListLive,
peekPendingFlows,
queueParsedFlows,
resetEngineForTests,
resolveServerId,
rollFlowRings,
setExporterResolveCtx,
type PendingFlowRow,
} from "./traffic-flow-engine.js"
import {
getTrafficFlowSettingsRow,
listHostPeers,
recordFlowListenerError,
recordFlowPacket,
resetFlowIngestCounters,
} from "./traffic-flow-settings.js"
import { ifaceCacheFresh, refreshServerIfaces, resolveIfaceName } from "./traffic-flow-ifaces.js"
import { applicationName } from "./traffic-flow-apps.js"
import { resolveIfaceName } from "./traffic-flow-ifaces.js"
export type { PendingFlowRow }
export interface FlowListenerState {
bound: boolean
address: string | null
}
export interface PendingFlowRow {
serverId: number
bucketAt: string
src: string
dst: string
proto: number
srcPort: number
dstPort: number
bytes: number
packets: number
inIface: string
outIface: string
export interface FlowWorkerHealth {
alive: boolean
bound: boolean
pendingSize: number
dropped: number
packetsReceived: number
}
const TICK_MS = 2_000
const RING_LEN = 60
let socket: Socket | null = null
let worker: Worker | null = null
let restartTimer: ReturnType<typeof setTimeout> | null = null
let restartAttempts = 0
let lastHeartbeat: CollectorHeartbeat | null = null
let state: FlowListenerState = { bound: false, address: null }
const pending = new Map<string, {
serverId: number
bucketAt: string
flow: ParsedFlow
bytes: number
packets: number
}>()
let flushTimer: ReturnType<typeof setInterval> | null = null
let wantListen = false
const tickAccum = new Map<string, { inBytes: number; outBytes: number }>()
const rings = new Map<string, { inBps: number[]; outBps: number[] }>()
attachEngineSqlite(sqliteDatabase)
export function getFlowListenerState(): FlowListenerState {
return state
function workerFileUrl(): URL {
const ts = import.meta.url.includes(".ts")
return new URL(
ts ? "./traffic-flow-collector-worker.ts" : "./traffic-flow-collector-worker.js",
import.meta.url,
)
}
function minuteBucketIso(at = Date.now()): string {
const d = new Date(at)
d.setSeconds(0, 0)
return d.toISOString()
}
function ringKey(serverId: number, iface: string): string {
return `${serverId}\0${iface || "__all__"}`
}
function bumpTick(key: string, inBytes: number, outBytes: number): void {
const prev = tickAccum.get(key) ?? { inBytes: 0, outBytes: 0 }
prev.inBytes += inBytes
prev.outBytes += outBytes
tickAccum.set(key, prev)
}
function addToTick(serverId: number, inIface: string, outIface: string, bytes: number): void {
bumpTick(ringKey(serverId, "__all__"), bytes, 0)
if (inIface) bumpTick(ringKey(serverId, inIface), bytes, 0)
if (outIface && outIface !== inIface) bumpTick(ringKey(serverId, outIface), 0, bytes)
}
function emptyRing(): { inBps: number[]; outBps: number[] } {
return { inBps: Array(RING_LEN).fill(0), outBps: Array(RING_LEN).fill(0) }
}
export function rollFlowRings(): void {
const keys = new Set([...tickAccum.keys(), ...rings.keys()])
const sec = TICK_MS / 1000
for (const key of keys) {
const acc = tickAccum.get(key) ?? { inBytes: 0, outBytes: 0 }
tickAccum.delete(key)
const inBps = (acc.inBytes * 8) / sec
const outBps = (acc.outBytes * 8) / sec
let ring = rings.get(key)
if (!ring) {
ring = emptyRing()
rings.set(key, ring)
}
ring.inBps.push(inBps)
ring.inBps.shift()
ring.outBps.push(outBps)
ring.outBps.shift()
}
}
export function getRingMbps(serverId: number, iface = "__all__"): {
rx: number[]
tx: number[]
rxNow: number
txNow: number
} {
const ring = rings.get(ringKey(serverId, iface))
const scale = 1_000_000
if (!ring) {
return { rx: Array(RING_LEN).fill(0), tx: Array(RING_LEN).fill(0), rxNow: 0, txNow: 0 }
}
return {
rx: ring.inBps.map((b) => b / scale),
tx: ring.outBps.map((b) => b / scale),
rxNow: (ring.inBps[RING_LEN - 1] ?? 0) / scale,
txNow: (ring.outBps[RING_LEN - 1] ?? 0) / scale,
}
}
function resolveServerId(exporterIp: string): number | null {
export function buildExporterMapPayload(): ExporterMapPayload {
const settings = getTrafficFlowSettingsRow()
const rows = db.select({
id: servers.id,
host: servers.host,
mgmtTunnelIp: servers.mgmtTunnelIp,
}).from(servers).all()
const byTunnelIp = new Map<string, number>()
const hostIps = new Map<string, number>()
const byTunnelIp: Array<[string, number]> = []
const hostIps: Array<[string, number]> = []
for (const row of rows) {
if (row.mgmtTunnelIp) byTunnelIp.set(row.mgmtTunnelIp, row.id)
if (/^\d{1,3}(?:\.\d{1,3}){3}$/.test(row.host)) hostIps.set(row.host, row.id)
if (row.mgmtTunnelIp) byTunnelIp.push([row.mgmtTunnelIp, row.id])
if (/^\d{1,3}(?:\.\d{1,3}){3}$/.test(row.host)) hostIps.push([row.host, row.id])
}
return pickServerIdForExporter({
exporterIp,
return {
overlayPrefix: settings.prefix,
byTunnelIp,
peers: listHostPeers(),
hostIps,
}
}
function applyExporterCtxFromDb(): void {
const payload = buildExporterMapPayload()
setExporterResolveCtx({
overlayPrefix: payload.overlayPrefix,
byTunnelIp: new Map(payload.byTunnelIp),
peers: payload.peers,
hostIps: new Map(payload.hostIps),
})
}
function queueFlows(exporterIp: string, flows: ParsedFlow[]): boolean {
const serverId = resolveServerId(exporterIp)
if (serverId == null) return false
const needsRefresh = !ifaceCacheFresh(serverId)
|| flows.some((f) => resolveIfaceName(serverId, f.inIface).name.startsWith("#"))
if (needsRefresh) void refreshServerIfaces(serverId, true)
const bucketAt = minuteBucketIso()
for (const flow of flows) {
addToTick(serverId, flow.inIface, flow.outIface, flow.bytes)
const key = `${serverId}\0${bucketAt}\0${flow.src}\0${flow.dst}\0${flow.proto}\0${flow.srcPort}\0${flow.dstPort}\0${flow.inIface}`
const prev = pending.get(key)
if (prev) {
prev.bytes += flow.bytes
prev.packets += flow.packets
} else {
pending.set(key, {
serverId,
bucketAt,
flow: { ...flow },
bytes: flow.bytes,
packets: flow.packets,
})
}
function postToWorker(msg: MainToWorker): void {
worker?.postMessage(msg)
}
function handleWorkerMessage(msg: WorkerToMain): void {
if (msg.type === "heartbeat") {
lastHeartbeat = msg.payload
state = { bound: msg.payload.bound, address: msg.payload.address }
applyRingSnapshot(msg.payload.rings)
restartAttempts = 0
return
}
if (msg.type === "error") {
lastHeartbeat = lastHeartbeat
? { ...lastHeartbeat, lastError: msg.payload.message, workerAlive: true }
: null
}
return true
}
export function peekPendingFlows(): PendingFlowRow[] {
return [...pending.values()].map((row) => ({
serverId: row.serverId,
bucketAt: row.bucketAt,
src: row.flow.src || "0.0.0.0",
dst: row.flow.dst || "0.0.0.0",
proto: row.flow.proto,
srcPort: row.flow.srcPort,
dstPort: row.flow.dstPort,
bytes: row.bytes,
packets: row.packets,
inIface: row.flow.inIface,
outIface: row.flow.outIface,
}))
}
function flushPending() {
if (pending.size === 0) return
function spawnWorker(): void {
stopWorkerProcess()
const settings = getTrafficFlowSettingsRow()
const topN = Math.max(20, settings.topN)
const cutoff = new Date(Date.now() - settings.retentionHours * 3600_000).toISOString()
const rows = [...pending.values()]
pending.clear()
configureEngine({ topN: settings.topN, retentionHours: settings.retentionHours })
applyExporterCtxFromDb()
const w = new Worker(workerFileUrl(), { execArgv: process.execArgv })
w.on("message", (msg: WorkerToMain) => handleWorkerMessage(msg))
w.on("error", (err) => {
state = { bound: false, address: null }
lastHeartbeat = lastHeartbeat
? { ...lastHeartbeat, workerAlive: false, lastError: err.message, bound: false }
: {
bound: false,
address: null,
packetsReceived: 0,
lastExporterIp: null,
lastError: err.message,
lastDatagramAt: null,
pendingSize: 0,
dropped: 0,
rowsStored: 0,
workerAlive: false,
rings: [],
}
})
w.on("exit", (code) => {
worker = null
state = { bound: false, address: null }
if (!wantListen) return
const delay = Math.min(30_000, 1000 * 2 ** restartAttempts)
restartAttempts += 1
restartTimer = setTimeout(() => {
if (wantListen) spawnWorker()
}, delay)
void code
})
worker = w
const host = process.env.FLOW_LISTEN_HOST?.trim() || settings.collectorIp || "127.0.0.1"
postToWorker({
type: "start",
payload: {
dbPath: env.DATABASE_PATH,
listenHost: host,
listenPort: settings.flowListenPort,
topN: settings.topN,
retentionHours: settings.retentionHours,
exporterMap: buildExporterMapPayload(),
},
})
}
for (const row of rows) {
function stopWorkerProcess(): void {
if (restartTimer) {
clearTimeout(restartTimer)
restartTimer = null
}
if (worker) {
try {
db.insert(flowBuckets).values({
serverId: row.serverId,
bucketAt: row.bucketAt,
src: row.flow.src || "0.0.0.0",
dst: row.flow.dst || "0.0.0.0",
proto: row.flow.proto,
srcPort: row.flow.srcPort,
dstPort: row.flow.dstPort,
bytes: row.bytes,
packets: row.packets,
inIface: row.flow.inIface,
}).onConflictDoUpdate({
target: [
flowBuckets.serverId,
flowBuckets.bucketAt,
flowBuckets.src,
flowBuckets.dst,
flowBuckets.proto,
flowBuckets.srcPort,
flowBuckets.dstPort,
flowBuckets.inIface,
],
set: {
bytes: sql`${flowBuckets.bytes} + excluded.bytes`,
packets: sql`${flowBuckets.packets} + excluded.packets`,
},
}).run()
postToWorker({ type: "stop" })
void worker.terminate()
} catch {
// ignore single-row failures
}
}
db.delete(flowBuckets).where(sql`${flowBuckets.bucketAt} < ${cutoff}`).run()
const latest = db.select({ bucketAt: flowBuckets.bucketAt }).from(flowBuckets)
.orderBy(desc(flowBuckets.bucketAt)).limit(1).all()[0]?.bucketAt
if (!latest) return
const latestRows = db.select().from(flowBuckets).where(eq(flowBuckets.bucketAt, latest)).all()
const byServer = new Map<number, typeof latestRows>()
for (const r of latestRows) {
const list = byServer.get(r.serverId) ?? []
list.push(r)
byServer.set(r.serverId, list)
}
for (const list of byServer.values()) {
if (list.length <= topN) continue
list.sort((a, b) => b.bytes - a.bytes)
for (const d of list.slice(topN)) {
db.delete(flowBuckets).where(eq(flowBuckets.id, d.id)).run()
/* ignore */
}
worker = null
}
}
function onTick() {
rollFlowRings()
flushPending()
export function reattachFlowSqlite(): void {
attachEngineSqlite(sqliteDatabase)
}
function onMessage(msg: Buffer, rinfo: { address: string }) {
try {
const flows = parseFlowPacket(msg, rinfo.address)
recordFlowPacket(rinfo.address)
if (!flows.length) return
if (!queueFlows(rinfo.address, flows)) {
recordFlowListenerError(
`IPFIX от ${rinfo.address}: нет jump-host с адресом wg-flow. Docker SNAT (172.x) при нескольких JH не различим.`,
)
return
}
recordFlowListenerError("")
} catch (e) {
recordFlowListenerError(e instanceof Error ? e.message : String(e))
}
export function applyHeartbeatForTests(payload: CollectorHeartbeat): void {
handleWorkerMessage({ type: "heartbeat", payload })
}
export function stopTrafficFlowListener() {
if (flushTimer) {
clearInterval(flushTimer)
flushTimer = null
}
flushPending()
if (socket) {
try { socket.close() } catch { /* ignore */ }
socket = null
}
export function simulateWorkerExitForTests(): number {
worker = null
state = { bound: false, address: null }
lastHeartbeat = lastHeartbeat ? { ...lastHeartbeat, workerAlive: false, bound: false } : null
if (!wantListen) return restartAttempts
restartAttempts += 1
return restartAttempts
}
export function setWantListenForTests(value: boolean): void {
wantListen = value
}
export function getFlowListenerState(): FlowListenerState {
return state
}
export function getFlowWorkerHealth(): FlowWorkerHealth {
const hb = lastHeartbeat
const mem = getEngineStats()
return {
alive: Boolean(worker) && (hb?.workerAlive ?? false),
bound: state.bound,
pendingSize: hb?.pendingSize ?? mem.pendingSize,
dropped: hb?.dropped ?? mem.dropped,
packetsReceived: hb?.packetsReceived ?? mem.packetsReceived,
}
}
export function getFlowRuntimeCounters() {
const settings = getTrafficFlowSettingsRow()
const hb = lastHeartbeat
return {
packetsReceived: hb?.packetsReceived ?? settings.packetsReceived,
lastExporterIp: hb?.lastExporterIp ?? settings.lastExporterIp ?? null,
lastError: (hb?.lastError ?? settings.lastError) || null,
lastDatagramAt: hb?.lastDatagramAt ?? settings.lastDatagramAt ?? null,
dropped: hb?.dropped ?? 0,
}
}
export function startTrafficFlowListener() {
stopTrafficFlowListener()
const settings = getTrafficFlowSettingsRow()
if (!settings.enabled) {
wantListen = false
state = { bound: false, address: null }
return
}
const host = process.env.FLOW_LISTEN_HOST?.trim() || settings.collectorIp || "127.0.0.1"
const port = settings.flowListenPort
const sock = createSocket("udp4")
sock.on("error", (err) => {
recordFlowListenerError(err.message)
state = { bound: false, address: null }
})
sock.on("message", onMessage)
sock.bind(port, host, () => {
state = { bound: true, address: `${host}:${port}` }
recordFlowListenerError("")
})
socket = sock
flushTimer = setInterval(onTick, TICK_MS)
wantListen = true
spawnWorker()
}
export function stopTrafficFlowListener() {
wantListen = false
stopWorkerProcess()
try {
flushPending()
} catch {
/* ignore */
}
state = { bound: false, address: null }
}
export function refreshFlowExporterMap(): void {
applyExporterCtxFromDb()
postToWorker({ type: "updateExporterMap", payload: buildExporterMapPayload() })
}
export function getRingMbps(serverId: number, iface = "__all__") {
return engineGetRingMbps(serverId, iface)
}
function mergeInto(map: Map<string, PendingFlowRow>, row: PendingFlowRow): void {
const key = `${row.serverId}|${row.bucketAt}|${row.src}|${row.dst}|${row.proto}|${row.srcPort}|${row.dstPort}|${row.inIface}`
const prev = map.get(key)
if (prev) {
prev.bytes += row.bytes
prev.packets += row.packets
if (row.outIface && !prev.outIface) prev.outIface = row.outIface
if (row.nextHop && !prev.nextHop) prev.nextHop = row.nextHop
if (row.flowStartMs && (!prev.flowStartMs || row.flowStartMs < prev.flowStartMs)) prev.flowStartMs = row.flowStartMs
if (row.flowEndMs > (prev.flowEndMs ?? 0)) prev.flowEndMs = row.flowEndMs
return
}
map.set(key, { ...row })
}
export function listLiveFlowRows(sinceIso: string): PendingFlowRow[] {
if (worker && lastHeartbeat?.workerAlive) {
return listStoredFlowRows(sinceIso)
}
return engineListLive(sinceIso)
}
export function listStoredFlowRows(sinceIso: string): PendingFlowRow[] {
const stored = db.select().from(flowBuckets).where(gte(flowBuckets.bucketAt, sinceIso)).all()
const settings = getTrafficFlowSettingsRow()
const cap = Math.max(20, settings.topN) * 60
const stored = db.select().from(flowBuckets)
.where(gte(flowBuckets.bucketAt, sinceIso))
.orderBy(sql`${flowBuckets.bytes} DESC`)
.limit(cap)
.all()
const merged = new Map<string, PendingFlowRow>()
for (const r of stored) {
const key = `${r.serverId}|${r.bucketAt}|${r.src}|${r.dst}|${r.proto}|${r.srcPort}|${r.dstPort}|${r.inIface}`
merged.set(key, {
mergeInto(merged, {
serverId: r.serverId,
bucketAt: r.bucketAt,
src: r.src,
@@ -322,27 +307,31 @@ export function listStoredFlowRows(sinceIso: string): PendingFlowRow[] {
bytes: r.bytes,
packets: r.packets,
inIface: r.inIface,
outIface: "",
outIface: r.outIface ?? "",
nextHop: r.nextHop ?? "",
flowStartMs: r.flowStartMs ?? 0,
flowEndMs: r.flowEndMs ?? 0,
})
}
for (const p of peekPendingFlows()) {
if (p.bucketAt < sinceIso) continue
const key = `${p.serverId}|${p.bucketAt}|${p.src}|${p.dst}|${p.proto}|${p.srcPort}|${p.dstPort}|${p.inIface}`
const prev = merged.get(key)
if (prev) {
prev.bytes += p.bytes
prev.packets += p.packets
} else {
merged.set(key, { ...p })
if (!worker) {
for (const p of peekPendingFlows()) {
if (p.bucketAt < sinceIso) continue
mergeInto(merged, p)
}
}
return [...merged.values()]
}
export function listFlowRowsForWindow(minutes: number): PendingFlowRow[] {
const sinceIso = new Date(Date.now() - minutes * 60_000).toISOString()
if (minutes <= 15 && !worker) return listLiveFlowRows(sinceIso)
return listStoredFlowRows(sinceIso)
}
export function listFlowTalkers(minutes = 5): FlowStatsDto {
const settings = getTrafficFlowSettingsRow()
const rangeStart = new Date(Date.now() - minutes * 60_000).toISOString()
const rows = listStoredFlowRows(rangeStart)
const runtime = getFlowRuntimeCounters()
const rows = listFlowRowsForWindow(minutes)
const serverRows = db.select().from(servers).all()
const nameById = new Map(serverRows.map((s) => [s.id, s.name || s.host]))
const agg = new Map<string, FlowTalkerDto & { rawBytes: number }>()
@@ -406,46 +395,124 @@ export function listFlowTalkers(minutes = 5): FlowStatsDto {
uniqueDst: dsts.size,
topProto,
talkers,
lastExporterIp: settings.lastExporterIp ?? null,
lastError: settings.lastError || null,
packetsReceived: settings.packetsReceived,
lastDatagramAt: settings.lastDatagramAt ?? null,
lastExporterIp: runtime.lastExporterIp,
lastError: runtime.lastError,
packetsReceived: runtime.packetsReceived,
lastDatagramAt: runtime.lastDatagramAt,
listenerBound: state.bound,
listenerAddress: state.address,
}
}
export function ingestParsedFlowsForTests(exporterIp: string, flows: ParsedFlow[]) {
queueFlows(exporterIp, flows)
export function ingestParsedFlowsForTests(exporterIp: string, flows: ParsedFlowInput[]) {
applyExporterCtxFromDb()
const serverId = resolveServerId(exporterIp)
if (serverId == null) return
queueParsedFlows(serverId, flows)
rollFlowRings()
flushPending()
}
/** Кладёт потоки в pending без flush в SQLite — для юнит-тестов аналитики. */
export function ingestParsedFlowsForServerForTests(serverId: number, flows: ParsedFlow[]) {
const bucketAt = minuteBucketIso()
for (const flow of flows) {
addToTick(serverId, flow.inIface, flow.outIface, flow.bytes)
const key = `${serverId}\0${bucketAt}\0${flow.src}\0${flow.dst}\0${flow.proto}\0${flow.srcPort}\0${flow.dstPort}\0${flow.inIface}`
const prev = pending.get(key)
if (prev) {
prev.bytes += flow.bytes
prev.packets += flow.packets
} else {
pending.set(key, {
serverId,
bucketAt,
flow: { ...flow },
bytes: flow.bytes,
packets: flow.packets,
})
}
}
rollFlowRings()
export function ingestParsedFlowsForServerForTests(serverId: number, flows: ParsedFlowInput[]) {
engineIngestForServer(serverId, flows)
}
export function resetFlowRingsForTests() {
tickAccum.clear()
rings.clear()
pending.clear()
resetEngineForTests()
attachEngineSqlite(sqliteDatabase)
lastHeartbeat = null
wantListen = false
restartAttempts = 0
}
export function lastFlushUsedTransactionForTests(): boolean {
return engineLastFlushTx()
}
export function flushPendingForTests(): void {
flushPending()
}
function tableCount(name: string): number {
const row = sqliteDatabase.prepare(`SELECT COUNT(*) AS n FROM ${name}`).get() as { n: number }
return Number(row?.n) || 0
}
function dbFileBytes(): number {
const resolved = path.resolve(process.cwd(), env.DATABASE_PATH)
if (!existsSync(resolved)) return 0
return statSync(resolved).size
}
async function stopWorkerProcessAsync(): Promise<void> {
if (restartTimer) {
clearTimeout(restartTimer)
restartTimer = null
}
if (!worker) return
const current = worker
worker = null
try {
current.postMessage({ type: "stop" })
await current.terminate()
} catch {
/* ignore */
}
}
/** Удаляет сессии, minute/daily rollup и сжимает SQLite. Ключи WG и пиры JH не трогает. */
export async function purgeTrafficFlowStore(): Promise<FlowPurgeDto> {
beginSqliteExclusiveOp()
try {
wantListen = false
await stopWorkerProcessAsync()
resetEngineForTests()
attachEngineSqlite(sqliteDatabase)
lastHeartbeat = null
state = { bound: false, address: null }
const fileBytesBefore = dbFileBytes()
const deleted = {
buckets: tableCount("flow_buckets"),
minuteStats: tableCount("flow_minute_stats"),
minuteDims: tableCount("flow_minute_dims"),
dailyDims: tableCount("flow_daily_dims"),
}
sqliteDatabase.exec(`
DELETE FROM flow_buckets;
DELETE FROM flow_minute_stats;
DELETE FROM flow_minute_dims;
DELETE FROM flow_daily_dims;
`)
resetFlowIngestCounters()
try {
sqliteDatabase.pragma("wal_checkpoint(TRUNCATE)")
} catch {
/* ignore */
}
let vacuumed = false
try {
sqliteDatabase.exec("VACUUM")
vacuumed = true
} catch {
vacuumed = false
}
return {
ok: true,
deleted,
fileBytesBefore,
fileBytesAfter: dbFileBytes(),
vacuumed,
}
} finally {
try {
startTrafficFlowListener()
} catch {
/* ingest мог остаться выключенным */
}
endSqliteExclusiveOp()
}
}
export { peekPendingFlows }
export { setPendingCapForTests } from "./traffic-flow-engine.js"
export { maybeRefreshIfaces, setRefreshIfacesForTests } from "./traffic-flow-ifaces.js"
@@ -0,0 +1,157 @@
import assert from "node:assert/strict"
import { rememberServerIfaces, resetIfaceCacheForTests } from "./traffic-flow-ifindex.js"
import {
ingestParsedFlowsForServerForTests,
resetFlowRingsForTests,
} from "./traffic-flow-ingest.js"
import { buildFlowMapHops } from "./traffic-flow-map-hops.js"
import { seedFlowTopologyForTests, type FlowTopology } from "./traffic-flow-topology.js"
import { disableCatalogFetchForTests, resetFlowCatalogForTests } from "./traffic-flow-classify.js"
import {
disableRipeEnqueueForTests,
disableRipePersistForTests,
resetRipeCacheForTests,
} from "./traffic-flow-ripe.js"
disableCatalogFetchForTests()
resetFlowCatalogForTests()
disableRipePersistForTests()
resetRipeCacheForTests()
disableRipeEnqueueForTests()
const topo: FlowTopology = {
clientIfaces: new Map([[7, new Set(["gre-client"])]]),
clientByIface: new Map([["7|gre-client", {
userId: "u1",
login: "alice",
name: "Alice",
serverId: 7,
interfaceName: "gre-client",
}]]),
enNodes: [{ id: 9, name: "NSK-EN", hosts: ["198.51.100.1"] }],
enHosts: new Set(["198.51.100.1"]),
jhHosts: new Set(["203.0.113.10"]),
wanIfaces: new Map([[3, new Set(["ether1-rt"])]]),
plane: {
clientIfaceNames: new Set(["gre-client"]),
enHosts: new Set(["198.51.100.1"]),
jhHosts: new Set(["203.0.113.10"]),
},
}
resetFlowRingsForTests()
resetIfaceCacheForTests()
seedFlowTopologyForTests(topo)
rememberServerIfaces(7, [
{ ".id": "*2", name: "gre-client" },
{ ".id": "*3", name: "gre-jh-en" },
{ ".id": "*A", name: "wg-flow" },
])
rememberServerIfaces(3, [
{ ".id": "*1", name: "ether1-rt" },
])
ingestParsedFlowsForServerForTests(7, [
{
src: "10.100.1.17",
dst: "8.8.8.8",
proto: 6,
srcPort: 51234,
dstPort: 443,
bytes: 12_000,
packets: 10,
inIface: "2",
outIface: "3",
nextHop: "198.51.100.1",
},
{
src: "203.0.113.10",
dst: "198.51.100.1",
proto: 47,
srcPort: 0,
dstPort: 0,
bytes: 5_000_000,
packets: 4000,
inIface: "3",
outIface: "3",
},
{
src: "10.100.1.17",
dst: "10.100.1.18",
proto: 6,
srcPort: 50000,
dstPort: 443,
bytes: 8000,
packets: 8,
inIface: "2",
outIface: "2",
},
{
src: "10.255.254.1",
dst: "10.255.254.2",
proto: 17,
srcPort: 4739,
dstPort: 2055,
bytes: 400,
packets: 2,
inIface: "10",
outIface: "",
},
])
ingestParsedFlowsForServerForTests(3, [
{
src: "192.168.1.10",
dst: "8.8.4.4",
proto: 6,
srcPort: 40000,
dstPort: 443,
bytes: 3000,
packets: 4,
inIface: "1",
outIface: "1",
},
])
try {
const def = buildFlowMapHops({ minutes: 5 })
assert.equal(def.excludeOverlayApplied, true)
assert.equal(def.excludeMeshApplied, true)
assert.equal(def.dedupApplied, true)
assert.equal(def.windowSec, 300)
const payloadGre = def.hops.find((h) => h.kind === "gre" && h.fromId === "7" && h.toId === "9")
assert.ok(payloadGre, "payload JH→EN hop")
assert.equal(payloadGre.bytes, 12_000)
assert.equal(payloadGre.bps, (12_000 * 8) / 300)
assert.equal(payloadGre.bpsFwd, (12_000 * 8) / 300)
assert.equal(payloadGre.iface, "gre-jh-en")
const greIface = def.hops.find((h) => h.kind === "iface" && h.iface === "gre-jh-en" && h.fromId === "7")
assert.ok(greIface)
assert.equal(greIface.bytes, 12_000)
assert.equal(greIface.bpsFwd, (12_000 * 8) / 300)
assert.ok(!def.hops.some((h) => h.bytes >= 5_000_000), "overlay GRE proto 47 excluded")
assert.ok(!def.hops.some((h) => h.iface === "wg-flow"), "mgmt wg-flow excluded")
const clientIngress = def.hops.find((h) => h.iface === "gre-client" && h.fromId === "7" && h.kind === "iface")
assert.ok(clientIngress, "payload ingress on client iface")
assert.equal(clientIngress.bytes, 12_000)
const wan = def.hops.find((h) => h.kind === "wan" && h.fromId === "3" && h.iface === "ether1-rt")
assert.ok(wan, "WAN hop from home-router")
assert.equal(wan.bytes, 3000)
const withAll = buildFlowMapHops({ minutes: 5, excludeOverlay: false, excludeMesh: false })
const overlayIface = withAll.hops.find((h) => h.iface === "gre-jh-en" && h.fromId === "7")
assert.ok(overlayIface && overlayIface.bytes >= 5_000_000)
const meshIface = withAll.hops.find((h) => h.iface === "gre-client" && h.fromId === "7" && h.kind === "iface")
assert.ok(meshIface && meshIface.bytes >= 20_000)
} finally {
seedFlowTopologyForTests(null)
resetFlowRingsForTests()
resetIfaceCacheForTests()
resetRipeCacheForTests()
resetFlowCatalogForTests()
}
console.log("traffic-flow-map-hops.test.ts: ok")
@@ -0,0 +1,216 @@
import { eq } from "drizzle-orm"
import type { FlowMapHop, FlowMapHopsDto } from "@mmapp/contracts/traffic-flow"
import { db } from "../db/index.js"
import { servers, userInterfaceBindings } from "../db/schema.js"
import { flowRowMatchesFilter } from "./traffic-flow-apps.js"
import { dedupFlowRowsMaxBytes } from "./traffic-flow-dedup.js"
import { getFlowListenerState, listFlowRowsForWindow } from "./traffic-flow-ingest.js"
import { resolveIfaceName } from "./traffic-flow-ifaces.js"
import { classifyFlowPlane, shouldKeepPlane } from "./traffic-flow-planes.js"
import { loadFlowTopology, resolveEn } from "./traffic-flow-topology.js"
export interface FlowMapHopsQuery {
minutes: number
serverId?: number
userId?: string
iface?: string
dedup?: boolean
excludeMesh?: boolean
excludeOverlay?: boolean
}
interface HopAcc {
fromId: string
fromLabel: string
toId: string
toLabel: string
kind: FlowMapHop["kind"]
iface?: string
bytes: number
bytesFwd: number
bytesRev: number
}
function userIfaceAllow(userId: string): Map<number, Set<string>> | null {
if (!userId) return null
const binds = db.select().from(userInterfaceBindings).where(eq(userInterfaceBindings.userId, userId)).all()
const allow = new Map<number, Set<string>>()
for (const b of binds) {
const set = allow.get(b.serverId) ?? new Set<string>()
set.add(b.interfaceName)
allow.set(b.serverId, set)
}
return allow
}
function ifaceUsable(name: string): boolean {
return Boolean(name) && name !== "—"
}
function bump(acc: Map<string, HopAcc>, key: string, seed: Omit<HopAcc, "bytes" | "bytesFwd" | "bytesRev">, bytes: number, dir: "fwd" | "rev" | "both"): void {
const prev = acc.get(key)
const addFwd = dir === "fwd" || dir === "both" ? bytes : 0
const addRev = dir === "rev" || dir === "both" ? bytes : 0
if (prev) {
prev.bytes += bytes
prev.bytesFwd += addFwd
prev.bytesRev += addRev
if (seed.iface && !prev.iface) prev.iface = seed.iface
return
}
acc.set(key, {
...seed,
bytes,
bytesFwd: addFwd,
bytesRev: addRev,
})
}
function toHop(a: HopAcc, windowSec: number): FlowMapHop {
return {
fromId: a.fromId,
fromLabel: a.fromLabel,
toId: a.toId,
toLabel: a.toLabel,
kind: a.kind,
...(a.iface ? { iface: a.iface } : {}),
bytes: a.bytes,
bps: (a.bytes * 8) / windowSec,
bpsFwd: (a.bytesFwd * 8) / windowSec,
bpsRev: (a.bytesRev * 8) / windowSec,
}
}
/** Hop-rates для карты сети: те же фильтры, что у общего NetFlow (dedup / mesh / overlay). */
export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
const windowSec = Math.max(60, q.minutes * 60)
const raw = listFlowRowsForWindow(q.minutes)
const allow = q.userId ? userIfaceAllow(q.userId) : null
const serverRows = db.select().from(servers).all()
const nameById = new Map(serverRows.map((s) => [s.id, s.name || s.host]))
const ifaceFilter = q.iface && q.iface !== "__all__" ? q.iface : ""
const wantDedup = q.dedup !== false && !ifaceFilter
const excludeMesh = q.excludeMesh !== false
const excludeOverlay = q.excludeOverlay !== false
const topo = loadFlowTopology()
const matched = []
for (const r of raw) {
const resolved = resolveIfaceName(r.serverId, r.inIface)
const outResolved = resolveIfaceName(r.serverId, r.outIface)
if (!flowRowMatchesFilter(r, resolved.name, q, allow)) continue
const plane = classifyFlowPlane({
src: r.src,
dst: r.dst,
proto: r.proto,
srcPort: r.srcPort,
dstPort: r.dstPort,
inIface: resolved.name,
outIface: outResolved.name,
}, topo.plane)
if (!shouldKeepPlane(plane, { excludeMesh, excludeOverlay })) continue
matched.push(r)
}
const working = wantDedup ? dedupFlowRowsMaxBytes(matched) : matched
const hops = new Map<string, HopAcc>()
for (const r of working) {
const inRes = resolveIfaceName(r.serverId, r.inIface)
const outRes = resolveIfaceName(r.serverId, r.outIface)
const inName = inRes.name
const outName = outRes.name
const fromId = String(r.serverId)
const fromLabel = nameById.get(r.serverId) ?? fromId
const wanSet = topo.wanIfaces.get(r.serverId)
const inOk = ifaceUsable(inName)
const outOk = ifaceUsable(outName)
const sameIface = inOk && outOk && inName.toLowerCase() === outName.toLowerCase()
if (sameIface) {
bump(hops, `iface|${fromId}|${inName.toLowerCase()}`, {
fromId,
fromLabel,
toId: "",
toLabel: "",
kind: "iface",
iface: inName,
}, r.bytes, "fwd")
} else {
if (inOk) {
bump(hops, `iface|${fromId}|${inName.toLowerCase()}`, {
fromId,
fromLabel,
toId: "",
toLabel: "",
kind: "iface",
iface: inName,
}, r.bytes, "rev")
}
if (outOk) {
bump(hops, `iface|${fromId}|${outName.toLowerCase()}`, {
fromId,
fromLabel,
toId: "",
toLabel: "",
kind: "iface",
iface: outName,
}, r.bytes, "fwd")
}
}
const enOut = ifaceUsable(outName) ? resolveEn(topo, r.nextHop, outName) : null
const enIn = ifaceUsable(inName) ? resolveEn(topo, "", inName) : null
const en = (enOut && enOut.id !== r.serverId ? enOut : null)
?? (enIn && enIn.id !== r.serverId ? enIn : null)
if (en) {
const toId = String(en.id)
const dir: "fwd" | "rev" = enOut && enOut.id === en.id ? "fwd" : "rev"
const greIface = dir === "fwd" && ifaceUsable(outName) ? outName : (ifaceUsable(inName) ? inName : undefined)
bump(hops, `gre|${fromId}|${toId}`, {
fromId,
fromLabel,
toId,
toLabel: en.name,
kind: "gre",
iface: greIface,
}, r.bytes, dir)
}
if (wanSet?.size) {
if (ifaceUsable(inName) && wanSet.has(inName)) {
bump(hops, `wan|${fromId}|${inName.toLowerCase()}`, {
fromId,
fromLabel,
toId: "",
toLabel: "",
kind: "wan",
iface: inName,
}, r.bytes, "rev")
}
if (ifaceUsable(outName) && wanSet.has(outName) && outName.toLowerCase() !== inName.toLowerCase()) {
bump(hops, `wan|${fromId}|${outName.toLowerCase()}`, {
fromId,
fromLabel,
toId: "",
toLabel: "",
kind: "wan",
iface: outName,
}, r.bytes, "fwd")
}
}
}
const listener = getFlowListenerState()
return {
hops: [...hops.values()]
.map((a) => toHop(a, windowSec))
.sort((a, b) => b.bytes - a.bytes),
live: listener.bound,
rangeMinutes: q.minutes,
windowSec,
dedupApplied: wantDedup,
excludeMeshApplied: excludeMesh,
excludeOverlayApplied: excludeOverlay,
}
}
+34 -1
View File
@@ -19,7 +19,7 @@ import {
getTrafficFlowSettingsRow,
upsertHostPeer,
} from "./traffic-flow-settings.js"
import { startTrafficFlowListener } from "./traffic-flow-ingest.js"
import { refreshFlowExporterMap, startTrafficFlowListener } from "./traffic-flow-ingest.js"
import { listTrafficFlowHostFiles } from "./traffic-flow-host-files.js"
const IFACE_NAME = "wg-flow"
@@ -97,6 +97,32 @@ async function ensureWgInputAccept(client: MikrotikClient, listenPort: number):
/** Официальный авто-source UDP IPFIX, не фильтр 0.0.0.0/0. */
export const FLOW_TARGET_SRC_AUTO = "0.0.0.0"
async function ensureIpfixFields(client: MikrotikClient): Promise<void> {
const body = toRosBody({
bytes: "yes",
packets: "yes",
"src-address": "yes",
"dst-address": "yes",
protocol: "yes",
"src-port": "yes",
"dst-port": "yes",
"in-interface": "yes",
"out-interface": "yes",
gateway: "yes",
"first-forwarded": "yes",
"last-forwarded": "yes",
"nat-src-address": "yes",
"nat-dst-address": "yes",
})
const rows = asRosArray<Record<string, unknown>>(await client.get("/ip/traffic-flow/ipfix"))
const id = rows[0] ? rosRowId(rows[0]) : ""
if (id) {
await patchRosPath(client, `/ip/traffic-flow/ipfix/${encodeRosId(id)}`, body)
return
}
await client.post("/ip/traffic-flow/ipfix/set", body)
}
async function ensureTrafficFlow(
client: MikrotikClient,
collectorIp: string,
@@ -116,6 +142,12 @@ async function ensureTrafficFlow(
await client.post("/ip/traffic-flow/set", body)
}
try {
await ensureIpfixFields(client)
} catch {
/* поля IPFIX опциональны на старых ROS */
}
const targets = asRosArray<Record<string, unknown>>(await client.get("/ip/traffic-flow/target"))
const existing = targets.find((t) => String(t["dst-address"] ?? "") === collectorIp)
const targetBody = toRosBody({
@@ -268,6 +300,7 @@ export async function applyFlowOverlay(
enableTrafficFlowIngest()
startTrafficFlowListener()
refreshFlowExporterMap()
steps.push("Коллектор IPFIX на MM включён")
return {
+103 -2
View File
@@ -1,5 +1,5 @@
import assert from "node:assert/strict"
import { parseFlowPacket, protoName, resetFlowTemplatesForTests } from "./traffic-flow-parse.js"
import { parseFlowPacket, protoName, resetFlowTemplatesForTests, templateExporterCountForTests } from "./traffic-flow-parse.js"
import { allocateOverlayAddress, FLOW_TARGET_SRC_AUTO, usablePublicHost } from "./traffic-flow-overlay.js"
function netflowV5One(): Buffer {
@@ -96,8 +96,109 @@ resetFlowTemplatesForTests()
parseFlowPacket(tpl, "10.255.254.3")
const named = parseFlowPacket(data, "10.255.254.3")
assert.equal(named.length, 1)
assert.equal(named[0]?.inIface, "ether1")
assert.equal(named[0]?.inIface, "13")
assert.equal(named[0]?.src, "10.1.1.8")
}
resetFlowTemplatesForTests()
{
const tpl = Buffer.alloc(16 + 20)
tpl.writeUInt16BE(10, 0)
tpl.writeUInt16BE(tpl.length, 2)
tpl.writeUInt16BE(2, 16)
tpl.writeUInt16BE(20, 18)
tpl.writeUInt16BE(256, 20)
tpl.writeUInt16BE(3, 22)
tpl.writeUInt16BE(8, 24)
tpl.writeUInt16BE(4, 26)
tpl.writeUInt16BE(12, 28)
tpl.writeUInt16BE(4, 30)
tpl.writeUInt16BE(82, 32)
tpl.writeUInt16BE(6, 34)
const data = Buffer.alloc(16 + 18)
data.writeUInt16BE(10, 0)
data.writeUInt16BE(data.length, 2)
data.writeUInt16BE(256, 16)
data.writeUInt16BE(18, 18)
data[20] = 10; data[21] = 1; data[22] = 1; data[23] = 8
data[24] = 8; data[25] = 8; data[26] = 8; data[27] = 8
data.write("ether1", 28)
parseFlowPacket(tpl, "10.255.254.4")
const namedOnly = parseFlowPacket(data, "10.255.254.4")
assert.equal(namedOnly[0]?.inIface, "ether1")
}
resetFlowTemplatesForTests()
{
const fieldSpecs: Array<[number, number]> = [
[8, 4],
[12, 4],
[10, 4],
[14, 4],
[15, 4],
[152, 8],
[153, 8],
[1, 4],
]
const tplSetLen = 4 + 4 + fieldSpecs.length * 4
const tpl = Buffer.alloc(16 + tplSetLen)
tpl.writeUInt16BE(10, 0)
tpl.writeUInt16BE(tpl.length, 2)
tpl.writeUInt16BE(2, 16)
tpl.writeUInt16BE(tplSetLen, 18)
tpl.writeUInt16BE(256, 20)
tpl.writeUInt16BE(fieldSpecs.length, 22)
let off = 24
for (const [type, len] of fieldSpecs) {
tpl.writeUInt16BE(type, off)
tpl.writeUInt16BE(len, off + 2)
off += 4
}
const recLen = fieldSpecs.reduce((n, [, len]) => n + len, 0)
const data = Buffer.alloc(16 + 4 + recLen)
data.writeUInt16BE(10, 0)
data.writeUInt16BE(data.length, 2)
data.writeUInt16BE(256, 16)
data.writeUInt16BE(4 + recLen, 18)
let d = 20
data[d] = 10; data[d + 1] = 100; data[d + 2] = 1; data[d + 3] = 17; d += 4
data[d] = 173; data[d + 1] = 194; data[d + 2] = 160; data[d + 3] = 163; d += 4
data.writeUInt32BE(13, d); d += 4
data.writeUInt32BE(42, d); d += 4
data[d] = 198; data[d + 1] = 51; data[d + 2] = 100; data[d + 3] = 1; d += 4
data.writeBigUInt64BE(1_700_000_000_000n, d); d += 8
data.writeBigUInt64BE(1_700_000_060_000n, d); d += 8
data.writeUInt32BE(1500, d)
parseFlowPacket(tpl, "10.255.254.5")
const extra = parseFlowPacket(data, "10.255.254.5")
assert.equal(extra.length, 1)
assert.equal(extra[0]?.src, "10.100.1.17")
assert.equal(extra[0]?.dst, "173.194.160.163")
assert.equal(extra[0]?.inIface, "13")
assert.equal(extra[0]?.outIface, "42")
assert.equal(extra[0]?.nextHop, "198.51.100.1")
assert.equal(extra[0]?.flowStartMs, 1_700_000_000_000)
assert.equal(extra[0]?.flowEndMs, 1_700_000_060_000)
assert.equal(extra[0]?.bytes, 1500)
}
resetFlowTemplatesForTests()
{
const tpl = Buffer.alloc(16 + 16 + 20)
tpl.writeUInt16BE(10, 0)
tpl.writeUInt16BE(tpl.length, 2)
tpl.writeUInt16BE(2, 16)
tpl.writeUInt16BE(16, 18)
tpl.writeUInt16BE(256, 20)
tpl.writeUInt16BE(2, 22)
tpl.writeUInt16BE(8, 24)
tpl.writeUInt16BE(4, 26)
tpl.writeUInt16BE(12, 28)
tpl.writeUInt16BE(4, 30)
for (let i = 0; i < 260; i++) {
parseFlowPacket(tpl, `203.0.${Math.floor(i / 250)}.${i % 250}`)
}
assert.ok(templateExporterCountForTests() <= 256)
}
console.log("traffic-flow-parse.test.ts: ok")
+116 -8
View File
@@ -8,6 +8,49 @@ export interface ParsedFlow {
packets: number
inIface: string
outIface: string
nextHop: string
flowStartMs: number
flowEndMs: number
natSrc: string
natDst: string
}
export type ParsedFlowInput = Partial<ParsedFlow> & Pick<ParsedFlow, "src" | "dst" | "proto" | "bytes">
export function emptyParsedFlow(): ParsedFlow {
return {
src: "",
dst: "",
proto: 0,
srcPort: 0,
dstPort: 0,
bytes: 0,
packets: 0,
inIface: "",
outIface: "",
nextHop: "",
flowStartMs: 0,
flowEndMs: 0,
natSrc: "",
natDst: "",
}
}
export function normalizeParsedFlow(flow: ParsedFlowInput): ParsedFlow {
return {
...emptyParsedFlow(),
...flow,
nextHop: flow.nextHop ?? "",
flowStartMs: flow.flowStartMs ?? 0,
flowEndMs: flow.flowEndMs ?? 0,
natSrc: flow.natSrc ?? "",
natDst: flow.natDst ?? "",
inIface: flow.inIface ?? "",
outIface: flow.outIface ?? "",
srcPort: flow.srcPort ?? 0,
dstPort: flow.dstPort ?? 0,
packets: flow.packets ?? 0,
}
}
interface FieldSpec {
@@ -19,8 +62,26 @@ interface Template {
fields: FieldSpec[]
}
const MAX_TEMPLATE_EXPORTERS = 256
const templatesByExporter = new Map<string, Map<number, Template>>()
function templatesForExporter(exporter: string): Map<number, Template> {
const existing = templatesByExporter.get(exporter)
if (existing) {
templatesByExporter.delete(exporter)
templatesByExporter.set(exporter, existing)
return existing
}
const created = new Map<number, Template>()
templatesByExporter.set(exporter, created)
while (templatesByExporter.size > MAX_TEMPLATE_EXPORTERS) {
const oldest = templatesByExporter.keys().next().value
if (oldest == null || oldest === exporter) break
templatesByExporter.delete(oldest)
}
return created
}
function ipv4(buf: Buffer, offset: number): string {
return `${buf[offset]}.${buf[offset + 1]}.${buf[offset + 2]}.${buf[offset + 3]}`
}
@@ -87,7 +148,7 @@ function parseNetflowV5(buf: Buffer): ParsedFlow[] {
const out: ParsedFlow[] = []
let off = 24
for (let i = 0; i < count && off + 48 <= buf.length; i++) {
out.push({
out.push(normalizeParsedFlow({
src: ipv4(buf, off),
dst: ipv4(buf, off + 4),
packets: buf.readUInt32BE(off + 16),
@@ -97,7 +158,7 @@ function parseNetflowV5(buf: Buffer): ParsedFlow[] {
proto: buf.readUInt8(off + 38),
inIface: String(buf.readUInt16BE(off + 12)),
outIface: String(buf.readUInt16BE(off + 14)),
})
}))
off += 48
}
return out
@@ -105,7 +166,7 @@ function parseNetflowV5(buf: Buffer): ParsedFlow[] {
function parseIpfixTemplates(exporter: string, buf: Buffer, setStart: number, setEnd: number, setId: number) {
let off = setStart + 4
const map = templatesByExporter.get(exporter) ?? new Map<number, Template>()
const map = templatesForExporter(exporter)
while (off + 4 <= setEnd) {
const templateId = buf.readUInt16BE(off)
const fieldCount = buf.readUInt16BE(off + 2)
@@ -148,6 +209,11 @@ function recordFromFields(
let inIface = ""
let outIface = ""
let ifaceName = ""
let nextHop = ""
let flowStartMs = 0
let flowEndMs = 0
let natSrc = ""
let natDst = ""
for (const f of fields) {
const field = consumeField(buf, off, f.length, limit)
if (!field) return null
@@ -165,11 +231,26 @@ function recordFromFields(
case 28:
if (data.length === 16 && !dst) dst = ipv6(data, 0)
break
case 15:
if (data.length === 4 && !nextHop) nextHop = ipv4(data, 0)
break
case 18:
if (data.length === 4 && !nextHop) nextHop = ipv4(data, 0)
break
case 62:
if (data.length === 16 && !nextHop) nextHop = ipv6(data, 0)
break
case 225:
if (data.length === 4 && !src) src = ipv4(data, 0)
if (data.length === 4) {
natSrc = ipv4(data, 0)
if (!src) src = natSrc
}
break
case 226:
if (data.length === 4 && !dst) dst = ipv4(data, 0)
if (data.length === 4) {
natDst = ipv4(data, 0)
if (!dst) dst = natDst
}
break
case 4:
proto = readUint(data, 0, data.length)
@@ -198,6 +279,24 @@ function recordFromFields(
case 14:
outIface = String(readUint(data, 0, data.length))
break
case 21:
if (!flowEndMs) flowEndMs = readUint(data, 0, data.length)
break
case 22:
if (!flowStartMs) flowStartMs = readUint(data, 0, data.length)
break
case 150:
if (!flowStartMs) flowStartMs = readUint(data, 0, data.length) * 1000
break
case 151:
if (!flowEndMs) flowEndMs = readUint(data, 0, data.length) * 1000
break
case 152:
flowStartMs = readUint(data, 0, data.length)
break
case 153:
flowEndMs = readUint(data, 0, data.length)
break
case 82:
ifaceName = data.toString("utf8").replace(/\0/g, "").trim()
break
@@ -206,8 +305,13 @@ function recordFromFields(
}
off = field.next
}
if (ifaceName) inIface = ifaceName
return { flow: { src, dst, proto, srcPort, dstPort, bytes, packets, inIface, outIface }, next: off }
if (ifaceName && !inIface) inIface = ifaceName
return {
flow: normalizeParsedFlow({
src, dst, proto, srcPort, dstPort, bytes, packets, inIface, outIface, nextHop, flowStartMs, flowEndMs, natSrc, natDst,
}),
next: off,
}
}
function parseDataRecords(
@@ -255,7 +359,7 @@ function parseNetflowV9(buf: Buffer, exporter: string): ParsedFlow[] {
const count = buf.readUInt16BE(2)
let off = 20
const out: ParsedFlow[] = []
const map = templatesByExporter.get(exporter) ?? new Map<number, Template>()
const map = templatesForExporter(exporter)
for (let s = 0; s < count && off + 4 <= buf.length; s++) {
const setId = buf.readUInt16BE(off)
const setLen = buf.readUInt16BE(off + 2)
@@ -308,3 +412,7 @@ export function protoName(proto: number): string {
export function resetFlowTemplatesForTests() {
templatesByExporter.clear()
}
export function templateExporterCountForTests(): number {
return templatesByExporter.size
}
@@ -0,0 +1,82 @@
import assert from "node:assert/strict"
import {
classifyFlowPlane,
classifyFlowPlaneLite,
flowBps,
shouldKeepPlane,
} from "./traffic-flow-planes.js"
const youtubeInner = {
src: "10.100.1.17",
dst: "173.194.160.163",
proto: 6,
srcPort: 51234,
dstPort: 443,
inIface: "gre-client",
outIface: "NSK-SERVHOST-RTK",
}
assert.equal(classifyFlowPlaneLite(youtubeInner), "payload")
assert.equal(classifyFlowPlane(youtubeInner), "payload")
const greOverlay = {
src: "203.0.113.10",
dst: "198.51.100.1",
proto: 47,
srcPort: 0,
dstPort: 0,
inIface: "ether1",
outIface: "NSK-SERVHOST-RTK",
}
assert.equal(classifyFlowPlaneLite(greOverlay), "overlay")
const espOverlay = { ...greOverlay, proto: 50 }
assert.equal(classifyFlowPlaneLite(espOverlay), "overlay")
const mesh = {
src: "10.100.1.17",
dst: "10.100.1.18",
proto: 6,
srcPort: 50000,
dstPort: 443,
inIface: "gre-a",
outIface: "gre-b",
}
assert.equal(classifyFlowPlaneLite(mesh), "client_mesh")
const mgmt = {
src: "10.255.254.2",
dst: "10.255.254.1",
proto: 17,
srcPort: 4739,
dstPort: 4739,
inIface: "wg-flow",
outIface: "",
}
assert.equal(classifyFlowPlaneLite(mgmt), "mgmt")
assert.equal(classifyFlowPlaneLite({ ...youtubeInner, outIface: "wg-flow" }), "payload")
assert.equal(shouldKeepPlane("mgmt", {}), false)
assert.equal(shouldKeepPlane("overlay", {}), false)
assert.equal(shouldKeepPlane("client_mesh", {}), false)
assert.equal(shouldKeepPlane("payload", {}), true)
assert.equal(shouldKeepPlane("overlay", { excludeOverlay: false }), true)
assert.equal(shouldKeepPlane("client_mesh", { excludeMesh: false }), true)
assert.equal(flowBps(1500, 1_000, 2_000, 300), (1500 * 8) / 1)
assert.equal(flowBps(1500, 0, 0, 300), (1500 * 8) / 300)
const publicJhEn = {
src: "203.0.113.10",
dst: "198.51.100.1",
proto: 6,
srcPort: 1000,
dstPort: 443,
inIface: "ether1",
outIface: "gre-en",
}
assert.equal(classifyFlowPlane(publicJhEn, {
clientIfaceNames: new Set(["gre-client"]),
enHosts: new Set(["198.51.100.1"]),
jhHosts: new Set(["203.0.113.10"]),
}), "overlay")
console.log("traffic-flow-planes.test.ts: ok")
+107
View File
@@ -0,0 +1,107 @@
export type FlowPlane = "payload" | "client_mesh" | "overlay" | "mgmt"
export const PLANE_LABEL: Record<FlowPlane, string> = {
payload: "Интернет",
client_mesh: "Клиенты",
overlay: "JH↔EN",
mgmt: "mgmt",
}
const WG_PORTS = new Set([51820, 13232, 51821])
const FLOW_PORTS = new Set([4739, 2055])
export function isRfc1918(ip: string): boolean {
const parts = String(ip ?? "").split(".").map((n) => Number.parseInt(n, 10))
if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) return false
const [a, b] = parts
if (a === 10) return true
if (a === 192 && b === 168) return true
if (a === 172 && b != null && b >= 16 && b <= 31) return true
if (a === 100 && b != null && b >= 64 && b <= 127) return true
return false
}
export function isPublicV4(ip: string): boolean {
const parts = String(ip ?? "").split(".").map((n) => Number.parseInt(n, 10))
if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n))) return false
const a = parts[0] ?? 0
if (a === 0 || a === 127 || a >= 224) return false
return !isRfc1918(ip)
}
export function isTunnelProto(proto: number, srcPort: number, dstPort: number): boolean {
if (proto === 47 || proto === 50) return true
if (proto === 17 && (WG_PORTS.has(srcPort) || WG_PORTS.has(dstPort))) return true
return false
}
function ifaceLooksMgmt(name: string): boolean {
const n = name.trim().toLowerCase()
return n === "wg-flow" || n.endsWith("/wg-flow") || n.includes("wg-flow")
}
export interface PlaneFlowInput {
src: string
dst: string
proto: number
srcPort: number
dstPort: number
inIface: string
outIface?: string
}
/** Быстрая классификация без топологии — для live ring на ingest. */
export function classifyFlowPlaneLite(flow: PlaneFlowInput): FlowPlane {
if (ifaceLooksMgmt(flow.inIface)) return "mgmt"
if (flow.proto === 17 && (FLOW_PORTS.has(flow.srcPort) || FLOW_PORTS.has(flow.dstPort))) return "mgmt"
if (isTunnelProto(flow.proto, flow.srcPort, flow.dstPort)) return "overlay"
if (isRfc1918(flow.src) && isRfc1918(flow.dst)) return "client_mesh"
return "payload"
}
export interface PlaneTopology {
clientIfaceNames: Set<string>
enHosts: Set<string>
jhHosts: Set<string>
}
function hostHit(ip: string, hosts: Set<string>): boolean {
return Boolean(ip) && hosts.has(ip)
}
export function classifyFlowPlane(
flow: PlaneFlowInput,
topo?: PlaneTopology | null,
): FlowPlane {
const lite = classifyFlowPlaneLite(flow)
if (!topo) return lite
if (lite === "mgmt") return "mgmt"
if (lite === "overlay") return "overlay"
const srcEn = hostHit(flow.src, topo.enHosts) || hostHit(flow.src, topo.jhHosts)
const dstEn = hostHit(flow.dst, topo.enHosts) || hostHit(flow.dst, topo.jhHosts)
if (srcEn && dstEn && isPublicV4(flow.src) && isPublicV4(flow.dst)) return "overlay"
if (lite === "client_mesh") {
const inClient = topo.clientIfaceNames.has(flow.inIface)
const outClient = Boolean(flow.outIface && topo.clientIfaceNames.has(flow.outIface))
if (inClient || outClient || (isRfc1918(flow.src) && isRfc1918(flow.dst))) return "client_mesh"
}
return "payload"
}
export function shouldKeepPlane(
plane: FlowPlane,
opts: { excludeMesh?: boolean; excludeOverlay?: boolean },
): boolean {
if (plane === "mgmt") return false
if (opts.excludeMesh !== false && plane === "client_mesh") return false
if (opts.excludeOverlay !== false && plane === "overlay") return false
return true
}
export function flowBps(bytes: number, startMs: number, endMs: number, windowSec: number): number {
if (startMs > 0 && endMs > startMs) {
const sec = Math.max(1, (endMs - startMs) / 1000)
return (bytes * 8) / sec
}
return (bytes * 8) / Math.max(1, windowSec)
}
@@ -0,0 +1,77 @@
import assert from "node:assert/strict"
import { mkdtempSync, rmSync } from "node:fs"
import os from "node:os"
import path from "node:path"
const dir = mkdtempSync(path.join(os.tmpdir(), "mm-flow-purge-"))
process.env.DATABASE_PATH = path.join(dir, "test.db")
const { sqliteDatabase } = await import("../db/index.js")
const {
getFlowRuntimeCounters,
purgeTrafficFlowStore,
stopTrafficFlowListener,
} = await import("./traffic-flow-ingest.js")
function count(name: string): number {
const row = sqliteDatabase.prepare(`SELECT COUNT(*) AS n FROM ${name}`).get() as { n: number }
return Number(row?.n) || 0
}
try {
sqliteDatabase.prepare(`
INSERT INTO servers (name, host) VALUES ('purge-test', '127.0.0.1')
`).run()
const serverId = Number(
(sqliteDatabase.prepare(`SELECT id FROM servers WHERE name = 'purge-test'`).get() as { id: number }).id,
)
sqliteDatabase.prepare(`
INSERT INTO flow_buckets (server_id, bucket_at, src, dst, proto, src_port, dst_port, bytes, packets, in_iface)
VALUES (?, '2026-01-01T00:00:00.000Z', '10.0.0.1', '8.8.8.8', 6, 50000, 443, 100, 1, 'wg-flow')
`).run(serverId)
sqliteDatabase.prepare(`
INSERT INTO flow_minute_stats (server_id, bucket_at, bytes, packets, unique_src, unique_dst, conversations)
VALUES (?, '2026-01-01T00:00:00.000Z', 100, 1, 1, 1, 1)
`).run(serverId)
sqliteDatabase.prepare(`
INSERT INTO flow_minute_dims (server_id, bucket_at, dim, key, bytes, packets)
VALUES (?, '2026-01-01T00:00:00.000Z', 'country', 'RU', 100, 1)
`).run(serverId)
sqliteDatabase.prepare(`
INSERT INTO flow_daily_dims (server_id, day, dim, key, bytes, packets)
VALUES (?, '2026-01-01', 'country', 'RU', 100, 1)
`).run(serverId)
sqliteDatabase.prepare(`
INSERT INTO flow_ip_meta (prefix, asn, country, holder, ok, fetched_at)
VALUES ('8.8.8.0/24', 15169, 'US', 'Google', 1, '2026-01-01T00:00:00.000Z')
`).run()
sqliteDatabase.prepare(`
UPDATE traffic_flow_settings SET packets_received = 42, last_exporter_ip = '10.255.254.3' WHERE id = 1
`).run()
const result = await purgeTrafficFlowStore()
stopTrafficFlowListener()
assert.equal(result.ok, true)
assert.equal(result.deleted.buckets, 1)
assert.equal(result.deleted.minuteStats, 1)
assert.equal(result.deleted.minuteDims, 1)
assert.equal(result.deleted.dailyDims, 1)
assert.equal(count("flow_buckets"), 0)
assert.equal(count("flow_minute_stats"), 0)
assert.equal(count("flow_minute_dims"), 0)
assert.equal(count("flow_daily_dims"), 0)
assert.equal(count("flow_ip_meta"), 1)
assert.equal(count("servers"), 1)
assert.equal(getFlowRuntimeCounters().packetsReceived, 0)
assert.equal(getFlowRuntimeCounters().lastExporterIp, null)
} finally {
try {
sqliteDatabase.close()
} catch {
/* ignore */
}
rmSync(dir, { recursive: true, force: true })
}
console.log("traffic-flow-purge.test.ts: ok")
@@ -69,4 +69,35 @@ enqueueRipeMisses(["203.0.113.50"])
await flushRipeQueueForTests()
assert.equal(ripeFetchCountForTests(), afterNeg)
resetRipeCacheForTests()
disableRipePersistForTests()
seedRipeCacheForTests({
prefix: "1.1.1.0/24",
asn: 13335,
country: "?",
lat: null,
lng: null,
holder: "CLOUDFLARENET, US",
ok: true,
fetchedAt: Date.now(),
})
assert.equal(lookupRipeCached("1.1.1.1")?.country, "US")
assert.ok(lookupRipeCached("1.1.1.1")?.country !== "?")
resetRipeCacheForTests()
disableRipePersistForTests()
setRipeFetchForTests(async (input) => {
const url = String(input)
const body = url.includes("network-info")
? { data: { prefix: "1.0.0.0/24", asns: ["13335"] } }
: url.includes("maxmind-geo-lite")
? { data: { located_resources: [{ locations: [{ country: "?" }] }] } }
: { data: { holder: "CLOUDFLARENET, US" } }
return new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } })
})
enqueueRipeMisses(["1.0.0.1"])
await flushRipeQueueForTests()
assert.equal(lookupRipeCached("1.0.0.1")?.country, "US")
assert.equal(lookupRipeCached("1.0.0.1")?.asn, 13335)
console.log("traffic-flow-ripe.test.ts: ok")
+14 -6
View File
@@ -1,5 +1,6 @@
import { sqliteDatabase } from "../db/index.js"
import { ipInCidrV4, ipv4ToInt, isNonPublicIp, parseCidrV4 } from "./traffic-flow-ip.js"
import { resolveRipeCountry } from "./traffic-flow-brands.js"
export interface FlowIpMeta {
prefix: string
@@ -15,6 +16,7 @@ export interface FlowIpMeta {
const HIT_TTL_MS = 24 * 60 * 60_000
const NEG_TTL_MS = 6 * 60 * 60_000
const MAX_NEW_PREFIX_PER_MIN = 30
const MAX_QUEUE = 90
const CONCURRENCY = 3
const RIPE_BASE = "https://stat.ripe.net/data"
const UA = "MikrotikManager-flow/1.0"
@@ -107,13 +109,15 @@ function loadSqlite(): void {
}>
for (const r of rows) {
const fetchedAt = Date.parse(r.fetched_at)
const asn = Number(r.asn ?? 0) || 0
const holder = r.holder || ""
mem.set(r.prefix, {
prefix: r.prefix,
asn: Number(r.asn ?? 0) || 0,
country: r.country || "—",
asn,
country: resolveRipeCountry(r.country || "", asn, holder) || "—",
lat: r.lat == null ? null : Number(r.lat),
lng: r.lng == null ? null : Number(r.lng),
holder: r.holder || "",
holder,
ok: r.ok !== 0,
fetchedAt: Number.isFinite(fetchedAt) ? fetchedAt : 0,
})
@@ -206,6 +210,8 @@ export function lookupRipeCached(ip: string): FlowIpMeta | null {
}
}
return best
? { ...best, country: resolveRipeCountry(best.country, best.asn, best.holder) || "—" }
: null
}
async function ripeJson(path: string, resource: string): Promise<unknown> {
@@ -247,7 +253,7 @@ function pickGeo(data: unknown): { country: string; lat: number | null; lng: num
}
}
const loc = d?.data?.located_resources?.[0]?.locations?.[0]
const country = String(loc?.country ?? "").trim().toUpperCase()
const country = resolveRipeCountry(String(loc?.country ?? ""), 0, "")
const lat = loc?.latitude == null ? null : Number(loc.latitude)
const lng = loc?.longitude == null ? null : Number(loc.longitude)
return {
@@ -299,14 +305,15 @@ async function resolveIp(ip: string): Promise<FlowIpMeta | null> {
/* best-effort */
}
}
const country = resolveRipeCountry(geo.country, asn, holder)
const entry: FlowIpMeta = {
prefix,
asn,
country: geo.country,
country: country || "—",
lat: geo.lat,
lng: geo.lng,
holder,
ok: Boolean(asn || (geo.country && geo.country !== "—")),
ok: Boolean(asn || country),
fetchedAt: Date.now(),
}
mem.set(prefix, entry)
@@ -360,6 +367,7 @@ export function enqueueRipeMisses(ips: Iterable<string>): void {
if (!enqueueEnabled) return
loadSqlite()
for (const raw of ips) {
if (queue.length >= MAX_QUEUE) break
const ip = String(raw ?? "").trim()
if (!ip || isNonPublicIp(ip)) continue
if (lookupRipeCached(ip)) continue
@@ -131,3 +131,13 @@ export function enableTrafficFlowIngest() {
export function listHostPeers(): FlowHostPeer[] {
return parsePeers(getTrafficFlowSettingsRow().peersJson)
}
export function resetFlowIngestCounters(): void {
db.update(trafficFlowSettings).set({
packetsReceived: 0,
lastDatagramAt: null,
lastExporterIp: null,
lastError: "",
updatedAt: nowIso(),
}).where(eq(trafficFlowSettings.id, 1)).run()
}
@@ -0,0 +1,164 @@
import { db, sqliteDatabase } from "../db/index.js"
import { appUsers, servers, userInterfaceBindings } from "../db/schema.js"
import { mapRosInterfaceType } from "../modules/users/iface-type.js"
import type { PlaneTopology } from "./traffic-flow-planes.js"
export interface FlowClientBinding {
userId: string
login: string
name: string
serverId: number
interfaceName: string
}
export interface FlowEnNode {
id: number
name: string
hosts: string[]
}
export interface FlowTopology {
clientIfaces: Map<number, Set<string>>
clientByIface: Map<string, FlowClientBinding>
enNodes: FlowEnNode[]
enHosts: Set<string>
jhHosts: Set<string>
wanIfaces: Map<number, Set<string>>
plane: PlaneTopology
}
let seeded: FlowTopology | null = null
function parseWanUplinks(raw: string): Array<{ iface?: string; ip?: string }> {
try {
const parsed = JSON.parse(raw || "[]") as unknown
return Array.isArray(parsed) ? parsed as Array<{ iface?: string; ip?: string }> : []
} catch {
return []
}
}
function ifaceKey(serverId: number, name: string): string {
return `${serverId}|${name}`
}
export function loadFlowTopology(): FlowTopology {
if (seeded) return seeded
const serverRows = db.select().from(servers).all()
const users = db.select().from(appUsers).all()
const binds = db.select().from(userInterfaceBindings).all()
const loginById = new Map(users.map((u) => [u.id, u]))
const clientIfaces = new Map<number, Set<string>>()
const clientByIface = new Map<string, FlowClientBinding>()
const allClientNames = new Set<string>()
for (const b of binds) {
const set = clientIfaces.get(b.serverId) ?? new Set<string>()
set.add(b.interfaceName)
clientIfaces.set(b.serverId, set)
allClientNames.add(b.interfaceName)
const user = loginById.get(b.userId)
clientByIface.set(ifaceKey(b.serverId, b.interfaceName), {
userId: b.userId,
login: user?.login || b.userId,
name: user?.name || user?.login || b.userId,
serverId: b.serverId,
interfaceName: b.interfaceName,
})
}
const enHosts = new Set<string>()
const jhHosts = new Set<string>()
const enNodes: FlowEnNode[] = []
const wanIfaces = new Map<number, Set<string>>()
for (const s of serverRows) {
const wans = parseWanUplinks(s.wanUplinks)
const hosts = [s.host, ...wans.map((w) => String(w.ip ?? "").trim())].filter(Boolean)
const wanSet = new Set(wans.map((w) => String(w.iface ?? "").trim()).filter(Boolean))
if (wanSet.size) wanIfaces.set(s.id, wanSet)
if (s.type === "exit-node") {
for (const h of hosts) enHosts.add(h)
enNodes.push({ id: s.id, name: s.name || s.host, hosts })
}
if (s.type === "jump-host") {
for (const h of hosts) jhHosts.add(h)
}
}
return {
clientIfaces,
clientByIface,
enNodes,
enHosts,
jhHosts,
wanIfaces,
plane: {
clientIfaceNames: allClientNames,
enHosts,
jhHosts,
},
}
}
export function seedFlowTopologyForTests(topo: FlowTopology | null): void {
seeded = topo
}
export function resolveClient(
topo: FlowTopology,
serverId: number,
inIfaceName: string,
): FlowClientBinding | null {
return topo.clientByIface.get(ifaceKey(serverId, inIfaceName)) ?? null
}
export function resolveEn(
topo: FlowTopology,
nextHop: string,
outIfaceName: string,
): FlowEnNode | null {
if (nextHop) {
const hit = topo.enNodes.find((n) => n.hosts.includes(nextHop))
if (hit) return hit
}
const needle = outIfaceName.trim().toLowerCase()
if (!needle) return null
return topo.enNodes.find((n) => {
const name = n.name.toLowerCase()
const host = (n.hosts[0] ?? "").toLowerCase()
return (name && needle.includes(name)) || (host && needle.includes(host.split(".")[0] ?? ""))
}) ?? null
}
export function enGreIfaceNames(topo: FlowTopology, serverId: number, ifaceNames: string[]): string[] {
const client = topo.clientIfaces.get(serverId) ?? new Set<string>()
return ifaceNames.filter((name) => {
if (client.has(name)) return false
if (name === "wg-flow") return false
return mapRosInterfaceType("", name) === "gre"
})
}
export function latestWireBps(serverId: number, ifaceNames: string[]): { bps: number; bytes: number } {
if (!ifaceNames.length) return { bps: 0, bytes: 0 }
const placeholders = ifaceNames.map(() => "?").join(",")
const rows = sqliteDatabase.prepare(`
SELECT interface_name AS name, rx_bps AS rxBps, tx_bps AS txBps, rx_bytes AS rxBytes, tx_bytes AS txBytes
FROM traffic_samples
WHERE server_id = ? AND interface_name IN (${placeholders})
ORDER BY sampled_at DESC
`).all(serverId, ...ifaceNames) as Array<{
name: string
rxBps: number
txBps: number
rxBytes: number
txBytes: number
}>
const seen = new Set<string>()
let bps = 0
let bytes = 0
for (const r of rows) {
if (seen.has(r.name)) continue
seen.add(r.name)
bps += (Number(r.rxBps) || 0) + (Number(r.txBps) || 0)
bytes += (Number(r.rxBytes) || 0) + (Number(r.txBytes) || 0)
}
return { bps, bytes }
}
+1 -1
View File
@@ -15,5 +15,5 @@
"sourceMap": true
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
"exclude": ["node_modules", "dist", "src/**/*.test.ts"]
}
@@ -28,12 +28,19 @@ function TrafficFlowsDataGrid({
}) {
const columns = useMemo<ColumnDef<FlowTalkerDto>[]>(
() => [
{
id: "client",
accessorFn: (r) => r.clientName ?? "",
header: () => <span className="text-xs font-medium text-muted-foreground">Клиент</span>,
cell: ({ row }) => <span className="text-xs">{row.original.clientName || "—"}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD_FIRST, cellClassName: DATA_GRID_CELL_PAD_FIRST },
},
{
id: "server",
accessorKey: "serverName",
header: () => <span className="text-xs font-medium text-muted-foreground">JH</span>,
cell: ({ row }) => <span className="text-sm font-medium">{row.original.serverName}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD_FIRST, cellClassName: DATA_GRID_CELL_PAD_FIRST },
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "src",
@@ -96,6 +103,20 @@ function TrafficFlowsDataGrid({
cell: ({ row }) => <span className="text-xs tabular-nums">{formatBytes(row.original.bytes)}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "en",
accessorFn: (r) => r.enName ?? "",
header: () => <span className="text-xs font-medium text-muted-foreground">EN</span>,
cell: ({ row }) => <span className="text-xs">{row.original.enName || "—"}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "plane",
accessorFn: (r) => r.plane ?? "",
header: () => <span className="text-xs font-medium text-muted-foreground">Плоскость</span>,
cell: ({ row }) => <span className="text-xs text-muted-foreground">{row.original.plane || "—"}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "iface",
accessorKey: "inIface",
+1
View File
@@ -46,6 +46,7 @@ function nearestCdnSize(px: number): number {
export function Flag({ code, size = 20, className }: FlagProps) {
if (!code) return null
const lower = code.toLowerCase()
if (!/^[a-z]{2}$/.test(lower)) return null
const name = countryName(code.toUpperCase())
const cdnSrc = nearestCdnSize(size)
const cdnSrc2x = nearestCdnSize(size * 2)
+224 -23
View File
@@ -2,10 +2,10 @@
import { useMemo, useState } from "react"
import { type ColumnDef, getCoreRowModel, useReactTable } from "@tanstack/react-table"
import type { FlowAnalyticsDto, FlowBreakdownRow, FlowEntityCard } from "@mmapp/contracts/traffic-flow"
import { ArrowDownIcon, ArrowUpIcon, GitBranchIcon, GlobeIcon, LayersIcon, UsersIcon } from "lucide-react"
import type { FlowAnalyticsDto, FlowBreakdownRow, FlowEntityCard, FlowPathRow, FlowTalkerDto } from "@mmapp/contracts/traffic-flow"
import { ArrowDownIcon, ArrowUpIcon, GitBranchIcon, GlobeIcon, LayersIcon, NetworkIcon, RouteIcon, ShieldIcon, UsersIcon } from "lucide-react"
import { Badge } from "@/components/reui/badge"
import { KpiStatGrid } from "@/components/reui-kit/kpi-stat-grid"
import { KpiStatGrid, type KpiStatItem } from "@/components/reui-kit/kpi-stat-grid"
import { TrafficRxTxChart } from "@/components/reui-kit/traffic-rx-tx-chart"
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
import { Switch } from "@/components/ui/switch"
@@ -30,13 +30,14 @@ function formatBytes(n: number): string {
return `${n} Б`
}
const RANGE_KEYS = ["5m", "15m", "1h", "4h", "24h"] as const
const RANGE_KEYS = ["5m", "15m", "1h", "4h", "24h", "30d"] as const
const RANGE_LABELS: Record<string, string> = {
"5m": "5м",
"15m": "15м",
"1h": "1ч",
"4h": "4ч",
"24h": "24ч",
"30d": "месяц",
}
function MiniAreaChart({ rx, tx, height = 44 }: { rx: number[]; tx: number[]; height?: number }) {
@@ -108,14 +109,38 @@ export function FlowEntityCardView({
)
}
type SessionFilter = {
kind: "application" | "category" | "service" | "asn" | "country" | "protocol" | "source" | "destination" | "iface" | "client" | "en"
value: string
label: string
}
function talkerMatchesFilter(row: FlowTalkerDto, filter: SessionFilter): boolean {
switch (filter.kind) {
case "application": return row.application === filter.value
case "category": return row.category === filter.value
case "service": return row.service === filter.value
case "asn": return String(row.dstAsn ?? "") === filter.value
case "country": return row.dstCountry === filter.value
case "protocol": return row.protoName === filter.value
case "source": return row.src === filter.value
case "destination": return row.dst === filter.value
case "iface": return row.inIface === filter.value
case "client": return (row.clientId || "unknown") === filter.value
case "en": return (row.enId || "") === filter.value
}
}
function FlowBreakdownGrid({
rows,
empty,
country,
onPick,
}: {
rows: FlowBreakdownRow[]
empty?: string
country?: boolean
onPick?: (row: FlowBreakdownRow) => void
}) {
const columns = useMemo<ColumnDef<FlowBreakdownRow>[]>(
() => [
@@ -178,7 +203,107 @@ function FlowBreakdownGrid({
})
return (
<DataGridShell table={table} recordCount={rows.length} emptyMessage={empty ?? "Нет данных за период"} />
<DataGridShell
table={table}
recordCount={rows.length}
emptyMessage={empty ?? "Нет данных за период"}
onRowClick={onPick}
/>
)
}
function FlowPathsGrid({
rows,
empty,
onPick,
}: {
rows: FlowPathRow[]
empty?: string
onPick?: (row: FlowPathRow) => void
}) {
const columns = useMemo<ColumnDef<FlowPathRow>[]>(
() => [
{
id: "client",
accessorKey: "clientName",
header: () => <span className="text-xs font-medium text-muted-foreground">Клиент</span>,
cell: ({ row }) => <span className="text-sm font-medium">{row.original.clientName}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD_FIRST, cellClassName: DATA_GRID_CELL_PAD_FIRST },
},
{
id: "ifaces",
accessorKey: "ifaces",
header: () => <span className="text-xs font-medium text-muted-foreground">Ifaces</span>,
cell: ({ row }) => <span className="font-mono text-xs text-muted-foreground">{row.original.ifaces}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "jh",
accessorKey: "serverName",
header: () => <span className="text-xs font-medium text-muted-foreground">JH</span>,
cell: ({ row }) => <span className="text-xs">{row.original.serverName}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "in",
accessorKey: "inIface",
header: () => <span className="text-xs font-medium text-muted-foreground">In</span>,
cell: ({ row }) => <span className="font-mono text-xs">{row.original.inIface}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "en",
accessorKey: "enName",
header: () => <span className="text-xs font-medium text-muted-foreground">EN</span>,
cell: ({ row }) => <span className="text-xs">{row.original.enName || "—"}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "dst",
accessorKey: "dst",
header: () => <span className="text-xs font-medium text-muted-foreground">Dest</span>,
cell: ({ row }) => (
<span className="flex min-w-0 flex-col gap-0.5 text-xs">
<span className="font-mono">{row.original.dst}</span>
<span className="text-[10px] text-muted-foreground truncate">
{[row.original.category, row.original.service].filter(Boolean).join(" · ")}
</span>
</span>
),
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "rate",
accessorFn: (r) => r.bps,
header: () => <span className="text-xs font-medium text-muted-foreground">Скорость</span>,
cell: ({ row }) => <span className="text-xs tabular-nums">{fmtRate(row.original.bps / 1_000_000)}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD, cellClassName: DATA_GRID_CELL_PAD },
},
{
id: "bytes",
accessorKey: "bytes",
header: () => <span className="text-xs font-medium text-muted-foreground">Байты</span>,
cell: ({ row }) => <span className="text-xs tabular-nums">{formatBytes(row.original.bytes)}</span>,
meta: { headerClassName: DATA_GRID_CELL_PAD_LAST, cellClassName: DATA_GRID_CELL_PAD_LAST },
},
],
[],
)
const table = useReactTable({
data: rows,
columns,
getCoreRowModel: getCoreRowModel(),
getRowId: (row) => row.id,
})
return (
<DataGridShell
table={table}
recordCount={rows.length}
emptyMessage={empty ?? "Нет путей"}
onRowClick={onPick}
/>
)
}
@@ -191,6 +316,10 @@ export function FlowAnalyticsDetail({
onIface,
dedup,
onDedup,
excludeMesh,
onExcludeMesh,
excludeOverlay,
onExcludeOverlay,
liveHint,
emptyHint,
}: {
@@ -202,17 +331,26 @@ export function FlowAnalyticsDetail({
onIface: (name: string) => void
dedup: boolean
onDedup: (value: boolean) => void
excludeMesh: boolean
onExcludeMesh: (value: boolean) => void
excludeOverlay: boolean
onExcludeOverlay: (value: boolean) => void
liveHint?: string
emptyHint?: string
}) {
const [slice, setSlice] = useState("applications")
const [mapCountry, setMapCountry] = useState<string | null>(null)
const [sessionFilter, setSessionFilter] = useState<SessionFilter | null>(null)
const rxNow = analytics ? analytics.bpsNow / 1_000_000 : (card?.rxNow ?? 0)
const bytes = analytics?.bytes ?? card?.bytes ?? 0
const sessionRows = (analytics?.conversationsList ?? []).filter((row) =>
mapCountry ? row.dstCountry === mapCountry : true,
sessionFilter ? talkerMatchesFilter(row, sessionFilter) : true,
)
function pickBreakdown(kind: SessionFilter["kind"], row: FlowBreakdownRow) {
setSessionFilter({ kind, value: row.id, label: row.label })
setSlice("sessions")
}
if (!card) {
return (
<p className="text-sm text-muted-foreground py-8 text-center">
@@ -243,6 +381,26 @@ export function FlowAnalyticsDetail({
Без дублей
</Label>
</div>
<div className="flex items-center gap-2">
<Switch
id="flow-exclude-overlay"
checked={excludeOverlay}
onCheckedChange={onExcludeOverlay}
/>
<Label htmlFor="flow-exclude-overlay" className="text-xs text-muted-foreground whitespace-nowrap">
Без overlay
</Label>
</div>
<div className="flex items-center gap-2">
<Switch
id="flow-exclude-mesh"
checked={excludeMesh}
onCheckedChange={onExcludeMesh}
/>
<Label htmlFor="flow-exclude-mesh" className="text-xs text-muted-foreground whitespace-nowrap">
Без mesh
</Label>
</div>
<div className="flex gap-1">
{RANGE_KEYS.map((r) => (
<button
@@ -311,7 +469,7 @@ export function FlowAnalyticsDetail({
<div className="mt-4 pt-4 border-t">
<KpiStatGrid
aria-label="Скорость потоков"
items={[
items={([
{
id: "bps-now",
label: "Скорость сейчас",
@@ -322,11 +480,38 @@ export function FlowAnalyticsDetail({
},
{
id: "bytes",
label: "Байт за период",
label: "Payload",
value: formatBytes(bytes),
hint: "inner IPFIX",
icon: <ArrowUpIcon className="size-4" />,
iconClassName: "text-info",
},
{
id: "overlay",
label: "JH↔EN overlay",
value: fmtRate((analytics?.bpsOverlay ?? 0) / 1_000_000),
hint: analytics?.bytesOverlay ? formatBytes(analytics.bytesOverlay) : undefined,
icon: <ShieldIcon className="size-4" />,
iconClassName: "text-warning",
},
{
id: "wire",
label: "Wire GRE",
value: fmtRate((analytics?.bpsWire ?? 0) / 1_000_000),
hint: "счётчик iface",
icon: <RouteIcon className="size-4" />,
iconClassName: "text-primary",
},
...(!excludeMesh
? [{
id: "mesh",
label: "Mesh",
value: formatBytes(analytics?.bytesMesh ?? 0),
hint: "клиент↔клиент",
icon: <NetworkIcon className="size-4" />,
iconClassName: "text-muted-foreground",
}]
: []),
{
id: "flows",
label: "Сессии",
@@ -351,7 +536,7 @@ export function FlowAnalyticsDetail({
icon: <LayersIcon className="size-4" />,
iconClassName: "text-primary",
},
]}
] satisfies KpiStatItem[])}
/>
</div>
@@ -371,51 +556,66 @@ export function FlowAnalyticsDetail({
<TabsTrigger value="protocols">Протоколы</TabsTrigger>
<TabsTrigger value="sources">Источники</TabsTrigger>
<TabsTrigger value="destinations">Назначения</TabsTrigger>
<TabsTrigger value="paths">Пути</TabsTrigger>
<TabsTrigger value="sessions">Сессии</TabsTrigger>
<TabsTrigger value="interfaces">Интерфейсы</TabsTrigger>
</TabsList>
<TabsContent value="applications">
<FlowBreakdownGrid rows={analytics?.applications ?? []} />
<FlowBreakdownGrid rows={analytics?.applications ?? []} onPick={(row) => pickBreakdown("application", row)} />
</TabsContent>
<TabsContent value="categories">
<FlowBreakdownGrid rows={analytics?.categories ?? []} />
<FlowBreakdownGrid rows={analytics?.categories ?? []} onPick={(row) => pickBreakdown("category", row)} />
</TabsContent>
<TabsContent value="services">
<FlowBreakdownGrid rows={analytics?.services ?? []} />
<FlowBreakdownGrid rows={analytics?.services ?? []} onPick={(row) => pickBreakdown("service", row)} />
</TabsContent>
<TabsContent value="asns">
<FlowBreakdownGrid rows={analytics?.asns ?? []} />
<FlowBreakdownGrid rows={analytics?.asns ?? []} onPick={(row) => pickBreakdown("asn", row)} />
</TabsContent>
<TabsContent value="countries">
<FlowBreakdownGrid rows={analytics?.countries ?? []} country />
<FlowBreakdownGrid rows={analytics?.countries ?? []} country onPick={(row) => pickBreakdown("country", row)} />
</TabsContent>
<TabsContent value="map">
<FlowTrafficMap
edges={analytics?.mapEdges ?? []}
onSelectCountry={(iso) => {
setMapCountry(iso)
setSessionFilter({ kind: "country", value: iso, label: iso })
setSlice("sessions")
}}
/>
</TabsContent>
<TabsContent value="protocols">
<FlowBreakdownGrid rows={analytics?.protocols ?? []} />
<FlowBreakdownGrid rows={analytics?.protocols ?? []} onPick={(row) => pickBreakdown("protocol", row)} />
</TabsContent>
<TabsContent value="sources">
<FlowBreakdownGrid rows={analytics?.sources ?? []} />
<FlowBreakdownGrid rows={analytics?.sources ?? []} onPick={(row) => pickBreakdown("source", row)} />
</TabsContent>
<TabsContent value="destinations">
<FlowBreakdownGrid rows={analytics?.destinations ?? []} />
<FlowBreakdownGrid rows={analytics?.destinations ?? []} onPick={(row) => pickBreakdown("destination", row)} />
</TabsContent>
<TabsContent value="paths">
<FlowPathsGrid
rows={analytics?.paths ?? []}
empty="Нет путей за период"
onPick={(row) => {
setSessionFilter({
kind: "client",
value: row.clientId,
label: `${row.clientName}${row.enName || row.dst}`,
})
setSlice("sessions")
}}
/>
</TabsContent>
<TabsContent value="sessions">
{mapCountry ? (
{sessionFilter ? (
<div className="flex items-center gap-2 mb-2">
<GlobeIcon className="size-3.5 text-muted-foreground" />
<span className="text-xs text-muted-foreground">фильтр страны {mapCountry}</span>
<Badge variant="secondary" size="sm">Фильтр: {sessionFilter.label}</Badge>
<button
type="button"
className="text-xs text-primary"
onClick={() => setMapCountry(null)}
onClick={() => setSessionFilter(null)}
>
сбросить
</button>
@@ -423,13 +623,14 @@ export function FlowAnalyticsDetail({
) : null}
<TrafficFlowsDataGrid
rows={sessionRows}
emptyHint={emptyHint}
emptyHint={emptyHint ?? "Нет сессий по выбранному фильтру"}
/>
</TabsContent>
<TabsContent value="interfaces">
<FlowBreakdownGrid
rows={analytics?.interfaces ?? []}
empty="Нет данных по интерфейсам"
onPick={(row) => pickBreakdown("iface", row)}
/>
</TabsContent>
</Tabs>
+9 -2
View File
@@ -72,7 +72,9 @@ function FlowTrafficMap({
header: () => <span className="text-xs font-medium text-muted-foreground">Назначение</span>,
cell: ({ row }) => (
<span className="flex items-center gap-1.5 text-sm">
<Flag code={row.original.toCountry} />
{/^[a-z]{2}$/i.test(row.original.toCountry)
? <Flag code={row.original.toCountry} />
: null}
{row.original.toCountry}
{row.original.toAsn ? <span className="font-mono text-[10px] text-muted-foreground">AS{row.original.toAsn}</span> : null}
</span>
@@ -176,7 +178,12 @@ function FlowTrafficMap({
) : null}
</FramePanel>
</Frame>
<DataGridShell table={table} recordCount={edges.length} emptyMessage="Нет рёбер с известной страной" />
<DataGridShell
table={table}
recordCount={edges.length}
emptyMessage="Нет рёбер с известной страной"
onRowClick={(row) => onSelectCountry?.(row.toCountry)}
/>
</div>
)
}
@@ -0,0 +1,72 @@
"use client"
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogMedia,
AlertDialogTitle,
} from "@/components/ui/alert-dialog"
import type { FlowPurgeDto } from "@mmapp/contracts/traffic-flow"
import { AlertCircleIcon, LoaderCircleIcon } from "lucide-react"
function formatDbFileBytes(n: number): string {
if (n < 1024) return `${n} Б`
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} КБ`
return `${(n / (1024 * 1024)).toFixed(1)} МБ`
}
export function formatFlowPurgeResult(result: FlowPurgeDto): string {
const rows =
result.deleted.buckets +
result.deleted.minuteStats +
result.deleted.minuteDims +
result.deleted.dailyDims
const vacuumHint = result.vacuumed ? "" : " VACUUM не выполнен."
return `Удалено строк: ${rows}. Файл ${formatDbFileBytes(result.fileBytesBefore)}${formatDbFileBytes(result.fileBytesAfter)}.${vacuumHint}`
}
export function NetflowPurgeConfirm({
open,
busy,
onConfirm,
onCancel,
}: {
open: boolean
busy?: boolean
onConfirm: () => void
onCancel: () => void
}) {
return (
<AlertDialog open={open} onOpenChange={(v) => { if (!v && !busy) onCancel() }}>
<AlertDialogContent size="default">
<AlertDialogHeader>
<AlertDialogMedia className="bg-destructive/10 text-destructive">
<AlertCircleIcon />
</AlertDialogMedia>
<AlertDialogTitle>Сбросить данные NetFlow?</AlertDialogTitle>
<AlertDialogDescription>
Будут удалены сессии и агрегаты (minute/daily) из SQLite, затем VACUUM.
Ключи WireGuard, пиры JH, настройки коллектора и кэш RIPE сохранятся.
На время операции приём IPFIX остановится.
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel disabled={busy} onClick={onCancel}>Отмена</AlertDialogCancel>
<AlertDialogAction
variant="destructive"
disabled={busy}
onClick={onConfirm}
>
{busy ? <LoaderCircleIcon className="size-4 animate-spin" /> : null}
{busy ? "Сброс…" : "Сбросить"}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
)
}
+45 -3
View File
@@ -14,8 +14,10 @@ import {
generateTrafficFlowKeys,
getTrafficFlowHostFiles,
getTrafficFlowSettings,
purgeTrafficFlowData,
putTrafficFlowSettings,
} from "@/shared/api/traffic-flow"
import { formatFlowPurgeResult, NetflowPurgeConfirm } from "@/components/traffic/netflow-purge-dialog"
import { KeyRoundIcon, DownloadIcon, InfoIcon } from "lucide-react"
const HOST_STEPS = [
@@ -46,6 +48,8 @@ function NetflowSettingsPanel({
const [retention, setRetention] = useState("24")
const [topN, setTopN] = useState("200")
const [ingestOn, setIngestOn] = useState(false)
const [purgeOpen, setPurgeOpen] = useState(false)
const [purgeBusy, setPurgeBusy] = useState(false)
const load = useCallback(async () => {
if (!enabled) return
@@ -102,6 +106,20 @@ function NetflowSettingsPanel({
}
}
async function handlePurgeConfirm() {
setPurgeBusy(true)
try {
const result = await purgeTrafficFlowData(backendUrl)
toast.success(formatFlowPurgeResult(result))
setPurgeOpen(false)
await load()
} catch (e) {
toast.error(e instanceof Error ? e.message : "Не удалось сбросить NetFlow")
} finally {
setPurgeBusy(false)
}
}
async function handleExport() {
setBusy(true)
try {
@@ -196,18 +214,35 @@ function NetflowSettingsPanel({
</div>
<div className="flex flex-wrap gap-2">
<Button size="sm" disabled={busy} onClick={() => { void handleSave() }}>
<Button size="sm" disabled={busy || purgeBusy} onClick={() => { void handleSave() }}>
Сохранить NetFlow
</Button>
<Button size="sm" variant="outline" disabled={busy} onClick={() => { void handleKeys() }}>
<Button size="sm" variant="outline" disabled={busy || purgeBusy} onClick={() => { void handleKeys() }}>
<KeyRoundIcon className="size-4" />
Ключи хоста
</Button>
<Button size="sm" variant="outline" disabled={busy} onClick={() => { void handleExport() }}>
<Button size="sm" variant="outline" disabled={busy || purgeBusy} onClick={() => { void handleExport() }}>
<DownloadIcon className="size-4" />
wg-quick / compose / firewall
</Button>
</div>
<div className="flex items-center justify-between gap-4 rounded-md border border-destructive/30 px-4 py-3">
<div className="min-w-0">
<p className="text-sm font-medium">Сбросить данные NetFlow</p>
<p className="text-xs text-muted-foreground mt-0.5">
Удалит сессии и агрегаты из SQLite, затем VACUUM. Ключи WG и пиры не трогает.
</p>
</div>
<Button
size="sm"
variant="destructive"
disabled={busy || purgeBusy}
onClick={() => setPurgeOpen(true)}
>
Сбросить
</Button>
</div>
</OpsPanel>
<CodeExportSheet
@@ -217,6 +252,13 @@ function NetflowSettingsPanel({
description="wg-quick, фрагмент compose и firewall. Хост, не контейнер backend."
formats={formats}
/>
<NetflowPurgeConfirm
open={purgeOpen}
busy={purgeBusy}
onConfirm={() => { void handlePurgeConfirm() }}
onCancel={() => { if (!purgeBusy) setPurgeOpen(false) }}
/>
</>
)
}
+8 -3
View File
@@ -23,6 +23,8 @@ export function useFlowLive(opts: {
userId?: string
iface?: string
dedup?: boolean
excludeMesh?: boolean
excludeOverlay?: boolean
}): { sample: FlowAnalyticsDto | null; error: string | null } {
const [sample, setSample] = useState<FlowAnalyticsDto | null>(null)
const [error, setError] = useState<string | null>(null)
@@ -43,6 +45,8 @@ export function useFlowLive(opts: {
userId: opts.userId,
iface: opts.iface,
dedup: opts.dedup,
excludeMesh: opts.excludeMesh,
excludeOverlay: opts.excludeOverlay,
})}`
const url = resolveApiUrl(opts.backendUrl, path)
@@ -71,8 +75,9 @@ export function useFlowLive(opts: {
if (!raw.trim() || raw.trim().startsWith(":")) continue
const ev = parseSseBlock(raw)
if (ev.event === "sample" && ev.data) {
setSample(JSON.parse(ev.data) as FlowAnalyticsDto)
setError(null)
const parsed = JSON.parse(ev.data) as FlowAnalyticsDto
setSample(parsed)
setError(parsed.degraded ? "Коллектор перегружен: упрощённая аналитика" : null)
} else if (ev.event === "error" && ev.data) {
const parsed = JSON.parse(ev.data) as { error?: string }
setError(parsed.error ?? "live error")
@@ -86,7 +91,7 @@ export function useFlowLive(opts: {
})()
return () => ac.abort()
}, [opts.enabled, opts.backendUrl, opts.range, opts.serverId, opts.userId, opts.iface, opts.dedup])
}, [opts.enabled, opts.backendUrl, opts.range, opts.serverId, opts.userId, opts.iface, opts.dedup, opts.excludeMesh, opts.excludeOverlay])
return { sample, error }
}
+103
View File
@@ -0,0 +1,103 @@
import type { FlowMapHop } from "@mmapp/contracts/traffic-flow"
import { fmtRate } from "@/lib/fmt-rate"
export interface MatchedNetflowHop {
bps: number
bpsFwd: number
bpsRev: number
bytes: number
}
function ifaceNorm(s: string | undefined): string {
return (s ?? "").trim().toLowerCase()
}
function pairKey(a: string, b: string): string {
const x = String(a)
const y = String(b)
return x <= y ? `${x}\t${y}` : `${y}\t${x}`
}
function mergeDirected(hops: FlowMapHop[], mapFromId: string): MatchedNetflowHop {
let bytes = 0
let bpsFwd = 0
let bpsRev = 0
const from = String(mapFromId)
for (const h of hops) {
bytes += h.bytes
if (h.fromId === from) {
bpsFwd += h.bpsFwd
bpsRev += h.bpsRev
} else {
bpsFwd += h.bpsRev
bpsRev += h.bpsFwd
}
}
return { bytes, bpsFwd, bpsRev, bps: bpsFwd + bpsRev }
}
export function hopHasRate(h: MatchedNetflowHop | undefined): h is MatchedNetflowHop {
return h != null && Number.isFinite(h.bps) && h.bps > 0
}
export function formatNetflowRate(hop: MatchedNetflowHop): string {
return fmtRate(hop.bps / 1_000_000)
}
export function formatNetflowDir(hop: MatchedNetflowHop): string {
return `${fmtRate(hop.bpsFwd / 1_000_000)}${fmtRate(hop.bpsRev / 1_000_000)}`
}
/** GRE: сначала имя интерфейса туннеля на любом конце, иначе пара узлов. */
export function matchNetflowForGreEdge(
edge: {
tunnel: { name: string }
fromServer: { id: string }
toServer: { id: string }
},
hops: FlowMapHop[],
): MatchedNetflowHop | undefined {
const name = ifaceNorm(edge.tunnel.name)
const fromId = String(edge.fromServer.id)
const toId = String(edge.toServer.id)
if (name) {
const ifaceHits = hops.filter((h) =>
h.kind === "iface"
&& ifaceNorm(h.iface) === name
&& (h.fromId === fromId || h.fromId === toId),
)
if (ifaceHits.length) return mergeDirected(ifaceHits, fromId)
const greNamed = hops.filter((h) =>
h.kind === "gre"
&& ifaceNorm(h.iface) === name
&& (h.fromId === fromId || h.fromId === toId || h.toId === fromId || h.toId === toId),
)
if (greNamed.length) return mergeDirected(greNamed, fromId)
}
const want = pairKey(fromId, toId)
const pairHits = hops.filter((h) =>
h.kind === "gre" && Boolean(h.toId) && pairKey(h.fromId, h.toId) === want,
)
if (pairHits.length) return mergeDirected(pairHits, fromId)
return undefined
}
/** WAN-аплинк HR: kind wan, иначе iface с тем же именем на homeId. */
export function matchNetflowForWan(
homeId: string,
wanIface: string,
hops: FlowMapHop[],
): MatchedNetflowHop | undefined {
const id = String(homeId)
const iface = ifaceNorm(wanIface)
if (!iface) return undefined
const wanHits = hops.filter((h) =>
h.kind === "wan" && h.fromId === id && ifaceNorm(h.iface) === iface,
)
if (wanHits.length) return mergeDirected(wanHits, id)
const ifaceHits = hops.filter((h) =>
h.kind === "iface" && h.fromId === id && ifaceNorm(h.iface) === iface,
)
if (ifaceHits.length) return mergeDirected(ifaceHits, id)
return undefined
}
+89
View File
@@ -80,11 +80,18 @@ export const flowTalkerDtoSchema = z.object({
bps: z.number().nonnegative(),
inIface: z.string(),
inIfaceIndex: z.string().optional(),
outIface: z.string().optional(),
nextHop: z.string().optional(),
application: z.string().optional(),
category: z.string().optional(),
service: z.string().optional(),
dstCountry: z.string().optional(),
dstAsn: z.number().int().optional(),
clientId: z.string().optional(),
clientName: z.string().optional(),
enId: z.string().optional(),
enName: z.string().optional(),
plane: z.string().optional(),
})
export const flowStatsDtoSchema = z.object({
@@ -148,6 +155,26 @@ export const flowMapEdgeSchema = z.object({
bps: z.number().nonnegative(),
})
export const flowPathRowSchema = z.object({
id: z.string(),
clientId: z.string(),
clientName: z.string(),
ifaces: z.string(),
serverId: z.string(),
serverName: z.string(),
inIface: z.string(),
outIface: z.string(),
enId: z.string(),
enName: z.string(),
dst: z.string(),
service: z.string(),
category: z.string(),
plane: z.string(),
bytes: z.number().nonnegative(),
packets: z.number().nonnegative(),
bps: z.number().nonnegative(),
})
export const flowAnalyticsDtoSchema = z.object({
bpsNow: z.number().nonnegative(),
bytes: z.number().nonnegative(),
@@ -171,9 +198,19 @@ export const flowAnalyticsDtoSchema = z.object({
services: z.array(flowBreakdownRowSchema).optional(),
mapEdges: z.array(flowMapEdgeSchema).optional(),
conversationsList: z.array(flowTalkerDtoSchema),
paths: z.array(flowPathRowSchema).optional(),
ifaces: z.array(flowIfaceChipSchema),
live: z.boolean(),
dedupApplied: z.boolean().optional(),
degraded: z.boolean().optional(),
bytesPayload: z.number().nonnegative().optional(),
bytesOverlay: z.number().nonnegative().optional(),
bytesMesh: z.number().nonnegative().optional(),
bytesWire: z.number().nonnegative().optional(),
bpsOverlay: z.number().nonnegative().optional(),
bpsWire: z.number().nonnegative().optional(),
excludeMeshApplied: z.boolean().optional(),
excludeOverlayApplied: z.boolean().optional(),
})
export const flowExportersDtoSchema = z.object({
@@ -190,12 +227,64 @@ export const flowClientsDtoSchema = z.object({
clients: z.array(flowEntityCardSchema),
})
export const flowMonthlyDtoSchema = z.object({
month: z.string(),
bytes: z.number().nonnegative(),
countries: z.array(flowBreakdownRowSchema),
services: z.array(flowBreakdownRowSchema),
asns: z.array(flowBreakdownRowSchema),
})
export const flowPurgeDtoSchema = z.object({
ok: z.literal(true),
deleted: z.object({
buckets: z.number().int().nonnegative(),
minuteStats: z.number().int().nonnegative(),
minuteDims: z.number().int().nonnegative(),
dailyDims: z.number().int().nonnegative(),
}),
fileBytesBefore: z.number().int().nonnegative(),
fileBytesAfter: z.number().int().nonnegative(),
vacuumed: z.boolean(),
})
export const flowMapHopKindSchema = z.enum(["gre", "wan", "iface"])
export const flowMapHopDtoSchema = z.object({
fromId: z.string(),
fromLabel: z.string(),
toId: z.string(),
toLabel: z.string(),
kind: flowMapHopKindSchema,
iface: z.string().optional(),
bytes: z.number().nonnegative(),
bps: z.number().nonnegative(),
bpsFwd: z.number().nonnegative(),
bpsRev: z.number().nonnegative(),
})
export const flowMapHopsDtoSchema = z.object({
hops: z.array(flowMapHopDtoSchema),
live: z.boolean(),
rangeMinutes: z.number().int().positive(),
windowSec: z.number().positive(),
dedupApplied: z.boolean(),
excludeMeshApplied: z.boolean(),
excludeOverlayApplied: z.boolean(),
})
export type FlowTalkerDto = z.infer<typeof flowTalkerDtoSchema>
export type FlowStatsDto = z.infer<typeof flowStatsDtoSchema>
export type FlowBreakdownRow = z.infer<typeof flowBreakdownRowSchema>
export type FlowIfaceChip = z.infer<typeof flowIfaceChipSchema>
export type FlowEntityCard = z.infer<typeof flowEntityCardSchema>
export type FlowMapEdge = z.infer<typeof flowMapEdgeSchema>
export type FlowPathRow = z.infer<typeof flowPathRowSchema>
export type FlowAnalyticsDto = z.infer<typeof flowAnalyticsDtoSchema>
export type FlowExportersDto = z.infer<typeof flowExportersDtoSchema>
export type FlowClientsDto = z.infer<typeof flowClientsDtoSchema>
export type FlowMonthlyDto = z.infer<typeof flowMonthlyDtoSchema>
export type FlowPurgeDto = z.infer<typeof flowPurgeDtoSchema>
export type FlowMapHopKind = z.infer<typeof flowMapHopKindSchema>
export type FlowMapHop = z.infer<typeof flowMapHopDtoSchema>
export type FlowMapHopsDto = z.infer<typeof flowMapHopsDtoSchema>
+55 -1
View File
@@ -2,6 +2,9 @@ import type {
FlowAnalyticsDto,
FlowClientsDto,
FlowExportersDto,
FlowMapHopsDto,
FlowMonthlyDto,
FlowPurgeDto,
FlowStatsDto,
TrafficFlowHostFile,
TrafficFlowOverlayResult,
@@ -60,6 +63,8 @@ function flowQuery(params: {
userId?: string
iface?: string
dedup?: boolean
excludeMesh?: boolean
excludeOverlay?: boolean
}): string {
const q = new URLSearchParams()
if (params.range) q.set("range", params.range)
@@ -68,6 +73,10 @@ function flowQuery(params: {
if (params.iface && params.iface !== "__all__") q.set("iface", params.iface)
if (params.dedup === false) q.set("dedup", "0")
else if (params.dedup === true) q.set("dedup", "1")
if (params.excludeMesh === false) q.set("excludeMesh", "0")
else if (params.excludeMesh === true) q.set("excludeMesh", "1")
if (params.excludeOverlay === false) q.set("excludeOverlay", "0")
else if (params.excludeOverlay === true) q.set("excludeOverlay", "1")
const s = q.toString()
return s ? `?${s}` : ""
}
@@ -80,11 +89,56 @@ export async function getFlowClients(baseUrl: string, range = "5m"): Promise<Flo
return requestJson<FlowClientsDto>(baseUrl, `/api/traffic/flow/clients?range=${encodeURIComponent(range)}`)
}
export async function getFlowMapHops(
baseUrl: string,
params: {
range?: string
serverId?: string
userId?: string
iface?: string
dedup?: boolean
excludeMesh?: boolean
excludeOverlay?: boolean
} = {},
): Promise<FlowMapHopsDto> {
return requestJson<FlowMapHopsDto>(baseUrl, `/api/traffic/flow/map-hops${flowQuery({
range: params.range ?? "5m",
serverId: params.serverId,
userId: params.userId,
iface: params.iface,
dedup: params.dedup,
excludeMesh: params.excludeMesh,
excludeOverlay: params.excludeOverlay,
})}`)
}
export async function getFlowAnalytics(
baseUrl: string,
params: { range?: string; serverId?: string; userId?: string; iface?: string; dedup?: boolean },
params: {
range?: string
serverId?: string
userId?: string
iface?: string
dedup?: boolean
excludeMesh?: boolean
excludeOverlay?: boolean
},
): Promise<FlowAnalyticsDto> {
return requestJson<FlowAnalyticsDto>(baseUrl, `/api/traffic/flow/analytics${flowQuery(params)}`)
}
export async function getFlowMonthly(
baseUrl: string,
params: { month: string; serverId?: string },
): Promise<FlowMonthlyDto> {
const q = new URLSearchParams()
q.set("month", params.month)
if (params.serverId) q.set("serverId", params.serverId)
return requestJson<FlowMonthlyDto>(baseUrl, `/api/traffic/flow/monthly?${q.toString()}`)
}
export async function purgeTrafficFlowData(baseUrl: string): Promise<FlowPurgeDto> {
return requestJson<FlowPurgeDto>(baseUrl, "/api/traffic/flow/purge", { method: "POST" })
}
export { flowQuery }