Compare commits

...
1 Commits
Author SHA1 Message Date
DenozordecandCursor 5aef419582 fix(statistics): считать уникальный payload без дублей hops
Docker images / prepare-release (push) Successful in 8s
Docker images / backend-test (push) Successful in 2m10s
Docker images / frontend-image (push) Successful in 3m13s
Docker images / updater-image (push) Successful in 46s
Docker images / backend-image (push) Successful in 2m52s
Docker images / notify-webhook (push) Skipped
Docker images / publish-release (push) Successful in 10s
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-11 01:27:40 +07:00
9 changed files with 375 additions and 122 deletions
+25 -8
View File
@@ -111,6 +111,10 @@ function readView(sp: URLSearchParams): "explore" | "pivot" {
return sp.get("view") === "pivot" ? "pivot" : "explore"
}
function readPlanes(sp: URLSearchParams): "unique" | "all" {
return sp.get("planes") === "all" ? "all" : "unique"
}
function readPivotDim(sp: URLSearchParams, key: string, fallback: StatisticsPivotDim): StatisticsPivotDim {
const v = sp.get(key)
return v && isStatisticsPivotDim(v) ? v : fallback
@@ -148,7 +152,7 @@ function filtersToSlices(filters: Filter[]): CubeSlices {
return next
}
function toQuery(range: DateRangeYmd, slices: CubeSlices): StatisticsQuery {
function toQuery(range: DateRangeYmd, slices: CubeSlices, planes: "unique" | "all"): StatisticsQuery {
const serverId = slices.serverId ? Number(slices.serverId) : undefined
const asn = slices.asn != null && slices.asn !== "" ? Number(slices.asn) : undefined
return {
@@ -160,6 +164,7 @@ function toQuery(range: DateRangeYmd, slices: CubeSlices): StatisticsQuery {
country: slices.country && slices.country.length === 2 ? slices.country : undefined,
service: slices.service,
asn: Number.isFinite(asn) ? asn : undefined,
planes,
}
}
@@ -260,6 +265,7 @@ export default function StatisticsPage() {
const filters = useMemo(() => slicesToFilters(slices), [slices])
const dim = useMemo(() => readDim(searchParams), [searchParams])
const view = useMemo(() => readView(searchParams), [searchParams])
const planes = useMemo(() => readPlanes(searchParams), [searchParams])
const pivotRow = useMemo(() => readPivotDim(searchParams, "pivotRow", "country"), [searchParams])
const pivotCol = useMemo(() => readPivotDim(searchParams, "pivotCol", "service"), [searchParams])
@@ -309,7 +315,7 @@ export default function StatisticsPage() {
setLoading(true)
setError(null)
try {
const query = toQuery(range, slices)
const query = toQuery(range, slices, planes)
const dto = await getStatistics(backendUrl, query)
if (!cancelled) setData(dto)
if (view === "pivot" && pivotRow !== pivotCol) {
@@ -334,14 +340,15 @@ export default function StatisticsPage() {
return () => {
cancelled = true
}
}, [backendUrl, isLive, prefsHydrated, range, slices, view, pivotRow, pivotCol])
}, [backendUrl, isLive, prefsHydrated, range, slices, view, pivotRow, pivotCol, planes])
const viewData = isLive ? data : EMPTY
const sliced = hasAnySlice(slices)
const emptyCube = !isLive || (!loading && viewData.kpis.bytes === 0)
const emptyCube = !isLive || (!loading && viewData.kpis.bytes === 0 && viewData.interfaces.length === 0)
function handleRowClick(kind: StatisticsSliceKind, row: StatisticsBreakdownRow) {
if (kind === "users" && row.id === STATISTICS_UNBOUND_USER_ID) return
if (kind === "interfaces" && row.label.includes("· дубль")) return
setSlices(applyDimValue(slices, kind, row.id))
}
@@ -395,9 +402,9 @@ export default function StatisticsPage() {
{!slices.serverId && isLive && !emptyCube ? (
<Alert>
<AlertTitle>Интернет сети</AlertTitle>
<AlertTitle>Уникальный объём</AlertTitle>
<AlertDescription>
KPI уникальный payload без overlay и транзита EN. Сумма WAN по серверам не равна интернету сети; для uplink откройте сервер.
Объём трафик клиентов на GRE/WG, без повторного учёта JHEN и WAN.
</AlertDescription>
</Alert>
) : null}
@@ -411,7 +418,7 @@ export default function StatisticsPage() {
id: "bytes",
label: "Объём",
value: formatBytes(kpis.bytes),
hint: kpis.topCountry ? `топ: ${kpis.topCountry}` : undefined,
hint: "GRE/WG клиентов, без hops",
icon: <DatabaseIcon />,
iconClassName: "text-muted-foreground",
},
@@ -443,7 +450,9 @@ export default function StatisticsPage() {
value: String(kpis.servers),
hint: slices.serverId
? (kpis.ifaces ? `${kpis.ifaces} iface` : undefined)
: "WAN — в слайсе сервера",
: planes === "all"
? "WAN и дубли в списке"
: "без WAN и overlay",
icon: <ServerIcon />,
iconClassName: "text-muted-foreground",
},
@@ -464,6 +473,14 @@ export default function StatisticsPage() {
{ value: "pivot", label: "Сводка" },
]}
/>
<SegmentedControl
value={planes}
onChange={(next) => replaceParams({ planes: next === "all" ? "all" : undefined })}
options={[
{ value: "unique", label: "Уникальный" },
{ value: "all", label: "Все плоскости" },
]}
/>
{view === "explore" && !sliced ? (
<DimensionSelect
label="Критерий"
+118 -22
View File
@@ -50,6 +50,7 @@ await ensurePartitionFor(pool, "flow_hour_facts", "day", new Date("2026-09-10T00
await dbQuery(`DELETE FROM flow_daily_facts WHERE server_id IN ($1, $2)`, [serverId, enId])
await dbQuery(`DELETE FROM flow_hour_facts WHERE server_id IN ($1, $2)`, [serverId, enId])
await dbQuery(`DELETE FROM user_interface_bindings WHERE server_id = $1`, [serverId])
await dbQuery(`DELETE FROM server_snapshots WHERE server_id IN ($1, $2)`, [serverId, enId])
await dbQuery(`DELETE FROM app_users WHERE id = 'u-stats-1'`)
await dbQuery(`
@@ -62,11 +63,38 @@ await dbQuery(`
VALUES ('bind-stats-1', 'u-stats-1', $1, 'gre-client', 'gre')
`, [serverId])
await dbQuery(`
INSERT INTO server_snapshots (server_id, polled_at, status, raw_interfaces)
VALUES
($1, '2026-09-10T12:00:00Z', 'online', $3::jsonb),
($2, '2026-09-10T12:00:00Z', 'online', $4::jsonb)
`, [
serverId,
enId,
JSON.stringify([
{ name: "gre-client", type: "gre-tunnel" },
{ name: "wan1", type: "ether" },
{ name: "gre-en", type: "gre-tunnel" },
{ name: "NSK-SERVHOST-RTK", type: "gre-tunnel" },
{ name: "wg-mesh", type: "wg" },
{ name: "wg-server", type: "wg" },
{ name: "wg-flow", type: "wg" },
]),
JSON.stringify([
{ name: "ether1", type: "ether" },
{ name: "gre-jh", type: "gre-tunnel" },
]),
])
resetIfaceCacheForTests()
rememberServerIfaces(serverId, [
{ name: "gre-client", ifindex: "2" },
{ name: "wan1", ifindex: "8" },
{ name: "gre-en", ifindex: "9" },
{ name: "NSK-SERVHOST-RTK" },
{ name: "wg-mesh" },
{ name: "wg-server" },
{ name: "wg-flow" },
])
rememberServerIfaces(enId, [
{ name: "ether1", ifindex: "2" },
@@ -83,53 +111,103 @@ await dbQuery(`
($1, '2026-09-10', 'wan1', 'NL', 'other', 0, 70, 1),
($1, '2026-09-10', '0', 'US', 'https', 0, 999, 3),
($1, '2026-09-10', 'gre-en', 'US', 'https', 15169, 400, 2),
($1, '2026-09-10', 'NSK-SERVHOST-RTK', 'US', 'https', 15169, 300, 2),
($1, '2026-09-10', 'wg-mesh', 'US', 'https', 0, 250, 2),
($1, '2026-09-10', 'wg-flow', 'US', 'https', 0, 80, 1),
($2, '2026-09-10', 'gre-jh', 'US', 'https', 15169, 500, 5),
($2, '2026-09-10', 'ether1', 'US', 'https', 15169, 200, 2)
`, [serverId, enId])
try {
const all = await getStatistics({ from: "2026-09-01", to: "2026-09-30" })
assert.equal(all.grain, "day")
assert.equal(all.kpis.bytes, 1070)
assert.equal(all.kpis.users, 1)
assert.ok(all.countries.some((r) => r.id === "US"))
assert.ok(all.users.some((r) => r.id === "u-stats-1"))
const unbound = all.users.find((r) => r.id === STATISTICS_UNBOUND_USER_ID)
assert.equal(unbound, undefined, "WAN не в Прочие / Без привязки")
assert.ok(all.servers.some((r) => r.id === String(serverId)))
assert.ok(!all.servers.some((r) => r.id === String(enId)), "EN-транзит не в сетевом KPI")
const greIface = all.interfaces.find((r) => r.label.includes("gre-client"))
const unique = await getStatistics({ from: "2026-09-01", to: "2026-09-30", planes: "unique" })
assert.equal(unique.grain, "day")
assert.equal(unique.kpis.bytes, 1000)
assert.equal(unique.kpis.users, 1)
assert.ok(unique.countries.some((r) => r.id === "US"))
assert.ok(unique.users.some((r) => r.id === "u-stats-1"))
assert.equal(unique.users.find((r) => r.id === STATISTICS_UNBOUND_USER_ID), undefined)
assert.ok(unique.servers.some((r) => r.id === String(serverId)))
assert.ok(!unique.servers.some((r) => r.id === String(enId)), "EN-транзит не в сетевом KPI")
const greIface = unique.interfaces.find((r) => r.label.includes("gre-client"))
assert.ok(greIface)
assert.equal(greIface.bytes, 1000)
assert.equal(greIface.id, `${serverId}:gre-client`)
assert.ok(!all.interfaces.some((r) => /· (?:#)?\d+$/.test(r.label)))
assert.ok(!all.interfaces.some((r) => r.label.includes(" · —") || r.label.endsWith("· —")))
assert.ok(!all.interfaces.some((r) => r.label.includes("gre-en")))
const wanRow = all.interfaces.find((r) => r.id === `${serverId}:wan1`)
assert.ok(!unique.interfaces.some((r) => /· (?:#)?\d+$/.test(r.label)))
assert.ok(!unique.interfaces.some((r) => r.label.includes(" · —") || r.label.endsWith("· —")))
assert.ok(!unique.interfaces.some((r) => r.label.includes("gre-en")))
assert.ok(!unique.interfaces.some((r) => r.label.includes("NSK-SERVHOST-RTK")))
assert.ok(!unique.interfaces.some((r) => r.label.includes("wg-mesh")))
assert.ok(!unique.interfaces.some((r) => r.label.includes("wg-flow")))
assert.equal(unique.interfaces.find((r) => r.id === `${serverId}:wan1`), undefined, "unique без WAN")
const allPlanes = await getStatistics({ from: "2026-09-01", to: "2026-09-30", planes: "all" })
assert.equal(allPlanes.kpis.bytes, 1000, "KPI unique и all одинаковый")
const wanRow = allPlanes.interfaces.find((r) => r.id === `${serverId}:wan1`)
assert.ok(wanRow)
assert.ok(wanRow.label.includes("WAN · интернет"))
assert.equal(wanRow.bytes, 70)
assert.equal(wanRow.percent, 0)
const overlayGre = allPlanes.interfaces.find((r) => r.id === `${serverId}:gre-en`)
assert.ok(overlayGre)
assert.ok(overlayGre.label.includes("дубль"))
assert.equal(overlayGre.percent, 0)
const overlayCustom = allPlanes.interfaces.find((r) => r.label.includes("NSK-SERVHOST-RTK"))
assert.ok(overlayCustom)
assert.ok(overlayCustom.label.includes("дубль"))
const overlayWg = allPlanes.interfaces.find((r) => r.label.includes("wg-mesh"))
assert.ok(overlayWg)
assert.ok(overlayWg.label.includes("дубль"))
assert.ok(!allPlanes.interfaces.some((r) => r.label.includes("wg-flow")))
const wanSlice = await getStatistics({
from: "2026-09-01",
to: "2026-09-30",
serverId,
iface: "wan1",
})
assert.equal(wanSlice.kpis.bytes, 70)
const nodeSlice = await getStatistics({
from: "2026-09-01",
to: "2026-09-30",
serverId,
planes: "unique",
})
assert.equal(nodeSlice.kpis.bytes, 1070)
const nodeWan = nodeSlice.interfaces.find((r) => r.id === `${serverId}:wan1`)
assert.ok(nodeWan)
assert.ok(nodeWan.label.includes("WAN · интернет"))
assert.ok(!nodeSlice.interfaces.some((r) => r.label.includes(" · —")))
assert.equal(nodeSlice.kpis.bytes, 1000)
assert.equal(nodeSlice.interfaces.find((r) => r.id === `${serverId}:wan1`), undefined)
assert.ok(!nodeSlice.users.some((r) => r.id === STATISTICS_UNBOUND_USER_ID))
const nodeAll = await getStatistics({
from: "2026-09-01",
to: "2026-09-30",
serverId,
planes: "all",
})
assert.equal(nodeAll.kpis.bytes, 1000)
const nodeWan = nodeAll.interfaces.find((r) => r.id === `${serverId}:wan1`)
assert.ok(nodeWan)
assert.equal(nodeWan.percent, 0)
assert.ok(nodeWan.label.includes("WAN · интернет"))
const enSlice = await getStatistics({
from: "2026-09-01",
to: "2026-09-30",
serverId: enId,
planes: "unique",
})
assert.equal(enSlice.kpis.bytes, 200)
assert.ok(enSlice.interfaces.some((r) => r.label.includes("WAN · интернет") && r.label.includes("ether1")))
assert.equal(enSlice.kpis.bytes, 0)
assert.ok(!enSlice.interfaces.some((r) => r.label.includes("gre-jh")))
assert.ok(!enSlice.interfaces.some((r) => r.label.includes("WAN · интернет")))
const enAll = await getStatistics({
from: "2026-09-01",
to: "2026-09-30",
serverId: enId,
planes: "all",
})
assert.equal(enAll.kpis.bytes, 0)
assert.ok(enAll.interfaces.some((r) => r.label.includes("WAN · интернет") && r.label.includes("ether1") && r.percent === 0))
assert.ok(enAll.interfaces.some((r) => r.label.includes("gre-jh") && r.label.includes("дубль")))
const sliced = await getStatistics({
from: "2026-09-01",
@@ -164,6 +242,22 @@ try {
assert.equal(us.cells.https, 800)
assert.equal(de.cells.dns, 200)
await dbQuery(`
INSERT INTO user_interface_bindings (id, user_id, server_id, interface_name, interface_type)
VALUES ('bind-stats-wg', 'u-stats-1', $1, 'wg-server', 'wg')
`, [serverId])
await dbQuery(`
INSERT INTO flow_daily_facts (server_id, day, iface, country, service, asn, bytes, packets)
VALUES ($1, '2026-09-10', 'wg-server', 'US', 'https', 15169, 150, 2)
`, [serverId])
invalidateFlowCatalogCache()
const withWg = await getStatistics({ from: "2026-09-01", to: "2026-09-30", planes: "unique" })
assert.equal(withWg.kpis.bytes, 1150)
assert.ok(withWg.interfaces.some((r) => r.label.includes("wg-server") && r.bytes === 150))
assert.ok(!withWg.interfaces.some((r) => r.label.includes("wg-flow")))
assert.ok(!withWg.interfaces.some((r) => r.label.includes("wg-mesh")))
await dbQuery(`
INSERT INTO flow_hour_facts (server_id, bucket_at, iface, country, service, asn, bytes, packets)
VALUES ($1, '2026-09-10T10:00:00Z', '2', 'US', 'https', 15169, 40, 2)
@@ -181,6 +275,8 @@ try {
invalidateFlowCatalogCache()
await dbQuery(`DELETE FROM flow_daily_facts WHERE server_id IN ($1, $2)`, [serverId, enId])
await dbQuery(`DELETE FROM flow_hour_facts WHERE server_id IN ($1, $2)`, [serverId, enId])
await dbQuery(`DELETE FROM user_interface_bindings WHERE server_id IN ($1, $2)`, [serverId, enId])
await dbQuery(`DELETE FROM server_snapshots WHERE server_id IN ($1, $2)`, [serverId, enId])
await dbQuery(`DELETE FROM servers WHERE id IN ($1, $2)`, [serverId, enId])
}
+121 -77
View File
@@ -21,8 +21,9 @@ import { refreshServerIfaces } from "./traffic-flow-ifaces.js"
import {
isDashDisplayIface,
isJunkFactIface,
isOverlayGreIface,
isOverlayTunnelIface,
isWanFactIface,
overlayDupLabel,
wanIfaceLabel,
} from "./traffic-flow-facts-filter.js"
import { getServerCatalog, loadFlowTopology, type FlowTopology } from "./traffic-flow-topology.js"
@@ -87,6 +88,8 @@ export function parseStatisticsPeriod(fromRaw: string, toRaw: string): ParsedPer
}
}
type FactScope = "unique" | "wan" | "overlay"
interface FilterCtx {
fromIso: string
toIso: string
@@ -97,6 +100,7 @@ interface FilterCtx {
country?: string
service?: string
asn?: number
planes: "unique" | "all"
userIfaces: Array<{ serverId: number; iface: string }> | null
unboundOnly: boolean
boundIfaces: Array<{ serverId: number; iface: string }>
@@ -134,7 +138,30 @@ function canonicalIfaceDimId(id: string): string {
return `${sid}:${displayFactIface(sid, id.slice(colon + 1))}`
}
function factWhere(alias: string, grain: "hour" | "day", ctx: FilterCtx): { sql: string; params: unknown[] } {
function pushIfaceTuples(
parts: string[],
params: unknown[],
alias: string,
tuples: Array<{ serverId: number; iface: string }>,
op: "IN" | "NOT IN",
): void {
if (!tuples.length) {
if (op === "IN") parts.push("FALSE")
return
}
const sql = tuples.map(() => "(?, ?)").join(", ")
parts.push(`(${alias}.server_id, ${alias}.iface) ${op} (${sql})`)
for (const t of tuples) {
params.push(t.serverId, t.iface)
}
}
function factWhere(
alias: string,
grain: "hour" | "day",
ctx: FilterCtx,
scope: FactScope = "unique",
): { sql: string; params: unknown[] } {
const params: unknown[] = []
const parts: string[] = []
if (grain === "hour") {
@@ -148,16 +175,6 @@ function factWhere(alias: string, grain: "hour" | "day", ctx: FilterCtx): { sql:
parts.push(`${alias}.server_id = ?`)
params.push(ctx.serverId)
}
if (ctx.iface) {
const aliases = ifaceFilterAliases(ctx.iface, ctx.serverId)
if (aliases.length <= 1) {
parts.push(`${alias}.iface = ?`)
params.push(aliases[0] ?? ctx.iface)
} else {
parts.push(`${alias}.iface IN (${aliases.map(() => "?").join(", ")})`)
params.push(...aliases)
}
}
if (ctx.country) {
parts.push(`${alias}.country = ?`)
params.push(ctx.country.toUpperCase())
@@ -170,39 +187,37 @@ function factWhere(alias: string, grain: "hour" | "day", ctx: FilterCtx): { sql:
parts.push(`${alias}.asn = ?`)
params.push(ctx.asn)
}
if (ctx.userIfaces) {
if (ctx.userIfaces.length === 0) {
parts.push("FALSE")
parts.push(`${alias}.iface NOT IN ('0', '—', '__unknown__', 'wg-flow', '')`)
if (scope === "wan") {
pushIfaceTuples(parts, params, alias, ctx.wanIfaces, "IN")
return { sql: parts.join(" AND "), params }
}
if (scope === "overlay") {
pushIfaceTuples(parts, params, alias, ctx.overlayIfaces, "IN")
return { sql: parts.join(" AND "), params }
}
if (ctx.iface) {
const aliases = ifaceFilterAliases(ctx.iface, ctx.serverId)
if (aliases.length <= 1) {
parts.push(`${alias}.iface = ?`)
params.push(aliases[0] ?? ctx.iface)
} else {
const tuples = ctx.userIfaces.map(() => "(?, ?)").join(", ")
parts.push(`(${alias}.server_id, ${alias}.iface) IN (${tuples})`)
for (const u of ctx.userIfaces) {
params.push(u.serverId, u.iface)
}
parts.push(`${alias}.iface IN (${aliases.map(() => "?").join(", ")})`)
params.push(...aliases)
}
return { sql: parts.join(" AND "), params }
}
if (ctx.userIfaces) {
pushIfaceTuples(parts, params, alias, ctx.userIfaces, "IN")
return { sql: parts.join(" AND "), params }
}
if (ctx.unboundOnly) {
const skip = [...ctx.boundIfaces, ...ctx.wanIfaces]
if (skip.length) {
const tuples = skip.map(() => "(?, ?)").join(", ")
parts.push(`(${alias}.server_id, ${alias}.iface) NOT IN (${tuples})`)
for (const u of skip) params.push(u.serverId, u.iface)
}
}
parts.push(`${alias}.iface NOT IN ('0', '—', '__unknown__', 'wg-flow', '')`)
const greLike = `(LOWER(${alias}.iface) LIKE 'gre%' OR LOWER(${alias}.iface) LIKE '%gre-tunnel%')`
if (ctx.boundIfaces.length) {
const tuples = ctx.boundIfaces.map(() => "(?, ?)").join(", ")
parts.push(`(NOT ${greLike} OR (${alias}.server_id, ${alias}.iface) IN (${tuples}))`)
for (const u of ctx.boundIfaces) params.push(u.serverId, u.iface)
} else {
parts.push(`NOT ${greLike}`)
}
if (ctx.overlayIfaces.length) {
const tuples = ctx.overlayIfaces.map(() => "(?, ?)").join(", ")
parts.push(`(${alias}.server_id, ${alias}.iface) NOT IN (${tuples})`)
for (const u of ctx.overlayIfaces) params.push(u.serverId, u.iface)
parts.push("FALSE")
return { sql: parts.join(" AND "), params }
}
pushIfaceTuples(parts, params, alias, ctx.boundIfaces, "IN")
if (ctx.excludeServerIds.length) {
parts.push(`${alias}.server_id NOT IN (${ctx.excludeServerIds.map(() => "?").join(", ")})`)
params.push(...ctx.excludeServerIds)
@@ -336,8 +351,10 @@ async function loadPayloadScope(serverId?: number): Promise<{
? [...wanSet]
: s.type === "home-router" ? [] : ["ether1"]
for (const name of wanNames) wanRaw.push({ serverId: s.id, iface: name })
for (const name of listCachedIfaceNames(s.id)) {
if (isOverlayGreIface(topo, s.id, name)) overlayRaw.push({ serverId: s.id, iface: name })
const names = new Set(listCachedIfaceNames(s.id))
for (const name of topo.tunnelIfaces?.get(s.id) ?? []) names.add(name)
for (const name of names) {
if (isOverlayTunnelIface(topo, s.id, name)) overlayRaw.push({ serverId: s.id, iface: name })
}
}
return {
@@ -356,6 +373,7 @@ async function buildFilterCtx(query: StatisticsQuery, period: ParsedPeriod): Pro
const unboundOnly = query.userId === STATISTICS_UNBOUND_USER_ID
const userIfaces = unboundOnly ? null : await resolveUserIfaces(query.userId)
if (userIfaces && userIfaces.length === 0) return null
if (unboundOnly) return null
const scope = await loadPayloadScope(query.serverId)
return {
...period,
@@ -364,6 +382,7 @@ async function buildFilterCtx(query: StatisticsQuery, period: ParsedPeriod): Pro
country: query.country,
service: query.service,
asn: query.asn,
planes: query.planes ?? "unique",
userIfaces,
unboundOnly,
boundIfaces,
@@ -453,11 +472,52 @@ export async function getStatistics(query: StatisticsQuery): Promise<StatisticsD
await warmIfaceCache(ifaceRowsRaw.filter((r) => looksLikeIfIndex(r.iface)).map((r) => r.serverId))
const ifaceRows = collapseServerIfaceRows(ifaceRowsRaw).filter((r) => {
if (isJunkFactIface(r.iface) || isDashDisplayIface(r.iface)) return false
if (ctx.topo && isOverlayGreIface(ctx.topo, r.serverId, r.iface)) return false
if (ctx.iface) return true
if (ctx.topo && isOverlayTunnelIface(ctx.topo, r.serverId, r.iface)) return false
if (ctx.topo && isWanFactIface(ctx.topo, r.serverId, r.iface)) return false
return true
})
const ifaceCount = ifaceRows.length
let dupeIfaceRows: Array<{ serverId: number; iface: string; bytes: number; packets: number; kind: "wan" | "overlay" }> = []
if (ctx.planes === "all" && !ctx.iface) {
const wanWhere = factWhere("f", period.grain, ctx, "wan")
const overlayWhere = factWhere("f", period.grain, ctx, "overlay")
const [wanRaw, overlayRaw] = await Promise.all([
dbAll<{ serverId: number; iface: string; bytes: number; packets: number }>(`
SELECT f.server_id AS "serverId", f.iface AS iface, SUM(f.bytes) AS bytes, SUM(f.packets) AS packets
FROM ${table} f
WHERE ${wanWhere.sql}
GROUP BY f.server_id, f.iface
`, wanWhere.params),
dbAll<{ serverId: number; iface: string; bytes: number; packets: number }>(`
SELECT f.server_id AS "serverId", f.iface AS iface, SUM(f.bytes) AS bytes, SUM(f.packets) AS packets
FROM ${table} f
WHERE ${overlayWhere.sql}
GROUP BY f.server_id, f.iface
`, overlayWhere.params),
])
await warmIfaceCache([
...wanRaw.filter((r) => looksLikeIfIndex(r.iface)).map((r) => r.serverId),
...overlayRaw.filter((r) => looksLikeIfIndex(r.iface)).map((r) => r.serverId),
])
const seen = new Set(ifaceRows.map((r) => `${r.serverId}:${r.iface}`))
for (const r of collapseServerIfaceRows(wanRaw)) {
if (isJunkFactIface(r.iface) || isDashDisplayIface(r.iface)) continue
const key = `${r.serverId}:${r.iface}`
if (seen.has(key)) continue
seen.add(key)
dupeIfaceRows.push({ ...r, kind: "wan" })
}
for (const r of collapseServerIfaceRows(overlayRaw)) {
if (isJunkFactIface(r.iface) || isDashDisplayIface(r.iface)) continue
const key = `${r.serverId}:${r.iface}`
if (seen.has(key)) continue
seen.add(key)
dupeIfaceRows.push({ ...r, kind: "overlay" })
}
}
let userRows: Array<{ id: string; bytes: number; packets: number }> = []
if (bindTuples.length && !ctx.unboundOnly) {
const join = userBindJoinSql(bindTuples)
@@ -526,7 +586,7 @@ export async function getStatistics(query: StatisticsQuery): Promise<StatisticsD
bytes,
period.windowSec,
)
const interfaces = toBreakdown(
const uniqueInterfaces = toBreakdown(
ifaceRows.map((r) => {
const serverName = serverNames.get(r.serverId) || String(r.serverId)
const wan = ctx.topo ? isWanFactIface(ctx.topo, r.serverId, r.iface) : false
@@ -540,6 +600,20 @@ export async function getStatistics(query: StatisticsQuery): Promise<StatisticsD
bytes,
period.windowSec,
)
const dupeInterfaces: StatisticsBreakdownRow[] = dupeIfaceRows.map((r) => {
const serverName = serverNames.get(r.serverId) || String(r.serverId)
const rowBytes = Number(r.bytes) || 0
const rowPackets = Number(r.packets) || 0
return {
id: `${r.serverId}:${r.iface}`,
label: r.kind === "wan" ? wanIfaceLabel(serverName, r.iface) : overlayDupLabel(serverName, r.iface),
bytes: rowBytes,
packets: rowPackets,
bps: (rowBytes * 8) / period.windowSec,
percent: 0,
}
})
const interfaces = [...uniqueInterfaces, ...dupeInterfaces]
const matchedUsers = toBreakdown(
userRows.map((r) => ({
id: r.id,
@@ -552,39 +626,6 @@ export async function getStatistics(query: StatisticsQuery): Promise<StatisticsD
)
const users = [...matchedUsers]
if (!ctx.unboundOnly && !ctx.userIfaces) {
let unboundBytes = 0
let unboundPackets = 0
const skipUnbound = [...ctx.boundIfaces, ...ctx.wanIfaces]
if (skipUnbound.length === 0) {
unboundBytes = bytes
unboundPackets = packets
} else {
const tuples = skipUnbound.map(() => "(?, ?)").join(", ")
const unboundParams = [...where.params]
for (const u of skipUnbound) unboundParams.push(u.serverId, u.iface)
const unboundRows = await dbAll<{ bytes: number; packets: number }>(`
SELECT COALESCE(SUM(f.bytes), 0) AS bytes, COALESCE(SUM(f.packets), 0) AS packets
FROM ${table} f
WHERE ${where.sql}
AND (f.server_id, f.iface) NOT IN (${tuples})
`, unboundParams)
unboundBytes = Number(unboundRows[0]?.bytes) || 0
unboundPackets = Number(unboundRows[0]?.packets) || 0
}
if (unboundBytes > 0) {
const denom = bytes || 1
users.push({
id: STATISTICS_UNBOUND_USER_ID,
label: "Прочие",
bytes: unboundBytes,
packets: unboundPackets,
bps: (unboundBytes * 8) / period.windowSec,
percent: (unboundBytes / denom) * 100,
})
users.sort((a, b) => b.bytes - a.bytes)
}
}
return {
from: period.fromIso,
@@ -828,6 +869,9 @@ async function loadPivotLabels(
if (Number.isFinite(sidNum) && isWanFactIface(topo, sidNum, name)) {
return wanIfaceLabel(serverName, name)
}
if (Number.isFinite(sidNum) && isOverlayTunnelIface(topo, sidNum, name)) {
return overlayDupLabel(serverName, name)
}
return `${serverName} · ${name}`
}
return id
@@ -2,6 +2,7 @@ import assert from "node:assert/strict"
import {
isJunkFactIface,
isOverlayGreIface,
isOverlayTunnelIface,
isWanFactIface,
shouldWriteFlowFact,
} from "./traffic-flow-facts-filter.js"
@@ -21,6 +22,7 @@ function topo(partial: Partial<FlowTopology> = {}): FlowTopology {
enHosts,
jhHosts,
wanIfaces,
tunnelIfaces: partial.tunnelIfaces,
plane: partial.plane ?? {
clientIfaceNames: new Set(["gre-client"]),
enHosts,
@@ -35,12 +37,28 @@ seedFlowTopologyForTests(topo())
assert.equal(isJunkFactIface("0"), true)
assert.equal(isJunkFactIface(""), true)
assert.equal(isJunkFactIface("wg-flow"), true)
assert.equal(isJunkFactIface("ether1"), false)
assert.equal(isWanFactIface(topo(), 1, "ether1"), true)
assert.equal(isOverlayGreIface(topo(), 1, "gre-en"), true)
assert.equal(isOverlayGreIface(topo(), 1, "gre-client"), false)
assert.equal(isOverlayGreIface(topo(), 1, "ether1"), false)
const typed = topo({
clientIfaces: new Map([[1, new Set(["gre-client", "wg-server"])]]),
tunnelIfaces: new Map([[1, new Set(["gre-en", "NSK-SERVHOST-RTK", "wg-jh-en", "wg-server"])]]),
plane: {
clientIfaceNames: new Set(["gre-client", "wg-server"]),
enHosts: new Set(["198.51.100.1"]),
jhHosts: new Set(["203.0.113.10"]),
},
})
assert.equal(isOverlayTunnelIface(typed, 1, "NSK-SERVHOST-RTK"), true, "кастомное GRE overlay по type")
assert.equal(isOverlayTunnelIface(typed, 1, "wg-jh-en"), true, "WG overlay по type")
assert.equal(isOverlayTunnelIface(typed, 1, "wg-server"), false, "клиентский WG с binding")
assert.equal(isOverlayTunnelIface(typed, 1, "wg-flow"), false, "wg-flow не overlay")
assert.equal(isOverlayTunnelIface(topo(), 1, "NSK-SERVHOST-RTK"), false, "без type в снимке — не overlay")
const overlayOuter = shouldWriteFlowFact({
serverId: 1,
serverType: "jump-host",
@@ -1,10 +1,13 @@
import { mapRosInterfaceType } from "../modules/users/iface-type.js"
import { STATISTICS_DUP_MARK, STATISTICS_WAN_MARK } from "@mmapp/contracts/statistics"
import { canonicalFactIface } from "./traffic-flow-ifindex.js"
import { classifyFlowPlane } from "./traffic-flow-planes.js"
import { resolveClient, type FlowTopology } from "./traffic-flow-topology.js"
const JUNK_IFACE = new Set(["", "0", "—", "__unknown__", "wg-flow"])
export { STATISTICS_WAN_MARK, STATISTICS_DUP_MARK }
export function isJunkFactIface(iface: string | null | undefined): boolean {
const n = String(iface ?? "").trim()
if (JUNK_IFACE.has(n)) return true
@@ -15,8 +18,23 @@ export function isDashDisplayIface(iface: string): boolean {
return String(iface ?? "").trim() === "—"
}
export function isGreIfaceName(name: string): boolean {
return mapRosInterfaceType("", name) === "gre"
export function isMgmtIface(name: string): boolean {
const n = String(name ?? "").trim().toLowerCase()
return n === "wg-flow" || n.endsWith("/wg-flow") || n.includes("wg-flow")
}
/** GRE или WG по снимку RouterOS, иначе по имени. */
export function isTunnelIfaceName(
topo: FlowTopology | null | undefined,
serverId: number,
iface: string,
): boolean {
const name = String(iface ?? "").trim()
if (!name || isJunkFactIface(name) || isMgmtIface(name)) return false
const typed = topo?.tunnelIfaces?.get(serverId)
if (typed && typed.size > 0) return typed.has(name)
const t = mapRosInterfaceType("", name)
return t === "gre" || t === "wg"
}
/** WAN uplink: `wanIfaces` топологии, иначе ether1 у JH/EN без wan_uplinks. */
@@ -32,17 +50,25 @@ export function isWanFactIface(
return /^ether1$/i.test(name)
}
/** GRE между своими серверами / транзит EN: gre-имя, не клиент, не WAN. */
export function isOverlayGreIface(
/** Overlay JH↔EN: GRE/WG не клиент, не WAN, не wg-flow. */
export function isOverlayTunnelIface(
topo: FlowTopology | null | undefined,
serverId: number,
iface: string,
): boolean {
const name = String(iface ?? "").trim()
if (!name || isWanFactIface(topo, serverId, name)) return false
if (!name || isWanFactIface(topo, serverId, name) || isMgmtIface(name)) return false
if (topo?.clientIfaces.get(serverId)?.has(name)) return false
if (name === "wg-flow") return false
return isGreIfaceName(name)
return isTunnelIfaceName(topo, serverId, name)
}
/** @deprecated используйте isOverlayTunnelIface (GRE и WG). */
export function isOverlayGreIface(
topo: FlowTopology | null | undefined,
serverId: number,
iface: string,
): boolean {
return isOverlayTunnelIface(topo, serverId, iface)
}
export function shouldWriteFlowFact(opts: {
@@ -75,11 +101,19 @@ export function shouldWriteFlowFact(opts: {
const client =
resolveClient(opts.topo, opts.serverId, inName)
?? (outName ? resolveClient(opts.topo, opts.serverId, outName) : null)
if (!client && isOverlayGreIface(opts.topo, opts.serverId, inName)) return false
if (!client && isOverlayTunnelIface(opts.topo, opts.serverId, inName)) return false
}
return true
}
export function wanIfaceLabel(serverName: string, iface: string): string {
return `${serverName} · ${iface} · WAN · интернет`
return `${serverName} · ${iface} · ${STATISTICS_WAN_MARK}`
}
export function overlayDupLabel(serverName: string, iface: string): string {
return `${serverName} · ${iface} · ${STATISTICS_DUP_MARK}`
}
export function isNonUniqueShareLabel(label: string): boolean {
return label.includes(STATISTICS_WAN_MARK) || label.includes(`· ${STATISTICS_DUP_MARK}`)
}
+30 -3
View File
@@ -1,7 +1,7 @@
import { db, dbAll } from "../db/index.js"
import { parseJsonArray } from "../db/json.js"
import { appUsers, servers, userInterfaceBindings } from "../db/schema.js"
import { mapRosInterfaceType } from "../modules/users/iface-type.js"
import { mapRosInterfaceType, parseRawInterfaces } from "../modules/users/iface-type.js"
import type { PlaneTopology } from "./traffic-flow-planes.js"
export interface FlowClientBinding {
@@ -25,6 +25,8 @@ export interface FlowTopology {
enHosts: Set<string>
jhHosts: Set<string>
wanIfaces: Map<number, Set<string>>
/** GRE/WG из последнего снимка RouterOS (`type`), без mgmt. */
tunnelIfaces?: Map<number, Set<string>>
plane: PlaneTopology
}
@@ -85,6 +87,25 @@ function ifaceKey(serverId: number, name: string): string {
return `${serverId}|${name}`
}
async function loadTunnelIfacesFromSnapshots(): Promise<Map<number, Set<string>>> {
const rows = await dbAll<{ serverId: number; rawInterfaces: unknown }>(`
SELECT DISTINCT ON (server_id) server_id AS "serverId", raw_interfaces AS "rawInterfaces"
FROM server_snapshots
ORDER BY server_id, polled_at DESC
`)
const map = new Map<number, Set<string>>()
for (const r of rows) {
const set = new Set<string>()
for (const iface of parseRawInterfaces(r.rawInterfaces)) {
if (iface.type !== "gre" && iface.type !== "wg") continue
if (iface.name.toLowerCase() === "wg-flow") continue
set.add(iface.name)
}
if (set.size) map.set(r.serverId, set)
}
return map
}
export async function loadFlowTopology(): Promise<FlowTopology> {
if (seeded) return seeded
const now = Date.now()
@@ -127,6 +148,7 @@ export async function loadFlowTopology(): Promise<FlowTopology> {
for (const h of hosts) jhHosts.add(h)
}
}
const tunnelIfaces = await loadTunnelIfacesFromSnapshots()
const topo: FlowTopology = {
clientIfaces,
clientByIface,
@@ -134,6 +156,7 @@ export async function loadFlowTopology(): Promise<FlowTopology> {
enHosts,
jhHosts,
wanIfaces,
tunnelIfaces,
plane: {
clientIfaceNames: allClientNames,
enHosts,
@@ -177,10 +200,14 @@ export function resolveEn(
export function enGreIfaceNames(topo: FlowTopology, serverId: number, ifaceNames: string[]): string[] {
const client = topo.clientIfaces.get(serverId) ?? new Set<string>()
const wan = topo.wanIfaces.get(serverId) ?? new Set<string>()
const typed = topo.tunnelIfaces?.get(serverId)
return ifaceNames.filter((name) => {
if (client.has(name)) return false
if (client.has(name) || wan.has(name)) return false
if (name === "wg-flow") return false
return mapRosInterfaceType("", name) === "gre"
if (typed && typed.size > 0) return typed.has(name)
const t = mapRosInterfaceType("", name)
return t === "gre" || t === "wg"
})
}
@@ -5,7 +5,12 @@ import { Flag } from "@/components/flag"
import { Badge } from "@/components/reui/badge"
import { fmtBps, formatBytes } from "@/lib/fmt-rate"
import { cn } from "@/lib/utils"
import { STATISTICS_UNBOUND_USER_ID, type StatisticsBreakdownRow } from "@mmapp/contracts/statistics"
import {
STATISTICS_DUP_MARK,
STATISTICS_UNBOUND_USER_ID,
STATISTICS_WAN_MARK,
type StatisticsBreakdownRow,
} from "@mmapp/contracts/statistics"
export type StatisticsSliceKind = "users" | "servers" | "interfaces" | "countries" | "services" | "asns"
@@ -72,7 +77,14 @@ export function StatisticsBreakdownDataGrid({
id: "percent",
header: "Доля",
accessorKey: "percent",
cell: (row) => <span className="tabular-nums">{row.percent.toFixed(1)}%</span>,
cell: (row) => (
<span className="tabular-nums">
{row.percent === 0
&& (row.label.includes(STATISTICS_WAN_MARK) || row.label.includes(`· ${STATISTICS_DUP_MARK}`))
? "—"
: `${row.percent.toFixed(1)}%`}
</span>
),
},
]
+3
View File
@@ -1,6 +1,8 @@
import { z } from "zod"
export const STATISTICS_UNBOUND_USER_ID = "__unbound__"
export const STATISTICS_WAN_MARK = "WAN · интернет"
export const STATISTICS_DUP_MARK = "дубль"
export const statisticsPivotDimSchema = z.enum([
"country",
@@ -45,6 +47,7 @@ export const statisticsQuerySchema = z.object({
country: z.string().min(2).max(2).optional(),
service: z.string().min(1).optional(),
asn: z.coerce.number().int().optional(),
planes: z.enum(["unique", "all"]).default("unique"),
})
export const statisticsDtoSchema = z.object({
+3 -1
View File
@@ -7,7 +7,7 @@ import type {
import { requestJson } from "@/shared/api/http-client"
export type { StatisticsDto, StatisticsQuery, StatisticsPivotDto, StatisticsPivotQuery }
export { STATISTICS_UNBOUND_USER_ID } from "@mmapp/contracts/statistics"
export { STATISTICS_UNBOUND_USER_ID, STATISTICS_WAN_MARK, STATISTICS_DUP_MARK } from "@mmapp/contracts/statistics"
export async function getStatistics(
baseUrl: string,
@@ -22,6 +22,7 @@ export async function getStatistics(
if (query.country) params.set("country", query.country)
if (query.service) params.set("service", query.service)
if (query.asn != null) params.set("asn", String(query.asn))
if (query.planes && query.planes !== "unique") params.set("planes", query.planes)
return requestJson<StatisticsDto>(baseUrl, `/api/statistics?${params.toString()}`)
}
@@ -41,5 +42,6 @@ export async function getStatisticsPivot(
if (query.country) params.set("country", query.country)
if (query.service) params.set("service", query.service)
if (query.asn != null) params.set("asn", String(query.asn))
if (query.planes && query.planes !== "unique") params.set("planes", query.planes)
return requestJson<StatisticsPivotDto>(baseUrl, `/api/statistics/pivot?${params.toString()}`)
}