denozord/MikrotikManager
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 104 Wiki Activity

fix(netflow): исключить специальные IPv4 из внешнего обогащения

Browse Source
This commit is contained in:
Denozordec committed 2026-10-11 12:59:57 +07:00
1 parent f3156b44cf
commit b619ef208e
14 files changed
+448 -81
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/src/services/traffic-flow-analytics.test.ts
+49 -37
View File
@@ -44,6 +44,12 @@ if (!(await withPgOrSkip())) {
process.exit(0)
}
const fixture = await dbQuery<{ id: number }>(
"INSERT INTO servers (name, host) VALUES ($1, $2) RETURNING id",
["analytics-test-" + Date.now(), "127.0.0.1"],
)
const serverId = fixture.rows[0]!.id
try {
disableCatalogFetchForTests()
resetFlowCatalogForTests()
disableRipePersistForTests()
@@ -60,13 +66,13 @@ seedFlowTopologyForTests({
wanIfaces: new Map(),
plane: { clientIfaceNames: new Set(), enHosts: new Set(), jhHosts: new Set() },
})
rememberServerIfaces(7, [
rememberServerIfaces(serverId, [
{ ".id": "*2", name: "ether1" },
{ ".id": "*B", name: "ether2" },
{ ".id": "*A", name: "wg-flow" },
])
ingestParsedFlowsForServerForTests(7, [
ingestParsedFlowsForServerForTests(serverId, [
{
src: "10.1.1.8",
dst: "8.8.8.8",
@@ -92,7 +98,7 @@ ingestParsedFlowsForServerForTests(7, [
])
try {
const all = await buildFlowAnalytics({ minutes: 5, serverId: 7 })
const all = await buildFlowAnalytics({ minutes: 5, serverId })
assert.equal(all.applications[0]?.label, "HTTPS")
assert.ok(all.protocols.some((p) => p.label === "TCP"))
assert.equal(all.ifaces[0]?.name, "ether1")
@@ -104,11 +110,11 @@ try {
assert.equal(conv.application, "HTTPS")
assert.ok(!/^\d+$/.test(conv.inIface))
const filtered = await buildFlowAnalytics({ minutes: 5, serverId: 7, iface: "ether1" })
const filtered = await buildFlowAnalytics({ minutes: 5, serverId, iface: "ether1" })
assert.ok(filtered.bytes >= 12_000)
assert.equal(filtered.ifaces[0]?.name, "ether1")
const miss = await buildFlowAnalytics({ minutes: 5, serverId: 7, iface: "wg-flow" })
const miss = await buildFlowAnalytics({ minutes: 5, serverId, iface: "wg-flow" })
assert.equal(miss.conversations, 0)
const other = await buildFlowAnalytics({ minutes: 5, serverId: 99 })
@@ -120,12 +126,12 @@ try {
resetFlowRingsForTests()
resetIfaceCacheForTests()
rememberServerIfaces(7, [
rememberServerIfaces(serverId, [
{ ".id": "*2", name: "ether1" },
{ ".id": "*B", name: "ether2" },
{ ".id": "*A", name: "wg-flow" },
])
ingestParsedFlowsForServerForTests(7, [
ingestParsedFlowsForServerForTests(serverId, [
{
src: "10.1.1.8",
dst: "8.8.8.8",
@@ -150,10 +156,10 @@ ingestParsedFlowsForServerForTests(7, [
},
])
try {
const summed = await buildFlowAnalytics({ minutes: 5, serverId: 7, dedup: false })
const summed = await buildFlowAnalytics({ minutes: 5, serverId, dedup: false })
assert.equal(summed.bytes, 21_000)
assert.equal(summed.conversations, 2)
const deduped = await buildFlowAnalytics({ minutes: 5, serverId: 7, dedup: true })
const deduped = await buildFlowAnalytics({ minutes: 5, serverId, dedup: true })
assert.equal(deduped.bytes, 12_000)
assert.equal(deduped.conversations, 1)
assert.equal(deduped.dedupApplied, true)
@@ -179,8 +185,8 @@ seedRipeCacheForTests({
seedFlowCatalogForTests({
cidrs: [{ cidr: "8.8.8.0/24", purpose: "steam-gaming" }],
})
rememberServerIfaces(7, [{ ".id": "*2", name: "ether1" }])
ingestParsedFlowsForServerForTests(7, [
rememberServerIfaces(serverId, [{ ".id": "*2", name: "ether1" }])
ingestParsedFlowsForServerForTests(serverId, [
{
src: "10.1.1.8",
dst: "8.8.8.8",
@@ -194,7 +200,7 @@ ingestParsedFlowsForServerForTests(7, [
},
])
try {
const geo = await buildFlowAnalytics({ minutes: 5, serverId: 7 })
const geo = await buildFlowAnalytics({ minutes: 5, serverId })
assert.equal(geo.categories?.[0]?.label, "Игры")
assert.ok(geo.asns?.some((r) => r.label.includes("AS15169")))
assert.equal(geo.countries?.[0]?.id, "US")
@@ -224,8 +230,8 @@ seedRipeCacheForTests({
ok: true,
fetchedAt: Date.now(),
})
rememberServerIfaces(7, [{ ".id": "*2", name: "ether1" }])
ingestParsedFlowsForServerForTests(7, [
rememberServerIfaces(serverId, [{ ".id": "*2", name: "ether1" }])
ingestParsedFlowsForServerForTests(serverId, [
{
src: "10.1.1.8",
dst: "1.1.1.1",
@@ -239,7 +245,7 @@ ingestParsedFlowsForServerForTests(7, [
},
])
try {
const cf = await buildFlowAnalytics({ minutes: 5, serverId: 7 })
const cf = await buildFlowAnalytics({ minutes: 5, serverId })
assert.equal(cf.countries?.[0]?.id, "US")
assert.ok(cf.mapEdges?.every((e) => e.toCountry !== "?"))
assert.equal(cf.services?.[0]?.label, "Cloudflare")
@@ -254,7 +260,7 @@ try {
}
{
ingestParsedFlowsForServerForTests(7, [
ingestParsedFlowsForServerForTests(serverId, [
{
src: "10.1.1.8",
dst: "8.8.8.8",
@@ -267,7 +273,7 @@ try {
outIface: "",
},
])
const degraded = await buildFlowAnalytics({ minutes: 5, serverId: 7, skipHeavy: true })
const degraded = await buildFlowAnalytics({ minutes: 5, serverId, skipHeavy: true })
assert.equal(degraded.degraded, true)
assert.equal(degraded.conversationsList.length, 0)
assert.ok((degraded.bytes ?? 0) >= 12_000)
@@ -283,22 +289,22 @@ try {
const { pool } = await import("../db/index.js")
const { ensurePartitionFor } = await import("../db/partitions.js")
await ensurePartitionFor(pool, "flow_daily_dims", "month", new Date("2026-09-01T00:00:00Z"))
await dbQuery(`DELETE FROM flow_daily_dims WHERE server_id = 7 AND day >= '2026-09-01' AND day < '2026-10-01'`)
await dbQuery(`DELETE FROM flow_daily_dims WHERE server_id = $1 AND day >= '2026-09-01' AND day < '2026-10-01'`, [serverId])
await dbQuery(`
INSERT INTO flow_daily_dims (server_id, day, dim, key, bytes, packets)
VALUES
(7, '2026-09-01', 'country', 'US', 1000, 10),
(7, '2026-09-02', 'country', 'US', 500, 5),
(7, '2026-09-01', 'service', 'steam', 800, 8),
(7, '2026-09-01', 'asn', '15169', 900, 9),
(7, '2026-09-01', 'asn', 'other', 100, 1)
`)
const monthly = await getFlowMonthly("2026-09", 7)
($1, '2026-09-01', 'country', 'US', 1000, 10),
($1, '2026-09-02', 'country', 'US', 500, 5),
($1, '2026-09-01', 'service', 'steam', 800, 8),
($1, '2026-09-01', 'asn', '15169', 900, 9),
($1, '2026-09-01', 'asn', 'other', 100, 1)
`, [serverId])
const monthly = await getFlowMonthly("2026-09", serverId)
assert.equal(monthly.bytes, 1500)
assert.equal(monthly.countries[0]?.id, "US")
assert.equal(monthly.countries[0]?.bytes, 1500)
assert.ok(monthly.asns.some((row) => row.id === "other"))
await dbQuery(`DELETE FROM flow_daily_dims WHERE server_id = 7 AND day >= '2026-09-01' AND day < '2026-10-01'`)
await dbQuery(`DELETE FROM flow_daily_dims WHERE server_id = $1 AND day >= '2026-09-01' AND day < '2026-10-01'`, [serverId])
}
}
@@ -308,18 +314,18 @@ try {
resetRipeCacheForTests()
resetFlowCatalogForTests()
disableRipeEnqueueForTests()
rememberServerIfaces(7, [
rememberServerIfaces(serverId, [
{ ".id": "*2", name: "gre-client" },
{ ".id": "*3", name: "NSK-SERVHOST-RTK" },
{ ".id": "*4", name: "gre-en-nsk" },
])
const topo: FlowTopology = {
clientIfaces: new Map([[7, new Set(["gre-client"])]]),
clientByIface: new Map([["7|gre-client", {
clientIfaces: new Map([[serverId, new Set(["gre-client"])]]),
clientByIface: new Map([[serverId + "|gre-client", {
userId: "u1",
login: "alice",
name: "Alice",
serverId: 7,
serverId,
interfaceName: "gre-client",
}]]),
enNodes: [{ id: 9, name: "NSK-SERVHOST-RTK", hosts: ["198.51.100.1"] }],
@@ -343,7 +349,7 @@ try {
ok: true,
fetchedAt: Date.now(),
})
ingestParsedFlowsForServerForTests(7, [
ingestParsedFlowsForServerForTests(serverId, [
{
src: "10.100.1.17",
dst: "173.194.160.163",
@@ -380,7 +386,7 @@ try {
},
])
try {
const def = await buildFlowAnalytics({ minutes: 5, serverId: 7 })
const def = await buildFlowAnalytics({ minutes: 5, serverId })
assert.equal(def.bytes, 12_000)
assert.equal(def.bytesPayload, 12_000)
assert.equal(def.bytesOverlay, 5_000_000)
@@ -398,7 +404,7 @@ try {
assert.equal(path.clientName, "Alice")
assert.equal(path.enName, "NSK-SERVHOST-RTK")
assert.equal(path.dst, "173.194.160.163")
const withAll = await buildFlowAnalytics({ minutes: 5, serverId: 7, excludeOverlay: false, excludeMesh: false })
const withAll = await buildFlowAnalytics({ minutes: 5, serverId, excludeOverlay: false, excludeMesh: false })
assert.equal(withAll.bytes, 12_000 + 5_000_000 + 8000)
assert.ok(withAll.conversationsList.some((r) => r.plane === "overlay"))
assert.ok(withAll.conversationsList.some((r) => r.plane === "client_mesh"))
@@ -446,8 +452,8 @@ try {
ok: true,
fetchedAt: Date.now(),
})
rememberServerIfaces(7, [{ ".id": "*2", name: "ether1" }])
ingestParsedFlowsForServerForTests(7, [
rememberServerIfaces(serverId, [{ ".id": "*2", name: "ether1" }])
ingestParsedFlowsForServerForTests(serverId, [
{
src: "74.125.104.196/32",
dst: "10.200.100.53/32",
@@ -483,7 +489,7 @@ try {
},
])
try {
const rev = await buildFlowAnalytics({ minutes: 5, serverId: 7 })
const rev = await buildFlowAnalytics({ minutes: 5, serverId })
const google = rev.conversationsList.find((r) => r.src === "74.125.104.196")
const cf = rev.conversationsList.find((r) => r.src === "104.18.35.51")
const fastly = rev.conversationsList.find((r) => r.src === "146.75.118.132")
@@ -515,7 +521,7 @@ try {
if (await withPgOrSkip()) {
resetFlowRingsForTests()
resetIfaceCacheForTests()
const sidRow = await dbQuery<{ id: number }>(`SELECT id FROM servers LIMIT 1`)
const sidRow = await dbQuery<{ id: number }>(`SELECT id FROM servers WHERE id = $1`, [serverId])
const sid = sidRow.rows[0]?.id
if (sid) {
const { pool } = await import("../db/index.js")
@@ -550,4 +556,10 @@ try {
}
}
} finally {
for (const table of ["flow_daily_dims", "flow_hour_facts", "flow_daily_facts", "flow_buckets", "traffic_samples", "user_interface_bindings", "server_snapshots"]) {
await dbQuery("DELETE FROM " + table + " WHERE server_id = $1", [serverId])
}
await dbQuery("DELETE FROM servers WHERE id = $1", [serverId])
}
console.log("traffic-flow-analytics.test.ts: ok")