feat(traffic-flow): enhance traffic classification and map display

- Added a new state variable `mapAsnLoaded` to track ASN loading status in the NetworkMapPage. - Updated the page to conditionally display a message when ASN data is not loaded, improving user feedback. - Enhanced traffic classification tests to include additional scenarios for Google and YouTube services, ensuring comprehensive coverage. - Introduced a new function `pickMapInternetDest` to improve destination selection logic in traffic flow processing. - Updated the traffic flow destination classification to handle new conditions and improve accuracy. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-12 01:07:26 +07:00
parent 60a0970d73
commit 88deb87e4e
9 changed files with 156 additions and 17 deletions
@@ -1103,6 +1103,7 @@ export default function NetworkMapPage() {
  const [mapNamedBytes, setMapNamedBytes] = useState(0)
  const [mapTotalBytes, setMapTotalBytes] = useState(0)
  const [mapWindowSec, setMapWindowSec] = useState(300)
+  const [mapAsnLoaded, setMapAsnLoaded] = useState(true)
  /** FQDN из GRE outer → IPv4 (ответ POST /api/network/resolve-hosts), для матчинга с WAN. */
  const [greResolvedIpv4ByHost, setGreResolvedIpv4ByHost] = useState<Record<string, string>>({})
  const [dataError, setDataError] = useState<string | null>(null)
@@ -1310,6 +1311,7 @@ export default function NetworkMapPage() {
          if (res.mapServiceMinSharePct != null) setMapSharePct(res.mapServiceMinSharePct)
          setMapNamedBytes(res.namedBytes ?? 0)
          setMapTotalBytes(res.totalBytes ?? 0)
+          if (res.asnLoaded != null) setMapAsnLoaded(res.asnLoaded)
          if (res.windowSec) setMapWindowSec(res.windowSec)
        })
        .catch((err: unknown) => {
@@ -2767,6 +2769,12 @@ export default function NetworkMapPage() {
                        </span>
                      </div>
                    )}
+                    {!mapAsnLoaded && (
+                      <div className="flex items-center justify-between py-2 border-b border-border/50">
+                        <span className="text-xs text-muted-foreground">GeoLite2 ASN</span>
+                        <span className="text-xs font-mono font-medium text-amber-500">не загружена</span>
+                      </div>
+                    )}
                    <div className="flex items-center justify-between py-2 border-b border-border/50">
                      <span className="text-xs text-muted-foreground">Скорость</span>
                      <span className="text-xs font-mono font-medium">
@@ -132,6 +132,29 @@ const greIgnore = classifyFlowDst("8.8.8.8", 47, 0, 0, {
  fetchedAt: Date.now(),
 }, { ignoreTunnelProto: true })
 assert.equal(greIgnore.service, "Google")
+const dnsGoogle = classifyFlowDst("8.8.8.8", 17, 53, 53000, {
+  prefix: "8.8.8.0/24",
+  asn: 15169,
+  country: "US",
+  lat: null,
+  lng: null,
+  holder: "GOOGLE",
+  ok: true,
+  fetchedAt: Date.now(),
+})
+assert.equal(dnsGoogle.service, "Google")
+assert.notEqual(dnsGoogle.service, "Прочее")
+const ipv6Yt = classifyFlowDst("2001:4860:4860::8888", 17, 443, 50000, {
+  prefix: "2001:4860:4860::8888/128",
+  asn: 15169,
+  country: "US",
+  lat: null,
+  lng: null,
+  holder: "GOOGLE",
+  ok: true,
+  fetchedAt: Date.now(),
+})
+assert.equal(ipv6Yt.service, "YouTube")
 const esp = classifyFlowDst("198.51.100.1", 50, 0, 0, null)
 assert.equal(esp.category, "Туннель")
 assert.equal(applicationName(17, 443, 50000), "QUIC")
@@ -149,6 +149,39 @@ assert.equal(mapInternetBrand("8.8.8.8", 6, 443, 51234, {
  fetchedAt: Date.now(),
 }).service, "Google")

+assert.equal(classifyInternetBrand("8.8.8.8", 17, 53, 53000, {
+  prefix: "8.8.8.0/24",
+  asn: 15169,
+  country: "US",
+  lat: null,
+  lng: null,
+  holder: "GOOGLE",
+  ok: true,
+  fetchedAt: Date.now(),
+})?.service, "Google")
+assert.equal(mapInternetBrand("8.8.8.8", 17, 53, 53000, {
+  prefix: "8.8.8.0/24",
+  asn: 15169,
+  country: "US",
+  lat: null,
+  lng: null,
+  holder: "GOOGLE",
+  ok: true,
+  fetchedAt: Date.now(),
+}).service, "Google")
+assert.equal(mapInternetBrand("2001:4860:4860::8888", 17, 443, 50000, {
+  prefix: "2001:4860:4860::8888/128",
+  asn: 15169,
+  country: "US",
+  lat: null,
+  lng: null,
+  holder: "GOOGLE",
+  ok: true,
+  fetchedAt: Date.now(),
+}).service, "YouTube")
+assert.equal(mapInternetBrand("64.233.161.1", 17, 443, 50000, null).service, "YouTube")
+assert.equal(mapInternetBrand("142.250.1.10", 6, 443, 1, null).service, "YouTube")
+
 resetEngineForTests()
 seedFlowTopologyForTests(null)
 resetRipeCacheForTests()
@@ -41,7 +41,14 @@ export function destCtxForIface(
  }
 }

-/** Бренд интернет-dest как на карте: GRE/ESP/WG — транспорт, не сервис. */
+const OTHER_BRAND: FlowClassification = { service: OTHER_SERVICE, category: OTHER_SERVICE }
+
+function isTunnelProto(proto: number, dstPort: number, srcPort: number): boolean {
+  if (proto === 47 || proto === 50) return true
+  return applicationName(proto, dstPort, srcPort) === "WireGuard"
+}
+
+/** Бренд интернет-dest: ASN/CIDR до skip DNS. GRE/ESP/WG — не сервис. */
 export function classifyInternetBrand(
  dst: string,
  proto: number,
@@ -49,17 +56,15 @@ export function classifyInternetBrand(
  srcPort: number,
  ripe: FlowIpMeta | null,
 ): FlowClassification | null {
-  if (proto === 47 || proto === 50) return null
-  const app = applicationName(proto, dstPort, srcPort)
-  if (app === "WireGuard" || app === "DNS" || app === "SSH" || app === "BGP") return null
+  if (isTunnelProto(proto, dstPort, srcPort)) return null
  const brand = resolveFlowBrand(dst, ripe?.asn ?? 0, ripe?.holder ?? "", proto, dstPort, srcPort)
-  if (!brand || !isNamedInternetService(brand.service, brand.category)) return null
-  return brand
+  if (brand && isNamedInternetService(brand.service, brand.category)) return brand
+  const app = applicationName(proto, dstPort, srcPort)
+  if (app === "DNS" || app === "SSH" || app === "BGP") return null
+  return null
 }

-const OTHER_BRAND: FlowClassification = { service: OTHER_SERVICE, category: OTHER_SERVICE }
-
-/** Бренд для карты: именованный сервис или «Прочее» (GRE/ESP не сервис). */
+/** Тот же классификатор, что аналитика (GeoLite2 ASN + catalog). Туннель → Прочее. */
 export function mapInternetBrand(
  dst: string,
  proto: number,
@@ -67,8 +72,10 @@ export function mapInternetBrand(
  srcPort: number,
  ripe: FlowIpMeta | null,
 ): FlowClassification {
-  if (proto === 47 || proto === 50) return OTHER_BRAND
-  return classifyInternetBrand(dst, proto, dstPort, srcPort, ripe) ?? OTHER_BRAND
+  if (isTunnelProto(proto, dstPort, srcPort)) return OTHER_BRAND
+  const classified = classifyFlowDst(dst, proto, dstPort, srcPort, ripe, { ignoreTunnelProto: true })
+  if (isNamedInternetService(classified.service, classified.category)) return classified
+  return OTHER_BRAND
 }

 export function resolveInternetDest(opts: {
@@ -21,6 +21,7 @@ import {
  minuteDimsSnapshotForTests,
 } from "./traffic-flow-engine.js"
 import { classifyFlowDst } from "./traffic-flow-classify.js"
+import { mapInternetBrand } from "./traffic-flow-dest.js"
 import { disableGeoipDbForTests } from "./geoip-settings.js"
 import {
  collectGeoipUpdateOnce,
@@ -102,6 +103,21 @@ assert.equal(lookupGeoip("6.6.6.6")?.country, "US")
 const classified = classifyFlowDst("8.8.8.8", 6, 443, 51504, hit)
 assert.equal(classified.service, "Google")

+setGeoipReadersForTests({
+  country: fakeCountryReader({ "8.8.8.8": "US", "2001:4860:4860::8888": "US" }),
+  asn: fakeAsnReader({
+    "8.8.8.8": { asn: 15169, org: "GOOGLE" },
+    "2001:4860:4860::8888": { asn: 15169, org: "GOOGLE" },
+    "64.233.161.1": { asn: 15169, org: "GOOGLE" },
+  }),
+})
+const v6meta = resolveFlowIp("2001:4860:4860::8888")
+assert.equal(v6meta?.asn, 15169)
+assert.equal(mapInternetBrand("2001:4860:4860::8888", 17, 443, 50000, v6meta).service, "YouTube")
+assert.notEqual(mapInternetBrand("2001:4860:4860::8888", 17, 443, 50000, v6meta).service, "Прочее")
+const cidrYt = mapInternetBrand("64.233.161.1", 17, 443, 50000, resolveFlowIp("64.233.161.1"))
+assert.equal(cidrYt.service, "YouTube")
+
 // ── движок: dims country/asn наполняются из geoip-ридеров ────────────────────
 resetEngineForTests()
 ingestParsedFlowsForServerForTests(1, [{
@@ -1,5 +1,5 @@
 import assert from "node:assert/strict"
-import { isNonPublicIp, pickInternetDest, pickInternetPeer } from "./traffic-flow-ip.js"
+import { isNonPublicIp, pickInternetDest, pickInternetPeer, pickMapInternetDest } from "./traffic-flow-ip.js"

 assert.equal(isNonPublicIp("10.200.100.53"), true)
 assert.equal(isNonPublicIp("173.194.151.65"), false)
@@ -78,5 +78,29 @@ assert.equal(
  "",
  "NAT 0.0.0.0 не dest",
 )
+assert.equal(
+  pickMapInternetDest("10.200.100.53", "10.200.100.1", 53880, 443, {
+    ...client,
+    natDst: "8.8.8.8",
+    natDstPort: 443,
+  }),
+  "8.8.8.8",
+  "карта: RFC1918 + NAT Google",
+)
+assert.equal(
+  pickMapInternetDest("10.200.100.53", "10.200.100.1", 53880, 443, client),
+  "",
+  "карта: RFC1918 без NAT → Прочее",
+)
+assert.equal(
+  pickMapInternetDest("173.194.151.65", "10.200.100.53", 12345, 57182, client),
+  "173.194.151.65",
+  "карта: реверс googlevideo не :443 — всё равно публичный src",
+)
+assert.equal(
+  pickMapInternetDest("203.0.113.10", "198.51.100.1", 0, 0, { ours }),
+  "",
+  "карта: JH ours → EN ours всё ещё не dest",
+)

 console.log("traffic-flow-ip.test.ts: ok")
@@ -132,6 +132,27 @@ export function pickInternetDest(
  return dst
 }

+/**
+ * Dest для карты: сначала pickInternetDest, иначе любой публичный src/dst/NAT
+ * (без well-known ловушки boundClient — реверс googlevideo не только :80/:443/:53).
+ */
+export function pickMapInternetDest(
+  srcRaw: string,
+  dstRaw: string,
+  srcPort: number,
+  dstPort: number,
+  ctx?: InternetDestCtx,
+): string {
+  const dest = pickInternetDest(srcRaw, dstRaw, srcPort, dstPort, ctx)
+  if (dest) return dest
+  const ours = ctx?.ours
+  const internet = (ip: string) => Boolean(ip) && !isLocalIp(ip, ours)
+  for (const ip of [usableIp(ctx?.natDst), usableIp(ctx?.natSrc), usableIp(dstRaw), usableIp(srcRaw)]) {
+    if (internet(ip)) return ip
+  }
+  return ""
+}
+
 /**
 * Интернет-сторона потока без топологии: у IPFIX сервис часто в src (Google:443 → RFC1918).
 * Для куба статистики используйте pickInternetDest.
@@ -3,14 +3,15 @@ import type { FlowMapHop, FlowMapHopsDto, FlowMapService, FlowMapServiceEdge, Fl
 import { db } from "../db/index.js"
 import { userInterfaceBindings } from "../db/schema.js"
 import { flowRowMatchesFilter } from "./traffic-flow-apps.js"
-import { OTHER_SERVICE, mapServiceNodeId } from "./traffic-flow-brands.js"
+import { OTHER_SERVICE, isNamedInternetService, mapServiceNodeId } from "./traffic-flow-brands.js"
+import { refreshFlowCatalogInBackground } from "./traffic-flow-classify.js"
 import { dedupFlowRowsAcrossExporters, dedupFlowRowsMaxBytes } from "./traffic-flow-dedup.js"
 import { getFlowListenerState, listFlowRowsForWindow } from "./traffic-flow-ingest.js"
 import { resolveIfaceName } from "./traffic-flow-ifaces.js"
 import { classifyFlowPlane, shouldKeepPlane } from "./traffic-flow-planes.js"
 import { destCtxForIface, mapInternetBrand } from "./traffic-flow-dest.js"
-import { pickInternetDest } from "./traffic-flow-ip.js"
-import { resolveFlowIp } from "./traffic-flow-geoip.js"
+import { pickMapInternetDest } from "./traffic-flow-ip.js"
+import { geoipReadersStatus, resolveFlowIp } from "./traffic-flow-geoip.js"
 import { getTrafficFlowSettingsRow } from "./traffic-flow-settings.js"
 import { loadFlowTopology, resolveClient, resolveEn, getServerCatalog, type FlowTopology } from "./traffic-flow-topology.js"
 import { flowDataEpoch } from "./traffic-flow-engine.js"
@@ -192,6 +193,7 @@ async function resolveMinSharePct(q: FlowMapHopsQuery): Promise<number> {
 }

 async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number): Promise<FlowMapHopsDto> {
+  refreshFlowCatalogInBackground()
  const windowSec = Math.max(60, q.minutes * 60)
  const raw = await listFlowRowsForWindow(q.minutes)
  const allow = q.userId ? await userIfaceAllow(q.userId) : null
@@ -334,7 +336,7 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number
    const inName = resolveIfaceName(r.serverId, r.inIface).name
    const outName = resolveIfaceName(r.serverId, r.outIface).name
    totalBytes += r.bytes
-    const dest = pickInternetDest(
+    const dest = pickMapInternetDest(
      r.src,
      r.dst,
      r.srcPort,
@@ -476,7 +478,7 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number
  }

  const namedBytes = [...svcTotals.values()]
-    .filter((s) => s.label !== OTHER_SERVICE)
+    .filter((s) => isNamedInternetService(s.label, s.category))
    .reduce((n, s) => n + s.bytes, 0)
  const unclassifiedBytes = Math.max(0, totalBytes - namedBytes)
  const shareBase = totalBytes > 0 ? totalBytes : namedBytes
@@ -528,6 +530,7 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number
    .sort((a, b) => b.bps - a.bps)

  const listener = getFlowListenerState()
+  const geo = geoipReadersStatus()
  return {
    hops: [...hops.values()]
      .map((a) => toHop(a, windowSec))
@@ -538,6 +541,8 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number
    totalBytes,
    namedBytes,
    unclassifiedBytes,
+    asnLoaded: geo.asnLoaded,
+    countryLoaded: geo.countryLoaded,
    services,
    serviceEdges,
    servicePaths,
@@ -320,6 +320,8 @@ export const flowMapHopsDtoSchema = z.object({
  totalBytes: z.number().nonnegative().optional(),
  namedBytes: z.number().nonnegative().optional(),
  unclassifiedBytes: z.number().nonnegative().optional(),
+  asnLoaded: z.boolean().optional(),
+  countryLoaded: z.boolean().optional(),
  services: z.array(flowMapServiceDtoSchema).optional(),
  serviceEdges: z.array(flowMapServiceEdgeDtoSchema).optional(),
  servicePaths: z.array(flowMapServicePathDtoSchema).optional(),