fix(geoip): проверить локальный кеш и ограничить загрузку баз

backend/package.json

@@ -34,7 +34,7 @@
     "test:evolution": "tsx src/services/evobgp-catalog.test.ts && tsx src/routes/evobgp-catalog.test.ts && tsx src/routes/evobgp-settings.test.ts && tsx src/plugins/auth.evolution-settings.test.ts",
     "test:netflow-purge": "tsx src/db/operation-lock.test.ts && tsx src/services/traffic-flow-purge-execution.test.ts && tsx src/routes/traffic-flow-purge.test.ts",
     "test:netflow-settings": "tsx src/services/traffic-flow-host-file-builders.test.ts && tsx src/services/traffic-flow-key-execution.test.ts && tsx src/routes/traffic-flow-settings.test.ts && tsx src/plugins/auth.netflow-settings.test.ts",
-    "test:geoip": "tsx src/services/traffic-flow-geoip.test.ts && tsx src/plugins/auth.geoip.test.ts",
+    "test:geoip": "tsx src/services/geoip-download.test.ts && tsx src/services/traffic-flow-geoip.test.ts && tsx src/plugins/auth.geoip.test.ts",
     "test:ospf-optimizer": "tsx src/services/ospf-optimizer-identity.test.ts && tsx src/services/ospf-optimizer-metrics.test.ts && tsx src/services/ospf-optimizer-plan.test.ts && tsx src/services/ospf-effective-disabled.test.ts && tsx src/routes/ospf-optimizer.test.ts",
     "test:containers": "tsx src/services/containers-state.test.ts && tsx src/routes/containers-state.test.ts",
     "test:certificates": "tsx src/services/certificate-issue-execution.test.ts && tsx src/services/certificate-issue-request.test.ts && tsx src/routes/certificates-issue.test.ts && tsx src/services/certificate-issue-dto.test.ts && tsx src/services/certificate-parse.test.ts && tsx src/services/certificates-state.test.ts && tsx src/routes/certificates-state.test.ts",

backend/scripts/acceptance/geoip-mirror.mjs

@@ -0,0 +1,66 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+const database = process.env.MMAPP_OWN_SQL_DB;
+assert(/^mm_accept_166_\d+$/.test(database));
+const url = new URL(process.env.DATABASE_URL); assert.equal(url.pathname, '/' + database); assert.equal(url.username, database);
+assert.equal(process.env.GEOIP_DIR, '/tmp/' + database + '/geoip');
+fs.mkdirSync('/tmp/' + database, { recursive: true }); process.chdir('/tmp/' + database);
+const { pool, closePool } = await import('/app/backend/dist/db/index.js');
+const { initDatabase } = await import('/app/backend/dist/db/bootstrap.js');
+const role = (await pool.query('SELECT rolsuper,rolcreatedb,rolcreaterole,rolbypassrls FROM pg_roles WHERE rolname=current_user')).rows[0];
+assert(Object.values(role).every(v => v === false)); await initDatabase();
+const { collectGeoipUpdateOnce, setGeoipFetchForTests, resetGeoipUpdateForTests } = await import('/app/backend/dist/services/geoip-update-collector.js');
+const { getGeoipSettings, updateGeoipSettings } = await import('/app/backend/dist/services/geoip-settings.js');
+const { geoipReadersStatus, lookupGeoip, GEOIP_COUNTRY_FILE, GEOIP_ASN_FILE } = await import('/app/backend/dist/services/traffic-flow-geoip.js');
+const { databaseBackgroundWork } = await import('/app/backend/dist/services/database-background-work.js');
+const report = { date: '2026-10-14', block: '12.5.242', passed: false, scope: 'Compiled GeoIP collector + owned PostgreSQL/files + actual public mirror; controlled negative bodies separately marked', mirrorRequests: [], checks: [], chrContacts: 0, mainDatabaseWrites: 0 };
+const file = name => path.join(process.env.GEOIP_DIR, name);
+const sha = name => createHash('sha256').update(fs.readFileSync(file(name))).digest('hex');
+const save = () => fs.writeFileSync('/checks/geoip-mirror242.json', JSON.stringify(report, null, 2) + '\n');
+const check = (name, detail = {}) => { report.checks.push({ name, ...detail }); save(); };
+const realFetch = globalThis.fetch.bind(globalThis);
+async function observedFetch(url, options) {
+ const started = Date.now(), response = await realFetch(url, options);
+ report.mirrorRequests.push({ file: new URL(String(url)).pathname.split('/').at(-1), conditional: Boolean(options?.headers?.['If-None-Match']), status: response.status, headersMs: Date.now() - started }); save(); return response;
+}
+try {
+ await updateGeoipSettings({ enabled: false }); setGeoipFetchForTests(observedFetch);
+ let snapshot = await collectGeoipUpdateOnce({ force: true });
+ assert(!snapshot.fatalError); assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.downloaded, 2);
+ assert(geoipReadersStatus().countryLoaded && geoipReadersStatus().asnLoaded);
+ const initial = { country: sha(GEOIP_COUNTRY_FILE), asn: sha(GEOIP_ASN_FILE) };
+ const lookup = lookupGeoip('8.8.8.8'); assert.equal(lookup?.country, 'US'); assert.equal(lookup?.asn, 15169);
+ check('actual-two-downloads-valid-mmdb-and-lookup', { bytes: snapshot.bytes, hashes: initial, settings: await getGeoipSettings(), lookup: { country: lookup.country, asn: lookup.asn } });
+ const first = collectGeoipUpdateOnce({ force: true });
+ await assert.rejects(collectGeoipUpdateOnce({ force: true }), e => e.statusCode === 409);
+ assert.equal((await collectGeoipUpdateOnce()).skipped, true);
+ snapshot = await first; assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.skippedUnchanged, 2); assert.equal(snapshot.downloaded, 0);
+ check('actual-conditional304-and-concurrent409', { skippedUnchanged: 2, hashesUnchanged: sha(GEOIP_COUNTRY_FILE) === initial.country && sha(GEOIP_ASN_FILE) === initial.asn });
+ fs.unlinkSync(file(GEOIP_COUNTRY_FILE));
+ const beforeMissing = report.mirrorRequests.length; snapshot = await collectGeoipUpdateOnce({ force: true });
+ assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.downloaded, 1); assert.equal(snapshot.skippedUnchanged, 1);
+ assert(report.mirrorRequests.slice(beforeMissing).some(r => r.file === GEOIP_COUNTRY_FILE && !r.conditional && r.status === 200));
+ assert.equal(sha(GEOIP_COUNTRY_FILE), initial.country);
+ check('missing-country-refetched-without-conditional-header');
+ fs.writeFileSync(file(GEOIP_COUNTRY_FILE), 'Own corrupt GeoIP fixture');
+ snapshot = await collectGeoipUpdateOnce({ force: true }); assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.downloaded, 1); assert.equal(sha(GEOIP_COUNTRY_FILE), initial.country);
+ check('corrupt-local-country-refetched-and-validated');
+ const lastSuccess = (await getGeoipSettings()).lastSuccessAt;
+ setGeoipFetchForTests(async (url, options) => String(url).endsWith(GEOIP_COUNTRY_FILE) ? new Response('Own invalid MMDB body', { status: 200 }) : observedFetch(url, options));
+ snapshot = await collectGeoipUpdateOnce({ force: true }); assert.equal(snapshot.errors.length, 1); assert.equal(snapshot.downloaded, 0); assert.equal(snapshot.skippedUnchanged, 1);
+ assert.equal(sha(GEOIP_COUNTRY_FILE), initial.country); assert.equal((await getGeoipSettings()).lastSuccessAt, lastSuccess);
+ assert((await getGeoipSettings()).lastError); assert(!fs.existsSync(file(GEOIP_COUNTRY_FILE) + '.tmp'));
+ check('controlled-invalid-body-preserves-file-and-last-success', { negativeCountryResponse: 'controlled-200-invalid-MMDB', realAsnStatus: 304 });
+ fs.unlinkSync(file(GEOIP_COUNTRY_FILE));
+ setGeoipFetchForTests(async (url, options) => String(url).endsWith(GEOIP_COUNTRY_FILE) ? new Response(null, { status: 304 }) : observedFetch(url, options));
+ snapshot = await collectGeoipUpdateOnce({ force: true }); assert.equal(snapshot.errors.length, 1); assert.equal(snapshot.skippedUnchanged, 1); assert(snapshot.errors[0].includes('304 без проверенной локальной базы'));
+ assert.equal((await getGeoipSettings()).lastSuccessAt, lastSuccess);
+ check('controlled-uncached304-is-error-with-last-success-retained');
+ setGeoipFetchForTests(observedFetch); snapshot = await collectGeoipUpdateOnce({ force: true }); assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.downloaded, 1);
+ assert.equal(sha(GEOIP_COUNTRY_FILE), initial.country); assert.equal(sha(GEOIP_ASN_FILE), initial.asn);
+ const beforeSkip = report.mirrorRequests.length; assert.equal((await collectGeoipUpdateOnce()).skipped, true); assert.equal(report.mirrorRequests.length, beforeSkip);
+ check('actual-recovery-and-disabled-background-no-fetch'); report.passed = true;
+} catch (e) { report.failure = { name: e.name }; process.exitCode = 1; }
+finally { resetGeoipUpdateForTests(); await databaseBackgroundWork.drain(); await closePool(); save(); console.log(JSON.stringify(report)); }

backend/scripts/acceptance/run-geoip-mirror.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euo pipefail
+umask 077
+cd /opt/mmapp-test
+mm242_db="mm_accept_166_$(date +%s%3N)"
+mm242_pass="$(openssl rand -hex 24)"
+mm242_role_created=false
+mm242_db_created=false
+cleanup242() {
+  docker rm mmapp-s3app242-owned >/dev/null 2>&1 || true
+  if $mm242_db_created; then docker exec mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1 -c "DROP DATABASE \"$mm242_db\" WITH (FORCE)"; fi
+  if $mm242_role_created; then docker exec mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1 -c "DROP ROLE \"$mm242_db\""; fi
+  remaining=$(docker exec mmapp-test-postgres psql -U mmapp -d mmapp -Atc "SELECT (SELECT count(*) FROM pg_database WHERE datname='$mm242_db')+(SELECT count(*) FROM pg_roles WHERE rolname='$mm242_db')")
+  [ "$remaining" = 0 ]
+  python3 - <<'PY'
+import json,pathlib
+p=pathlib.Path('/opt/mmapp-test/artifacts/backup242-checks/geoip-mirror242.json')
+if p.exists():
+ r=json.loads(p.read_text());r['ownedDatabaseAndRoleRemoved']=True;p.write_text(json.dumps(r,indent=2)+'\n')
+PY
+  printf 'OWNED_DATABASE_AND_ROLE_REMOVED\n'
+  unset mm242_pass
+}
+trap cleanup242 EXIT
+[[ "$mm242_db" =~ ^mm_accept_166_[0-9]+$ ]]
+existing=$(docker exec mmapp-test-postgres psql -U mmapp -d mmapp -Atc "SELECT (SELECT count(*) FROM pg_database WHERE datname='$mm242_db')+(SELECT count(*) FROM pg_roles WHERE rolname='$mm242_db')")
+[ "$existing" = 0 ]
+printf "CREATE ROLE \"%s\" LOGIN PASSWORD '%s' NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS;\n" "$mm242_db" "$mm242_pass" | docker exec -i mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1
+mm242_role_created=true
+docker exec mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1 -c "CREATE DATABASE \"$mm242_db\" OWNER \"$mm242_db\""
+mm242_db_created=true
+docker run --name mmapp-s3app242-owned --network mmapp-test_default --cap-drop ALL --security-opt no-new-privileges -e "MMAPP_ORIGIN_BASELINE=${2:-false}" -e NODE_ENV=test -e "DATABASE_URL=postgres://$mm242_db:$mm242_pass@postgres:5432/$mm242_db" -e "MMAPP_OWN_SQL_DB=$mm242_db" -e "DATABASE_PATH=/tmp/$mm242_db/legacy.db" -e "GEOIP_DIR=/tmp/$mm242_db/geoip" -v /opt/mmapp-test/artifacts/backup242-checks:/checks --entrypoint node "${1:-mmapp-test/backend:ux-geoip242}" /checks/geoip-mirror.mjs

backend/src/services/geoip-download.test.ts

@@ -0,0 +1,27 @@
+import assert from "node:assert/strict"
+import { createServer } from "node:http"
+import { downloadGeoipDatabase } from "./geoip-download.js"
+
+let stalledClosed = false
+const server = createServer((req, res) => {
+ if (req.url === "/slow") { res.writeHead(200); res.write("partial"); res.on("close", () => { stalledClosed = true }) }
+ else if (req.url === "/chunked") { res.writeHead(200); res.write("1234"); res.end("5678") }
+ else if (req.url === "/oversized") { res.writeHead(200, { "content-length": "9999" }); res.end("oversized") }
+ else if (req.url === "/304") { res.writeHead(304); res.end() }
+ else { res.writeHead(200); res.end("valid fixture") }
+})
+await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve))
+const address = server.address(); assert(address && typeof address !== "string")
+const base = `http://127.0.0.1:${address.port}`
+try {
+ const valid = await downloadGeoipDatabase(base, {}, fetch, 2000, 100)
+ assert.equal(valid.body.toString(), "valid fixture")
+ assert.equal((await downloadGeoipDatabase(base + "/304", {}, fetch, 2000, 100)).response.status, 304)
+ await assert.rejects(downloadGeoipDatabase(base + "/slow", {}, fetch, 120, 100), error => (error as Error).name === "AbortError")
+ await assert.rejects(downloadGeoipDatabase(base + "/oversized", {}, fetch, 2000, 5), /превышает/)
+ await assert.rejects(downloadGeoipDatabase(base + "/chunked", {}, fetch, 2000, 5), /превышает/)
+ // A failed request leaves no persistent timer or lock preventing the next successful download.
+ assert.equal((await downloadGeoipDatabase(base, {}, fetch, 2000, 100)).body.toString(), "valid fixture")
+ await new Promise(resolve => setTimeout(resolve, 20)); assert(stalledClosed)
+ console.log("GeoIP HTTP: whole-body timeout, declared/chunked limits,304 and recovery passed")
+} finally { server.closeAllConnections(); await new Promise<void>(resolve => server.close(() => resolve())) }

backend/src/services/geoip-download.ts

@@ -0,0 +1,38 @@
+const DOWNLOAD_TIMEOUT_MS = 120_000
+const MAX_DOWNLOAD_BYTES = 64 * 1024 * 1024
+
+/** A single deadline covers headers and streaming; a mirror cannot allocate an unbounded body. */
+export async function downloadGeoipDatabase(
+ url: string, headers: Record<string, string>, fetcher: typeof fetch,
+ timeoutMs = DOWNLOAD_TIMEOUT_MS, maxBytes = MAX_DOWNLOAD_BYTES,
+): Promise<{ response: Response; body: Buffer }> {
+ const controller = new AbortController()
+ const timer = setTimeout(() => controller.abort(), timeoutMs)
+ try {
+  const response = await fetcher(url, { headers, signal: controller.signal })
+  if (!response.ok || !response.body) {
+   await response.body?.cancel().catch(() => {})
+   return { response, body: Buffer.alloc(0) }
+  }
+  const size = Number(response.headers.get("content-length"))
+  if (Number.isFinite(size) && size > maxBytes) {
+   await response.body.cancel().catch(() => {})
+   throw new Error("Размер базы GeoIP превышает допустимый предел")
+  }
+  const reader = response.body.getReader(), chunks: Buffer[] = []
+  let received = 0
+  try {
+   for (;;) {
+    const { value, done } = await reader.read()
+    if (done) break
+    received += value.byteLength
+    if (received > maxBytes) {
+     await reader.cancel().catch(() => {})
+     throw new Error("Размер базы GeoIP превышает допустимый предел")
+    }
+    chunks.push(Buffer.from(value))
+   }
+   return { response, body: Buffer.concat(chunks, received) }
+  } finally { reader.releaseLock() }
+ } finally { clearTimeout(timer) }
+}

backend/src/services/geoip-update-collector.ts

@@ -4,6 +4,7 @@ import { open, type AsnResponse, type CountryResponse } from "maxmind"
 import type { GeoipUpdateRunSnapshot } from "../types/scheduler-run-snapshot.js"
 import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
 import { getGeoipEtags, getGeoipSettings, touchGeoipRunMeta } from "./geoip-settings.js"
+import { downloadGeoipDatabase } from "./geoip-download.js"
 import {
   GEOIP_ASN_FILE,
   GEOIP_COUNTRY_FILE,
@@ -13,7 +14,6 @@ import {
 
 /** Зеркало GeoLite2 без регистрации и ключей (см. README: GeoIP). */
 const MIRROR_BASE = "https://github.com/P3TERX/GeoLite.mmdb/raw/download"
-const DOWNLOAD_TIMEOUT_MS = 120_000
 /** Пробный IP для валидации скачанной базы: Google DNS. */
 const PROBE_IP = "8.8.8.8"
 
@@ -83,11 +83,7 @@ export async function collectGeoipUpdateOnce(
     return emptySnapshot(sampledAt, true)
   }
 
-  const settings = await getGeoipSettings()
-  if (!settings.enabled && !opts.force) {
-    return emptySnapshot(sampledAt, true)
-  }
-
+  // Acquire before the first await: concurrent requests must not both pass the gate.
   updating = true
   const snapshot: GeoipUpdateRunSnapshot = {
     v: SCHEDULER_RUN_SNAPSHOT_VERSION,
@@ -101,6 +97,8 @@ export async function collectGeoipUpdateOnce(
   }
 
   try {
+    const settings = await getGeoipSettings()
+    if (!settings.enabled && !opts.force) return emptySnapshot(sampledAt, true)
     const dir = geoipDir()
     await mkdir(dir, { recursive: true })
     const storedEtags = await getGeoipEtags()
@@ -114,25 +112,26 @@ export async function collectGeoipUpdateOnce(
       const tmp = `${target}.tmp`
       const prevEtag = storedEtags[file]
       try {
-        const ac = new AbortController()
-        const timer = setTimeout(() => ac.abort(), DOWNLOAD_TIMEOUT_MS)
-        let res: Response
-        try {
-          res = await fetchImpl(`${MIRROR_BASE}/${file}`, {
-            headers: prevEtag ? { "If-None-Match": prevEtag } : {},
-            signal: ac.signal,
-          })
-        } finally {
-          clearTimeout(timer)
+        let cachedFileValid = false
+        if (prevEtag) {
+          try {
+            await (kind === "country" ? validateCountry(target) : validateAsn(target))
+            cachedFileValid = true
+          } catch { /* A missing or invalid local cache requires a full download. */ }
         }
+        const { response: res, body } = await downloadGeoipDatabase(
+          `${MIRROR_BASE}/${file}`,
+          prevEtag && cachedFileValid ? { "If-None-Match": prevEtag } : {},
+          fetchImpl,
+        )
         if (res.status === 304) {
+          if (!cachedFileValid || !prevEtag) throw new Error("Зеркало вернуло HTTP 304 без проверенной локальной базы")
           snapshot.skippedUnchanged += 1
           if (prevEtag) etags[file] = prevEtag
           continue
         }
         if (!res.ok) throw new Error(`HTTP ${res.status}`)
         const etag = res.headers.get("etag") ?? ""
-        const body = Buffer.from(await res.arrayBuffer())
         snapshot.bytes += body.byteLength
         await writeFile(tmp, body)
         buildAt[kind] =
@@ -161,7 +160,7 @@ export async function collectGeoipUpdateOnce(
 
     await touchGeoipRunMeta({
       lastCheckAt: sampledAt,
-      lastSuccessAt: snapshot.errors.length ? null : sampledAt,
+      ...(snapshot.errors.length ? {} : { lastSuccessAt: sampledAt }),
       lastError: snapshot.errors.length ? snapshot.errors.join("; ") : null,
       countryBuildAt: buildAt.country,
       asnBuildAt: buildAt.asn,

backend/src/services/traffic-flow-geoip.test.ts

@@ -141,7 +141,7 @@ const dims = minuteDimsSnapshotForTests()
 assert.equal(dims.get("country")?.get("US")?.bytes, 1000)
 assert.equal(dims.get("asn")?.get("15169")?.bytes, 1000)
 
-// ── коллектор: 304 → обе базы без изменений ───────────────────────────────────
+// ── коллектор: 304 без локального cache не является успехом ──────────────────
 disableGeoipDbForTests()
 resetGeoipUpdateForTests()
 const geoipDir = mkdtempSync(path.join(tmpdir(), "mm-geoip-test-"))
@@ -149,7 +149,9 @@ process.env.GEOIP_DIR = geoipDir
 
 setGeoipFetchForTests(async () => new Response(null, { status: 304 }))
 let snap = await collectGeoipUpdateOnce({ force: true })
-assert.equal(snap.skippedUnchanged, 2)
+assert.equal(snap.skippedUnchanged, 0)
+assert.equal(snap.errors.length, 2)
+assert(snap.errors.every(message => message.includes("304 без проверенной локальной базы")))
 assert.equal(snap.downloaded, 0)
 assert.equal(existsSync(path.join(geoipDir, "GeoLite2-Country.mmdb")), false)
 
@@ -195,6 +197,17 @@ assert.equal(readFileSync(asnPath, "utf8"), "good-asn", "asn не тронут")
 assert.equal(existsSync(`${countryPath}.tmp`), false)
 assert.equal(existsSync(`${asnPath}.tmp`), false)
 
+// Lock is acquired before settings lookup yields; a concurrent forced run rejects.
+let releaseFetch!: () => void
+const heldFetch = new Promise<void>(resolve => { releaseFetch = resolve })
+setGeoipFetchForTests(async () => { await heldFetch; return new Response(null, { status: 503 }) })
+const firstRun = collectGeoipUpdateOnce({ force: true })
+await assert.rejects(collectGeoipUpdateOnce({ force: true }), error => (error as { statusCode?: number }).statusCode === 409)
+assert.equal((await collectGeoipUpdateOnce()).skipped, true)
+releaseFetch()
+await firstRun
+assert.equal((await collectGeoipUpdateOnce({ force: true })).errors.length, 2, "lock released after errors")
+
 rmSync(geoipDir, { recursive: true, force: true })
 delete process.env.GEOIP_DIR
 resetGeoipUpdateForTests()