fix(settings): сохранять результат операций базы при вторичных сбоях

backend/src/db/operation-lock.ts

@@ -3,13 +3,13 @@ type Client = {
   release: (destroy?: boolean) => void
 }
 /** Nonblocking session lock: a second destructive operation is rejected rather than queued. */
-export async function withTrafficFlowPurgeLock<T>(connect: () => Promise<Client>, key: number, run: () => Promise<T>): Promise<T> {
+export async function withOperationLock<T>(connect: () => Promise<Client>, key: number, run: () => Promise<T>, busyMessage = "Сброс NetFlow уже выполняется"): Promise<T> {
   const client = await connect()
   let owned = false, destroy = false
   try {
     destroy = true
     const result = await client.query("SELECT pg_try_advisory_lock($1) AS acquired", [key])
-    if (result.rows[0]?.acquired === false) { destroy = false; throw new Error("Сброс NetFlow уже выполняется") }
+    if (result.rows[0]?.acquired === false) { destroy = false; throw new Error(busyMessage) }
     if (result.rows[0]?.acquired !== true) throw new Error("Блокировка сброса NetFlow не подтверждена")
     owned = true; destroy = false
     return await run()
@@ -24,3 +24,5 @@ export async function withTrafficFlowPurgeLock<T>(connect: () => Promise<Client>
     client.release(destroy)
   }
 }
+
+export const withTrafficFlowPurgeLock = withOperationLock

backend/src/index.ts

@@ -86,6 +86,7 @@ export async function buildApp(opts?: {
 
   await app.register(cors, {
     origin: env.CORS_ORIGIN,
+    exposedHeaders: ["Content-Disposition", "X-System-Database-Outcome"],
     methods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
   })
 

backend/src/plugins/auth.system-database.test.ts

@@ -0,0 +1,16 @@
+import assert from "node:assert/strict"
+import Fastify from "fastify"
+import {SignJWT} from "jose"
+process.env.AUTH_REQUIRED="true";process.env.AUTH_JWT_SECRET="evolution-fixture-secret";process.env.AUTH_ISSUER="https://auth.test.local";process.env.NODE_ENV="test"
+const {default:authPlugin,requireAuth}=await import("./auth.js")
+const {createSystemDatabaseRoutes}=await import("../routes/system-database.js")
+const app=Fastify();await app.register(authPlugin);app.addHook("preHandler",requireAuth)
+let exports=0,restores=0
+await app.register(createSystemDatabaseRoutes({exportBackup:async()=>{exports++;return {filename:"fixture.dump",buffer:Buffer.from("PGDMPfixture")}},restoreBackup:async()=>{restores++;return {}},event:async()=>undefined as never}),{prefix:"/api"})
+for(const permission of ["mm:settings:admin","mm:settings:write","mm:traffic:admin","mm:dashboard:read"]){
+ const token=await new SignJWT({sub:"fixture",apps:["mm"],permissions:[permission]}).setProtectedHeader({alg:"HS256"}).setIssuer("https://auth.test.local").setExpirationTime("1h").sign(new TextEncoder().encode("evolution-fixture-secret"))
+ for(const [method,url] of [["GET","/api/system/database/backup"],["POST","/api/system/database/restore"]] as const){const response=await app.inject({method,url,headers:{authorization:"Bearer "+token,"content-type":"application/octet-stream"},...(method==="POST"?{payload:Buffer.from("PGDMPfixture")}:{})});assert.equal(response.statusCode,permission==="mm:settings:admin"?200:403)}
+}
+assert.equal(exports,1);assert.equal(restores,1)
+for(const [method,url] of [["GET","/api/system/database/backup"],["POST","/api/system/database/restore"]] as const)assert.equal((await app.inject({method,url})).statusCode,401)
+assert.equal(exports,1);assert.equal(restores,1);await app.close();console.log("System DB JWT: actual backup/restore handlers require settings admin before operations")

backend/src/routes/system-database.test.ts

@@ -0,0 +1,17 @@
+import assert from "node:assert/strict"
+import Fastify from "fastify"
+import {createSystemDatabaseRoutes} from "./system-database.js"
+import {withOperationLock} from "../db/operation-lock.js"
+const app=Fastify(),dump=Buffer.from("PGDMPfixture")
+let exports=0,restores=0,events=0,eventFails=true,coreFails=false,held=false
+let gate:Promise<void>|undefined
+let entered:(()=>void)|undefined
+const lock=async()=>({query:async(sql:string)=>{if(sql.includes("pg_try")){const acquired=!held;if(acquired)held=true;return {rows:[{acquired}]}}held=false;return {rows:[{released:true}]}},release:()=>{}})
+await app.register(createSystemDatabaseRoutes({exportBackup:async()=>{exports++;if(coreFails)throw Error("dump failed");return {filename:"fixture.dump",buffer:dump,listenerError:"listener warning"}},restoreBackup:async()=>withOperationLock(lock,1,async()=>{restores++;entered?.();if(gate)await gate;if(coreFails)throw Error("restore failed");return {schedulerError:"scheduler warning",cleanupError:"cleanup warning"}},"Операция с базой приложения уже выполняется"),event:async()=>{events++;if(eventFails)throw Error("event fixture secret");return undefined as never}}),{prefix:"/api"})
+const restore=()=>app.inject({method:"POST",url:"/api/system/database/restore",headers:{"content-type":"application/octet-stream"},payload:dump})
+let response=await app.inject({method:"GET",url:"/api/system/database/backup"});assert.equal(response.statusCode,200);assert.deepEqual(response.rawPayload,dump);let warnings=JSON.parse(decodeURIComponent(response.headers["x-system-database-outcome"] as string));assert.equal(warnings.listenerError,"listener warning");assert.match(warnings.eventError,/событие/);assert.equal(response.body.includes("secret"),false)
+response=await restore();assert.equal(response.statusCode,200);assert.equal(response.json().ok,true);assert.equal(response.json().cleanupError,"cleanup warning");assert.match(response.json().eventError,/событие/)
+coreFails=true;const before=events;response=await restore();assert.equal(response.statusCode,400);assert.equal(response.json().ok,undefined);assert.equal(events,before);response=await app.inject({method:"GET",url:"/api/system/database/backup"});assert.equal(response.statusCode,500);assert.equal(events,before)
+coreFails=false;eventFails=false;let release!:()=>void;gate=new Promise<void>(resolve=>release=resolve);const started=new Promise<void>(resolve=>entered=resolve);const first=restore();await started;const runs=restores;response=await restore();assert.equal(response.statusCode,409);assert.equal(restores,runs);release();assert.equal((await first).statusCode,200);gate=undefined
+response=await app.inject({method:"POST",url:"/api/system/database/restore",headers:{"content-type":"application/octet-stream"},payload:Buffer.alloc(0)});assert.equal(response.statusCode,400);assert.equal(restores,runs)
+assert.equal(exports,2);await app.close();console.log("System DB HTTP: confirmed binary/restore, separate audit warnings, failures, nonqueued conflict and no retry")

backend/src/routes/system-database.ts

@@ -5,7 +5,12 @@ import {
   restoreSystemDatabaseBackup,
 } from "../services/system-database-backup.js"
 
-const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
+interface Dependencies {
+  exportBackup: typeof exportSystemDatabaseBackup
+  restoreBackup: typeof restoreSystemDatabaseBackup
+  event: typeof appendEvent
+}
+export function createSystemDatabaseRoutes(deps: Dependencies): FastifyPluginAsync { return async (app) => {
   app.addContentTypeParser(
     "application/octet-stream",
     { parseAs: "buffer", bodyLimit: 512 * 1024 * 1024 },
@@ -16,8 +21,10 @@ const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
 
   app.get("/system/database/backup", async (_req, reply) => {
     try {
-      const { filename, buffer } = await exportSystemDatabaseBackup()
-      await appendEvent({
+      const { filename, buffer, ...warnings } = await deps.exportBackup()
+      let eventError: string | undefined
+      try {
+        await deps.event({
         level: "info",
         eventType: "system.database.backup",
         sourceModule: "system",
@@ -30,12 +37,15 @@ const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
           sizeBytes: buffer.length,
         },
       })
+      } catch { eventError = "Не удалось записать событие резервного копирования" }
+      const outcome = { ...warnings, ...(eventError ? { eventError } : {}) }
+      reply.header("X-System-Database-Outcome", encodeURIComponent(JSON.stringify(outcome)))
       reply.header("Content-Type", "application/octet-stream")
       reply.header("Content-Disposition", `attachment; filename="${filename}"`)
       return reply.send(buffer)
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err)
-      return reply.status(500).send({ error: message })
+      return reply.status(message.includes("уже выполняется") ? 409 : 500).send({ error: message })
     }
   })
 
@@ -45,8 +55,10 @@ const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
       return reply.status(400).send({ error: "Ожидается тело запроса с файлом pg_dump (custom, PGDMP)" })
     }
     try {
-      const outcome = await restoreSystemDatabaseBackup(body)
-      await appendEvent({
+      const outcome = await deps.restoreBackup(body)
+      let eventError: string | undefined
+      try {
+        await deps.event({
         level: "warning",
         eventType: "system.database.restore",
         sourceModule: "system",
@@ -58,7 +70,8 @@ const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
           sizeBytes: body.length,
         },
       })
-      return reply.send({ ok: true, ...outcome })
+      } catch { eventError = "Не удалось записать событие восстановления" }
+      return reply.send({ ok: true, ...outcome, ...(eventError ? { eventError } : {}) })
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err)
       const status = message.includes("уже выполняется") ? 409 : 400
@@ -67,4 +80,6 @@ const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
   })
 }
 
-export default systemDatabaseRoutes
+}
+
+export default createSystemDatabaseRoutes({ exportBackup: exportSystemDatabaseBackup, restoreBackup: restoreSystemDatabaseBackup, event: appendEvent })

backend/src/services/system-database-backup.ts

@@ -5,7 +5,9 @@ import os from "node:os"
 import path from "node:path"
 import { executeSystemDatabaseRestore, validateSystemDatabaseRestoreArchive, type SystemDatabaseRestoreOutcome } from "./system-database-restore-execution.js"
 import { env } from "../config.js"
-import { withAdvisoryLock } from "../db/index.js"
+import { pool } from "../db/index.js"
+import { withOperationLock } from "../db/operation-lock.js"
+import { executeDatabaseOperation, type DatabaseOperationWarnings } from "./system-database-operation.js"
 import { refreshScheduler, stopScheduler } from "./scheduler.js"
 import {
   startTrafficFlowListener,
@@ -34,7 +36,7 @@ function spawnCollected(command: string, args: string[], opts?: { stdin?: Buffer
     child.on("error", reject)
     child.on("close", (code) => {
       if (code !== 0) {
-        reject(new Error(err.trim() || `${command} завершился с кодом ${code}`))
+        reject(new Error(err.trim().replaceAll(env.DATABASE_URL, "[DATABASE_URL]") || `${command} завершился с кодом ${code}`))
         return
       }
       resolve(Buffer.concat(chunks))
@@ -44,20 +46,16 @@ function spawnCollected(command: string, args: string[], opts?: { stdin?: Buffer
   })
 }
 
-async function withDatabaseOperation<T>(fn: () => Promise<T>): Promise<T> {
-  return await withAdvisoryLock(BACKUP_LOCK_KEY, async () => {
-    await stopTrafficFlowListener()
-    stopScheduler()
-    try {
-      return await fn()
-    } finally {
-      await startTrafficFlowListener()
-      await refreshScheduler()
-    }
-  })
+async function withDatabaseOperation<T extends object>(fn: () => Promise<T>): Promise<T & DatabaseOperationWarnings> {
+  return withOperationLock(() => pool.connect(), BACKUP_LOCK_KEY, () => executeDatabaseOperation(fn, {
+    stopListener: stopTrafficFlowListener,
+    stopScheduler,
+    startListener: startTrafficFlowListener,
+    startScheduler: refreshScheduler,
+  }), "Операция с базой приложения уже выполняется")
 }
 
-export async function exportSystemDatabaseBackup(): Promise<{ filename: string; buffer: Buffer }> {
+export async function exportSystemDatabaseBackup(): Promise<{ filename: string; buffer: Buffer } & DatabaseOperationWarnings> {
   return await withDatabaseOperation(async () => {
     const buffer = await spawnCollected("pg_dump", ["-Fc", env.DATABASE_URL])
     return {
@@ -67,7 +65,7 @@ export async function exportSystemDatabaseBackup(): Promise<{ filename: string;
   })
 }
 
-export async function restoreSystemDatabaseBackup(buffer: Buffer): Promise<SystemDatabaseRestoreOutcome> {
+export async function restoreSystemDatabaseBackup(buffer: Buffer): Promise<SystemDatabaseRestoreOutcome & DatabaseOperationWarnings> {
   validateSystemDatabaseRestoreArchive(buffer)
   return await withDatabaseOperation(() => executeSystemDatabaseRestore(buffer, env.DATABASE_URL, {
     allocate: async () => {

backend/src/services/system-database-operation.test.ts

@@ -0,0 +1,23 @@
+import assert from "node:assert/strict"
+import {test} from "node:test"
+import {executeDatabaseOperation} from "./system-database-operation.js"
+function fixture(fail: string[] = []) {
+  const calls:string[]=[]
+  const step=(name:string)=>async()=>{calls.push(name);if(fail.includes(name))throw Error(name+" failed")}
+  return {calls,deps:{stopListener:step("stop listener"),stopScheduler:step("stop scheduler"),startListener:step("start listener"),startScheduler:step("start scheduler")},run:async()=>{await step("command")();return {cleanupError:"cleanup warning"}}}
+}
+test("confirmed command survives independent restart failures",async()=>{
+ for(const fail of [[],["start listener"],["start scheduler"],["start listener","start scheduler"]]){
+  const f=fixture(fail);const result=await executeDatabaseOperation(f.run,f.deps)
+  assert.equal(result.cleanupError,"cleanup warning");assert.equal(!!result.listenerError,fail.includes("start listener"));assert.equal(!!result.schedulerError,fail.includes("start scheduler"))
+  assert.deepEqual(f.calls,["stop listener","stop scheduler","command","start listener","start scheduler"])
+ }
+})
+test("core failure survives restart failures without command retry",async()=>{
+ const f=fixture(["command","start listener","start scheduler"])
+ await assert.rejects(executeDatabaseOperation(f.run,f.deps),/command failed/)
+ assert.equal(f.calls.filter(s=>s==="command").length,1);assert.deepEqual(f.calls.slice(-2),["start listener","start scheduler"])
+})
+test("stop failure prevents command and still attempts both restarts",async()=>{
+ for(const step of ["stop listener","stop scheduler"]){const f=fixture([step,"start listener"]);await assert.rejects(executeDatabaseOperation(f.run,f.deps),new RegExp(step+" failed"));assert.equal(f.calls.includes("command"),false);assert.deepEqual(f.calls.slice(-2),["start listener","start scheduler"])}
+})

backend/src/services/system-database-operation.ts

@@ -0,0 +1,23 @@
+export type DatabaseOperationWarnings = { listenerError?: string; schedulerError?: string }
+interface Dependencies {
+  stopListener: () => Promise<void> | void
+  stopScheduler: () => Promise<void> | void
+  startListener: () => Promise<void> | void
+  startScheduler: () => Promise<void> | void
+}
+/** Restart both consumers independently; secondary errors cannot erase a completed command. */
+export async function executeDatabaseOperation<T extends object>(run: () => Promise<T>, deps: Dependencies): Promise<T & DatabaseOperationWarnings> {
+  let result: T | undefined, failed = false, failure: unknown
+  const warnings: DatabaseOperationWarnings = {}
+  try {
+    await deps.stopListener()
+    await deps.stopScheduler()
+    result = await run()
+  } catch (error) { failed = true; failure = error }
+  finally {
+    try { await deps.startListener() } catch { warnings.listenerError = "Не удалось перезапустить приём NetFlow" }
+    try { await deps.startScheduler() } catch { warnings.schedulerError = "Не удалось перезапустить планировщик" }
+  }
+  if (failed) throw failure
+  return { ...result!, ...warnings }
+}