fix(settings): восстанавливать базу транзакционно и сохранять результат операции

This commit is contained in:
Denozordec committed 2026-10-03 14:45:51 +07:00
1 parent ff8b04bddf
commit af5dbb0519
12 files changed
+86 -40

No files matched your search

+3 -2
View File
@@ -27,14 +27,15 @@
"test:scheduler": "tsx src/services/scheduler-recorded-run.test.ts && tsx src/routes/scheduler.test.ts",
"test:backups": "tsx src/services/backup-scheduler-run.test.ts && tsx src/services/backup-restore.test.ts && tsx src/services/backup-deletion.test.ts && tsx src/services/backup-dto.test.ts && tsx src/services/backup-storage-update.test.ts && tsx src/services/backup-job-execution.test.ts && tsx src/routes/backups-operations.test.ts && tsx src/services/s3-backup-client.test.ts",
"test:live-maps": "tsx src/services/ospf-route-parse.test.ts && tsx src/services/vxlan-live.test.ts && tsx src/services/containers-live.test.ts && tsx src/services/firewall-poll.test.ts && tsx src/services/gre-catalog-poll.test.ts && tsx src/routes/filters-gre-catalog.test.ts && tsx src/routes/bgp-catalog.test.ts && tsx src/services/bgp-catalog-parse.test.ts && tsx src/routes/ospf-catalog.test.ts && tsx src/services/ospf-observed-values.test.ts",
"test": "npm run test:probes && npm run test:exec && npm run test:alert-engine && npm run test:auth && npm run test:wireguard && npm run test:traffic-rate && npm run test:traffic-flow && npm run test:users && npm run test:pg && npm run test:backups && npm run test:scheduler && npm run test:live-maps && npm run test:ospf-optimizer && npm run test:recursive-routes && npm run test:config-sync && npm run test:ros-file && npm run test:containers && npm run test:certificates && npm run test:netflow-settings && npm run test:netflow-purge && npm run test:evolution",
"test": "npm run test:probes && npm run test:exec && npm run test:alert-engine && npm run test:auth && npm run test:wireguard && npm run test:traffic-rate && npm run test:traffic-flow && npm run test:users && npm run test:pg && npm run test:backups && npm run test:scheduler && npm run test:live-maps && npm run test:ospf-optimizer && npm run test:recursive-routes && npm run test:config-sync && npm run test:ros-file && npm run test:containers && npm run test:certificates && npm run test:netflow-settings && npm run test:netflow-purge && npm run test:evolution && npm run test:system-database",
"test:evolution": "tsx src/services/evobgp-catalog.test.ts && tsx src/routes/evobgp-catalog.test.ts && tsx src/routes/evobgp-settings.test.ts && tsx src/plugins/auth.evolution-settings.test.ts",
"test:netflow-purge": "tsx src/db/operation-lock.test.ts && tsx src/services/traffic-flow-purge-execution.test.ts && tsx src/routes/traffic-flow-purge.test.ts",
"test:netflow-settings": "tsx src/services/traffic-flow-host-file-builders.test.ts && tsx src/services/traffic-flow-key-execution.test.ts && tsx src/routes/traffic-flow-settings.test.ts && tsx src/plugins/auth.netflow-settings.test.ts",
"test:geoip": "tsx src/services/traffic-flow-geoip.test.ts && tsx src/plugins/auth.geoip.test.ts",
"test:ospf-optimizer": "tsx src/services/ospf-optimizer-identity.test.ts && tsx src/services/ospf-optimizer-metrics.test.ts && tsx src/services/ospf-optimizer-plan.test.ts && tsx src/routes/ospf-optimizer.test.ts",
"test:containers": "tsx src/services/containers-state.test.ts && tsx src/routes/containers-state.test.ts",
"test:certificates": "tsx src/services/certificate-issue-execution.test.ts && tsx src/services/certificate-issue-request.test.ts && tsx src/routes/certificates-issue.test.ts && tsx src/services/certificate-issue-dto.test.ts && tsx src/services/certificate-parse.test.ts && tsx src/services/certificates-state.test.ts && tsx src/routes/certificates-state.test.ts"
"test:certificates": "tsx src/services/certificate-issue-execution.test.ts && tsx src/services/certificate-issue-request.test.ts && tsx src/routes/certificates-issue.test.ts && tsx src/services/certificate-issue-dto.test.ts && tsx src/services/certificate-parse.test.ts && tsx src/services/certificates-state.test.ts && tsx src/routes/certificates-state.test.ts",
"test:system-database": "tsx src/services/system-database-restore-execution.test.ts"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.888.0",
+2 -2
View File
@@ -45,7 +45,7 @@ const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
return reply.status(400).send({ error: "Ожидается тело запроса с файлом pg_dump (custom, PGDMP)" })
}
try {
await restoreSystemDatabaseBackup(body)
const outcome = await restoreSystemDatabaseBackup(body)
await appendEvent({
level: "warning",
eventType: "system.database.restore",
@@ -58,7 +58,7 @@ const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
sizeBytes: body.length,
},
})
return reply.send({ ok: true })
return reply.send({ ok: true, ...outcome })
} catch (err) {
const message = err instanceof Error ? err.message : String(err)
const status = message.includes("уже выполняется") ? 409 : 400
+14 -29
View File
@@ -3,6 +3,7 @@ import { spawn } from "node:child_process"
import { mkdir, rm, writeFile } from "node:fs/promises"
import os from "node:os"
import path from "node:path"
import { executeSystemDatabaseRestore, validateSystemDatabaseRestoreArchive, type SystemDatabaseRestoreOutcome } from "./system-database-restore-execution.js"
import { env } from "../config.js"
import { withAdvisoryLock } from "../db/index.js"
import { refreshScheduler, stopScheduler } from "./scheduler.js"
@@ -11,8 +12,6 @@ import {
stopTrafficFlowListener,
} from "./traffic-flow-ingest.js"
const PG_DUMP_MAGIC = Buffer.from("PGDMP")
const MAX_RESTORE_BYTES = 512 * 1024 * 1024
const BACKUP_LOCK_KEY = 8_723_102
function fmtTimestamp(date = new Date()): string {
@@ -20,15 +19,6 @@ function fmtTimestamp(date = new Date()): string {
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}_${pad(date.getHours())}-${pad(date.getMinutes())}-${pad(date.getSeconds())}`
}
function assertPgDumpFile(buffer: Buffer): void {
if (buffer.length < PG_DUMP_MAGIC.length) {
throw new Error("Файл слишком маленький для дампа PostgreSQL")
}
if (!buffer.subarray(0, PG_DUMP_MAGIC.length).equals(PG_DUMP_MAGIC)) {
throw new Error("Файл не похож на резервную копию PostgreSQL (ожидается формат custom, PGDMP)")
}
}
function spawnCollected(command: string, args: string[], opts?: { stdin?: Buffer }): Promise<Buffer> {
return new Promise((resolve, reject) => {
const child = spawn(command, args, {
@@ -56,7 +46,7 @@ function spawnCollected(command: string, args: string[], opts?: { stdin?: Buffer
async function withDatabaseOperation<T>(fn: () => Promise<T>): Promise<T> {
return await withAdvisoryLock(BACKUP_LOCK_KEY, async () => {
stopTrafficFlowListener()
await stopTrafficFlowListener()
stopScheduler()
try {
return await fn()
@@ -77,21 +67,16 @@ export async function exportSystemDatabaseBackup(): Promise<{ filename: string;
})
}
export async function restoreSystemDatabaseBackup(buffer: Buffer): Promise<void> {
if (buffer.length > MAX_RESTORE_BYTES) {
throw new Error("Файл бэкапа слишком большой")
}
assertPgDumpFile(buffer)
await withDatabaseOperation(async () => {
const tempDir = path.join(os.tmpdir(), "mmapp-db-restore")
await mkdir(tempDir, { recursive: true })
const tempPath = path.join(tempDir, `restore-${randomUUID()}.dump`)
try {
await writeFile(tempPath, buffer)
await spawnCollected("pg_restore", ["--clean", "--if-exists", "-d", env.DATABASE_URL, tempPath])
} finally {
await rm(tempPath, { force: true })
}
})
export async function restoreSystemDatabaseBackup(buffer: Buffer): Promise<SystemDatabaseRestoreOutcome> {
validateSystemDatabaseRestoreArchive(buffer)
return await withDatabaseOperation(() => executeSystemDatabaseRestore(buffer, env.DATABASE_URL, {
allocate: async () => {
const tempDir = path.join(os.tmpdir(), "mmapp-db-restore")
await mkdir(tempDir, { recursive: true })
return path.join(tempDir, `restore-${randomUUID()}.dump`)
},
write: (file, data) => writeFile(file, data),
run: async args => { await spawnCollected("pg_restore", args) },
remove: file => rm(file, { force: true }),
}))
}
@@ -0,0 +1,9 @@
import assert from "node:assert/strict"
import {test} from "node:test"
import {executeSystemDatabaseRestore} from "./system-database-restore-execution.js"
const dump=Buffer.from("PGDMPfixture")
function fixture(){const calls:string[]=[];let failWrite=false,failRun=false,failRemove=false;const deps={allocate:async()=>{calls.push("allocate");return "fixture.dump"},write:async()=>{calls.push("write");if(failWrite)throw Error("write failed")},run:async(args:string[])=>{calls.push("run");assert.ok(args.includes("--single-transaction"));assert.deepEqual(args.slice(-3),["-d","postgres://fixture","fixture.dump"]);if(failRun)throw Error("restore failed")},remove:async()=>{calls.push("remove");if(failRemove)throw Error("cleanup failed")}};return {calls,deps,setWrite:()=>failWrite=true,setRun:()=>failRun=true,setRemove:()=>failRemove=true}}
test("invalid archive never allocates or runs command",async()=>{const f=fixture();for(const data of [Buffer.alloc(0),Buffer.from("sqlite")])await assert.rejects(executeSystemDatabaseRestore(data,"postgres://fixture",f.deps),/PGDMP/);assert.deepEqual(f.calls,[])})
test("restore runs one transactional command and cleans file after success",async()=>{const f=fixture();assert.deepEqual(await executeSystemDatabaseRestore(dump,"postgres://fixture",f.deps),{});assert.deepEqual(f.calls,["allocate","write","run","remove"])})
test("write or restore failure always cleans temporary file and never retries",async()=>{for(const step of ["write","run"]){const f=fixture();if(step==="write")f.setWrite();else f.setRun();await assert.rejects(executeSystemDatabaseRestore(dump,"postgres://fixture",f.deps),/failed/);assert.deepEqual(f.calls,step==="write"?["allocate","write","remove"]:["allocate","write","run","remove"])}})
test("cleanup failure keeps confirmed restore result; main failure is preserved",async()=>{const f=fixture();f.setRemove();assert.deepEqual(await executeSystemDatabaseRestore(dump,"postgres://fixture",f.deps),{cleanupError:"Не удалось удалить временный файл восстановления"});const bad=fixture();bad.setRun();bad.setRemove();await assert.rejects(executeSystemDatabaseRestore(dump,"postgres://fixture",bad.deps),/restore failed; Не удалось удалить/)})
@@ -0,0 +1,23 @@
export type SystemDatabaseRestoreOutcome={cleanupError?:string}
interface Dependencies {
allocate:()=>Promise<string>
write:(path:string,buffer:Buffer)=>Promise<void>
run:(args:string[])=>Promise<void>
remove:(path:string)=>Promise<void>
}
export function validateSystemDatabaseRestoreArchive(buffer:Buffer):void{
if(buffer.length>512*1024*1024)throw new Error("Файл бэкапа слишком большой")
if(buffer.length<5||!buffer.subarray(0,5).equals(Buffer.from("PGDMP")))throw new Error("Файл не похож на резервную копию PostgreSQL (PGDMP)")
}
export async function executeSystemDatabaseRestore(buffer:Buffer,databaseUrl:string,deps:Dependencies):Promise<SystemDatabaseRestoreOutcome>{
validateSystemDatabaseRestoreArchive(buffer)
const file=await deps.allocate()
let failed=false,failure:unknown,cleanupError:string|undefined
try{
await deps.write(file,buffer)
await deps.run(["--single-transaction","--clean","--if-exists","-d",databaseUrl,file])
}catch(error){failed=true;failure=error}
finally{try{await deps.remove(file)}catch{cleanupError="Не удалось удалить временный файл восстановления"}}
if(failed){const message=failure instanceof Error?failure.message:"Восстановление PostgreSQL не подтверждено";throw new Error(message+(cleanupError?"; "+cleanupError:""))}
return cleanupError?{cleanupError}:{}
}