fix(settings): восстанавливать базу транзакционно и сохранять результат операции
This commit is contained in:
1 parent
ff8b04bddf
commit
af5dbb0519
12 files changed
+86
-40
No files matched your search
@@ -27,14 +27,15 @@
|
||||
"test:scheduler": "tsx src/services/scheduler-recorded-run.test.ts && tsx src/routes/scheduler.test.ts",
|
||||
"test:backups": "tsx src/services/backup-scheduler-run.test.ts && tsx src/services/backup-restore.test.ts && tsx src/services/backup-deletion.test.ts && tsx src/services/backup-dto.test.ts && tsx src/services/backup-storage-update.test.ts && tsx src/services/backup-job-execution.test.ts && tsx src/routes/backups-operations.test.ts && tsx src/services/s3-backup-client.test.ts",
|
||||
"test:live-maps": "tsx src/services/ospf-route-parse.test.ts && tsx src/services/vxlan-live.test.ts && tsx src/services/containers-live.test.ts && tsx src/services/firewall-poll.test.ts && tsx src/services/gre-catalog-poll.test.ts && tsx src/routes/filters-gre-catalog.test.ts && tsx src/routes/bgp-catalog.test.ts && tsx src/services/bgp-catalog-parse.test.ts && tsx src/routes/ospf-catalog.test.ts && tsx src/services/ospf-observed-values.test.ts",
|
||||
"test": "npm run test:probes && npm run test:exec && npm run test:alert-engine && npm run test:auth && npm run test:wireguard && npm run test:traffic-rate && npm run test:traffic-flow && npm run test:users && npm run test:pg && npm run test:backups && npm run test:scheduler && npm run test:live-maps && npm run test:ospf-optimizer && npm run test:recursive-routes && npm run test:config-sync && npm run test:ros-file && npm run test:containers && npm run test:certificates && npm run test:netflow-settings && npm run test:netflow-purge && npm run test:evolution",
|
||||
"test": "npm run test:probes && npm run test:exec && npm run test:alert-engine && npm run test:auth && npm run test:wireguard && npm run test:traffic-rate && npm run test:traffic-flow && npm run test:users && npm run test:pg && npm run test:backups && npm run test:scheduler && npm run test:live-maps && npm run test:ospf-optimizer && npm run test:recursive-routes && npm run test:config-sync && npm run test:ros-file && npm run test:containers && npm run test:certificates && npm run test:netflow-settings && npm run test:netflow-purge && npm run test:evolution && npm run test:system-database",
|
||||
"test:evolution": "tsx src/services/evobgp-catalog.test.ts && tsx src/routes/evobgp-catalog.test.ts && tsx src/routes/evobgp-settings.test.ts && tsx src/plugins/auth.evolution-settings.test.ts",
|
||||
"test:netflow-purge": "tsx src/db/operation-lock.test.ts && tsx src/services/traffic-flow-purge-execution.test.ts && tsx src/routes/traffic-flow-purge.test.ts",
|
||||
"test:netflow-settings": "tsx src/services/traffic-flow-host-file-builders.test.ts && tsx src/services/traffic-flow-key-execution.test.ts && tsx src/routes/traffic-flow-settings.test.ts && tsx src/plugins/auth.netflow-settings.test.ts",
|
||||
"test:geoip": "tsx src/services/traffic-flow-geoip.test.ts && tsx src/plugins/auth.geoip.test.ts",
|
||||
"test:ospf-optimizer": "tsx src/services/ospf-optimizer-identity.test.ts && tsx src/services/ospf-optimizer-metrics.test.ts && tsx src/services/ospf-optimizer-plan.test.ts && tsx src/routes/ospf-optimizer.test.ts",
|
||||
"test:containers": "tsx src/services/containers-state.test.ts && tsx src/routes/containers-state.test.ts",
|
||||
"test:certificates": "tsx src/services/certificate-issue-execution.test.ts && tsx src/services/certificate-issue-request.test.ts && tsx src/routes/certificates-issue.test.ts && tsx src/services/certificate-issue-dto.test.ts && tsx src/services/certificate-parse.test.ts && tsx src/services/certificates-state.test.ts && tsx src/routes/certificates-state.test.ts"
|
||||
"test:certificates": "tsx src/services/certificate-issue-execution.test.ts && tsx src/services/certificate-issue-request.test.ts && tsx src/routes/certificates-issue.test.ts && tsx src/services/certificate-issue-dto.test.ts && tsx src/services/certificate-parse.test.ts && tsx src/services/certificates-state.test.ts && tsx src/routes/certificates-state.test.ts",
|
||||
"test:system-database": "tsx src/services/system-database-restore-execution.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.888.0",
|
||||
|
||||
@@ -45,7 +45,7 @@ const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
|
||||
return reply.status(400).send({ error: "Ожидается тело запроса с файлом pg_dump (custom, PGDMP)" })
|
||||
}
|
||||
try {
|
||||
await restoreSystemDatabaseBackup(body)
|
||||
const outcome = await restoreSystemDatabaseBackup(body)
|
||||
await appendEvent({
|
||||
level: "warning",
|
||||
eventType: "system.database.restore",
|
||||
@@ -58,7 +58,7 @@ const systemDatabaseRoutes: FastifyPluginAsync = async (app) => {
|
||||
sizeBytes: body.length,
|
||||
},
|
||||
})
|
||||
return reply.send({ ok: true })
|
||||
return reply.send({ ok: true, ...outcome })
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
const status = message.includes("уже выполняется") ? 409 : 400
|
||||
|
||||
@@ -3,6 +3,7 @@ import { spawn } from "node:child_process"
|
||||
import { mkdir, rm, writeFile } from "node:fs/promises"
|
||||
import os from "node:os"
|
||||
import path from "node:path"
|
||||
import { executeSystemDatabaseRestore, validateSystemDatabaseRestoreArchive, type SystemDatabaseRestoreOutcome } from "./system-database-restore-execution.js"
|
||||
import { env } from "../config.js"
|
||||
import { withAdvisoryLock } from "../db/index.js"
|
||||
import { refreshScheduler, stopScheduler } from "./scheduler.js"
|
||||
@@ -11,8 +12,6 @@ import {
|
||||
stopTrafficFlowListener,
|
||||
} from "./traffic-flow-ingest.js"
|
||||
|
||||
const PG_DUMP_MAGIC = Buffer.from("PGDMP")
|
||||
const MAX_RESTORE_BYTES = 512 * 1024 * 1024
|
||||
const BACKUP_LOCK_KEY = 8_723_102
|
||||
|
||||
function fmtTimestamp(date = new Date()): string {
|
||||
@@ -20,15 +19,6 @@ function fmtTimestamp(date = new Date()): string {
|
||||
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}_${pad(date.getHours())}-${pad(date.getMinutes())}-${pad(date.getSeconds())}`
|
||||
}
|
||||
|
||||
function assertPgDumpFile(buffer: Buffer): void {
|
||||
if (buffer.length < PG_DUMP_MAGIC.length) {
|
||||
throw new Error("Файл слишком маленький для дампа PostgreSQL")
|
||||
}
|
||||
if (!buffer.subarray(0, PG_DUMP_MAGIC.length).equals(PG_DUMP_MAGIC)) {
|
||||
throw new Error("Файл не похож на резервную копию PostgreSQL (ожидается формат custom, PGDMP)")
|
||||
}
|
||||
}
|
||||
|
||||
function spawnCollected(command: string, args: string[], opts?: { stdin?: Buffer }): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, {
|
||||
@@ -56,7 +46,7 @@ function spawnCollected(command: string, args: string[], opts?: { stdin?: Buffer
|
||||
|
||||
async function withDatabaseOperation<T>(fn: () => Promise<T>): Promise<T> {
|
||||
return await withAdvisoryLock(BACKUP_LOCK_KEY, async () => {
|
||||
stopTrafficFlowListener()
|
||||
await stopTrafficFlowListener()
|
||||
stopScheduler()
|
||||
try {
|
||||
return await fn()
|
||||
@@ -77,21 +67,16 @@ export async function exportSystemDatabaseBackup(): Promise<{ filename: string;
|
||||
})
|
||||
}
|
||||
|
||||
export async function restoreSystemDatabaseBackup(buffer: Buffer): Promise<void> {
|
||||
if (buffer.length > MAX_RESTORE_BYTES) {
|
||||
throw new Error("Файл бэкапа слишком большой")
|
||||
}
|
||||
assertPgDumpFile(buffer)
|
||||
|
||||
await withDatabaseOperation(async () => {
|
||||
const tempDir = path.join(os.tmpdir(), "mmapp-db-restore")
|
||||
await mkdir(tempDir, { recursive: true })
|
||||
const tempPath = path.join(tempDir, `restore-${randomUUID()}.dump`)
|
||||
try {
|
||||
await writeFile(tempPath, buffer)
|
||||
await spawnCollected("pg_restore", ["--clean", "--if-exists", "-d", env.DATABASE_URL, tempPath])
|
||||
} finally {
|
||||
await rm(tempPath, { force: true })
|
||||
}
|
||||
})
|
||||
export async function restoreSystemDatabaseBackup(buffer: Buffer): Promise<SystemDatabaseRestoreOutcome> {
|
||||
validateSystemDatabaseRestoreArchive(buffer)
|
||||
return await withDatabaseOperation(() => executeSystemDatabaseRestore(buffer, env.DATABASE_URL, {
|
||||
allocate: async () => {
|
||||
const tempDir = path.join(os.tmpdir(), "mmapp-db-restore")
|
||||
await mkdir(tempDir, { recursive: true })
|
||||
return path.join(tempDir, `restore-${randomUUID()}.dump`)
|
||||
},
|
||||
write: (file, data) => writeFile(file, data),
|
||||
run: async args => { await spawnCollected("pg_restore", args) },
|
||||
remove: file => rm(file, { force: true }),
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import assert from "node:assert/strict"
|
||||
import {test} from "node:test"
|
||||
import {executeSystemDatabaseRestore} from "./system-database-restore-execution.js"
|
||||
const dump=Buffer.from("PGDMPfixture")
|
||||
function fixture(){const calls:string[]=[];let failWrite=false,failRun=false,failRemove=false;const deps={allocate:async()=>{calls.push("allocate");return "fixture.dump"},write:async()=>{calls.push("write");if(failWrite)throw Error("write failed")},run:async(args:string[])=>{calls.push("run");assert.ok(args.includes("--single-transaction"));assert.deepEqual(args.slice(-3),["-d","postgres://fixture","fixture.dump"]);if(failRun)throw Error("restore failed")},remove:async()=>{calls.push("remove");if(failRemove)throw Error("cleanup failed")}};return {calls,deps,setWrite:()=>failWrite=true,setRun:()=>failRun=true,setRemove:()=>failRemove=true}}
|
||||
test("invalid archive never allocates or runs command",async()=>{const f=fixture();for(const data of [Buffer.alloc(0),Buffer.from("sqlite")])await assert.rejects(executeSystemDatabaseRestore(data,"postgres://fixture",f.deps),/PGDMP/);assert.deepEqual(f.calls,[])})
|
||||
test("restore runs one transactional command and cleans file after success",async()=>{const f=fixture();assert.deepEqual(await executeSystemDatabaseRestore(dump,"postgres://fixture",f.deps),{});assert.deepEqual(f.calls,["allocate","write","run","remove"])})
|
||||
test("write or restore failure always cleans temporary file and never retries",async()=>{for(const step of ["write","run"]){const f=fixture();if(step==="write")f.setWrite();else f.setRun();await assert.rejects(executeSystemDatabaseRestore(dump,"postgres://fixture",f.deps),/failed/);assert.deepEqual(f.calls,step==="write"?["allocate","write","remove"]:["allocate","write","run","remove"])}})
|
||||
test("cleanup failure keeps confirmed restore result; main failure is preserved",async()=>{const f=fixture();f.setRemove();assert.deepEqual(await executeSystemDatabaseRestore(dump,"postgres://fixture",f.deps),{cleanupError:"Не удалось удалить временный файл восстановления"});const bad=fixture();bad.setRun();bad.setRemove();await assert.rejects(executeSystemDatabaseRestore(dump,"postgres://fixture",bad.deps),/restore failed; Не удалось удалить/)})
|
||||
@@ -0,0 +1,23 @@
|
||||
export type SystemDatabaseRestoreOutcome={cleanupError?:string}
|
||||
interface Dependencies {
|
||||
allocate:()=>Promise<string>
|
||||
write:(path:string,buffer:Buffer)=>Promise<void>
|
||||
run:(args:string[])=>Promise<void>
|
||||
remove:(path:string)=>Promise<void>
|
||||
}
|
||||
export function validateSystemDatabaseRestoreArchive(buffer:Buffer):void{
|
||||
if(buffer.length>512*1024*1024)throw new Error("Файл бэкапа слишком большой")
|
||||
if(buffer.length<5||!buffer.subarray(0,5).equals(Buffer.from("PGDMP")))throw new Error("Файл не похож на резервную копию PostgreSQL (PGDMP)")
|
||||
}
|
||||
export async function executeSystemDatabaseRestore(buffer:Buffer,databaseUrl:string,deps:Dependencies):Promise<SystemDatabaseRestoreOutcome>{
|
||||
validateSystemDatabaseRestoreArchive(buffer)
|
||||
const file=await deps.allocate()
|
||||
let failed=false,failure:unknown,cleanupError:string|undefined
|
||||
try{
|
||||
await deps.write(file,buffer)
|
||||
await deps.run(["--single-transaction","--clean","--if-exists","-d",databaseUrl,file])
|
||||
}catch(error){failed=true;failure=error}
|
||||
finally{try{await deps.remove(file)}catch{cleanupError="Не удалось удалить временный файл восстановления"}}
|
||||
if(failed){const message=failure instanceof Error?failure.message:"Восстановление PostgreSQL не подтверждено";throw new Error(message+(cleanupError?"; "+cleanupError:""))}
|
||||
return cleanupError?{cleanupError}:{}
|
||||
}
|
||||
Reference in new issue
Block a user