fix(backups): отделить результат операций от ошибок журнала
This commit is contained in:
1 parent
3dcc7c9ea6
commit
5ce21e9a47
10 files changed
+113
-67
No files matched your search
@@ -0,0 +1,25 @@
|
||||
import assert from "node:assert/strict"
|
||||
import Fastify from "fastify"
|
||||
import { validatorCompiler, serializerCompiler } from "@fastify/type-provider-zod"
|
||||
import { backupJobResponseSchema } from "@mmapp/contracts/backups"
|
||||
import { createBackupsRoutes } from "./backups.js"
|
||||
import { permissionForRequest } from "../lib/permissions.js"
|
||||
const app = Fastify(); app.setValidatorCompiler(validatorCompiler); app.setSerializerCompiler(serializerCompiler)
|
||||
let created = 0, restored = 0, eventCalls = 0, restoreFail = false
|
||||
await app.register(createBackupsRoutes({ listServers: async () => [{ id: 1 }, { id: 2 }], create: async id => { created++; if (id === "2") throw new Error("RouterOS unavailable"); return { id: "backup-1", serverId: 1, serverName: "router", filename: "config.rsc", sizeBytes: 0, createdAt: "2026-10-01T00:00:00Z", kind: "manual", storage: "local" } }, restore: async () => { restored++; if (restoreFail) throw new Error("import failed"); return { filename: "config.rsc", serverName: "router" } }, appendEvent: async () => { eventCalls++; throw new Error("event DB unavailable") } }), { prefix: "/api" })
|
||||
const post = (payload: unknown) => app.inject({ method: "POST", url: "/api/backups/create", payload: payload as Record<string, unknown> })
|
||||
for (const serverIds of [[], ["1", "missing"], ["1", "3"], ["1", "1"], ["0"], ["1junk"], [9007199254740992]]) assert.equal((await post({ serverIds })).statusCode, 400)
|
||||
assert.equal(created, 0)
|
||||
const ack = await post({ serverIds: [1, "2"] }); assert.equal(ack.statusCode, 202)
|
||||
const id = ack.json().jobId
|
||||
for (let attempt = 0; attempt < 20; attempt++) {
|
||||
const observation = await app.inject({ method: "GET", url: "/api/backups/jobs/" + id })
|
||||
const job = backupJobResponseSchema.parse(observation.json())
|
||||
if (job.status === "done") { assert.equal(job.completed, 2); assert.equal(job.created.length, 1); assert.deepEqual(job.failures, [{ serverId: "2", error: "RouterOS unavailable" }]); break }
|
||||
await new Promise(resolve => setImmediate(resolve)); assert.ok(attempt < 19)
|
||||
}
|
||||
assert.equal(created, 2); assert.ok(eventCalls >= 2)
|
||||
const success = await app.inject({ method: "POST", url: "/api/backups/backup-1/restore" }); assert.equal(success.statusCode, 200); assert.deepEqual(success.json(), { filename: "config.rsc", serverName: "router" }); assert.equal(restored, 1)
|
||||
restoreFail = true; const failure = await app.inject({ method: "POST", url: "/api/backups/backup-1/restore" }); assert.equal(failure.statusCode, 400); assert.equal(failure.json().error, "import failed")
|
||||
assert.equal(permissionForRequest("POST", "/api/backups/create"), "mm:backups:write"); assert.equal(permissionForRequest("POST", "/api/backups/backup-1/restore"), "mm:backups:write"); assert.equal(permissionForRequest("GET", "/api/backups/jobs/job"), "mm:backups:read")
|
||||
await app.close(); console.log("Backups HTTP: event failure does not relabel created/restore, invalid targets have no side effects, partial results and RBAC")
|
||||
@@ -2,6 +2,8 @@ import { randomUUID } from "node:crypto"
|
||||
import { z } from "zod"
|
||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import {
|
||||
createBackupRequestSchema,
|
||||
type BackupJobStatusResponse,
|
||||
putBackupScheduleSettingsSchema,
|
||||
putBackupStorageSettingsSchema,
|
||||
} from "@mmapp/contracts/backups"
|
||||
@@ -21,28 +23,11 @@ import {
|
||||
testBackupStorageConnection,
|
||||
updateBackupScheduleSettings,
|
||||
updateBackupStorageSettings,
|
||||
type BackupMeta,
|
||||
} from "../services/backup-service.js"
|
||||
|
||||
type BackupJobStatus = "queued" | "running" | "done" | "failed"
|
||||
type BackupJob = {
|
||||
id: string
|
||||
status: BackupJobStatus
|
||||
requestedAt: string
|
||||
startedAt?: string
|
||||
finishedAt?: string
|
||||
total: number
|
||||
completed: number
|
||||
created: BackupMeta[]
|
||||
failures: Array<{ serverId: string; error: string }>
|
||||
}
|
||||
import { processBackupJob, recordBackupEvent } from "../services/backup-job-execution.js"
|
||||
|
||||
const backupJobs = new Map<string, BackupJob>()
|
||||
|
||||
const CreateBackupBodySchema = z.object({
|
||||
serverIds: z.array(z.union([z.string(), z.number()])).min(1),
|
||||
notes: z.string().max(500).optional(),
|
||||
})
|
||||
type BackupJob = BackupJobStatusResponse
|
||||
|
||||
const BackupIdParamSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
@@ -51,40 +36,10 @@ const BackupJobIdParamSchema = z.object({
|
||||
jobId: z.string().min(1),
|
||||
})
|
||||
|
||||
async function processBackupJob(job: BackupJob, ids: string[], notes?: string) {
|
||||
job.status = "running"
|
||||
job.startedAt = new Date().toISOString()
|
||||
for (const id of ids) {
|
||||
try {
|
||||
const meta = await runBackupForServer(id, "manual", notes)
|
||||
job.created.push(meta)
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
job.failures.push({ serverId: id, error: message })
|
||||
} finally {
|
||||
job.completed += 1
|
||||
}
|
||||
}
|
||||
job.status = "done"
|
||||
job.finishedAt = new Date().toISOString()
|
||||
await appendEvent({
|
||||
level: job.failures.length > 0 ? "warning" : "info",
|
||||
eventType: "backups.job.done",
|
||||
sourceModule: "backups",
|
||||
title: job.failures.length > 0 ? "Бэкап завершен с ошибками" : "Бэкап завершен",
|
||||
message: `Создано: ${job.created.length}, ошибок: ${job.failures.length}`,
|
||||
entityType: "backup_job",
|
||||
entityId: job.id,
|
||||
payload: {
|
||||
total: job.total,
|
||||
completed: job.completed,
|
||||
failures: job.failures,
|
||||
notes: notes ?? null,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
export function createBackupsRoutes(deps: { listServers?: () => Promise<Array<{ id: string | number }>>; create?: typeof runBackupForServer; restore?: typeof restoreBackupToDevice; appendEvent?: typeof appendEvent } = {}): FastifyPluginAsyncZod {
|
||||
return async (app) => {
|
||||
const backupJobs = new Map<string, BackupJob>()
|
||||
const event = (input: Parameters<typeof appendEvent>[0]) => recordBackupEvent(input, deps.appendEvent ?? appendEvent, error => app.log.warn({ err: error }, "Не удалось записать событие резервных копий"))
|
||||
app.get("/backups", async (_req, reply) => {
|
||||
return reply.send(await listBackups())
|
||||
})
|
||||
@@ -136,12 +91,12 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
}
|
||||
})
|
||||
|
||||
app.post("/backups/create", { schema: { body: CreateBackupBodySchema } }, async (req, reply) => {
|
||||
app.post("/backups/create", { schema: { body: createBackupRequestSchema } }, async (req, reply) => {
|
||||
const inputIds = req.body.serverIds.map((x) => String(x))
|
||||
const notes = req.body.notes?.trim() || undefined
|
||||
const existingServers = new Set((await listServersRead()).map((s) => String(s.id)))
|
||||
const ids = [...new Set(inputIds)].filter((id) => existingServers.has(id))
|
||||
if (ids.length === 0) return reply.status(400).send({ error: "Не выбраны валидные серверы" })
|
||||
const existingServers = new Set((await (deps.listServers ?? listServersRead)()).map((s) => String(s.id)))
|
||||
const ids = [...new Set(inputIds)]
|
||||
if (ids.some(id => !existingServers.has(id))) return reply.status(400).send({ error: "Выбранный сервер отсутствует в каталоге; обновите выбор" })
|
||||
|
||||
const jobId = randomUUID()
|
||||
const job: BackupJob = {
|
||||
@@ -154,7 +109,7 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
failures: [],
|
||||
}
|
||||
backupJobs.set(jobId, job)
|
||||
await appendEvent({
|
||||
await event({
|
||||
level: "info",
|
||||
eventType: "backups.job.started",
|
||||
sourceModule: "backups",
|
||||
@@ -168,7 +123,7 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
},
|
||||
})
|
||||
queueMicrotask(async () => {
|
||||
void processBackupJob(job, ids, notes).catch(async (err) => {
|
||||
void processBackupJob(job, ids, notes, { create: deps.create ?? runBackupForServer, event }).catch(async (err) => {
|
||||
job.status = "failed"
|
||||
job.finishedAt = new Date().toISOString()
|
||||
const failure = {
|
||||
@@ -176,7 +131,7 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
}
|
||||
job.failures.push(failure)
|
||||
await appendEvent({
|
||||
await event({
|
||||
level: "critical",
|
||||
eventType: "backups.job.failed",
|
||||
sourceModule: "backups",
|
||||
@@ -220,8 +175,8 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
|
||||
app.post("/backups/:id/restore", { schema: { params: BackupIdParamSchema } }, async (req, reply) => {
|
||||
try {
|
||||
const result = await restoreBackupToDevice(req.params.id)
|
||||
await appendEvent({
|
||||
const result = await (deps.restore ?? restoreBackupToDevice)(req.params.id)
|
||||
await event({
|
||||
level: "warning",
|
||||
eventType: "backups.restore",
|
||||
sourceModule: "backups",
|
||||
@@ -243,6 +198,7 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
if (!hit) return reply.status(404).send({ error: "Бэкап не найден" })
|
||||
return reply.status(204).send()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default backupsRoutes
|
||||
export default createBackupsRoutes()
|
||||
@@ -0,0 +1,22 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import type { BackupJobStatusResponse } from "@mmapp/contracts/backups"
|
||||
import { backupJobResponseSchema } from "@mmapp/contracts/backups"
|
||||
import { processBackupJob, recordBackupEvent } from "./backup-job-execution.js"
|
||||
const item = { id: "backup", serverId: 1, serverName: "router", filename: "config.rsc", sizeBytes: 0, createdAt: "2026-10-01T00:00:00Z", kind: "manual" as const, storage: "local" as const }
|
||||
const job = (): BackupJobStatusResponse => ({ id: "job", status: "queued", requestedAt: "2026-10-01T00:00:00Z", total: 2, completed: 0, created: [], failures: [] })
|
||||
test("event failure preserves done and actual created files", async () => {
|
||||
const row = job(), warnings: unknown[] = []
|
||||
await processBackupJob(row, ["1", "2"], undefined, { create: async id => ({ ...item, id, serverId: Number(id) }), event: input => recordBackupEvent(input, async () => { throw new Error("event unavailable") }, error => { warnings.push(error) }) })
|
||||
assert.equal(row.status, "done"); assert.equal(row.created.length, 2); assert.equal(row.failures.length, 0); assert.equal(warnings.length, 1); backupJobResponseSchema.parse(row)
|
||||
})
|
||||
test("one failed device keeps partial files and completes every target", async () => {
|
||||
const row = job()
|
||||
await processBackupJob(row, ["1", "2"], undefined, { create: async id => { if (id === "2") throw new Error("device unavailable"); return item }, event: async input => { assert.equal(input.level, "warning") } })
|
||||
assert.equal(row.status, "done"); assert.equal(row.completed, 2); assert.equal(row.created.length, 1); assert.deepEqual(row.failures, [{ serverId: "2", error: "device unavailable" }]); backupJobResponseSchema.parse(row)
|
||||
})
|
||||
test("S3 upload error remains a local copy and a warning rather than full success", async () => {
|
||||
const row = job(); row.total = 1
|
||||
await processBackupJob(row, ["1"], undefined, { create: async () => ({ ...item, uploadError: "S3 unavailable" }), event: async input => { assert.equal(input.level, "warning"); assert.match(input.message, /ошибок загрузки: 1/) } })
|
||||
assert.equal(row.status, "done"); assert.equal(row.created[0].storage, "local"); assert.equal(row.created[0].uploadError, "S3 unavailable"); backupJobResponseSchema.parse(row)
|
||||
})
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { BackupJobStatusResponse } from "@mmapp/contracts/backups"
|
||||
import type { AppendEventBody } from "@mmapp/contracts/events"
|
||||
import type { BackupMeta } from "./backup-service.js"
|
||||
|
||||
export async function recordBackupEvent(input: AppendEventBody, append: (input: AppendEventBody) => Promise<void>, warn: (error: unknown) => void): Promise<void> {
|
||||
try { await append(input) } catch (error) { warn(error) }
|
||||
}
|
||||
export async function processBackupJob(job: BackupJobStatusResponse, ids: string[], notes: string | undefined, deps: {
|
||||
create: (id: string, kind: "manual", notes?: string) => Promise<BackupMeta>
|
||||
event: (input: AppendEventBody) => Promise<void>
|
||||
}) {
|
||||
job.status = "running"
|
||||
job.startedAt = new Date().toISOString()
|
||||
for (const id of ids) {
|
||||
try { job.created.push(await deps.create(id, "manual", notes)) }
|
||||
catch (error) { job.failures.push({ serverId: id, error: error instanceof Error ? error.message : String(error) }) }
|
||||
finally { job.completed += 1 }
|
||||
}
|
||||
job.status = "done"
|
||||
job.finishedAt = new Date().toISOString()
|
||||
const uploadFailures = job.created.filter(item => item.uploadError)
|
||||
const partial = job.failures.length > 0 || uploadFailures.length > 0
|
||||
await deps.event({ level: partial ? "warning" : "info", eventType: "backups.job.done", sourceModule: "backups", title: partial ? "Бэкап завершен с ошибками" : "Бэкап завершен", message: "Создано: " + job.created.length + ", ошибок устройств: " + job.failures.length + ", ошибок загрузки: " + uploadFailures.length, entityType: "backup_job", entityId: job.id, payload: { total: job.total, completed: job.completed, failures: job.failures, uploadFailures: uploadFailures.map(item => ({ id: item.id, error: item.uploadError })), notes: notes ?? null } })
|
||||
}
|
||||
Reference in new issue
Block a user