fix(backups): сохранить достоверность плановых операций при сбоях

backend/src/services/backup-prune.ts

@@ -0,0 +1,8 @@
+export class BackupPruneError extends Error {
+ constructor(public readonly pruned: number, cause: unknown) { super("Очистка резервных копий прервана после " + pruned + " удалений: " + (cause instanceof Error ? cause.message : String(cause)), { cause }); this.name = "BackupPruneError" }
+}
+export async function pruneBackupRecords(ids: string[], remove: (id: string) => Promise<unknown>): Promise<number> {
+ let pruned = 0
+ for (const id of ids) { try { if (await remove(id)) pruned++ } catch (cause) { throw new BackupPruneError(pruned, cause) } }
+ return pruned
+}

backend/src/services/backup-scheduler-collector.ts

@@ -1,136 +1,6 @@
 import { appendEvent } from "../modules/events/service/events-service.js"
-import type { BackupsRunSnapshot } from "../types/scheduler-run-snapshot.js"
-import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
-import {
-  getBackupScheduleSettings,
-  isBackupDue,
-  pruneBackupsForServer,
-  resolveBackupServerIds,
-  runBackupForServer,
-  touchBackupScheduleRunMeta,
-} from "./backup-service.js"
-
-let collecting = false
-
-export function getBackupSchedulerCollectorState(): { running: boolean } {
-  return { running: collecting }
-}
-
-export async function collectScheduledBackupsOnce(): Promise<BackupsRunSnapshot> {
-  const sampledAt = new Date().toISOString()
-  const settings = await getBackupScheduleSettings()
-  const snapshot: BackupsRunSnapshot = {
-    v: SCHEDULER_RUN_SNAPSHOT_VERSION,
-    job: "backups",
-    sampledAt,
-    due: false,
-    created: 0,
-    failures: 0,
-    pruned: 0,
-    errors: [],
-  }
-
-  if (collecting) {
-    snapshot.skipped = true
-    return snapshot
-  }
-
-  if (!settings.enabled) {
-    snapshot.skipped = true
-    return snapshot
-  }
-
-  const due = isBackupDue(new Date(), settings, settings.lastRunAt)
-  snapshot.due = due
-  if (!due) {
-    return snapshot
-  }
-
-  if (settings.format !== "rsc") {
-    snapshot.skipped = true
-    snapshot.errors = ["Формат backup пока не поддерживается, используйте rsc"]
-    await touchBackupScheduleRunMeta({
-      lastRunAt: sampledAt,
-      lastDurationMs: 0,
-      lastError: snapshot.errors[0],
-    })
-    return snapshot
-  }
-
-  collecting = true
-  const started = Date.now()
-  const serverIds = await resolveBackupServerIds(settings)
-
-  await appendEvent({
-    level: "info",
-    eventType: "backups.job.started",
-    sourceModule: "backups",
-    title: "Запущен плановый бэкап",
-    message: `Серверов в очереди: ${serverIds.length}`,
-    entityType: "backup_job",
-    entityId: "scheduler",
-    payload: { serverIds, scheduled: true },
-  })
-
-  try {
-    for (const id of serverIds) {
-      try {
-        await runBackupForServer(id, "auto")
-        snapshot.created += 1
-      } catch (e) {
-        const message = e instanceof Error ? e.message : String(e)
-        snapshot.failures += 1
-        snapshot.errors?.push(`${id}: ${message}`)
-      }
-    }
-
-    for (const id of serverIds) {
-      snapshot.pruned += await pruneBackupsForServer(id, settings.keepCount)
-    }
-
-    await touchBackupScheduleRunMeta({
-      lastRunAt: sampledAt,
-      lastDurationMs: Date.now() - started,
-      lastError: snapshot.errors?.length ? snapshot.errors.join("; ") : null,
-    })
-
-    await appendEvent({
-      level: snapshot.failures > 0 ? "warning" : "info",
-      eventType: "backups.job.done",
-      sourceModule: "backups",
-      title: snapshot.failures > 0 ? "Плановый бэкап завершен с ошибками" : "Плановый бэкап завершен",
-      message: `Создано: ${snapshot.created}, ошибок: ${snapshot.failures}`,
-      entityType: "backup_job",
-      entityId: "scheduler",
-      payload: {
-        total: serverIds.length,
-        completed: snapshot.created + snapshot.failures,
-        failures: snapshot.errors,
-        pruned: snapshot.pruned,
-      },
-    })
-
-    return snapshot
-  } catch (e) {
-    const message = e instanceof Error ? e.message : String(e)
-    snapshot.fatalError = message
-    snapshot.errors?.push(message)
-    await touchBackupScheduleRunMeta({
-      lastRunAt: sampledAt,
-      lastDurationMs: Date.now() - started,
-      lastError: message,
-    })
-    await appendEvent({
-      level: "critical",
-      eventType: "backups.job.failed",
-      sourceModule: "backups",
-      title: "Плановый бэкап прерван",
-      message,
-      entityType: "backup_job",
-      entityId: "scheduler",
-    })
-    return snapshot
-  } finally {
-    collecting = false
-  }
-}
+import { createBackupSchedulerCollector } from "./backup-scheduler-run.js"
+import { getBackupScheduleSettings, isBackupDue, pruneBackupsForServer, resolveBackupServerIds, runBackupForServer, touchBackupScheduleRunMeta } from "./backup-service.js"
+const collector = createBackupSchedulerCollector({ settings: getBackupScheduleSettings, due: isBackupDue, resolveServers: resolveBackupServerIds, create: runBackupForServer, prune: pruneBackupsForServer, metadata: touchBackupScheduleRunMeta, event: appendEvent })
+export const getBackupSchedulerCollectorState = collector.state
+export const collectScheduledBackupsOnce = collector.collect

backend/src/services/backup-scheduler-run.test.ts

@@ -0,0 +1,56 @@
+import assert from "node:assert/strict"
+import { test } from "node:test"
+import type { AppendEventBody } from "@mmapp/contracts/events"
+import { createBackupSchedulerCollector } from "./backup-scheduler-run.js"
+import { BackupPruneError, pruneBackupRecords } from "./backup-prune.js"
+type Dependencies = Parameters<typeof createBackupSchedulerCollector>[0]
+const settings = { enabled: true, frequency: "daily" as const, hour: 0, minute: 0, weekDay: 0, monthDay: 1, keepCount: 7, format: "rsc" as const, serverIds: [] }
+function fixture(patch: Partial<Dependencies> = {}) {
+ const events: AppendEventBody[] = [], created: string[] = [], cleaned: string[] = [], metadata: unknown[] = []
+ const deps: Dependencies = { settings: async () => settings, due: () => true, resolveServers: async () => ["1", "2"], create: async id => { created.push(id); return {} }, prune: async id => { cleaned.push(id); return 0 }, metadata: async value => { metadata.push(value) }, event: async value => { events.push(value) }, now: () => new Date("2026-10-01T00:00:00Z"), ...patch }
+ return { collector: createBackupSchedulerCollector(deps), events, created, cleaned, metadata }
+}
+test("settings/catalog failure releases lock and preserves truthful fatal setup snapshot", async () => {
+ for (const phase of ["settings", "catalog"]) {
+  let fail = true
+  const f = fixture(phase === "settings" ? { settings: async () => { if (fail) throw new Error("settings unavailable"); return settings } } : { resolveServers: async () => { if (fail) throw new Error("catalog unavailable"); return ["1"] } })
+  const first = await f.collector.collect(); assert.match(first.fatalError ?? "", /unavailable/); assert.equal(first.created, 0); assert.equal(f.collector.state().running, false)
+  if (phase === "settings") assert.equal(f.metadata.length, 0)
+  fail = false; const second = await f.collector.collect(); assert.equal(second.fatalError, undefined); assert.ok(second.created > 0); assert.equal(f.collector.state().running, false)
+ }
+})
+test("concurrent collection is skipped before reading settings again", async () => {
+ let calls = 0, release!: () => void; const gate = new Promise<void>(resolve => release = resolve)
+ const f = fixture({ settings: async () => { calls++; await gate; return settings } })
+ const running = f.collector.collect(); assert.equal(f.collector.state().running, true)
+ const skipped = await f.collector.collect(); assert.equal(skipped.skipped, true); assert.equal(calls, 1)
+ release(); assert.equal((await running).created, 2); assert.equal(f.collector.state().running, false)
+})
+test("start/done event failure does not cancel confirmed backups or metadata", async () => {
+ const f = fixture({ event: async () => { throw new Error("event DB unavailable") } }), result = await f.collector.collect()
+ assert.equal(result.created, 2); assert.equal(result.failures, 0); assert.equal(result.fatalError, undefined); assert.equal(result.eventErrors?.length, 2); assert.equal(f.metadata.length, 1); assert.equal(f.cleaned.length, 2); assert.equal(f.collector.state().running, false)
+})
+test("device and upload failures are distinct while every target is attempted", async () => {
+ const f = fixture({ create: async id => { if (id === "2") throw new Error("RouterOS offline"); return { uploadError: "S3 denied" } } }), result = await f.collector.collect()
+ assert.equal(result.created, 1); assert.equal(result.failures, 1); assert.equal(result.uploadFailures, 1); assert.equal(result.fatalError, undefined)
+ assert.match(result.errors?.join(";") ?? "", /S3 denied.*RouterOS offline/); assert.equal(f.events.at(-1)?.level, "warning"); assert.equal(f.events.at(-1)?.payload?.completed, 2)
+})
+test("partial pruning counts acknowledged deletions and continues the other device", async () => {
+ const f = fixture({ prune: async id => { if (id === "1") throw new BackupPruneError(2, new Error("S3 unavailable")); return 3 } }), result = await f.collector.collect()
+ assert.equal(result.created, 2); assert.equal(result.failures, 0); assert.equal(result.pruned, 5); assert.equal(result.pruneFailures, 1); assert.equal(result.fatalError, undefined); assert.equal(f.events.at(-1)?.eventType, "backups.job.done")
+})
+test("metadata failure preserves result and the in-process attempt prevents repeating the same slot", async () => {
+ const f = fixture({ metadata: async () => { throw new Error("metadata DB unavailable") }, due: (_now, _settings, previous) => previous !== "2026-10-01T00:00:00.000Z" })
+ const first = await f.collector.collect(); assert.equal(first.created, 2); assert.equal(first.failures, 0); assert.equal(first.metadataError, "metadata DB unavailable"); assert.equal(first.fatalError, undefined)
+ const second = await f.collector.collect(); assert.equal(second.due, false); assert.equal(second.created, 0); assert.equal(f.created.length, 2); assert.equal(f.collector.state().running, false)
+})
+test("disabled, not due and unsupported format keep creation side effects off", async () => {
+ for (const patch of [{ settings: async () => ({ ...settings, enabled: false }) }, { due: () => false }, { settings: async () => ({ ...settings, format: "backup" as const }), metadata: async () => { throw new Error("metadata denied") } }]) {
+  const f = fixture(patch), result = await f.collector.collect(); assert.equal(result.created, 0); assert.equal(f.created.length, 0); assert.equal(f.cleaned.length, 0); assert.equal(f.collector.state().running, false)
+ }
+})
+test("prune helper retains partial acknowledged count and retries already missing rows", async () => {
+ const calls: string[] = []
+ await assert.rejects(pruneBackupRecords(["1", "2", "3"], async id => { calls.push(id); if (id === "2") throw new Error("delete failed"); return {} }), error => error instanceof BackupPruneError && error.pruned === 1)
+ assert.deepEqual(calls, ["1", "2"]); assert.equal(await pruneBackupRecords(["1", "2", "3"], async id => id === "1" ? null : {}), 2)
+})

backend/src/services/backup-scheduler-run.ts

@@ -0,0 +1,61 @@
+import type { BackupScheduleSettingsDto } from "@mmapp/contracts/backups"
+import type { AppendEventBody } from "@mmapp/contracts/events"
+import { SCHEDULER_RUN_SNAPSHOT_VERSION, type BackupsRunSnapshot } from "../types/scheduler-run-snapshot.js"
+import { BackupPruneError } from "./backup-prune.js"
+export function createBackupSchedulerCollector(deps: {
+ settings: () => Promise<BackupScheduleSettingsDto>
+ due: (now: Date, settings: BackupScheduleSettingsDto, lastRunAt?: string | null) => boolean
+ resolveServers: (settings: BackupScheduleSettingsDto) => Promise<string[]>
+ create: (id: string, kind: "auto") => Promise<{ uploadError?: string | null }>
+ prune: (id: string, keepCount: number) => Promise<number>
+ metadata: (patch: { lastRunAt: string; lastDurationMs: number; lastError: string | null }) => Promise<void>
+ event: (event: AppendEventBody) => Promise<void>
+ now?: () => Date
+}) {
+ let collecting = false
+ let lastAttemptAt: string | undefined
+ const now = deps.now ?? (() => new Date())
+ const message = (error: unknown) => error instanceof Error ? error.message : String(error)
+ async function collect(): Promise<BackupsRunSnapshot> {
+  const sampledAt = now().toISOString(), started = now().getTime()
+  const snapshot: BackupsRunSnapshot = { v: SCHEDULER_RUN_SNAPSHOT_VERSION, job: "backups", sampledAt, due: false, created: 0, failures: 0, pruned: 0, errors: [] }
+  if (collecting) return { ...snapshot, skipped: true }
+  collecting = true
+  let attempted = false
+  const event = async (body: AppendEventBody) => { try { await deps.event(body) } catch (error) { (snapshot.eventErrors ??= []).push(body.eventType + ": " + message(error)) } }
+  const metadata = async () => {
+   try { await deps.metadata({ lastRunAt: sampledAt, lastDurationMs: Math.max(0, now().getTime() - started), lastError: snapshot.errors?.length ? snapshot.errors.join("; ") : null }) }
+   catch (error) { snapshot.metadataError = message(error); snapshot.errors?.push("Метаданные расписания: " + snapshot.metadataError) }
+  }
+  try {
+   const settings = await deps.settings()
+   if (!settings.enabled) return { ...snapshot, skipped: true }
+   const storedTime = settings.lastRunAt ? Date.parse(settings.lastRunAt) : NaN
+   const previous = lastAttemptAt && (!Number.isFinite(storedTime) || Date.parse(lastAttemptAt) >= storedTime) ? lastAttemptAt : settings.lastRunAt
+   snapshot.due = deps.due(now(), settings, previous)
+   if (!snapshot.due) return snapshot
+   attempted = true; lastAttemptAt = sampledAt
+   if (settings.format !== "rsc") { snapshot.skipped = true; snapshot.errors?.push("Формат backup не поддерживается, используйте rsc"); await metadata(); return snapshot }
+   const ids = await deps.resolveServers(settings)
+   await event({ level: "info", eventType: "backups.job.started", sourceModule: "backups", title: "Запущен плановый бэкап", message: "Серверов в очереди: " + ids.length, entityType: "backup_job", entityId: "scheduler", payload: { serverIds: ids, scheduled: true } })
+   for (const id of ids) {
+    try { const result = await deps.create(id, "auto"); snapshot.created++; if (result.uploadError) { snapshot.uploadFailures = (snapshot.uploadFailures ?? 0) + 1; snapshot.errors?.push(id + ": загрузка S3: " + result.uploadError) } }
+    catch (error) { snapshot.failures++; snapshot.errors?.push(id + ": " + message(error)) }
+   }
+   for (const id of ids) {
+    try { snapshot.pruned += await deps.prune(id, settings.keepCount) }
+    catch (error) { if (error instanceof BackupPruneError) snapshot.pruned += error.pruned; snapshot.pruneFailures = (snapshot.pruneFailures ?? 0) + 1; snapshot.errors?.push(id + ": очистка: " + message(error)) }
+   }
+   await metadata()
+   const partial = !!snapshot.errors?.length || !!snapshot.eventErrors?.length
+   await event({ level: partial ? "warning" : "info", eventType: "backups.job.done", sourceModule: "backups", title: partial ? "Плановый бэкап завершен с ограничениями" : "Плановый бэкап завершен", message: "Создано: " + snapshot.created + ", ошибок устройств: " + snapshot.failures + ", ошибок загрузки: " + (snapshot.uploadFailures ?? 0) + ", ошибок очистки: " + (snapshot.pruneFailures ?? 0), entityType: "backup_job", entityId: "scheduler", payload: { total: ids.length, completed: snapshot.created + snapshot.failures, errors: snapshot.errors, eventErrors: snapshot.eventErrors, metadataError: snapshot.metadataError, pruned: snapshot.pruned } })
+   return snapshot
+  } catch (error) {
+   snapshot.fatalError = message(error); snapshot.errors?.push(snapshot.fatalError)
+   if (attempted) await metadata()
+   await event({ level: "critical", eventType: "backups.job.failed", sourceModule: "backups", title: "Плановый бэкап прерван", message: snapshot.fatalError, entityType: "backup_job", entityId: "scheduler" })
+   return snapshot
+  } finally { collecting = false }
+ }
+ return { collect, state: () => ({ running: collecting }) }
+}

backend/src/services/backup-service.ts

@@ -8,6 +8,7 @@ import type {
   PutBackupStorageSettings,
 } from "@mmapp/contracts/backups"
 import { mapBackupEntry } from "./backup-dto.js"
+import { pruneBackupRecords } from "./backup-prune.js"
 import { importBackupOnce } from "./backup-restore.js"
 import { deleteStoredBackup } from "./backup-deletion.js"
 import { planBackupStorageUpdate } from "./backup-storage-update.js"
@@ -448,10 +449,7 @@ export async function pruneBackupsForServer(serverId: string, keepCount: number)
     .orderBy(desc(backupEntries.createdAt))
   if (rows.length <= keepCount) return 0
   const toDelete = rows.slice(keepCount)
-  for (const hit of toDelete) {
-    await deleteBackupRecord(hit.id)
-  }
-  return toDelete.length
+  return pruneBackupRecords(toDelete.map(row => row.id), deleteBackupRecord)
 }
 
 export async function readBackupContent(meta: BackupMeta): Promise<Buffer> {