fix(backups): защитить подключение S3 и согласовать операции каталога

This commit is contained in:
Denozordec committed 2026-10-14 11:05:38 +07:00
1 parent 7cef0cc133
commit faf452d149
18 files changed
+1385 -14

No files matched your search

+7 -1
View File
@@ -27,6 +27,7 @@ import {
} from "../services/backup-service.js"
import { BackupRestoreUnknownError } from "../services/backup-restore.js"
import { BackupStorageConflictError } from "../services/backup-storage-operation.js"
import { processBackupJob, recordBackupEvent } from "../services/backup-job-execution.js"
type BackupJob = BackupJobStatusResponse
@@ -69,7 +70,12 @@ export function createBackupsRoutes(deps: { listServers?: () => Promise<Array<{
if (!parsed.success) {
return reply.status(400).send({ error: "Некорректное тело запроса", details: parsed.error.flatten() })
}
return reply.send(await updateBackupStorageSettings(parsed.data))
try {
return reply.send(await updateBackupStorageSettings(parsed.data))
} catch (error) {
if (error instanceof BackupStorageConflictError) return reply.status(409).send({ error: error.message })
throw error
}
})
app.post("/backups/storage/test", async (_req, reply) => {
+34 -2
View File
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto"
import { mkdir, rm, stat, writeFile, readFile } from "node:fs/promises"
import path from "node:path"
import { and, desc, eq } from "drizzle-orm"
import { and, desc, eq, isNotNull } from "drizzle-orm"
import type {
BackupScheduleSettingsDto,
BackupStorageSettingsDto,
@@ -11,7 +11,8 @@ import { mapBackupEntry } from "./backup-dto.js"
import { pruneBackupRecords } from "./backup-prune.js"
import { importBackupOnce } from "./backup-restore.js"
import { deleteStoredBackup } from "./backup-deletion.js"
import { planBackupStorageUpdate } from "./backup-storage-update.js"
import { assertBackupStorageConnectionChange, planBackupStorageUpdate } from "./backup-storage-update.js"
import { withBackupStorageOperation, withBackupStorageSettingsUpdate } from "./backup-storage-operation.js"
import { db } from "../db/index.js"
import { parseJsonArray } from "../db/json.js"
import { backupEntries, backupScheduleSettings, backupStorageSettings } from "../db/schema.js"
@@ -67,6 +68,10 @@ export async function getBackupById(id: string): Promise<BackupMeta | null> {
}
export async function insertBackup(meta: BackupMeta): Promise<void> {
return withBackupStorageOperation(() => insertBackupRecord(meta))
}
async function insertBackupRecord(meta: BackupMeta): Promise<void> {
await db.insert(backupEntries).values({
id: meta.id,
serverId: meta.serverId,
@@ -88,6 +93,10 @@ async function getBackupRow(id: string): Promise<BackupRow | undefined> {
}
export async function deleteBackupRecord(id: string): Promise<BackupMeta | null> {
return withBackupStorageOperation(() => deleteStoredBackupRecord(id))
}
async function deleteStoredBackupRecord(id: string): Promise<BackupMeta | null> {
const row = await getBackupRow(id)
if (!row) return null
const meta = rowToMeta(row)
@@ -245,9 +254,15 @@ export async function getBackupStorageSettings(): Promise<BackupStorageSettingsD
export async function updateBackupStorageSettings(
patch: PutBackupStorageSettings,
): Promise<BackupStorageSettingsDto> {
return withBackupStorageSettingsUpdate(() => updateStorageSettings(patch))
}
async function updateStorageSettings(patch: PutBackupStorageSettings): Promise<BackupStorageSettingsDto> {
const prev = await getBackupStorageSettingsRow()
const now = new Date().toISOString()
const next = planBackupStorageUpdate(prev, patch, now)
const remote = await db.select({ id: backupEntries.id }).from(backupEntries).where(isNotNull(backupEntries.s3Key)).limit(1)
assertBackupStorageConnectionChange(prev, next, remote.length > 0)
if ((await db.select().from(backupStorageSettings).where(eq(backupStorageSettings.id, SETTINGS_ID)).limit(1))[0]) {
await db.update(backupStorageSettings).set(next).where(eq(backupStorageSettings.id, SETTINGS_ID))
} else {
@@ -375,6 +390,10 @@ export async function runBackupForServer(
kind: BackupMeta["kind"],
notes?: string,
): Promise<BackupMeta> {
return withBackupStorageOperation(() => createBackupForServer(id, kind, notes))
}
async function createBackupForServer(id: string, kind: BackupMeta["kind"], notes?: string): Promise<BackupMeta> {
const serverIdNum = Number.parseInt(id, 10)
if (!Number.isFinite(serverIdNum)) {
throw new Error("Невалидный id сервера")
@@ -453,6 +472,15 @@ export async function pruneBackupsForServer(serverId: string, keepCount: number)
}
export async function readBackupContent(meta: BackupMeta): Promise<Buffer> {
return withBackupStorageOperation(async () => {
// The route may have read metadata before a concurrent deletion changed the connection.
const current = await getBackupById(meta.id)
if (!current) throw new Error("Бэкап не найден")
return readStoredBackupContent(current)
})
}
async function readStoredBackupContent(meta: BackupMeta): Promise<Buffer> {
if ((meta.storage === "s3" || meta.storage === "both") && meta.s3Key) {
try {
const cfg = await getS3ConfigIfEnabled()
@@ -481,6 +509,10 @@ export async function restoreBackupToDevice(id: string): Promise<{ backupId: str
}
export async function syncBackupsFromS3(): Promise<{ imported: number; skipped: number }> {
return withBackupStorageOperation(syncStoredBackupsFromS3)
}
async function syncStoredBackupsFromS3(): Promise<{ imported: number; skipped: number }> {
const cfg = await getS3ConfigIfEnabled()
if (!cfg) throw new Error("S3 не настроен")
const client = createS3ClientFromConfig(cfg)
@@ -0,0 +1,21 @@
/** Serializes storage changes with catalog-producing operations in one backend process. */
let tail: Promise<unknown> = Promise.resolve()
let pending = 0
export function withBackupStorageOperation<T>(operation: () => Promise<T>): Promise<T> {
pending += 1
const result = tail.then(operation).finally(() => { pending -= 1 })
tail = result.catch(() => {})
return result
}
export async function withBackupStorageSettingsUpdate<T>(operation: () => Promise<T>): Promise<T> {
if (pending > 0) throw new BackupStorageConflictError("Хранилище занято созданием или синхронизацией резервных копий. Дождитесь завершения операции и повторите сохранение.")
return withBackupStorageOperation(operation)
}
export class BackupStorageConflictError extends Error {
constructor(message = "Смена подключения S3 недоступна: в каталоге есть резервные копии из текущего хранилища. Сохраните исходное подключение для скачивания и удаления этих файлов.") {
super(message)
this.name = "BackupStorageConflictError"
}
}
@@ -1,6 +1,7 @@
import assert from "node:assert/strict"
import { test } from "node:test"
import { planBackupStorageUpdate } from "./backup-storage-update.js"
import { assertBackupStorageConnectionChange, planBackupStorageUpdate } from "./backup-storage-update.js"
import { BackupStorageConflictError, withBackupStorageOperation, withBackupStorageSettingsUpdate } from "./backup-storage-operation.js"
const row = { provider: "s3" as const, s3Endpoint: "https://s3.example.test", s3Region: "us-east-1", s3Bucket: "bucket", s3Prefix: "prefix", s3AccessKeyId: "test-key", s3SecretAccessKey: "test-secret", s3ForcePathStyle: false, keepLocalCopy: true, lastTestAt: "2026-10-01T00:00:00Z", lastTestError: null, updatedAt: "2026-10-01T00:00:00Z" }
const now = "2026-10-01T00:01:00Z"
test("unchanged credentials and blank key preserve secret and last receipt", () => {
@@ -18,3 +19,35 @@ test("local retention does not invent a connection change or discard a failed te
const result = planBackupStorageUpdate({ ...row, lastTestError: "failed" }, { keepLocalCopy: false }, now)
assert.equal(result.keepLocalCopy, false); assert.equal(result.lastTestAt, row.lastTestAt); assert.equal(result.lastTestError, "failed")
})
test("remote catalog prevents addressing existing keys through another S3 connection", () => {
for (const patch of [{ provider: "local" as const }, { s3Endpoint: "https://other.example.test" }, { s3Region: "new" }, { s3Bucket: "other" }, { s3ForcePathStyle: true }]) {
const next = planBackupStorageUpdate(row, patch, now)
assert.throws(() => assertBackupStorageConnectionChange(row, next, true), BackupStorageConflictError)
assert.doesNotThrow(() => assertBackupStorageConnectionChange(row, next, false))
}
for (const patch of [{ s3Prefix: "new" }, { s3AccessKeyId: "rotated" }, { s3SecretAccessKey: "rotated" }, { keepLocalCopy: false }]) {
assert.doesNotThrow(() => assertBackupStorageConnectionChange(row, planBackupStorageUpdate(row, patch, now), true))
}
assert.doesNotThrow(() => assertBackupStorageConnectionChange({ ...row, provider: "local" }, row, true))
assert.throws(() => assertBackupStorageConnectionChange({ ...row, provider: "local" }, { ...row, s3Bucket: "other" }, true), BackupStorageConflictError)
})
test("storage update rejects before mutation during upload, then sees the published catalog", async () => {
let release!: () => void
const uploaded = new Promise<void>(resolve => { release = resolve })
let hasRemoteEntries = false, updateStarted = false
const create = withBackupStorageOperation(async () => { await uploaded; hasRemoteEntries = true })
const update = withBackupStorageSettingsUpdate(async () => {
updateStarted = true
assertBackupStorageConnectionChange(row, planBackupStorageUpdate(row, { s3Bucket: "other" }, now), hasRemoteEntries)
})
const rejected = assert.rejects(update, BackupStorageConflictError)
await Promise.resolve(); assert.equal(updateStarted, false)
release(); await create; await rejected
assert.equal(updateStarted, false)
await assert.rejects(withBackupStorageSettingsUpdate(async () => {
assertBackupStorageConnectionChange(row, planBackupStorageUpdate(row, { s3Bucket: "other" }, now), hasRemoteEntries)
}), BackupStorageConflictError)
assert.equal(await withBackupStorageOperation(async () => "next operation"), "next operation")
})
@@ -1,5 +1,12 @@
import type { BackupStorageSettingsDto, PutBackupStorageSettings } from "@mmapp/contracts/backups"
import { BackupStorageConflictError } from "./backup-storage-operation.js"
type StorageRecord = Omit<BackupStorageSettingsDto, "secretConfigured"> & { s3SecretAccessKey: string }
export function assertBackupStorageConnectionChange(prev: StorageRecord, next: StorageRecord, hasRemoteEntries: boolean): void {
const addressFields = ["s3Endpoint", "s3Region", "s3Bucket", "s3ForcePathStyle"] as const
// Older installations may have disabled S3 while retaining its connection and catalog.
if (hasRemoteEntries && (next.provider !== "s3" || addressFields.some(key => next[key] !== prev[key]))) throw new BackupStorageConflictError()
}
/** A test of previous credentials cannot confirm the new configuration. */
export function planBackupStorageUpdate(prev: StorageRecord, patch: PutBackupStorageSettings, updatedAt: string) {
const next = {