fix(backups): защитить подключение S3 и согласовать операции каталога
This commit is contained in:
1 parent
7cef0cc133
commit
faf452d149
18 files changed
+1385
-14
No files matched your search
@@ -27,6 +27,7 @@ import {
|
||||
} from "../services/backup-service.js"
|
||||
|
||||
import { BackupRestoreUnknownError } from "../services/backup-restore.js"
|
||||
import { BackupStorageConflictError } from "../services/backup-storage-operation.js"
|
||||
import { processBackupJob, recordBackupEvent } from "../services/backup-job-execution.js"
|
||||
|
||||
type BackupJob = BackupJobStatusResponse
|
||||
@@ -69,7 +70,12 @@ export function createBackupsRoutes(deps: { listServers?: () => Promise<Array<{
|
||||
if (!parsed.success) {
|
||||
return reply.status(400).send({ error: "Некорректное тело запроса", details: parsed.error.flatten() })
|
||||
}
|
||||
return reply.send(await updateBackupStorageSettings(parsed.data))
|
||||
try {
|
||||
return reply.send(await updateBackupStorageSettings(parsed.data))
|
||||
} catch (error) {
|
||||
if (error instanceof BackupStorageConflictError) return reply.status(409).send({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
})
|
||||
|
||||
app.post("/backups/storage/test", async (_req, reply) => {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { mkdir, rm, stat, writeFile, readFile } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { and, desc, eq } from "drizzle-orm"
|
||||
import { and, desc, eq, isNotNull } from "drizzle-orm"
|
||||
import type {
|
||||
BackupScheduleSettingsDto,
|
||||
BackupStorageSettingsDto,
|
||||
@@ -11,7 +11,8 @@ import { mapBackupEntry } from "./backup-dto.js"
|
||||
import { pruneBackupRecords } from "./backup-prune.js"
|
||||
import { importBackupOnce } from "./backup-restore.js"
|
||||
import { deleteStoredBackup } from "./backup-deletion.js"
|
||||
import { planBackupStorageUpdate } from "./backup-storage-update.js"
|
||||
import { assertBackupStorageConnectionChange, planBackupStorageUpdate } from "./backup-storage-update.js"
|
||||
import { withBackupStorageOperation, withBackupStorageSettingsUpdate } from "./backup-storage-operation.js"
|
||||
import { db } from "../db/index.js"
|
||||
import { parseJsonArray } from "../db/json.js"
|
||||
import { backupEntries, backupScheduleSettings, backupStorageSettings } from "../db/schema.js"
|
||||
@@ -67,6 +68,10 @@ export async function getBackupById(id: string): Promise<BackupMeta | null> {
|
||||
}
|
||||
|
||||
export async function insertBackup(meta: BackupMeta): Promise<void> {
|
||||
return withBackupStorageOperation(() => insertBackupRecord(meta))
|
||||
}
|
||||
|
||||
async function insertBackupRecord(meta: BackupMeta): Promise<void> {
|
||||
await db.insert(backupEntries).values({
|
||||
id: meta.id,
|
||||
serverId: meta.serverId,
|
||||
@@ -88,6 +93,10 @@ async function getBackupRow(id: string): Promise<BackupRow | undefined> {
|
||||
}
|
||||
|
||||
export async function deleteBackupRecord(id: string): Promise<BackupMeta | null> {
|
||||
return withBackupStorageOperation(() => deleteStoredBackupRecord(id))
|
||||
}
|
||||
|
||||
async function deleteStoredBackupRecord(id: string): Promise<BackupMeta | null> {
|
||||
const row = await getBackupRow(id)
|
||||
if (!row) return null
|
||||
const meta = rowToMeta(row)
|
||||
@@ -245,9 +254,15 @@ export async function getBackupStorageSettings(): Promise<BackupStorageSettingsD
|
||||
export async function updateBackupStorageSettings(
|
||||
patch: PutBackupStorageSettings,
|
||||
): Promise<BackupStorageSettingsDto> {
|
||||
return withBackupStorageSettingsUpdate(() => updateStorageSettings(patch))
|
||||
}
|
||||
|
||||
async function updateStorageSettings(patch: PutBackupStorageSettings): Promise<BackupStorageSettingsDto> {
|
||||
const prev = await getBackupStorageSettingsRow()
|
||||
const now = new Date().toISOString()
|
||||
const next = planBackupStorageUpdate(prev, patch, now)
|
||||
const remote = await db.select({ id: backupEntries.id }).from(backupEntries).where(isNotNull(backupEntries.s3Key)).limit(1)
|
||||
assertBackupStorageConnectionChange(prev, next, remote.length > 0)
|
||||
if ((await db.select().from(backupStorageSettings).where(eq(backupStorageSettings.id, SETTINGS_ID)).limit(1))[0]) {
|
||||
await db.update(backupStorageSettings).set(next).where(eq(backupStorageSettings.id, SETTINGS_ID))
|
||||
} else {
|
||||
@@ -375,6 +390,10 @@ export async function runBackupForServer(
|
||||
kind: BackupMeta["kind"],
|
||||
notes?: string,
|
||||
): Promise<BackupMeta> {
|
||||
return withBackupStorageOperation(() => createBackupForServer(id, kind, notes))
|
||||
}
|
||||
|
||||
async function createBackupForServer(id: string, kind: BackupMeta["kind"], notes?: string): Promise<BackupMeta> {
|
||||
const serverIdNum = Number.parseInt(id, 10)
|
||||
if (!Number.isFinite(serverIdNum)) {
|
||||
throw new Error("Невалидный id сервера")
|
||||
@@ -453,6 +472,15 @@ export async function pruneBackupsForServer(serverId: string, keepCount: number)
|
||||
}
|
||||
|
||||
export async function readBackupContent(meta: BackupMeta): Promise<Buffer> {
|
||||
return withBackupStorageOperation(async () => {
|
||||
// The route may have read metadata before a concurrent deletion changed the connection.
|
||||
const current = await getBackupById(meta.id)
|
||||
if (!current) throw new Error("Бэкап не найден")
|
||||
return readStoredBackupContent(current)
|
||||
})
|
||||
}
|
||||
|
||||
async function readStoredBackupContent(meta: BackupMeta): Promise<Buffer> {
|
||||
if ((meta.storage === "s3" || meta.storage === "both") && meta.s3Key) {
|
||||
try {
|
||||
const cfg = await getS3ConfigIfEnabled()
|
||||
@@ -481,6 +509,10 @@ export async function restoreBackupToDevice(id: string): Promise<{ backupId: str
|
||||
}
|
||||
|
||||
export async function syncBackupsFromS3(): Promise<{ imported: number; skipped: number }> {
|
||||
return withBackupStorageOperation(syncStoredBackupsFromS3)
|
||||
}
|
||||
|
||||
async function syncStoredBackupsFromS3(): Promise<{ imported: number; skipped: number }> {
|
||||
const cfg = await getS3ConfigIfEnabled()
|
||||
if (!cfg) throw new Error("S3 не настроен")
|
||||
const client = createS3ClientFromConfig(cfg)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
/** Serializes storage changes with catalog-producing operations in one backend process. */
|
||||
let tail: Promise<unknown> = Promise.resolve()
|
||||
let pending = 0
|
||||
export function withBackupStorageOperation<T>(operation: () => Promise<T>): Promise<T> {
|
||||
pending += 1
|
||||
const result = tail.then(operation).finally(() => { pending -= 1 })
|
||||
tail = result.catch(() => {})
|
||||
return result
|
||||
}
|
||||
|
||||
export async function withBackupStorageSettingsUpdate<T>(operation: () => Promise<T>): Promise<T> {
|
||||
if (pending > 0) throw new BackupStorageConflictError("Хранилище занято созданием или синхронизацией резервных копий. Дождитесь завершения операции и повторите сохранение.")
|
||||
return withBackupStorageOperation(operation)
|
||||
}
|
||||
|
||||
export class BackupStorageConflictError extends Error {
|
||||
constructor(message = "Смена подключения S3 недоступна: в каталоге есть резервные копии из текущего хранилища. Сохраните исходное подключение для скачивания и удаления этих файлов.") {
|
||||
super(message)
|
||||
this.name = "BackupStorageConflictError"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { planBackupStorageUpdate } from "./backup-storage-update.js"
|
||||
import { assertBackupStorageConnectionChange, planBackupStorageUpdate } from "./backup-storage-update.js"
|
||||
import { BackupStorageConflictError, withBackupStorageOperation, withBackupStorageSettingsUpdate } from "./backup-storage-operation.js"
|
||||
const row = { provider: "s3" as const, s3Endpoint: "https://s3.example.test", s3Region: "us-east-1", s3Bucket: "bucket", s3Prefix: "prefix", s3AccessKeyId: "test-key", s3SecretAccessKey: "test-secret", s3ForcePathStyle: false, keepLocalCopy: true, lastTestAt: "2026-10-01T00:00:00Z", lastTestError: null, updatedAt: "2026-10-01T00:00:00Z" }
|
||||
const now = "2026-10-01T00:01:00Z"
|
||||
test("unchanged credentials and blank key preserve secret and last receipt", () => {
|
||||
@@ -18,3 +19,35 @@ test("local retention does not invent a connection change or discard a failed te
|
||||
const result = planBackupStorageUpdate({ ...row, lastTestError: "failed" }, { keepLocalCopy: false }, now)
|
||||
assert.equal(result.keepLocalCopy, false); assert.equal(result.lastTestAt, row.lastTestAt); assert.equal(result.lastTestError, "failed")
|
||||
})
|
||||
|
||||
test("remote catalog prevents addressing existing keys through another S3 connection", () => {
|
||||
for (const patch of [{ provider: "local" as const }, { s3Endpoint: "https://other.example.test" }, { s3Region: "new" }, { s3Bucket: "other" }, { s3ForcePathStyle: true }]) {
|
||||
const next = planBackupStorageUpdate(row, patch, now)
|
||||
assert.throws(() => assertBackupStorageConnectionChange(row, next, true), BackupStorageConflictError)
|
||||
assert.doesNotThrow(() => assertBackupStorageConnectionChange(row, next, false))
|
||||
}
|
||||
for (const patch of [{ s3Prefix: "new" }, { s3AccessKeyId: "rotated" }, { s3SecretAccessKey: "rotated" }, { keepLocalCopy: false }]) {
|
||||
assert.doesNotThrow(() => assertBackupStorageConnectionChange(row, planBackupStorageUpdate(row, patch, now), true))
|
||||
}
|
||||
assert.doesNotThrow(() => assertBackupStorageConnectionChange({ ...row, provider: "local" }, row, true))
|
||||
assert.throws(() => assertBackupStorageConnectionChange({ ...row, provider: "local" }, { ...row, s3Bucket: "other" }, true), BackupStorageConflictError)
|
||||
})
|
||||
|
||||
test("storage update rejects before mutation during upload, then sees the published catalog", async () => {
|
||||
let release!: () => void
|
||||
const uploaded = new Promise<void>(resolve => { release = resolve })
|
||||
let hasRemoteEntries = false, updateStarted = false
|
||||
const create = withBackupStorageOperation(async () => { await uploaded; hasRemoteEntries = true })
|
||||
const update = withBackupStorageSettingsUpdate(async () => {
|
||||
updateStarted = true
|
||||
assertBackupStorageConnectionChange(row, planBackupStorageUpdate(row, { s3Bucket: "other" }, now), hasRemoteEntries)
|
||||
})
|
||||
const rejected = assert.rejects(update, BackupStorageConflictError)
|
||||
await Promise.resolve(); assert.equal(updateStarted, false)
|
||||
release(); await create; await rejected
|
||||
assert.equal(updateStarted, false)
|
||||
await assert.rejects(withBackupStorageSettingsUpdate(async () => {
|
||||
assertBackupStorageConnectionChange(row, planBackupStorageUpdate(row, { s3Bucket: "other" }, now), hasRemoteEntries)
|
||||
}), BackupStorageConflictError)
|
||||
assert.equal(await withBackupStorageOperation(async () => "next operation"), "next operation")
|
||||
})
|
||||
@@ -1,5 +1,12 @@
|
||||
import type { BackupStorageSettingsDto, PutBackupStorageSettings } from "@mmapp/contracts/backups"
|
||||
import { BackupStorageConflictError } from "./backup-storage-operation.js"
|
||||
type StorageRecord = Omit<BackupStorageSettingsDto, "secretConfigured"> & { s3SecretAccessKey: string }
|
||||
|
||||
export function assertBackupStorageConnectionChange(prev: StorageRecord, next: StorageRecord, hasRemoteEntries: boolean): void {
|
||||
const addressFields = ["s3Endpoint", "s3Region", "s3Bucket", "s3ForcePathStyle"] as const
|
||||
// Older installations may have disabled S3 while retaining its connection and catalog.
|
||||
if (hasRemoteEntries && (next.provider !== "s3" || addressFields.some(key => next[key] !== prev[key]))) throw new BackupStorageConflictError()
|
||||
}
|
||||
/** A test of previous credentials cannot confirm the new configuration. */
|
||||
export function planBackupStorageUpdate(prev: StorageRecord, patch: PutBackupStorageSettings, updatedAt: string) {
|
||||
const next = {
|
||||
|
||||
Reference in new issue
Block a user