192 lines
13 KiB
TypeScript
192 lines
13 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { randomUUID, createHmac } from "node:crypto"
|
|
import { readFile, writeFile } from "node:fs/promises"
|
|
import path from "node:path"
|
|
import type { PoolClient } from "pg"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import { pool, closePool } from "../../db/index.js"
|
|
import { mcpConnectionCreateSchema } from "@mmapp/contracts/mcp"
|
|
import { type AuthUser } from "../../lib/permissions.js"
|
|
import { accessGeneration, mcpStateDirectory } from "./security.js"
|
|
import { createMcpConnection, authenticateMcpToken, updateMcpSettings } from "./store.js"
|
|
import { invokeMcpTool, approvePlan, applyPlan } from "./runtime.js"
|
|
import { uploadRestoreArchive, readRestoreReceipt, executeMcpDatabaseRestore } from "./database-restore.js"
|
|
import { exportSystemDatabaseBackup, restoreSystemDatabaseBackup } from "../../services/system-database-backup.js"
|
|
import { initializeAutomaticSchedulerRoot } from "../../services/scheduler-automatic-root.js"
|
|
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
|
|
import { stopSchedulerForDatabaseOperation } from "../../services/scheduler.js"
|
|
import { stopTrafficFlowListenerForDatabaseOperation } from "../../services/traffic-flow-ingest.js"
|
|
import { acceptPortalRevocation, assertPortalClaimsCurrent } from "./portal-revocation.js"
|
|
|
|
// This destructive test is only invoked by the private temporary-cluster runner.
|
|
const url = new URL(process.env.DATABASE_URL ?? "postgresql://invalid")
|
|
assert.equal(process.env.MMAPP_OWN_TERMINAL_RESTORE, "true")
|
|
assert.equal(url.hostname, "127.0.0.1")
|
|
assert.equal(url.pathname.slice(1), process.env.MMAPP_OWN_SQL_DB)
|
|
assert.match(mcpStateDirectory(), /mm-mcp-test-/)
|
|
// Standalone owned runner has no buildApp bootstrap; initialize before actor/request scopes.
|
|
initializeAutomaticSchedulerRoot()
|
|
const actor: AuthUser = { id: "restore-owner", email: "restore@example.invalid", name: "Restore acceptance", apps: ["mm"], permissions: ["mm:settings:admin"], expiresAt: Date.now() + 3600000 }
|
|
const verifyData = process.env.MMAPP_OWN_TERMINAL_DATA_EQUIVALENCE === "true"
|
|
let dataExpectation: Awaited<ReturnType<typeof import("./database-restore-data-equivalence.sql-helper.js")["prepareOwnedRestoreDataExpectation"]>> | undefined
|
|
const originalConnect = pool.connect
|
|
let migrationCaptures = 0
|
|
async function waitForDatabaseRetirement() {
|
|
let timer: ReturnType<typeof setTimeout> | undefined
|
|
try {
|
|
await Promise.race([sharedDatabaseGate.whenOpen(), new Promise<never>((_, reject) => {
|
|
timer = setTimeout(() => reject(Error("Terminal restore did not retire its database admission")), 10000)
|
|
})])
|
|
} finally { clearTimeout(timer) }
|
|
}
|
|
|
|
try {
|
|
// Every owned restore profile archives inert domain collectors, not only the
|
|
// data-equivalence profile. Otherwise restart can launch unrelated downloads
|
|
// or router polling and race the final scheduler drain. Lifecycle SQL remains
|
|
// real; production scheduler configuration and drain checks are unchanged.
|
|
for (const table of ["traffic_settings", "uptime_settings", "traffic_flow_settings", "servers_api_ping_settings", "certificate_renew_settings", "backup_schedule_settings", "geoip_settings", "internet_path_settings", "router_log_settings"]) await pool.query(`UPDATE ${table} SET enabled=false WHERE id=1`)
|
|
await pool.query("UPDATE uptime_settings SET resources_enabled=false,ping_enabled=false,speed_enabled=false WHERE id=1")
|
|
await pool.query("DELETE FROM mcp_protections")
|
|
await updateMcpSettings({ enabled: true }, actor)
|
|
await pool.query("CREATE TABLE mcp_restore_acceptance_marker (value text NOT NULL)")
|
|
await pool.query("INSERT INTO mcp_restore_acceptance_marker VALUES('before')")
|
|
const dump = await exportSystemDatabaseBackup()
|
|
assert.equal(dump.buffer.subarray(0, 5).toString(), "PGDMP")
|
|
if (verifyData) dataExpectation = await (await import("./database-restore-data-equivalence.sql-helper.js")).prepareOwnedRestoreDataExpectation(dump.buffer)
|
|
const revokedOwner = { id: "restore-revocation-after-backup", issuedAt: Date.now() - 60000 }
|
|
const portalSecret = "restore-acceptance-only-portal-secret-32", stamp = String(Math.floor(Date.now() / 1000))
|
|
const portalEvent = Buffer.from(JSON.stringify({ eventId: randomUUID(), userId: revokedOwner.id, type: "permissions.changed" }))
|
|
await acceptPortalRevocation(portalEvent, stamp, "sha256=" + createHmac("sha256", portalSecret).update(stamp + ".").update(portalEvent).digest("hex"), portalSecret)
|
|
const source = await uploadRestoreArchive(dump.buffer, actor)
|
|
const encrypted = await readFile(path.join(mcpStateDirectory(), "restore-vault", source.archiveRef + ".archive.bin"))
|
|
assert.ok(!encrypted.equals(dump.buffer))
|
|
assert.ok(!encrypted.subarray(0, 5).equals(Buffer.from("PGDMP")))
|
|
await assert.rejects(uploadRestoreArchive(dump.buffer, { ...actor, permissions: ["mm:mcp:admin"] }))
|
|
await assert.rejects(uploadRestoreArchive(Buffer.from("invalid"), actor))
|
|
const serverIds = (await pool.query("SELECT id FROM servers")).rows.map(row => Number(row.id))
|
|
// Combined owned profiles retain their evidence/disabled synthetic servers.
|
|
// The human restore fixture grants this exact catalog instead of inheriting
|
|
// the ordinary five-server default; production policy guards stay unchanged.
|
|
assert(serverIds.length <= 1000)
|
|
const issue = (owner = actor) => createMcpConnection(mcpConnectionCreateSchema.parse({ name: "Database restore acceptance", mode: "approval", expiresAt: new Date(Date.now() + 1800000).toISOString(), permissions: ["mm:settings:admin"], scope: { allowGlobal: true, serverIds, operationIds: ["database.restore"] }, limits: { serversPerPlan: Math.max(1, serverIds.length) } }), owner)
|
|
const issued = await issue(), principal = await authenticateMcpToken(issued.token)
|
|
const other = await issue({ ...actor, id: "another-owner" }), otherPrincipal = await authenticateMcpToken(other.token)
|
|
await assert.rejects(invokeMcpTool(otherPrincipal, "mm_database_restore_plan", { archiveRef: source.archiveRef, expectedSha256: source.sha256 }))
|
|
await assert.rejects(invokeMcpTool(principal, "mm_database_restore_plan", { archiveRef: source.archiveRef, expectedSha256: "0".repeat(64) }))
|
|
const prepared = await invokeMcpTool(principal, "mm_database_restore_plan", { archiveRef: source.archiveRef, expectedSha256: source.sha256 }) as { planId: string; state: string }
|
|
assert.equal(prepared.state, "awaiting-approval")
|
|
await assert.rejects(applyPlan(principal, prepared.planId, "restore-acceptance"))
|
|
await approvePlan(prepared.planId, actor, true)
|
|
|
|
// Legacy direct restore must respect protected objects in every domain.
|
|
const protection = randomUUID()
|
|
await pool.query("INSERT INTO mcp_protections(id,object_ref,reason,created_by) VALUES($1,$2::jsonb,'fixture',$3)", [protection, JSON.stringify({ kind: "user", id: "protected-user", protection: "deny" }), actor.id])
|
|
await assert.rejects(restoreSystemDatabaseBackup(dump.buffer))
|
|
await pool.query("DELETE FROM mcp_protections WHERE id=$1", [protection])
|
|
await pool.query("UPDATE mcp_restore_acceptance_marker SET value='after'")
|
|
const generation = await accessGeneration()
|
|
|
|
// Emulates the admitted HTTP apply request: runner must wait, never deadlock it.
|
|
const restoreStartedAt = Date.now()
|
|
if (dataExpectation) {
|
|
// Owned-test observation only: capture after migrations release their
|
|
// advisory lease, before the production restore resumes consumers.
|
|
pool.connect = function (...args: unknown[]) {
|
|
if (args.length) return Reflect.apply(originalConnect, pool, args)
|
|
return (Reflect.apply(originalConnect, pool, []) as Promise<PoolClient>).then(client => {
|
|
let migrationClient = false
|
|
return new Proxy(client, {
|
|
get(target, property) {
|
|
if (property === "query") return (...queryArgs: unknown[]) => {
|
|
const sql = queryArgs[0]
|
|
if (typeof sql === "string" && sql.includes("CREATE TABLE IF NOT EXISTS schema_migrations")) migrationClient = true
|
|
const result = Reflect.apply(target.query, target, queryArgs)
|
|
if (migrationClient && sql === "RESET statement_timeout" && queryArgs.length === 1) {
|
|
return Promise.resolve(result).then(async value => {
|
|
assert.equal(++migrationCaptures, 1)
|
|
await dataExpectation!.compareBeforeConsumers()
|
|
return value
|
|
})
|
|
}
|
|
return result
|
|
}
|
|
const value = Reflect.get(target, property, target)
|
|
return typeof value === "function" ? value.bind(target) : value
|
|
},
|
|
})
|
|
})
|
|
} as typeof pool.connect
|
|
}
|
|
const operation = await sharedDatabaseGate.request(() => applyPlan(principal, prepared.planId, "restore-acceptance"))
|
|
let receipt: Record<string, unknown> = {}
|
|
for (let i = 0; i < 600; i++) {
|
|
receipt = await readRestoreReceipt(operation.id, actor).catch(() => ({}))
|
|
if (["applied", "unknown", "blocked"].includes(String(receipt.state))) break
|
|
await delay(100)
|
|
}
|
|
assert.equal(receipt.state, "applied", JSON.stringify(receipt))
|
|
assert.equal(receipt.accessRevoked, true)
|
|
pool.connect = originalConnect
|
|
if (dataExpectation) {
|
|
await stopSchedulerForDatabaseOperation()
|
|
await stopTrafficFlowListenerForDatabaseOperation()
|
|
await (await import("../../services/database-background-work.js")).databaseBackgroundWork.drain()
|
|
const dataInput = { operationId: operation.id, receipt, actorId: actor.id, startedAt: restoreStartedAt }
|
|
await assert.rejects(dataExpectation.compareAfterRestore({ ...dataInput, actorId: "foreign-owner" }))
|
|
await assert.rejects(dataExpectation.compareAfterRestore({ ...dataInput, operationId: randomUUID() }))
|
|
await assert.rejects(dataExpectation.compareAfterRestore({ ...dataInput, receipt: { ...receipt, accessRevoked: false } }))
|
|
console.log("Owned terminal restore data equivalence:", JSON.stringify(await dataExpectation.compareAfterRestore(dataInput)))
|
|
assert.equal(migrationCaptures, 1)
|
|
assert.equal((await dataExpectation.compareAfterRestore(dataInput)).auditRecordsObserved, 1)
|
|
}
|
|
assert.equal((await pool.query("SELECT value FROM mcp_restore_acceptance_marker")).rows[0].value, "before")
|
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE id=$1", [operation.id])).rows[0].n, 0)
|
|
assert.notEqual(await accessGeneration(), generation)
|
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_owner_revocations WHERE owner_id=$1", [revokedOwner.id])).rows[0].n, 0)
|
|
await assert.rejects(assertPortalClaimsCurrent(revokedOwner))
|
|
await assert.rejects(authenticateMcpToken(issued.token))
|
|
await assert.rejects(readRestoreReceipt(operation.id, { ...actor, permissions: ["mm:mcp:read"] }))
|
|
// The applied receipt is durable before the terminal callback releases its
|
|
// last SQL lease/exclusive admission. Await that existing owner, rather than
|
|
// racing its cleanup with the stale-callback/no-replay assertion below.
|
|
await waitForDatabaseRetirement()
|
|
// A stale callback or process replay cannot invoke pg_restore a second time.
|
|
await pool.query("UPDATE mcp_restore_acceptance_marker SET value='after-receipt'")
|
|
await executeMcpDatabaseRestore(operation.id, principal)
|
|
assert.equal((await pool.query("SELECT value FROM mcp_restore_acceptance_marker")).rows[0].value, "after-receipt")
|
|
assert.equal((await readRestoreReceipt(operation.id, actor)).state, "applied")
|
|
if (dataExpectation) {
|
|
console.log("Owned restore data negative cases: foreign owner, wrong operation and unconfirmed receipt refused; repeat validation writes no audit; full equality observed before consumers resume")
|
|
}
|
|
console.log("✓ Terminal restore: real pg_dump/pg_restore transaction, encrypted human upload, approval, global protection, request drain, external receipt, revoked tokens, preserved external owner revocation and no replay")
|
|
|
|
// Archive authentication is checked before a terminal attempt can replace data.
|
|
await updateMcpSettings({ enabled: true }, actor)
|
|
const next = await issue(), nextPrincipal = await authenticateMcpToken(next.token), tampered = await uploadRestoreArchive(dump.buffer, actor)
|
|
const badPlan = await invokeMcpTool(nextPrincipal, "mm_database_restore_plan", { archiveRef: tampered.archiveRef, expectedSha256: tampered.sha256 }) as { planId: string }
|
|
await approvePlan(badPlan.planId, actor, true)
|
|
await writeFile(path.join(mcpStateDirectory(), "restore-vault", tampered.archiveRef + ".archive.bin"), Buffer.from("corrupted"))
|
|
const generationBeforeCorruption = await accessGeneration()
|
|
const bad = await applyPlan(nextPrincipal, badPlan.planId, "corrupted-archive")
|
|
// Admission, durable receipt writes and authority checks are asynchronous.
|
|
// Use the same terminal-state budget as the successful restore above; never
|
|
// interpret a still-running receipt as a rejection or dispatch apply again.
|
|
for (let i = 0; i < 600; i++) {
|
|
receipt = await readRestoreReceipt(bad.id, actor).catch(() => ({}))
|
|
if (["blocked", "applied", "unknown"].includes(String(receipt.state))) break
|
|
await delay(100)
|
|
}
|
|
assert.equal(receipt.state, "blocked", JSON.stringify(receipt))
|
|
await waitForDatabaseRetirement()
|
|
assert.equal(await accessGeneration(), generationBeforeCorruption)
|
|
assert.equal((await pool.query("SELECT value FROM mcp_restore_acceptance_marker")).rows[0].value, "after-receipt")
|
|
console.log("✓ Tampered archive is blocked before data replacement")
|
|
} finally {
|
|
pool.connect = originalConnect
|
|
await stopSchedulerForDatabaseOperation()
|
|
await stopTrafficFlowListenerForDatabaseOperation()
|
|
await dataExpectation?.close()
|
|
await closePool()
|
|
}
|