Files
MikrotikManager/backend/src/modules/mcp/routing-community-scoped-acceptance.ts
T

1419 lines
122 KiB
TypeScript

import { createRoutingReservationStore } from "./routing-reservation-store.js"
import { observeCommittedScopedCheckpoint } from "../../test/committed-scoped-checkpoint-observer.js"
import { prepareRoutingControllerBaseline, advanceRoutingControllerBaseline } from "./routing-controller-baseline.js"
import { loadScopedConfigurationPacket, loadScopedNetworkCheckpoint } from "./routing-community-scoped-checkpoint.js"
import { assertScopedCommunityRequest } from "./routing-community-scoped-request.js"
import { originalEffectReportSchema, originalEffectSelectionSchema } from "./routing-original-effects.js"
import { tickRoutingProfiles } from "./routing-profiles.js"
import { saveControllerProfileCas } from "./routing-controller-fence.js"
import { withApplicationMutationLock, withConfigMutationLock } from "../../services/config-mutation-lock.js"
import { performance } from "node:perf_hooks"
import assert from "node:assert/strict"
import { createServer, type RequestListener } from "node:http"
import { randomUUID } from "node:crypto"
import type { PoolClient } from "pg"
import { setTimeout as delay } from "node:timers/promises"
import { mcpConnectionCreateSchema } from "@mmapp/contracts/mcp"
import { routeAiProfileSchema } from "@mmapp/contracts/mcp-routing"
import { pool } from "../../db/index.js"
import type { AuthUser } from "../../lib/permissions.js"
import { managedComment } from "../../managed-markers.js"
import { compileManagedFilterRules } from "../../services/filter-rules-write-model.js"
import { attestRoutingFixture } from "../../test/mcp-routing-fixture.js"
import { createMcpConnection, authenticateMcpToken, setConnectionStatus, effectiveMcpConnection } from "./store.js"
import { invokeMcpTool, approvePlan, applyPlan, publicPlan, loadPlan } from "./runtime.js"
import { accessGeneration, decryptSecret, encryptSecret, fingerprint, McpError } from "./security.js"
import type { McpPrincipal } from "./store.js"
import { loadScopedRecoverySource } from "./routing-community-scoped-recovery-source.js"
import { findMcpService } from "./service-registry.js"
import { prepareScopedCommunitySwitch } from "./routing-community-scoped-service.js"
import { resolveRoutingNetworkConfigurationContext, consumeRoutingNetworkConfigurationContext, closeRoutingNetworkConfigurationContext } from "./routing-network-configuration-bridge.js"
import { createRoutingNetworkNativeSession } from "./routing-network-native-session.js"
import { requireRecentPath } from "./path-verification.js"
import { registerManagementChannel } from "./management.js"
import { isSchedulerJobsPaused, pauseSchedulerJobsAndWait, resumeSchedulerJobs } from "../../services/scheduler-running.js"
/** Disposable SQL + loopback HTTP only. Does not access or configure CHR. */
export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: { originalEffectsOnly?: boolean; controllerFailoverOnly?: boolean; originalCredentialRevocationOnly?: boolean; forwardRoundsOnly?: boolean; manualInverseChargeOnly?: boolean; manualInverseChargeCase?: "positive" | "quota" | "concurrent" | "unknown" | "revoked" | "nonpersistent" | "mixed-forward"; lateInverseAuthorityChange?: "edit" | "disable" | "grant" } = {}) {
const sid = options.lateInverseAuthorityChange ? { edit: 131, disable: 132, grant: 133 }[options.lateInverseAuthorityChange] : 131
const writer: AuthUser = { ...actor, permissions: [...new Set([...actor.permissions, "mm:filters:write"])] }
const initial = compileManagedFilterRules([{ community: "65000:10", gateway: "192.0.2.1", outIface: "", action: "route", description: "" }, { community: "65000:20", gateway: "192.0.2.5", outIface: "", action: "route", description: "" }])
const filters = ["a", "b"].map(key => ({ ".id": "*f" + key, chain: "custom-" + key, rule: initial, disabled: "false", comment: managedComment("NEVER_SCOPED_COMMENT_" + key), opaque: "NEVER_SCOPED_OPAQUE" }))
let mode: "normal" | "ignored" | "lost" | "collateral" | "post-failed" | "post-unknown" = "normal", writes = 0, requests = 0
let controllerProbeUnknown = false
let controllerFailoverProbeSources: string[] | undefined
let controllerFailoverReserveHealthy = true
let pendingPostCheckpoint = false, postCheckpointsObserved = 0
let afterForwardWrite: (() => Promise<void>) | undefined
let afterConfigurationCheckpoint: ((operationId: string) => Promise<void>) | undefined
let forwardRoundBulk = 0, afterRoundLogRead: (() => Promise<void>) | undefined
let primaryDown = false, primaryControlRequests = 0, pendingInversePost = false, inversePostChecks = 0
const observedVrfs = new Set<string>()
const paddedEffectResources = new Set(["/routing/route", "/routing/bgp/session", "/routing/bfd/session", "/log"])
// Persistent fixtures need an original RIB selection. These are declared
// synthetic RouterOS responses, never actual CHR facts.
let originalEffectCase: "transition" | "no-op" | "ambiguous" = "transition"
let exposeEffectCatalogue = false
const originalEffectWire: Array<{ method: string; path: string; afterPatch: boolean }> = []
const originalEffectChanged = () => originalEffectCase === "no-op" || filters[0].rule !== initial
const effectRoutes = () => {
const changed = originalEffectChanged()
const row = { ".id": "*effect1", afi: "ip4", "routing-table": "tenant-a", "dst-address": "203.0.113.0/24",
bgp: "true", connect: "false", static: "false", active: "true", contribution: "active", ecmp: "false",
disabled: "false", filtered: "false", unreachable: "false", blackhole: "false", "bgp.communities": "65000:10",
gateway: changed ? "192.0.2.9" : "192.0.2.1", "immediate-gw": (changed ? "203.0.113.9" : "203.0.113.1") + "%wan-a" }
return originalEffectCase === "ambiguous" ? [row, { ...row, ".id": "*effect2" }] : [row]
}
const effectConnection = { ".id": "*ca", name: "peer-a", instance: "default", vrf: "main", "routing-table": "tenant-a", afi: "ip",
"input.filter": "custom-a", "output.filter-chain": "", templates: "", disabled: "false", "use-bfd": "false",
"local.address": "198.18.0.1", "remote.address": "198.18.0.2", "remote.as": "65001" }
const { vrf: effectTransportVrf, ...effectSessionFields } = effectConnection
const effectSession = { ...effectSessionFields, ".id": "*sa", "local.afi": "ip", "remote.afi": "ip", established: "true",
"local.address": `${effectConnection["local.address"]}@${effectTransportVrf}`, "remote.address": `${effectConnection["remote.address"]}@${effectTransportVrf}` }
const collections = (path: string): Record<string, unknown>[] => ({
"/system/resource": [{ version: "7.20.1" }],
"/routing/table": ["main", "tenant-a", "tenant-b", "probe-a", "probe-b"].map((name, index) => ({ ".id": "*t" + index, name, fib: "true", disabled: "false", invalid: "false" })),
"/ip/vrf": ["a", "b"].map(key => ({ ".id": "*v" + key, name: "probe-" + key, interfaces: "wan-" + key, disabled: "false" })).concat([{ ".id": "*vm", name: "main", interfaces: "all", disabled: "false" }]),
"/routing/bgp/connection": [effectConnection, { ...effectConnection, ".id": "*cb", name: "peer-b", "routing-table": "tenant-b", "input.filter": "custom-b" }],
"/routing/bgp/instance": [{ ".id": "*i", name: "default", vrf: "main", "routing-table": "main", disabled: "false" }],
"/routing/bgp/template": [], "/routing/filter/rule": filters,
"/ip/route": (controllerFailoverProbeSources ? [1, 9, 10, 11] : [1, 9]).map(value => ({ ".id": "*r" + value, "dst-address": "192.0.2." + value + "/32", "routing-table": "main", disabled: "false",
"immediate-gw": "203.0.113." + value + "%wan-a", comment: managedComment("recursive"), opaque: "NEVER_SCOPED_ROUTE" })),
"/interface": ["wan-a", "wan-b", "hop", "target"].map((name, index) => ({ ".id": "*if" + index, name, disabled: "false", running: "true" })),
"/ip/address": [
...["wan-a", "wan-b", "hop", "target"].map((name, index) => ({ ".id": "*ad" + index, interface: name, address: "198.51.100." + (index + 1) + "/32", disabled: "false", invalid: "false" })),
...(controllerFailoverProbeSources ? [11, 12, 13].map(index => ({ ".id": "*ad" + index, interface: "wan-a", address: "198.51.100." + index + "/32", disabled: "false", invalid: "false" })) : []),
],
...(options.originalEffectsOnly ? {
"/routing/table": ["main", "tenant-a", "tenant-b", "probe-a", "probe-b"].map((name, index) => ({ ".id": "*t" + index, name, fib: "", disabled: "false", invalid: "false" })),
"/routing/bgp/connection": [effectConnection],
"/log": [{ ".id": "*l1", topics: "bgp,route,info", message: "Declared synthetic localhost effect observation; interval coverage unknown" }],
} : {}),
...(exposeEffectCatalogue || options.originalEffectsOnly ? {
"/routing/route": effectRoutes(),
"/routing/bgp/session": [effectSession],
} : {}),
} as Record<string, Record<string, unknown>[]>)[path] ?? []
let requestFailure: { error: unknown } | undefined
const handler: RequestListener = async (req, res) => {
requests++; res.setHeader("content-type", "application/json")
if (options.originalEffectsOnly) {
assert.equal(req.socket.localAddress, "127.0.0.1")
assert.equal(req.headers.authorization, "Basic " + Buffer.from("admin:NEVER_SCOPED_PASSWORD").toString("base64"))
}
let raw = ""; for await (const chunk of req) raw += chunk
const args = raw ? JSON.parse(raw) : {}, path = decodeURIComponent((req.url ?? "").replace(/^\/rest/, ""))
if (options.originalEffectsOnly || options.forwardRoundsOnly) originalEffectWire.push({ method: req.method ?? "", path, afterPatch: filters[0].rule !== initial })
if (req.socket.localAddress === "127.0.0.1") {
if (/^\/(routing|tool)\//.test(path) || path === "/system/license") primaryControlRequests++
if (primaryDown) { req.socket.destroy(); return }
}
if (req.method === "GET" && path === "/system/license") { res.end(JSON.stringify([{ "software-id": "NEVER_SCOPED_LICENSE" }])); return }
if (req.method === "GET") {
const payload = collections(path)
if (path === "/log" && afterRoundLogRead) { const hook = afterRoundLogRead; afterRoundLogRead = undefined; await hook() }
// Force the cumulative 10,000-row budget without exceeding the native
// cap of 1,000 rows on any individual response.
const padding = forwardRoundBulk && paddedEffectResources.has(path)
? Array.from({ length: 1000 - payload.length }, (_, index) => ({ ".id": "*padding" + index })) : []
res.end(JSON.stringify(padding.length ? [...payload, ...padding] : payload)); return
}
if (req.method === "POST" && path.endsWith("/get")) { const row = collections(path.slice(0, -4)).find(row => row[".id"] === args[".id"]), value = row?.[args["value-name"]]; res.end(JSON.stringify(value === undefined ? [] : { ret: value })); return }
if (req.method === "POST" && path === "/tool/ping") {
if (pendingInversePost) {
assert.ok(connectionId)
const intents = (await pool.query("SELECT i.source_operation_id FROM mcp_scoped_recovery_intents i JOIN mcp_operations o ON o.id=i.recovery_operation_id WHERE i.state='intent' AND o.connection_id=$1", [connectionId])).rows
assert.equal(intents.length, 1, "Post-inverse probe must follow one durable single-use inverse intent")
pendingInversePost = false; inversePostChecks++
}
if (pendingPostCheckpoint) {
assert.ok(connectionId)
const checkpoint = (await pool.query("SELECT c.configuration_sealed,c.configuration_proof FROM mcp_operation_checkpoints c JOIN mcp_operations o ON o.id=c.operation_id WHERE o.state='running' AND c.configuration_sealed IS NOT NULL AND o.connection_id=$1", [connectionId])).rows
assert.equal(checkpoint.length, 1, "Exact CONFIG_APPLIED must commit before the post-change network probe")
const packet = JSON.parse(await decryptSecret(checkpoint[0].configuration_sealed))
assert.equal(packet.phase, "CONFIG_APPLIED"); assert.equal(packet.originalRule, initial)
assert.equal(packet.originalCandidateId, "a"); assert.equal(packet.desiredCandidateId, controllerFailoverProbeSources ? "d" : "b")
assert.equal(packet.desiredRule, filters.find(row => row[".id"] === packet.nativeId)!.rule)
assert.ok(checkpoint[0].configuration_proof); pendingPostCheckpoint = false; postCheckpointsObserved++
const sharedCheck = afterConfigurationCheckpoint; afterConfigurationCheckpoint = undefined
await sharedCheck?.(packet.operationId)
}
observedVrfs.add(args.vrf)
if (controllerFailoverProbeSources) controllerFailoverProbeSources.push(args["src-address"])
const probeFailed = controllerFailoverProbeSources && (!controllerFailoverReserveHealthy || args["src-address"] !== "198.51.100.13") || mode === "post-failed" && writes
res.end(JSON.stringify(controllerProbeUnknown || mode === "post-unknown" && writes ? [{ message: "unsupported fixture output" }] : [{ sent: "3", received: probeFailed ? "0" : "3", "packet-loss": probeFailed ? "100" : "0", "avg-rtt": "1ms" }]))
return
}
if (req.method === "POST" && path === "/tool/traceroute") { observedVrfs.add(args.vrf); res.end(JSON.stringify([3, 4].map(index => ({ address: "198.51.100." + index, sent: "1", loss: "0" })))); return }
if (req.method === "PATCH") {
assert.deepEqual(Object.keys(args), ["rule"]); const target = filters.find(row => path === "/routing/filter/rule/" + row[".id"]); assert.ok(target)
writes++; if (mode !== "ignored") target.rule = args.rule
pendingPostCheckpoint = args.rule !== initial; pendingInversePost = args.rule === initial
if (mode === "collateral") filters.find(row => row !== target)!.opaque = "changed collateral"
if (mode === "lost") { req.socket.destroy(); return }
if (args.rule !== initial && afterForwardWrite) { const change = afterForwardWrite; afterForwardWrite = undefined; await change() }
res.end("{}"); return
}
assert.fail("Unsupported fixture request " + req.method + " " + path)
}
// EventEmitter does not await an async RequestListener. Preserve the first
// original hook/assertion error for the awaited fixture rather than crash.
const listener: RequestListener = (req,res) => {
void Promise.resolve().then(()=>handler(req,res)).catch(error=>{
requestFailure??={error}; req.socket.destroy()
})
}
const router = createServer(listener), alternate = createServer(listener)
await new Promise<void>(resolve => router.listen(0, "127.0.0.1", resolve))
await new Promise<void>(resolve => alternate.listen(0, "127.0.0.2", resolve))
let connectionId: string | undefined
let insertedServer = false
const persistentProfileIds: string[] = [], persistentCases: string[] = []
const schedulerWasPaused = isSchedulerJobsPaused()
let schedulerDrained = false
let workerFixtureLease:PoolClient|undefined, workerLocked=false, workerUnsafe=false
const originalQuery = pool.query.bind(pool)
let afterIntent: (() => Promise<void>) | undefined
pool.query = (async (...args: unknown[]) => {
const result = await (originalQuery as (...args: unknown[]) => Promise<unknown>)(...args)
const parameters = args[1] as unknown[] | undefined
if (typeof args[0] === "string" && args[0].includes("INSERT INTO mcp_audit") && parameters?.[1] === "configuration.intent" && afterIntent) { const hook = afterIntent; afterIntent = undefined; await hook() }
return result
}) as typeof pool.query
try {
// Full integration may already have periodic GRE/BGP collection running.
// Drain it before registering this router: its reads must not be attributed
// to the inverse transport or race exact zero-request assertions.
await pauseSchedulerJobsAndWait(); schedulerDrained = true
// Manual non-main authority rows are intentionally unsupported controller
// profiles. Drain a real tick by owning its existing session key, then keep
// periodic workers out until all fixture rows are disabled/cleaned up.
workerFixtureLease=await pool.connect()
const workerDrainDeadline=performance.now()+60000
do {
try { workerLocked=(await workerFixtureLease.query("SELECT pg_try_advisory_lock(hashtextextended('mcp-workers',0)) AS locked")).rows[0].locked===true }
catch(error){workerUnsafe=true;throw error}
if(workerLocked)break
assert(performance.now()<workerDrainDeadline,"Active MCP worker did not drain before manual fixture setup")
await delay(25)
} while(!workerLocked)
await pool.query("INSERT INTO servers(id,name,host,port,use_ssl,password) VALUES($1,'scoped-route','127.0.0.1',$2,false,'NEVER_SCOPED_PASSWORD')", [sid, (router.address() as { port: number }).port])
insertedServer = true
const access = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: "Scoped manual", mode: "approval", expiresAt: new Date(Date.now() + 1800000).toISOString(), permissions: ["mm:network:read", "mm:network:write", "mm:filters:write"], scope: { serverIds: [sid], operationIds: ["routing.analyze", "routing.path.verify", "routing.community.switch", "routing.profiles.save", "operations.reconcile", "operations.recover"], allowGlobal: false }, limits: { callsPerMinute: 1000, mutationsPerHour: 300 } }), writer)
connectionId = access.connection.id; const principal = await authenticateMcpToken(access.token)
const profile = (key: string) => routeAiProfileSchema.parse({ name: "independent " + key, maxLossPct: 5, serverId: sid, family: "ipv4", community: "65000:10", table: "tenant-" + key, probeVrf: "probe-" + key, bgpContext: { connectionId: "*c" + key, instance: "default", vrf: "main", inputChain: "custom-" + key }, candidates: [1, 9].map(value => ({ id: value === 1 ? "a" : "b", jumpHostId: sid, exitNodeId: sid, wanInterface: "wan-" + key, downstreamInterface: "hop", gateway: "192.0.2." + value, gatewayRef: { resource: "ipv4_route", id: "*r" + value }, controlProbe: { targetInterface: "target", sourceAddress: key === "a" ? "198.51.100.1" : "198.51.100.2", expectedHops: [{ serverId: sid, interface: "hop" }, { serverId: sid, interface: "target" }] } })) })
for (const key of ["a", "b"]) for (const iface of ["wan-" + key, "hop"]) {
const id = randomUUID(); await pool.query("INSERT INTO mcp_routing_observations(id,src_server_id,dst_server_id,src_interface,dst_interface,family,routing_table,ping_ms,loss_pct,dl_mbps,ul_mbps,ping_at,bandwidth_at,context_verified,source) VALUES($1,$2,$2,$3,'peer','ipv4',$4,1,0,100,100,now(),now(),true,'fixture')", [id, sid, iface, "tenant-" + key]); await attestRoutingFixture(id)
}
await pool.query("INSERT INTO filter_rules(server_id,sort_order,community,community_name,action,gateway,gateway_tunnel_id,description) VALUES($1,77,'65000:10','NEVER_SCOPED_SQL_NAME','route','203.0.113.99','NEVER_SCOPED_SQL_TUNNEL','NEVER_SCOPED_SQL_DESCRIPTION')", [sid])
const dbBefore = (await pool.query("SELECT * FROM filter_rules WHERE server_id=$1 ORDER BY id", [sid])).rows
const prepare = async (key = "a", persistent?: { id: string; version: number }, exactProfile?: ReturnType<typeof profile>, assertApprovalRequired = false, approver: AuthUser | null = writer) => {
exposeEffectCatalogue = !!persistent
const current = exactProfile === undefined ? profile(key) : routeAiProfileSchema.parse(exactProfile)
assert.equal((await invokeMcpTool(principal, "mm_route_path_verify", { profile: current, candidateId: "b" }) as { status: string }).status, "passed")
const plan = await invokeMcpTool(principal, "mm_route_optimizer_plan", { profile: current, candidateId: "b" }, undefined,
persistent ? { persistentRoutingProfile: persistent } : undefined) as { planId: string }
const projection = await publicPlan(await loadPlan(plan.planId)); assert.ok(!JSON.stringify(projection).includes("NEVER_"))
const row = (await pool.query("SELECT evidence FROM mcp_plans WHERE id=$1", [plan.planId])).rows[0]
const evidence = JSON.parse(await decryptSecret(row.evidence.sealed)); const input = evidence.steps[0].input
assert.equal(input.proofRevision, persistent ? 3 : 2)
if (persistent) {
assert.equal(input.persistentProfileAuthority.packet.profileId, persistent.id)
assert.equal(input.persistentProfileAuthority.packet.profileVersion, persistent.version)
assert.equal(input.persistentProfileAuthority.packet.credentialId, principal.credentialId)
assert(!JSON.stringify(projection).includes("persistentProfileAuthority"), "Persistent authority remains private")
}
assert.equal(evidence.requiresHuman, true); assert.equal(evidence.mutationCost, 2); assert.ok(!JSON.stringify(input).includes("NEVER_")); assert.equal(input.compiledBody, undefined)
if (assertApprovalRequired) {
const beforeApproval = { writes, requests }
await assert.rejects(applyPlan(principal, plan.planId, randomUUID()), (error: unknown) => error instanceof McpError && error.code === "APPROVAL_REQUIRED")
assert.deepEqual({ writes, requests }, beforeApproval)
}
if (approver) await approvePlan(plan.planId, approver, true)
return { plan, input, evidence }
}
const waitForOperation = async (operationId: string) => {
let state = ""
const deadline = Date.now() + 140000
do { state = (await pool.query("SELECT state FROM mcp_operations WHERE id=$1", [operationId])).rows[0].state; if (!["pending", "running"].includes(state)) break; await delay(20) } while (Date.now() < deadline)
if(requestFailure)throw requestFailure.error
assert.ok(!["pending", "running"].includes(state), "The bounded executor must reach a terminal state before assertions or fixture cleanup")
return state
}
const execute = async (planId: string, executingPrincipal: McpPrincipal = principal) => {
const op = await applyPlan(executingPrincipal, planId, randomUUID())
return { ...op, state: await waitForOperation(op.id) }
}
const reconcile = async (operationId: string) => (await invokeMcpTool(principal, "mm_operation_reconcile", { operationId }) as { results: Array<{ matchesDesiredState: boolean; configurationMatchesDesired: boolean; configurationCheckpointAvailable: boolean }> }).results[0]
const reset = () => { filters.forEach(row => { row.rule = initial; row.opaque = "NEVER_SCOPED_OPAQUE" }); mode = "normal"; pendingPostCheckpoint = false; pendingInversePost = false; primaryDown = false }
const refresh = async () => {
await pool.query("UPDATE mcp_routing_observations SET ping_at=now(),bandwidth_at=now(),loss_pct=0 WHERE src_server_id=$1", [sid])
for (const row of (await pool.query("SELECT id FROM mcp_routing_observations WHERE src_server_id=$1", [sid])).rows) await attestRoutingFixture(row.id)
}
// Seed authority for fault cases. The public save/approval path is
// exercised separately below without inserting a profile directly.
const persistentFixture = async (config = profile("a")) => {
const id = randomUUID(), current = await effectiveMcpConnection(principal)
const expiry = new Date(Math.min(Date.now() + 600000, current.owner_valid_until.getTime(), Date.parse(principal.credentialExpiresAt), Date.parse(current.policy.expiresAt)))
await withApplicationMutationLock(() => withConfigMutationLock([sid], async () => {
// Unsupported autoRecovery is a deliberate legacy negative fixture: it
// receives NO baseline and can never acquire native write authority.
const baseline=config.autoRecovery?undefined:await prepareRoutingControllerBaseline(id,1,config,expiry.toISOString(),{principal,connection:current})
const saved=await saveControllerProfileCas(pool,{profileId:id,connectionId,expectedVersion:null,profile:config,enabled:true,expiresAt:expiry.toISOString(),credentialId:principal.credentialId,generation:await accessGeneration(),policyVersion:current.policy.version,baseline})
assert.equal(saved?.version,1)
}))
persistentProfileIds.push(id)
return { id, version: 1 }
}
const disablePersistent = (id: string) => pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [id])
const executeWithCheckpointObservation = async (planId: string, check: (operationId: string) => Promise<void>) => {
const observer = observeCommittedScopedCheckpoint(pool, check, {
databaseUrl: process.env.DATABASE_URL ?? "", stateDir: process.env.MCP_STATE_DIR ?? "",
})
try {
const operation = await execute(planId)
observer.assertCompleted()
return operation
} finally { observer.close() }
}
const acceptControllerFailover = async () => {
reset(); await refresh(); exposeEffectCatalogue = true; originalEffectCase = "transition"
controllerFailoverProbeSources = []
const profileId = randomUUID(), initialWrites = writes
persistentProfileIds.push(profileId)
try {
const configured = profile("a"), original = configured.candidates[0]
configured.candidates = [1, 10, 11, 9].map((gateway, index) => ({ ...original,
id: ["a", "b", "c", "d"][index], gateway: "192.0.2." + gateway,
gatewayRef: { resource: "ipv4_route", id: "*r" + gateway },
controlProbe: { ...original.controlProbe!, sourceAddress: "198.51.100." + [1, 11, 12, 13][index] },
}))
const analysis = await invokeMcpTool(principal, "mm_route_optimizer_analyze", { profile: configured }) as { currentCandidateId: string; ranking: Array<{ eligible: boolean; candidate: { id: string } }> }
assert.equal(analysis.currentCandidateId, "a")
assert.deepEqual(analysis.ranking.filter(row => row.eligible).map(row => row.candidate.id), ["a", "b", "c", "d"])
const enrollment = await invokeMcpTool(principal, "mm_route_profile_save_plan", {
profileId, profile: configured, enabled: true, expiresAt: new Date(Date.now() + 600000).toISOString(),
}) as { planId: string }
await approvePlan(enrollment.planId, writer, true)
assert.equal((await execute(enrollment.planId)).state, "applied")
const readController = async () => (await pool.query("SELECT enabled,state FROM mcp_routing_profiles WHERE id=$1", [profileId])).rows[0]
const failures = new Set<string>()
let switchPlanId: string | undefined
for (const [index, alternate] of [12, 13, 13, 12, 13].entries()) {
// Drive the normal scheduled entry point without waiting a minute.
await pool.query("UPDATE mcp_routing_profiles SET state=state-'probedAt' WHERE id=$1", [profileId])
controllerFailoverProbeSources.length = 0
// The previously passing reserve must be rechecked, and its prior
// pass cannot authorize failover if the current check fails.
controllerFailoverReserveHealthy = index !== 2
await tickRoutingProfiles()
const row = await readController()
assert.equal(row.enabled, true, String(row.state.pausedReason))
assert.deepEqual(controllerFailoverProbeSources, [1, 11, alternate].map(last => "198.51.100." + last))
failures.add(row.state.failureKey)
assert.equal(failures.size, index + 1, "Each counted failure must come from a distinct path check")
assert.equal(writes, initialWrites)
if (index < 4) {
assert.equal(row.state.failureCount, index + 1)
assert.equal(row.state.planId, undefined, "Earlier healthy reserve evidence cannot replace a fresh passing probe")
} else {
assert.equal(row.state.candidateId, "d")
switchPlanId = row.state.planId
}
}
assert.ok(switchPlanId)
assert.equal((await loadPlan(switchPlanId)).state, "awaiting-approval")
await tickRoutingProfiles()
assert.equal(writes, initialWrites)
await approvePlan(switchPlanId, writer, true)
await tickRoutingProfiles()
const operationId: string = (await readController()).state.operationId
assert.ok(operationId)
assert.equal(await waitForOperation(operationId), "applied")
await tickRoutingProfiles()
const completed = await readController()
assert.equal(completed.enabled, true, String(completed.state.pausedReason))
assert.equal(completed.state.baseline.packet.candidateId, "d")
assert.equal(completed.state.baseline.packet.transition.operationId, operationId)
assert.equal(completed.state.failoverReserveCursor, undefined)
assert.equal(writes, initialWrites + 1)
const result = { accepted: true, stableRanking: ["a", "b", "c", "d"], selectedReserve: "d", probeCallsPerTick: 3,
distinctFailuresBeforePlan: failures.size, priorPassedProbeNotReused: true, humanApproval: true, nativeWrites: 1 }
console.log("Bounded controller reserve SQL/HTTP:", JSON.stringify(result))
return result
} finally { await disablePersistent(profileId); controllerFailoverProbeSources = undefined; controllerFailoverReserveHealthy = true }
}
if (options.controllerFailoverOnly) return { ...await acceptControllerFailover(), persistentAuthorityCases: 0 }
if (options.originalEffectsOnly) {
const cases: Array<Record<string, unknown>> = []
let publicManualRevision2: Record<string, unknown> | undefined
const delegatedAdmin: AuthUser = { ...writer, id: "scoped-delegated-approval-admin", permissions: [...new Set([...writer.permissions, "mm:mcp:admin", "mm:mcp:approve"])] }
const snapshotApprovalAndAdmission = async (planId: string) => (await pool.query(`SELECT
(SELECT count(*)::int FROM mcp_operations WHERE plan_id=$1) AS operations,
(SELECT count(*)::int FROM mcp_routing_forward_charges WHERE source_plan_id=$1) AS forward_charges,
(SELECT COALESCE(jsonb_object_agg(bucket,used),'{}'::jsonb) FROM mcp_budget_buckets WHERE connection_id=$2 AND kind='mutations') AS mutation_buckets,
(SELECT count(*)::int FROM mcp_audit WHERE connection_id=$2 AND action='plan.approve' AND payload->>'planId'=$1::text) AS approval_audit`, [planId, connectionId])).rows[0]
const assertOwnerApproval = async (planId: string) => {
const nativeBefore = { requests, writes }
const before = await snapshotApprovalAndAdmission(planId)
assert.equal(before.operations, 0); assert.equal(before.forward_charges, 0); assert.equal(before.approval_audit, 0)
await assert.rejects(approvePlan(planId, delegatedAdmin, true), (error: unknown) => error instanceof McpError && error.code === "PERMISSION_DENIED")
const denied = await loadPlan(planId)
assert.equal(denied.state, "awaiting-approval"); assert.equal(denied.approved_by, null)
assert.deepEqual(await snapshotApprovalAndAdmission(planId), before)
// Simulate a delegated approval stored by an older deployment.
await pool.query("UPDATE mcp_plans SET state='approved',approved_by=$2,approved_at=now() WHERE id=$1", [planId, delegatedAdmin.id])
try {
await assert.rejects(applyPlan(principal, planId, randomUUID()), (error: unknown) => error instanceof McpError && error.code === "APPROVAL_REQUIRED")
const stale = await loadPlan(planId)
assert.equal(stale.state, "approved"); assert.equal(stale.approved_by, delegatedAdmin.id)
assert.deepEqual(await snapshotApprovalAndAdmission(planId), before)
assert.deepEqual({ requests, writes }, nativeBefore)
} finally {
await pool.query("UPDATE mcp_plans SET state='awaiting-approval',approved_by=NULL,approved_at=NULL WHERE id=$1", [planId])
}
await approvePlan(planId, writer, true)
}
for (const scenario of ["transition", "no-op", "ambiguous", "baseline-drift", "public-manual-rev2"] as const) {
const manual = scenario === "public-manual-rev2"
const effectCase = manual || scenario === "baseline-drift" ? "transition" : scenario
reset(); await refresh(); originalEffectCase = effectCase
const config = profile("a")
assert.equal(config.autoRecovery, false)
// The baseline and persistent authority are issued by the SAME actual
// fixed readers/CAS used by existing tests, not hand-written proofs.
const persistent = manual ? undefined : await persistentFixture(config)
try {
const { plan, input, evidence } = await prepare("a", persistent, config, manual, manual ? delegatedAdmin : scenario === "transition" ? null : writer)
if (scenario === "transition") await assertOwnerApproval(plan.planId)
assert.equal(input.manualEffectPolicy, manual ? "observational-v1" : undefined)
if (manual) assert.equal(input.persistentProfileAuthority, undefined)
const sealedSelection = originalEffectSelectionSchema.parse(input.originalEffectSelection)
assert.equal(sealedSelection.status, effectCase === "ambiguous" ? "unknown" : "prepared")
const storedPlan = await loadPlan(plan.planId)
assert.equal(storedPlan.approved_by, manual ? delegatedAdmin.id : writer.id)
if (manual) assert.equal(evidence.validUntil, undefined, "Manual scoped plans keep their existing preparation lifetime")
if (scenario === "transition") {
type ProbeRow = { created_at: Date; evidence: { observedAt: string } }
const probe: ProbeRow = (await pool.query<ProbeRow>(
"SELECT created_at,evidence FROM mcp_path_checks WHERE id=$1 AND connection_id=$2", [input.pathCheckId, connectionId])).rows[0]
const probeDeadline = Math.min(probe.created_at.getTime(), Date.parse(probe.evidence.observedAt)) + 90_000
const authorityDeadline = Date.parse(input.persistentProfileAuthority.packet.validUntil)
assert(probeDeadline < authorityDeadline, "The fixture must exercise the probe cap, not the authority cap")
assert.equal(evidence.validUntil, new Date(probeDeadline).toISOString())
assert.equal(storedPlan.expires_at.getTime(), probeDeadline)
// A newer successful observation cannot extend an already approved plan.
const newerProbe = await invokeMcpTool(principal, "mm_route_path_verify", { profile: config, candidateId: "b" }) as { id: string; status: string; observedAt: string }
assert.equal(newerProbe.status, "passed")
assert.notEqual(newerProbe.id, input.pathCheckId)
assert(Date.parse(newerProbe.observedAt) + 90_000 > probeDeadline + 1)
const nativeBefore = { requests, writes }, admissionBefore = await snapshotApprovalAndAdmission(plan.planId)
const refuseAfterProbeDeadline = async (code: string) => {
const realNow = Date.now
try {
Date.now = () => probeDeadline + 1
await assert.rejects(applyPlan(principal, plan.planId, randomUUID()), (error: unknown) => error instanceof McpError && error.code === code)
} finally { Date.now = realNow }
assert.deepEqual(await snapshotApprovalAndAdmission(plan.planId), admissionBefore)
assert.deepEqual({ requests, writes }, nativeBefore)
assert.equal((await loadPlan(plan.planId)).state, "approved")
}
await refuseAfterProbeDeadline("PLAN_EXPIRED")
// The old deployment stored only the authority deadline in evidence.
const legacyValidity = input.persistentProfileAuthority.packet.validUntil as string
const legacyEvidence = { sealed: await encryptSecret(JSON.stringify({ ...evidence, validUntil: legacyValidity })) }
await pool.query("UPDATE mcp_plans SET expires_at=$2,evidence=$3::jsonb WHERE id=$1", [plan.planId, legacyValidity, JSON.stringify(legacyEvidence)])
try {
await refuseAfterProbeDeadline("MEASUREMENT_UNKNOWN")
} finally {
await pool.query("UPDATE mcp_plans SET expires_at=$2,evidence=$3::jsonb WHERE id=$1", [plan.planId, storedPlan.expires_at, JSON.stringify(storedPlan.evidence)])
}
cases.push({ case: "expired-approved-probe", newerProbeCannotRenew: true, scopedPlanCode: "PLAN_EXPIRED", legacyPlanCode: "MEASUREMENT_UNKNOWN", nativeRequests: 0, nativeWrites: 0, admissions: 0 })
}
const publicProjection = JSON.stringify(await publicPlan(storedPlan))
assert(!publicProjection.includes("originalEffectSelection"))
assert(!publicProjection.includes("manualEffectPolicy"))
const beforeWrites = writes, wireStart = originalEffectWire.length
const configurationIntents = async () => (await pool.query(`SELECT count(*)::int AS n FROM mcp_audit
WHERE connection_id=$1 AND action='configuration.intent'`, [connectionId])).rows[0].n
const intentsBefore = await configurationIntents()
if (scenario === "baseline-drift") originalEffectCase = "no-op"
const operation = await execute(plan.planId)
if (scenario === "ambiguous" || scenario === "baseline-drift") {
assert.equal(operation.state, "unknown")
const { rows: [refusal] } = await pool.query("SELECT error_code FROM mcp_operations WHERE id=$1", [operation.id])
assert.equal(refusal.error_code, "MEASUREMENT_UNKNOWN")
assert.equal(writes, beforeWrites)
assert.equal(filters[0].rule, initial)
assert.equal(await configurationIntents(), intentsBefore)
const receiptCount: number = (await pool.query<{ n: number }>(`SELECT count(*)::int AS n FROM mcp_audit
WHERE connection_id=$1 AND action='routing.scoped.attempt' AND payload->>'receiptId'=$2`, [connectionId, input.receiptId])).rows[0].n
assert.equal(receiptCount, 0)
const runningWire = originalEffectWire.slice(wireStart)
assert.equal(runningWire.filter(row => row.method === "GET" && row.path === "/routing/route").length, scenario === "baseline-drift" ? 1 : 0)
cases.push({ case: scenario, state: operation.state, errorCode: refusal.error_code, nativeWrites: 0, attemptReceipts: 0 })
continue
}
assert.equal(operation.state, "applied")
assert.equal(writes - beforeWrites, 1)
const rows: Array<{ created_at: Date; state: string; operation_id: string; operation_version: number; step_state: string;
result: { configurationVerified: boolean; controlProbeVerified: boolean; originalEffects: unknown } }> = (await pool.query(`SELECT o.created_at,o.state,p.operation_id,p.operation_version,s.state AS step_state,s.result
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id JOIN mcp_operation_steps s ON s.operation_id=o.id
WHERE o.id=$1 AND o.connection_id=$2 AND o.plan_id=$3 AND s.step_index=0`, [operation.id, connectionId, plan.planId])).rows
assert.equal(rows.length, 1); assert.equal(rows[0].operation_id, "routing.community.switch")
assert.equal(rows[0].state, "applied"); assert.equal(rows[0].step_state, "verified")
assert.equal(rows[0].result.configurationVerified, true); assert.equal(rows[0].result.controlProbeVerified, true)
const report = originalEffectReportSchema.parse(rows[0].result.originalEffects)
const requestSeal = await assertScopedCommunityRequest(input)
if (manual) {
const changedInput = { ...input, manualEffectPolicy: "unapproved-policy" }
await assert.rejects(assertScopedCommunityRequest(changedInput), (error: unknown) => error instanceof McpError && error.code === "STATE_CONFLICT")
}
const configuration = await loadScopedConfigurationPacket(input, requestSeal, { operationId: operation.id, stepIndex: 0 })
assert(configuration); assert.equal(configuration.packet.operationId, operation.id)
assert.equal(configuration.packet.revision, manual ? 1 : 2); assert.equal(configuration.packet.phase, "CONFIG_APPLIED")
const network = await loadScopedNetworkCheckpoint(input, requestSeal)
assert(network); assert.equal(network.status, "passed"); assert.deepEqual(network.originalEffects, report)
const runningWire = originalEffectWire.slice(wireStart)
const ribBefore = runningWire.filter(row => row.method === "GET" && row.path === "/routing/route" && !row.afterPatch).length
const ribAfter = runningWire.filter(row => row.method === "GET" && row.path === "/routing/route" && row.afterPatch).length
console.log("Owned original effect diagnostic:", JSON.stringify({ case: scenario, state: operation.state,
status: report.status, reason: report.reason, rounds: report.rounds.length, budget: report.budget ?? null,
ribBefore, ribAfter, runtimeNativeRequests: runningWire.length, lastNativePaths: runningWire.slice(-8).map(row => row.path) }))
assert.equal(ribBefore, 1)
assert.equal(ribAfter, 2)
assert.equal(report.status, effectCase === "transition" ? "observed-expected" : "unknown")
assert.equal(report.switchEffectObserved, effectCase === "transition")
assert.equal(report.logCoverage, "unknown"); assert.equal(report.routeOriginVerified, false)
assert.equal(report.forwardingVerified, false); assert.equal(report.semanticProof, false); assert.equal(report.autoRecoveryAllowed, false)
if (sealedSelection.status === "prepared") {
assert.equal(report.selectionIdentity, fingerprint(sealedSelection)); assert.equal(report.rounds.length, 2)
assert(report.budget && report.budget.calls > 18 && report.budget.calls <= 256)
assert(report.budget.bytes > 0 && report.budget.bytes <= 8 * 1048576)
assert(report.budget.rows > 0 && report.budget.rows <= 10000)
assert(report.budget.deadline <= rows[0].created_at.getTime() + 105000, "Original120s horizon is narrowed by15s publication reserve")
assert(report.rounds.every(round => Date.parse(round.observedAt) < report.budget!.deadline && round.controlProbe === "passed" && round.bgp === "observed-expected" && round.bfd === "not-configured"))
assert.deepEqual(report.transition?.changedPrefixes, effectCase === "transition" ? ["203.0.113.0/24"] : [])
assert.equal(report.transitionAttribution, "unproven")
}
if (manual) {
const repeatReads = requests
const repeated = await applyPlan(principal, plan.planId, randomUUID())
assert.equal(repeated.id, operation.id)
assert.equal(repeated.state, "applied")
assert.equal(requests, repeatReads)
assert.equal(writes - beforeWrites, 1)
assert.equal(Number((await pool.query("SELECT count(*) AS n FROM mcp_routing_profiles WHERE enabled")).rows[0].n), 0)
assert.equal(Number((await pool.query("SELECT count(*) AS n FROM mcp_scoped_recovery_intents WHERE source_operation_id=$1", [operation.id])).rows[0].n), 0)
publicManualRevision2 = { accepted: true, humanApprovalRequired: true, proofRevision: input.proofRevision,
configurationCheckpointRevision: configuration.packet.revision, effectStatus: report.status,
rounds: report.rounds.length, nativeRibBefore: ribBefore, nativeRibAfter: ribAfter, sharedBudget: report.budget,
autoRecoveryAllowed: report.autoRecoveryAllowed, scopedControllerEnabled: false, nativeWrites: 1 }
continue
}
assert(persistent)
const row = (await pool.query("SELECT state,grant_expires_at FROM mcp_routing_profiles WHERE id=$1 AND version=1", [persistent.id])).rows[0]
assert(row?.state.baseline)
const advance = () => advanceRoutingControllerBaseline(row.state.baseline, persistent.id, 1, config, row.grant_expires_at.toISOString(),
{ principal, connection: principalConnection }, plan.planId, operation.id)
const principalConnection = await effectiveMcpConnection(principal), beforeAdvanceReads = requests
if (effectCase === "transition") {
const advanced = await advance(); assert.equal(advanced.packet.candidateId, "b")
assert.equal(advanced.packet.transition?.operationId, operation.id); assert.equal(advanced.packet.transition?.checkpointProof, configuration.checkpointProof)
} else {
await assert.rejects(advance, (error: unknown) => error instanceof McpError && error.code === "STATE_CONFLICT")
assert.equal(requests, beforeAdvanceReads, "Unknown/no-transition effect gate refuses before another native context read")
}
// Check actual HMAC loading, not only a caller-visible result object.
if (effectCase === "transition") {
const audit: Array<{ id: string; payload: Record<string, unknown> }> = (await pool.query("SELECT id,payload FROM mcp_audit WHERE connection_id=$1 AND action='routing.scoped.network-check' AND payload->>'receiptId'=$2", [connectionId, input.receiptId])).rows
assert.equal(audit.length, 1)
try {
await pool.query("UPDATE mcp_audit SET payload=jsonb_set(payload,'{originalEffects,switchEffectObserved}','false'::jsonb) WHERE id=$1", [audit[0].id])
assert.equal(await loadScopedNetworkCheckpoint(input, requestSeal), null, "Modified effect vector cannot reuse its original network HMAC")
} finally { await pool.query("UPDATE mcp_audit SET payload=$2::jsonb WHERE id=$1", [audit[0].id, JSON.stringify(audit[0].payload)]) }
assert.deepEqual((await loadScopedNetworkCheckpoint(input, requestSeal))?.originalEffects, report)
}
const repeatReads = requests
const repeated = await applyPlan(principal, plan.planId, randomUUID())
assert.equal(repeated.id, operation.id, "Completed plan returns its original operation without replay")
assert.equal(repeated.state, "applied")
assert.equal(Number((await pool.query("SELECT count(*) AS n FROM mcp_operations WHERE plan_id=$1", [plan.planId])).rows[0].n), 1)
assert.equal(writes - beforeWrites, 1); assert.equal(requests, repeatReads, "Completed approved plan cannot repeat native work")
// Link the applied operation as the controller does when it issues
// a plan. This checks both outcomes at the real worker boundary.
await pool.query("UPDATE mcp_routing_profiles SET state=state || $2::jsonb WHERE id=$1", [persistent.id, JSON.stringify({ planId: plan.planId, candidateId: "b" })])
await tickRoutingProfiles()
const completed = (await pool.query("SELECT enabled,state FROM mcp_routing_profiles WHERE id=$1", [persistent.id])).rows[0]
if (effectCase === "transition") {
assert.equal(completed.enabled, true)
assert.equal(completed.state.baseline.packet.transition.operationId, operation.id)
assert.equal(completed.state.expectedCandidateId, "b")
} else {
assert.equal(completed.enabled, false)
assert.equal(completed.state.pausedReason, "STATE_CONFLICT")
assert.deepEqual(completed.state.baseline, row.state.baseline)
}
cases.push({ case: effectCase, manualState: operation.state, configurationCheckpoint: true, actualPrivateExecutionOrigin: true,
effectStatus: report.status, observedFibTransition: report.switchEffectObserved, nativeRibBefore: ribBefore, nativeRibAfter: ribAfter,
sharedBudget: report.budget ?? null, effectBaselineAdvanceAccepted: effectCase === "transition", effectGateNativeReads: effectCase === "transition" ? "fresh-native-checked" : 0,
tickEffectPauseAccepted: effectCase !== "transition", publicManualRevision2EffectOwnerAccepted: false })
} finally { if (persistent) await disablePersistent(persistent.id) }
}
assert(publicManualRevision2)
reset(); await refresh(); originalEffectCase = "transition"; exposeEffectCatalogue = true
const controllerConfig = profile("a")
controllerConfig.confirmationSamples = 1
controllerConfig.weights = { ping: 100, speed: 0, loss: 0 }
controllerConfig.candidates[1].downstreamInterface = "target"
const fast = randomUUID()
await pool.query("UPDATE mcp_routing_observations SET ping_ms=100 WHERE src_server_id=$1 AND routing_table='tenant-a' AND src_interface='hop'", [sid])
await pool.query("INSERT INTO mcp_routing_observations(id,src_server_id,dst_server_id,src_interface,dst_interface,family,routing_table,ping_ms,loss_pct,dl_mbps,ul_mbps,ping_at,bandwidth_at,context_verified,source) VALUES($1,$2,$2,'target','peer','ipv4','tenant-a',1,0,100,100,now(),now(),true,'fixture')", [fast, sid])
await refresh()
const activeProfileId = randomUUID(), beforeControllerWrites = writes
persistentProfileIds.push(activeProfileId)
const saveProfile = (config: typeof controllerConfig) => invokeMcpTool(principal, "mm_route_profile_save_plan", {
profileId: activeProfileId, profile: config, enabled: true, expiresAt: new Date(Date.now() + 600000).toISOString(),
}) as Promise<{ planId: string }>
for (const invalid of [
{ ...controllerConfig, bgpContext: undefined },
{ ...controllerConfig, autoRecovery: true },
{ ...controllerConfig, candidates: controllerConfig.candidates.map(candidate => ({ ...candidate, controlProbe: { ...candidate.controlProbe!, sourceAddress: undefined } })) },
]) {
await assert.rejects(saveProfile(invalid), (error: unknown) => error instanceof McpError && error.code === "UNSUPPORTED_CAPABILITY")
}
const savedProfile = await saveProfile(controllerConfig)
assert.equal((await loadPlan(savedProfile.planId)).state, "awaiting-approval")
await approvePlan(savedProfile.planId, writer, true)
assert.equal((await execute(savedProfile.planId)).state, "applied")
const readController = async () => (await pool.query("SELECT enabled,state FROM mcp_routing_profiles WHERE id=$1", [activeProfileId])).rows[0]
const enrolled = await readController()
assert.equal(enrolled.enabled, true); assert.equal(enrolled.state.baseline.packet.candidateId, "a")
controllerProbeUnknown = true
try { await tickRoutingProfiles() } finally { controllerProbeUnknown = false }
const waitingForProbe = await readController()
assert.equal(waitingForProbe.enabled, true, String(waitingForProbe.state.pausedReason))
assert.equal(waitingForProbe.state.planId, undefined)
assert.equal(waitingForProbe.state.pendingCandidateId, "b", String(waitingForProbe.state.reason))
assert.equal(writes, beforeControllerWrites)
// Let the next tick run without waiting for the production polling interval.
await pool.query("UPDATE mcp_routing_profiles SET state=state-'probedAt' WHERE id=$1", [activeProfileId])
await tickRoutingProfiles()
const proposed = await readController()
assert.equal(proposed.enabled, true, String(proposed.state.pausedReason))
assert.equal(proposed.state.candidateId, "b", String(proposed.state.reason))
const switchPlanId: string = proposed.state.planId
assert.equal((await loadPlan(switchPlanId)).state, "awaiting-approval")
await tickRoutingProfiles()
assert.equal(writes, beforeControllerWrites)
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE plan_id=$1", [switchPlanId])).rows[0].n, 0)
await approvePlan(switchPlanId, writer, true)
await tickRoutingProfiles()
const pendingSwitch = await readController()
const switchOperationId: string = pendingSwitch.state.operationId
assert.ok(switchOperationId, String(pendingSwitch.state.pausedReason))
assert.equal(await waitForOperation(switchOperationId), "applied")
await tickRoutingProfiles()
const switched = await readController()
assert.equal(switched.enabled, true, String(switched.state.pausedReason))
assert.equal(switched.state.planId, undefined)
assert.equal(switched.state.expectedCandidateId, "b")
assert.equal(switched.state.baseline.packet.candidateId, "b")
assert.equal(switched.state.baseline.packet.previousProof, enrolled.state.baseline.proof)
assert.equal(switched.state.baseline.packet.transition.operationId, switchOperationId)
assert.equal(switched.state.baseline.packet.transition.planId, switchPlanId)
assert.equal(writes - beforeControllerWrites, 1)
await disablePersistent(activeProfileId)
const boundedReserveFailover = await acceptControllerFailover()
assert.deepEqual((await pool.query("SELECT * FROM filter_rules WHERE server_id=$1 ORDER BY id", [sid])).rows, dbBefore)
const result = { accepted: true, cases, publicManualRevision2, boundedReserveFailover, persistentAuthorityCases: 0, realMcpDispatcher: true, realSql: true, routerOsTransport: "declared-synthetic-localhost-http",
genuineApprovedRunningOrigin: true, actualConfigurationCheckpoint: true, signedEffectTamperRejected: true,
originalAutoRecoveryEnabled: false, fullRoutingAccepted: false, scopedControllerEnabled: true, publicScopedControllerLifecycle: true, nativeChrAccepted: false, forwardingVerified: false }
console.log("Original effect SQL/runtime component:", JSON.stringify(result))
return result
}
const assertOriginalCredentialRevocation = async () => {
console.log(JSON.stringify({stage:'original-credential-revocation:init'}))
reset(); await refresh()
console.log(JSON.stringify({stage:'original-credential-revocation:prepare-source'}))
const persistent = await persistentFixture(), { plan, input } = await prepare("a", persistent)
mode = "post-failed"; const source = await execute(plan.planId); assert.equal(source.state, "partial"); mode = "normal"
const newerId = randomUUID(), newerExpiry = principal.credentialExpiresAt
await pool.query("INSERT INTO mcp_credentials(id,connection_id,digest,kind,expires_at) VALUES($1,$2,$3,'bearer',$4)", [newerId, connectionId, randomUUID(), newerExpiry])
const newer: McpPrincipal = { ...principal, credentialId: newerId }
const loaded = await loadScopedRecoverySource(source.id, { principal: newer, connection: await effectiveMcpConnection(newer) })
assert.equal((loaded.input.principal as McpPrincipal).credentialId, principal.credentialId)
assert.equal(fingerprint(loaded.configuration.packet.revision === 2 && loaded.configuration.packet.persistentProfileAuthority), fingerprint(input.persistentProfileAuthority))
const recovery = await invokeMcpTool(newer, "mm_operation_recovery_plan", { operationId: source.id }) as { planId: string }
await approvePlan(recovery.planId, writer, true)
const before = writes, beforeReads = requests
const snapshot = async () => (await pool.query(`SELECT
(SELECT state FROM mcp_plans WHERE id=$1) AS plan_state,
(SELECT count(*)::int FROM mcp_operations WHERE connection_id=$2) AS operations,
(SELECT count(*)::int FROM mcp_operations WHERE plan_id=$1) AS recovery_operations,
(SELECT COALESCE(jsonb_agg(jsonb_build_object('bucket',bucket,'used',used) ORDER BY bucket),'[]'::jsonb) FROM mcp_budget_buckets WHERE connection_id=$2 AND kind='mutations') AS mutation_buckets,
(SELECT count(*)::int FROM mcp_routing_manual_inverse_charges WHERE connection_id=$2) AS inverse_charges,
(SELECT count(*)::int FROM mcp_scoped_recovery_intents WHERE source_operation_id=$3) AS inverse_intents
`,[recovery.planId,connectionId,source.id])).rows[0]
const admittedBefore=await snapshot()
assert.equal(admittedBefore.plan_state,'approved');assert.equal(admittedBefore.recovery_operations,0);assert.equal(admittedBefore.inverse_intents,0)
try {
await pool.query("UPDATE mcp_credentials SET revoked_at=now() WHERE id=$1", [principal.credentialId])
const credentials=(await pool.query("SELECT id,revoked_at IS NOT NULL AS revoked,expires_at>now() AS unexpired FROM mcp_credentials WHERE id=ANY($1::uuid[])",[[principal.credentialId,newerId]])).rows
assert.equal(credentials.find(row=>row.id===principal.credentialId)?.revoked,true)
assert.equal(credentials.find(row=>row.id===newerId)?.revoked,false);assert(credentials.every(row=>row.unexpired))
assert.equal((await effectiveMcpConnection(newer)).id,connectionId)
console.log(JSON.stringify({stage:'original-credential-revocation:expect-admission-refusal'}))
await assert.rejects(applyPlan(newer,recovery.planId,randomUUID()),(error:unknown)=>error instanceof McpError && error.code==='UNAUTHORIZED' && error.statusCode===401)
assert.deepEqual(await snapshot(),admittedBefore,"Revoked original authority must roll back operation, executing plan, mutation debit, charge and intent admission")
assert.equal(writes, before)
assert.equal(requests, beforeReads, "Newer recovery credential cannot replace revoked original authority before native reads")
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_scoped_recovery_intents WHERE source_operation_id=$1", [source.id])).rows[0].n, 0)
persistentCases.push("recovery-preserves-original-authority")
console.log(JSON.stringify({stage:'original-credential-revocation-complete',unauthorized401:true,fullAdmissionRollback:true,nativeReads:requests-beforeReads,nativeWrites:writes-before,wholeSuiteAccepted:false}))
} finally {
await pool.query("UPDATE mcp_credentials SET revoked_at=NULL WHERE id=$1", [principal.credentialId])
await pool.query("UPDATE mcp_credentials SET revoked_at=now() WHERE id=$1", [newerId])
await disablePersistent(persistent.id)
}
}
if (options.originalCredentialRevocationOnly) {
await assertOriginalCredentialRevocation()
return {accepted:true,originalCredentialRevocation:true,unauthorized401:true,fullAdmissionRollback:true,nativeReads:0,nativeWrites:0,realSql:true,allWriterLedgerComplete:false}
}
if (options.manualInverseChargeOnly || options.lateInverseAuthorityChange) {
const chargeCase = options.manualInverseChargeCase ?? "positive"
if (options.lateInverseAuthorityChange) assert.equal(chargeCase, "positive")
const persistent = chargeCase !== "nonpersistent"
// Same genuine fixed-reader baseline/CAS used by the existing persistent
// acceptance below. No hand-written baseline, sealed plan or authority.
const id = randomUUID(), current = await effectiveMcpConnection(principal), config = profile("a")
assert.equal(config.autoRecovery, false)
if (chargeCase === "quota") config.maxSwitchesPerHour = 1
if (chargeCase === "mixed-forward") config.maxSwitchesPerHour = 2
const expiry = new Date(Math.min(Date.now() + 600000, current.owner_valid_until.getTime(), Date.parse(principal.credentialExpiresAt), Date.parse(current.policy.expiresAt)))
await withApplicationMutationLock(() => withConfigMutationLock([sid], async () => {
const baseline = await prepareRoutingControllerBaseline(id, 1, config, expiry.toISOString(), { principal, connection: current })
const saved = await saveControllerProfileCas(pool, { profileId: id, connectionId, expectedVersion: null, profile: config, enabled: persistent,
expiresAt: expiry.toISOString(), credentialId: principal.credentialId, generation: await accessGeneration(), policyVersion: current.policy.version, baseline })
assert.equal(saved?.version, 1)
}))
persistentProfileIds.push(id)
const ledger = createRoutingReservationStore()
let snapshotStage = "initial"
const snapshot = async () => {
console.log(JSON.stringify({stage:"manual-inverse:snapshot-hourly",chargeCase,snapshotStage}))
const hourly = await ledger.readHourlyCharges(id, principal)
console.log(JSON.stringify({stage:"manual-inverse:snapshot-generic",chargeCase,snapshotStage}))
const generic = Number((await pool.query("SELECT COALESCE(sum(used),0)::int AS used FROM mcp_budget_buckets WHERE connection_id=$1 AND kind='mutations'", [connectionId])).rows[0].used)
console.log(JSON.stringify({stage:"manual-inverse:snapshot-complete",chargeCase,snapshotStage}))
return { forward: hourly.forwardCharges, manualInverse: hourly.manualInverseCharges, reserved: hourly.reservedCharges, legacy: hourly.legacyCharges, total: hourly.totalCharges, generic, writes }
}
console.log(JSON.stringify({ stage: "manual-inverse:before-forward" }))
const before = await snapshot()
assert.deepEqual(before, { forward: 0, manualInverse: 0, reserved: 0, legacy: 0, total: 0, generic: 0, writes: 0 })
if (!persistent) {
const active = (await pool.query(`SELECT count(*)::int AS n FROM (
SELECT 1 FROM mcp_routing_profiles WHERE enabled AND (config->>'serverId'=$1 OR EXISTS(
SELECT 1 FROM jsonb_array_elements(config->'candidates') AS candidate
WHERE candidate->>'jumpHostId'=$1 OR candidate->>'exitNodeId'=$1)) LIMIT 1
) exact_active_writer_guard`, [String(sid)])).rows[0].n
assert.equal(active,0,"A genuine nonpersistent request2 fixture must not compete with an enabled persistent server profile")
console.log(JSON.stringify({stage:"manual-inverse:nonpersistent-active-profile-zero",chargeCase,activeProfiles:active}))
}
console.log(JSON.stringify({stage:"manual-inverse:forward-prepare",chargeCase}))
const { plan, input } = await prepare("a", persistent ? { id, version: 1 } : undefined, config)
// Prepare the competing forward while the ORIGINAL native baseline is
// current. Admission must charge it even if execution later refuses the
// changed network state. No invented request/authority or reserved lane.
const competingForward = chargeCase === "mixed-forward" ? await prepare("a", { id, version: 1 }, config) : undefined
// Existing known partial configuration path: genuine committed CONFIG_APPLIED,
// then failed control probe, then genuine manual inverse positive to A.
mode = "post-failed"
console.log(JSON.stringify({stage:"manual-inverse:forward-apply",chargeCase}))
const source = await execute(plan.planId)
// Read-only terminal diagnostics for this exact operation/connection.
// Nothing private is logged; malformed/missing diagnostics never replace
// the original expected partial assertion or make unknown acceptable.
try {
const rows = (await pool.query(`SELECT
CASE WHEN o.state IN('pending','running','applied','partial','unknown','rejected') THEN o.state ELSE 'other' END AS operation_state,
CASE WHEN o.error_code IN('MUTATION_OUTCOME_UNKNOWN','VERIFICATION_FAILED','STATE_CONFLICT','READ_CANCELLED','PLAN_EXPIRED','PERMISSION_DENIED','DEVICE_UNAVAILABLE','RESET_REQUIRED','PROCESS_INTERRUPTED') THEN o.error_code ELSE 'other' END AS error_code,
CASE WHEN s.state IN('intent','applied','verified','unknown','failed') THEN s.state ELSE 'other' END AS step_state,
s.result IS NOT NULL AS step_result_present,c.configuration_sealed IS NOT NULL AS configuration_present,
c.configuration_proof IS NOT NULL AS configuration_proof_present,
CASE WHEN octet_length(c.configuration_sealed)<=524288 THEN c.configuration_sealed END AS private_configuration,
(SELECT count(*)::int FROM (SELECT 1 FROM mcp_audit a WHERE a.connection_id=o.connection_id AND a.action='routing.scoped.network-check' AND a.payload->>'receiptId'=$3 LIMIT 2) bounded_network) AS network_receipt_rows_at_most2
FROM mcp_operations o LEFT JOIN mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
LEFT JOIN mcp_operation_checkpoints c ON c.operation_id=o.id AND c.step_index=0
WHERE o.id=$1 AND o.connection_id=$2 LIMIT 2`, [source.id,connectionId,input.receiptId])).rows
let checkpointRevision: number | null = null, checkpointPhaseApplied = false, checkpointIdentityMatches = false
if (rows.length===1 && rows[0].private_configuration) {
try {
const packet = JSON.parse(await decryptSecret(rows[0].private_configuration))
checkpointRevision = packet.revision===1 || packet.revision===2 ? packet.revision : null
checkpointPhaseApplied = packet.phase==="CONFIG_APPLIED"
checkpointIdentityMatches = packet.operationId===source.id && packet.stepIndex===0 && packet.connectionId===connectionId && packet.receiptId===input.receiptId
} catch { /* fixed unavailable flags below; no private exception */ }
}
console.log(JSON.stringify({stage:"manual-inverse:forward-terminal",chargeCase,exactOperationFound:rows.length===1,
operationState:rows[0]?.operation_state,errorCode:rows[0]?.error_code,stepState:rows[0]?.step_state,
stepResultPresent:rows[0]?.step_result_present,configurationPresent:rows[0]?.configuration_present,
configurationProofPresent:rows[0]?.configuration_proof_present,networkReceiptRowsAtMost2:rows[0]?.network_receipt_rows_at_most2,
requestRevision:input.proofRevision===2 ? 2 : input.proofRevision===3 ? 3 : null,checkpointRevision,checkpointPhaseApplied,checkpointIdentityMatches,
writes,postCheckpointsObserved,inversePostChecks}))
} catch { console.log(JSON.stringify({stage:"manual-inverse:forward-terminal-diagnostic-unavailable",chargeCase})) }
assert.equal(source.state, "partial"); mode = "normal"
console.log(JSON.stringify({ stage: "manual-inverse:after-forward" }))
snapshotStage = "after-forward"
const afterForward = await snapshot()
assert.equal(afterForward.forward, persistent ? 1 : 0); assert.equal(afterForward.reserved, 0); assert.equal(afterForward.legacy, 0); assert.equal(afterForward.total, persistent ? 1 : 0)
assert.equal(afterForward.writes, 1)
assert.equal(postCheckpointsObserved, 1)
console.log(JSON.stringify({ stage: "manual-inverse:reload-original-authority" }))
const loaded = await loadScopedRecoverySource(source.id, { principal, connection: await effectiveMcpConnection(principal) })
assert.equal(fingerprint(loaded.input.persistentProfileAuthority), fingerprint(input.persistentProfileAuthority))
assert.equal(loaded.configuration.packet.revision, persistent ? 2 : 1)
if (persistent) assert.equal(fingerprint(loaded.configuration.packet.revision === 2 && loaded.configuration.packet.persistentProfileAuthority), fingerprint(input.persistentProfileAuthority))
console.log(JSON.stringify({ stage: "manual-inverse:human-recovery-prepare" }))
const recovery = await invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: source.id }) as { planId: string; state: string }
assert.equal(recovery.state, "awaiting-approval")
assert.equal((await loadPlan(recovery.planId)).operation_id, "operations.recover")
assert.equal((await loadPlan(recovery.planId)).operation_version, 8)
await assert.rejects(applyPlan(principal, recovery.planId, randomUUID()))
assert.deepEqual(await snapshot(), afterForward, "No approval means no debit/charge/native write")
await approvePlan(recovery.planId, writer, true)
if (chargeCase === "positive" && !options.lateInverseAuthorityChange) {
await pool.query("UPDATE mcp_plans SET operation_version=7 WHERE id=$1", [recovery.planId])
try {
await assert.rejects(applyPlan(principal, recovery.planId, randomUUID()), { code: "PLAN_EXPIRED" })
assert.equal((await pool.query("SELECT id FROM mcp_operations WHERE plan_id=$1", [recovery.planId])).rowCount, 0)
assert.deepEqual(await snapshot(), afterForward, "Superseded approved inverse cannot create an operation, debit or native write")
} finally {
await pool.query("UPDATE mcp_plans SET operation_version=8 WHERE id=$1", [recovery.planId])
}
}
if (chargeCase === "mixed-forward") {
assert(competingForward)
const attempts = [
{ lane: "forward", planId: competingForward.plan.planId, key: randomUUID(), cost: 2 },
{ lane: "manual-inverse", planId: recovery.planId, key: randomUUID(), cost: 1 },
] as const
const initialPlans = await Promise.all(attempts.map(attempt => loadPlan(attempt.planId)))
assert(initialPlans.every(value => value.state === "approved"))
const admitted = await Promise.allSettled(attempts.map(attempt => applyPlan(principal, attempt.planId, attempt.key)))
const winners = admitted.flatMap((value, index) => value.status === "fulfilled" ? [index] : [])
assert.equal(winners.length, 1, "Two writer lanes must compete for the same last profile slot")
const winnerIndex = winners[0], loserIndex = 1 - winnerIndex
const winner = admitted[winnerIndex], loser = admitted[loserIndex]
assert(winner.status === "fulfilled" && loser.status === "rejected")
assert(loser.reason instanceof McpError && loser.reason.code === "STATE_CONFLICT", "The losing lane must fail profile admission")
assert.equal(fingerprint(await loadPlan(attempts[loserIndex].planId)), fingerprint(initialPlans[loserIndex]), "Losing plan is completely unchanged")
assert.equal(Number((await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE plan_id=$1", [attempts[loserIndex].planId])).rows[0].n), 0)
assert.equal(Number((await pool.query("SELECT count(*)::int AS n FROM mcp_routing_forward_charges WHERE source_plan_id=$1", [attempts[loserIndex].planId])).rows[0].n), 0)
assert.equal(Number((await pool.query("SELECT count(*)::int AS n FROM mcp_routing_manual_inverse_charges WHERE recovery_plan_id=$1", [attempts[loserIndex].planId])).rows[0].n), 0)
const deadline = Date.now() + 140000
let terminalState = "pending"
while (["pending", "running"].includes(terminalState) && Date.now() < deadline) {
terminalState = (await pool.query("SELECT state FROM mcp_operations WHERE id=$1", [winner.value.id])).rows[0].state
if (["pending", "running"].includes(terminalState)) await delay(20)
}
assert(!["pending", "running"].includes(terminalState), "Winner must retire before fixture cleanup")
if (requestFailure) throw requestFailure.error
snapshotStage = "mixed-writer-terminal"
const afterMixed = await snapshot()
assert.equal(afterMixed.total, 2)
assert.equal(afterMixed.forward, winnerIndex === 0 ? 2 : 1)
assert.equal(afterMixed.manualInverse, winnerIndex === 1 ? 1 : 0)
assert.equal(afterMixed.reserved, 0); assert.equal(afterMixed.legacy, 0)
assert.equal(afterMixed.generic, afterForward.generic + attempts[winnerIndex].cost, "Losing writer cannot debit generic quota")
if (winnerIndex === 1) {
const charges = (await pool.query("SELECT source_operation_id,source_step_index,recovery_operation_id FROM mcp_routing_manual_inverse_charges WHERE recovery_operation_id=$1", [winner.value.id])).rows
assert.equal(charges.length, 1); assert.equal(charges[0].source_operation_id, source.id)
assert.equal(charges[0].source_step_index, 0); assert.equal(charges[0].recovery_operation_id, winner.value.id)
assert.equal(terminalState, "applied"); assert.equal(afterMixed.writes, 2)
}
const beforeReplay = fingerprint(await snapshot())
assert.equal((await applyPlan(principal, attempts[winnerIndex].planId, attempts[winnerIndex].key)).id, winner.value.id)
await delay(30)
assert.equal(fingerprint(await snapshot()), beforeReplay, "Same-key observation cannot recharge or replay either lane")
return { accepted: true, mixedWriterAdmission: true, winnerLane: attempts[winnerIndex].lane, before, afterForward, afterMixed,
exactlyOneAdmission: true, losingPlanOperationChargeAndGenericRollback: true, sameKeyReplayUnchanged: true,
realSql: true, allWriterLedgerComplete: false, reservedDispatchEnabled: false, autoRecoveryEnabled: false }
}
console.log(JSON.stringify({ stage: "manual-inverse:human-recovery-apply" }))
const lateChange = options.lateInverseAuthorityChange
if (lateChange) {
const priorIntent = (await pool.query("SELECT coalesce(max(id),0)::text AS id FROM mcp_audit WHERE connection_id=$1 AND action='configuration.intent'", [connectionId])).rows[0].id
let writeId: string | undefined
afterIntent = async () => {
const intents = (await pool.query(`SELECT payload FROM mcp_audit WHERE connection_id=$1 AND id>$2::bigint
AND action='configuration.intent' AND payload->>'path'=$3`,
[connectionId, priorIntent, "/routing/filter/rule/" + loaded.configuration.packet.nativeId])).rows
assert.equal(intents.length, 1)
assert.equal(intents[0].payload.method, "PATCH")
assert.equal(intents[0].payload.serverId, sid)
writeId = intents[0].payload.writeId
assert.match(writeId ?? "", /^[0-9a-f-]{36}$/)
const sql = {
edit: "UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late inverse edit\"') WHERE id=$1",
disable: "UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1",
grant: "UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1",
}[lateChange]
assert.equal((await pool.query(sql, [id])).rowCount, 1)
}
const inverse = await execute(recovery.planId)
assert.equal(afterIntent, undefined)
assert.ok(writeId)
assert.notEqual(inverse.state, "applied")
assert.equal(writes, afterForward.writes)
assert.equal(inversePostChecks, 0)
const auditRows = (await pool.query(`SELECT action FROM mcp_audit WHERE connection_id=$1 AND payload->>'writeId'=$2
ORDER BY id`, [connectionId, writeId])).rows
assert.deepEqual(auditRows.map(row => row.action), ["configuration.intent", "configuration.unconfirmed"])
const refusal = (await pool.query(`SELECT payload->>'phase' AS phase, payload->>'code' AS code
FROM mcp_audit WHERE connection_id=$1 AND action='routing.scoped.recovery-unconfirmed'`, [connectionId])).rows
assert.deepEqual(refusal, [{ phase: "before-dispatch-context", code: "STATE_CONFLICT" }])
assert.deepEqual((await pool.query(`SELECT state,verified_sealed FROM mcp_scoped_recovery_intents
WHERE source_operation_id=$1 AND recovery_operation_id=$2`, [source.id, inverse.id])).rows,
[{ state: "unknown", verified_sealed: null }])
assert.equal((await pool.query(`SELECT count(*)::int AS n FROM mcp_operation_checkpoints
WHERE operation_id=$1 AND configuration_sealed IS NOT NULL`, [inverse.id])).rows[0].n, 0)
assert.equal((await pool.query("SELECT error_code FROM mcp_operations WHERE id=$1", [inverse.id])).rows[0].error_code,
"MUTATION_OUTCOME_UNKNOWN")
return { accepted: true, lateInverseAuthorityChange: lateChange, durableIntents: 1, inverseNativeWrites: 0,
realSql: true, realMcpDispatcher: true, fullInverseLifecycleRepeated: false }
}
if (chargeCase === "quota" || chargeCase === "revoked") {
if (chargeCase === "revoked") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [id])
const beforeOperations = (await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE connection_id=$1", [connectionId])).rows[0].n
await assert.rejects(applyPlan(principal, recovery.planId, randomUUID()))
assert.deepEqual(await snapshot(), afterForward, "Refused inverse admission rolls back operation/plan/generic debit/profile charge before intent/PATCH")
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE connection_id=$1", [connectionId])).rows[0].n, beforeOperations)
assert.equal((await loadPlan(recovery.planId)).state, "approved")
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_scoped_recovery_intents WHERE source_operation_id=$1", [source.id])).rows[0].n, 0)
return { accepted:true,manualInverseChargeFix:true,chargeCase,before,afterForward,afterInverse:afterForward,
admissionRefusedBeforeIntent:true,fullRollback:true,realSql:true,allWriterLedgerComplete:false }
}
let inverse: Awaited<ReturnType<typeof execute>>
if (chargeCase === "concurrent") {
const competitor = await invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: source.id }) as {planId:string}
await approvePlan(competitor.planId, writer, true)
const results = await Promise.allSettled([execute(recovery.planId), execute(competitor.planId)])
const winners = results.filter((result): result is PromiseFulfilledResult<Awaited<ReturnType<typeof execute>>> => result.status === "fulfilled")
assert.equal(winners.length, 1); assert.equal(results.filter(result => result.status === "rejected").length, 1)
inverse = winners[0].value
} else {
if (chargeCase === "unknown") mode = "lost"
inverse = await execute(recovery.planId); mode = "normal"
}
if (chargeCase !== "unknown") assert.equal(inverse.state, "applied")
else assert.notEqual(inverse.state, "applied")
console.log(JSON.stringify({ stage: "manual-inverse:after-inverse" }))
snapshotStage = "after-inverse"
const afterInverse = await snapshot()
assert.equal(afterInverse.generic, afterForward.generic + 1)
assert.equal(afterInverse.forward, persistent ? 1 : 0); assert.equal(afterInverse.manualInverse, persistent ? 1 : 0); assert.equal(afterInverse.total, persistent ? 2 : 0)
assert.equal(afterInverse.reserved, 0); assert.equal(afterInverse.legacy, 0)
assert.equal(afterInverse.writes, 2); if (chargeCase !== "unknown") assert.equal(inversePostChecks, 1)
const charges = (await pool.query("SELECT recovery_operation_id,recovery_plan_id,source_operation_id,profile_id,charge_units,binding_sealed FROM mcp_routing_manual_inverse_charges WHERE source_operation_id=$1", [source.id])).rows
assert.equal(charges.length, persistent ? 1 : 0);
if (persistent) {
assert.equal(charges[0].recovery_operation_id, inverse.id); assert.equal(charges[0].recovery_plan_id, inverse.plan_id); assert.equal(charges[0].profile_id,id); assert.equal(charges[0].charge_units,1)
const binding = JSON.parse(await decryptSecret(charges[0].binding_sealed))
assert.equal(binding.operationVersion, 8)
assert.equal(binding.sourceVersion, 7)
}
const intent = (await pool.query("SELECT state,verified_sealed,recovery_operation_id,authority_profile_id,authority_profile_version,reservation_id FROM mcp_scoped_recovery_intents WHERE source_operation_id=$1 AND source_step_index=0", [source.id])).rows
assert.equal(intent.length, 1); assert.equal(intent[0].state, chargeCase === "unknown" ? "unknown" : "verified"); assert.equal(intent[0].recovery_operation_id, inverse.id)
assert.equal(intent[0].authority_profile_id, null); assert.equal(intent[0].authority_profile_version, persistent ? 1 : null); assert.equal(intent[0].reservation_id, null)
if (chargeCase !== "unknown") {
if (persistent) {
for (const sql of ["UPDATE mcp_routing_manual_inverse_charges SET charge_units=1 WHERE source_operation_id=$1",
"DELETE FROM mcp_routing_manual_inverse_charges WHERE source_operation_id=$1"]) {
await assert.rejects(pool.query(sql, [source.id]), (error:unknown) => (error as {code?:string}).code === "P0001")
}
await assert.rejects(pool.query("TRUNCATE mcp_routing_manual_inverse_charges"), (error:unknown) => (error as {code?:string}).code === "P0001")
}
const verified = JSON.parse(await decryptSecret(intent[0].verified_sealed))
assert.equal(verified.phase, "RECOVERY_VERIFIED"); assert.equal(verified.originalRule, initial)
assert.equal(filters[0].rule, initial); assert.equal(filters[1].rule, initial)
assert.equal((await reconcile(inverse.id)).matchesDesiredState, true)
}
assert.equal((await applyPlan(principal, inverse.plan_id, randomUUID())).id, inverse.id)
await assert.rejects(invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: source.id }))
assert.deepEqual(await snapshot(), afterInverse, "Known terminal same-plan replay/new recovery plan cannot debit or dispatch again")
assert.deepEqual((await pool.query("SELECT * FROM filter_rules WHERE server_id=$1 ORDER BY id", [sid])).rows, dbBefore)
return { accepted: true, manualInverseChargeFix: true, chargeCase, before, afterForward, afterInverse,
genuineConfigApplied: true, originalPersistentAuthorityReloaded: true, genuineRecoveryVerified: chargeCase !== "unknown", humanApprovalRequired: true,
profileInverseChargeObserved: persistent ? 1 : 0, inverseGenericDebitObserved: 1, terminalReplayUnchanged: true,
realSql: true, routerOsTransport: "loopback-http-fixture", forwardingVerified: false, allWriterLedgerComplete: false }
}
// Genuine fixed-reader baseline A plus signed persistent authority remain
// valid, while the external native body changes to B BEFORE preparation.
// Public ingress cannot set internal profile context; this trusted-host
// preparation case proves the native gate independently binds enrollment.
{
const driftId=randomUUID(),driftCurrent=await effectiveMcpConnection(principal),driftExpiry=new Date(Math.min(Date.now()+600000,driftCurrent.owner_valid_until.getTime(),Date.parse(principal.credentialExpiresAt),Date.parse(driftCurrent.policy.expiresAt)))
await withApplicationMutationLock(()=>withConfigMutationLock([sid],async()=>{
const baseline=await prepareRoutingControllerBaseline(driftId,1,profile("a"),driftExpiry.toISOString(),{principal,connection:driftCurrent})
assert.equal((await saveControllerProfileCas(pool,{profileId:driftId,connectionId,expectedVersion:null,profile:profile("a"),enabled:true,expiresAt:driftExpiry.toISOString(),credentialId:principal.credentialId,generation:await accessGeneration(),policyVersion:driftCurrent.policy.version,baseline}))?.version,1)
}))
persistentProfileIds.push(driftId)
try {
filters[0].rule=compileManagedFilterRules([{community:"65000:10",gateway:"192.0.2.9",outIface:"",action:"route",description:""},{community:"65000:20",gateway:"192.0.2.5",outIface:"",action:"route",description:""}])
assert.equal((await invokeMcpTool(principal,"mm_route_path_verify",{profile:profile("a"),candidateId:"a"}) as {status:string}).status,"passed")
const driftPlan=await invokeMcpTool(principal,"mm_route_optimizer_plan",{profile:profile("a"),candidateId:"a"},undefined,{persistentRoutingProfile:{id:driftId,version:1}}) as {planId:string}
await approvePlan(driftPlan.planId,writer,true)
const beforeDrift=writes,operation=await execute(driftPlan.planId)
assert.notEqual(operation.state,"applied")
assert.equal(writes,beforeDrift,"Authenticated private scoped input prepared after external A->B drift must send0nativewrites")
assert.equal((await pool.query("SELECT configuration_sealed FROM mcp_operation_checkpoints WHERE operation_id=$1",[operation.id])).rows[0].configuration_sealed,null)
console.log("TARGETED_SCOPED_PRIVATE_DRIFT_NATIVE0_PASS")
}finally{filters[0].rule=initial;pendingPostCheckpoint=false;pendingInversePost=false;await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1",[driftId])}
}
if (options.forwardRoundsOnly) {
const accepted: string[] = []
for (const scenario of ["two-rounds", "native-drift", "checkpoint-tamper", "late-revoke", "row-budget"] as const) {
reset(); await refresh()
const persistent = await persistentFixture(), { plan, input } = await prepare("a", persistent)
const beforeWrites = writes, wireStart = originalEffectWire.length
let faultReached = false
try {
if (scenario === "row-budget") forwardRoundBulk = 1000
const operation = await executeWithCheckpointObservation(plan.planId, async operationId => {
const reads = requests, connection = await effectiveMcpConnection(principal)
await assert.rejects(resolveRoutingNetworkConfigurationContext(operationId, {
principal: { ...principal, connectionId: randomUUID() }, connection,
}))
assert.equal(requests, reads, "A mismatched principal cannot read native state")
if (scenario !== "two-rounds") afterRoundLogRead = async () => {
faultReached = true
if (scenario === "native-drift") filters[0].rule = "reject;"
if (scenario === "checkpoint-tamper") await pool.query("UPDATE mcp_operation_checkpoints SET configuration_proof=$2 WHERE operation_id=$1", [operationId, "0".repeat(64)])
if (scenario === "late-revoke") await disablePersistent(persistent.id)
}
})
assert.equal(faultReached, scenario !== "two-rounds")
const wire = originalEffectWire.slice(wireStart)
const originalRib = wire.filter(row => row.method === "GET" && row.path === "/routing/route" && !row.afterPatch)
const forwardRib = wire.filter(row => row.method === "GET" && row.path === "/routing/route" && row.afterPatch)
assert.equal(originalRib.length, 1); assert.equal(writes, beforeWrites + 1)
const { rows: [terminal] } = await pool.query("SELECT o.error_code,s.result FROM mcp_operations o JOIN mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0 WHERE o.id=$1", [operation.id])
if (scenario === "two-rounds" || scenario === "row-budget") {
assert.equal(operation.state, "applied"); assert.equal(forwardRib.length, 2)
const report = originalEffectReportSchema.parse(terminal.result.originalEffects)
const network = await loadScopedNetworkCheckpoint(input, await assertScopedCommunityRequest(input))
assert(network); assert.equal(network.status, "passed"); assert.deepEqual(network.originalEffects, report)
assert.equal(report.semanticProof, false); assert.equal(report.forwardingVerified, false); assert.equal(report.logCoverage, "unknown")
if (scenario === "two-rounds") {
assert.equal(report.status, "observed-expected"); assert.equal(report.rounds.length, 2)
assert(report.budget && report.budget.calls > 0 && report.budget.calls <= 256 && report.budget.rows <= 10000)
} else {
assert.equal(report.status, "unknown"); assert.equal(report.rounds.length, 0)
assert.equal(report.reason, "original-effect-rounds-unavailable:LIMIT_EXCEEDED")
assert.equal(report.budget, undefined)
}
} else {
assert.equal(operation.state, "unknown"); assert.equal(forwardRib.length, 1)
assert.equal(terminal.error_code, "MUTATION_OUTCOME_UNKNOWN")
}
const readsBeforeReplay = requests
const replay = await applyPlan(principal, plan.planId, randomUUID())
assert.equal(replay.id, operation.id); assert.equal(requests, readsBeforeReplay); assert.equal(writes, beforeWrites + 1)
accepted.push(scenario)
console.log(JSON.stringify({ scenario, state: operation.state, errorCode: terminal.error_code, forwardRibReads: forwardRib.length, nativeWrites: 1 }))
} finally {
afterRoundLogRead = undefined; forwardRoundBulk = 0
await disablePersistent(persistent.id)
}
}
console.log(JSON.stringify({ forwardRoundCases: accepted, actualGenuineConfigApplied: true, sharedBudget: true,
securityProof: false, logCoverage: "unknown", forwardingVerified: false }))
return { accepted: true, forwardRoundCases: accepted, securityProof: false, forwardingVerified: false }
}
for (const key of ["a", "b"]) {
const before = structuredClone(filters), { plan, input } = await prepare(key), count = writes
const op = await execute(plan.planId); assert.equal(op.state, "applied"); assert.equal(writes, count + 1)
assert.equal((await reconcile(op.id)).configurationCheckpointAvailable, true)
const privateCheckpoint = (await pool.query("SELECT configuration_sealed,configuration_proof FROM mcp_operation_checkpoints WHERE operation_id=$1", [op.id])).rows[0]
assert.ok(!JSON.stringify(privateCheckpoint).includes(initial), "Native bodies are encrypted at rest")
for (const tamper of ["proof", "ciphertext"]) {
await pool.query("UPDATE mcp_operation_checkpoints SET configuration_sealed=$2,configuration_proof=$3 WHERE operation_id=$1", [op.id, tamper === "ciphertext" ? privateCheckpoint.configuration_sealed.slice(0, -8) + "AAAAAAAA" : privateCheckpoint.configuration_sealed, tamper === "proof" ? "0".repeat(64) : privateCheckpoint.configuration_proof])
const checked = await reconcile(op.id); assert.equal(checked.configurationCheckpointAvailable, false); assert.equal(checked.matchesDesiredState, false)
await pool.query("UPDATE mcp_operation_checkpoints SET configuration_sealed=$2,configuration_proof=$3 WHERE operation_id=$1", [op.id, privateCheckpoint.configuration_sealed, privateCheckpoint.configuration_proof])
}
const networkReceipt = (await pool.query("SELECT id,payload FROM mcp_audit WHERE action='routing.scoped.network-check' AND payload->>'receiptId'=$1", [input.receiptId])).rows[0]
assert.ok(networkReceipt)
await pool.query("UPDATE mcp_audit SET payload=jsonb_set(payload,'{check,status}','\"failed\"'::jsonb) WHERE id=$1", [networkReceipt.id])
assert.equal((await reconcile(op.id)).matchesDesiredState, false, "A generic VERIFIED result cannot replace the signed post-check evidence")
await pool.query("UPDATE mcp_audit SET payload=$2::jsonb WHERE id=$1", [networkReceipt.id, JSON.stringify(networkReceipt.payload)])
assert.equal((await reconcile(op.id)).matchesDesiredState, true)
const index = key === "a" ? 0 : 1; assert.deepEqual({ ...filters[index], rule: before[index].rule }, before[index]); assert.deepEqual(filters[1 - index], before[1 - index])
assert.equal((await reconcile(op.id)).matchesDesiredState, true)
const checkpoint = (await pool.query("SELECT intended_state,result FROM mcp_operation_steps WHERE operation_id=$1", [op.id])).rows[0]
await pool.query("UPDATE mcp_operation_steps SET state='applied' WHERE operation_id=$1", [op.id]); assert.equal((await reconcile(op.id)).matchesDesiredState, false)
await pool.query("UPDATE mcp_operation_steps SET state='verified',intended_state=jsonb_set(intended_state,'{input,receiptId}',to_jsonb($2::text)) WHERE operation_id=$1", [op.id, randomUUID()]); assert.equal((await reconcile(op.id)).matchesDesiredState, false)
await pool.query("UPDATE mcp_operation_steps SET intended_state=$2::jsonb WHERE operation_id=$1", [op.id, JSON.stringify(checkpoint.intended_state)])
assert.equal((await reconcile(op.id)).matchesDesiredState, true)
assert.equal((await applyPlan(principal, plan.planId, randomUUID())).id, op.id, "already applied plan returns its operation without replay")
await assert.rejects(findMcpService("routing.community.scoped")!.current({ ...input, receiptId: randomUUID() })); assert.equal(writes, count + 1)
}
assert.deepEqual([...observedVrfs].sort(), ["probe-a", "probe-b"])
for (const scenario of ["ignored", "lost", "collateral"] as const) {
reset(); const { plan } = await prepare(); mode = scenario; const count = writes
const op = await execute(plan.planId); assert.ok(["unknown", "partial", "failed"].includes(op.state), op.state); assert.equal(writes, count + 1)
assert.equal((await pool.query("SELECT configuration_sealed FROM mcp_operation_checkpoints WHERE operation_id=$1", [op.id])).rows[0].configuration_sealed, null, "Lost/ignored/collateral forward must not produce CONFIG_APPLIED")
const beforeRecoveryRequests = requests
await assert.rejects(invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: op.id }))
assert.equal(requests, beforeRecoveryRequests, "Unknown or unverified source is rejected before native recovery reads")
if (scenario !== "collateral") { const checked = await reconcile(op.id); assert.equal(checked.matchesDesiredState, false); if (scenario === "lost") assert.equal(checked.configurationMatchesDesired, true) }
assert.equal(writes, count + 1)
}
for (const scenario of ["post-failed", "post-unknown"] as const) for (const slaChanged of ["none", "stale", "failed"]) {
reset(); await refresh(); const { plan } = await prepare(), count = writes
mode = scenario
if (slaChanged !== "none") afterForwardWrite = async () => {
await pool.query(slaChanged === "stale" ? "UPDATE mcp_routing_observations SET ping_at='2000-01-01' WHERE src_server_id=$1" : "UPDATE mcp_routing_observations SET loss_pct=100 WHERE src_server_id=$1", [sid])
for (const row of (await pool.query("SELECT id FROM mcp_routing_observations WHERE src_server_id=$1", [sid])).rows) await attestRoutingFixture(row.id)
}
const op = await execute(plan.planId)
assert.equal(op.state, scenario === "post-failed" ? "partial" : "unknown", scenario)
assert.equal(writes, count + 1)
assert.equal(afterForwardWrite, undefined)
const checked = await reconcile(op.id)
assert.equal(checked.configurationMatchesDesired, true); assert.equal(checked.configurationCheckpointAvailable, true); assert.equal(checked.matchesDesiredState, false)
assert.equal((await applyPlan(principal, plan.planId, randomUUID())).id, op.id)
assert.equal(writes, count + 1, "Neither failed nor unknown post-check authorizes replay or inverse")
if (scenario === "post-failed") {
const result = (await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1", [op.id])).rows[0].result
assert.equal(result.postProbeStatus, "failed"); assert.equal(result.configurationVerified, true); assert.equal(result.controlProbeVerified, false)
} else {
const beforeRecoveryRequests = requests
await assert.rejects(invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: op.id }))
assert.equal(requests, beforeRecoveryRequests, "Unknown network outcome does not authorize inverse")
}
}
reset(); await refresh()
{
const { plan } = await prepare(), count = writes
afterForwardWrite = async () => {
await pool.query("UPDATE mcp_routing_observations SET ping_at='2000-01-01' WHERE src_server_id=$1", [sid])
for (const row of (await pool.query("SELECT id FROM mcp_routing_observations WHERE src_server_id=$1", [sid])).rows) await attestRoutingFixture(row.id)
}
const op = await execute(plan.planId); assert.equal(op.state, "applied"); assert.equal(writes, count + 1)
const checked = await reconcile(op.id)
assert.equal(checked.configurationMatchesDesired, true); assert.equal(checked.configurationCheckpointAvailable, true); assert.equal(checked.matchesDesiredState, true)
assert.equal((await applyPlan(principal, plan.planId, randomUUID())).id, op.id); assert.equal(writes, count + 1)
}
await refresh()
console.log("Scoped post-write SLA: fresh admission remains mandatory; signed SQL samples becoming stale after PATCH preserve CONFIG_APPLIED and failed/unknown probe receipts, read-only reconciliation and no replay")
for (const change of ["protection", "epoch", "pause", "path-proof"]) {
reset(); await refresh()
const {plan,input}=await prepare(), before=writes, protection=randomUUID()
afterForwardWrite=async()=>{
if(change==="protection")await pool.query("INSERT INTO mcp_protections(id,object_ref,created_by) VALUES($1,$2::jsonb,$3)",[protection,JSON.stringify({kind:"routing_filter",id:"*fa",serverId:sid,protection:"deny"}),writer.id])
if(change==="epoch")await pool.query("UPDATE servers SET password='post-write-source-change' WHERE id=$1",[sid])
if(change==="pause")await setConnectionStatus(connectionId!,"paused",writer)
if(change==="path-proof")await pool.query("UPDATE mcp_path_checks SET evidence=jsonb_set(evidence,'{untrustedChange}','true'::jsonb) WHERE id=$1",[input.pathCheckId])
}
const op=await execute(plan.planId);assert.notEqual(op.state,"applied",change);assert.equal(writes,before+1)
assert.equal((await pool.query("SELECT configuration_sealed FROM mcp_operation_checkpoints WHERE operation_id=$1",[op.id])).rows[0].configuration_sealed,null,"Post-write authority or immutable source drift cannot produce CONFIG_APPLIED")
await pool.query("DELETE FROM mcp_protections WHERE id=$1",[protection])
if(change==="epoch")await pool.query("UPDATE servers SET password='NEVER_SCOPED_PASSWORD' WHERE id=$1",[sid])
if(change==="pause")await setConnectionStatus(connectionId!,"active",writer)
}
console.log("Scoped post-write authority: protection, endpoint epoch, policy pause and immutable path-row tampering still stop configuration confirmation; exact one PATCH and no replay")
assert.ok(postCheckpointsObserved >= 4)
console.log("Scoped CONFIG_APPLIED SQL/HTTP: encrypted exact operation checkpoint precedes post-probe, native body/candidate/proofs bound, tamper blocked, failed control probe partial, unknown probe unknown, no inverse or replay")
for (const scenario of ["normal", "concurrent", "lost", "ignored", "collateral", "late-protection", "late-epoch", "primary-down"] as const) {
console.log("Scoped inverse scenario:", scenario)
reset(); await refresh()
if (scenario === "primary-down") await registerManagementChannel({ serverId: sid, host: "127.0.0.2", port: (alternate.address() as { port: number }).port, useSsl: false, verifySsl: true, username: "admin", password: "NEVER_SCOPED_ALTERNATE", attestation: "Owned loopback fixture: independent address with the same fixed license identity", expiresAt: new Date(Date.now() + 900000).toISOString() }, writer)
const oldPath = await invokeMcpTool(principal, "mm_route_path_verify", { profile: profile("a"), candidateId: "a" }) as { status: string; id: string }
assert.equal(oldPath.status, "passed")
if (scenario === "normal") {
const savedTime = (await pool.query("SELECT created_at::text AS value FROM mcp_path_checks WHERE id=$1", [oldPath.id])).rows[0].value
for (const shift of ["1 minute", "-91 seconds"]) {
await pool.query("UPDATE mcp_path_checks SET created_at=clock_timestamp()+$2::interval WHERE id=$1", [oldPath.id, shift])
await assert.rejects(requireRecentPath(profile("a"), "a", connectionId!))
}
await pool.query("UPDATE mcp_path_checks SET created_at=$2::timestamptz WHERE id=$1", [oldPath.id, savedTime])
assert.equal(await requireRecentPath(profile("a"), "a", connectionId!), oldPath.id)
}
const { plan: forward } = await prepare()
mode = "post-failed"
const original = await execute(forward.planId)
assert.equal(original.state, "partial")
mode = "normal"
await assert.rejects(requireRecentPath(profile("a"), "a", connectionId!), "The pre-switch A proof must not authorize inverse in B context")
const beforeRecovery: number = writes, primaryBefore: number = primaryControlRequests
if (scenario === "primary-down") primaryDown = true
if (scenario === "normal") {
const saved = (await pool.query("SELECT configuration_sealed FROM mcp_operation_checkpoints WHERE operation_id=$1", [original.id])).rows[0]
await pool.query("UPDATE mcp_operation_checkpoints SET configuration_sealed='invalid-checkpoint' WHERE operation_id=$1", [original.id])
const count: number = requests
await assert.rejects(invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: original.id }))
assert.equal(requests, count, "A tampered source packet is rejected before native reads")
await pool.query("UPDATE mcp_operation_checkpoints SET configuration_sealed=$2 WHERE operation_id=$1", [original.id, saved.configuration_sealed])
}
const recovery = await invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: original.id }) as { planId: string; state: string }
const competitor = scenario === "concurrent" ? await invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: original.id }) as { planId: string } : undefined
assert.equal(recovery.state, "awaiting-approval")
const projected = await publicPlan(await loadPlan(recovery.planId))
assert.ok(!JSON.stringify(projected).includes("NEVER_")); assert.ok(!JSON.stringify(projected).includes("managementBindingCiphertext")); assert.ok(!JSON.stringify(projected).includes("requestProof"))
const beforeApproval: number = writes
await assert.rejects(applyPlan(principal, recovery.planId, randomUUID())); assert.equal(writes, beforeApproval)
await approvePlan(recovery.planId, writer, true)
if (competitor) await approvePlan(competitor.planId, writer, true)
const protection = randomUUID()
if (scenario === "lost" || scenario === "ignored" || scenario === "collateral") mode = scenario
if (scenario === "late-protection" || scenario === "late-epoch") afterIntent = async () => {
if (scenario === "late-protection") await pool.query("INSERT INTO mcp_protections(id,object_ref,created_by) VALUES($1,$2::jsonb,$3)", [protection, JSON.stringify({ kind: "routing_filter", id: "*fa", serverId: sid, protection: "deny" }), writer.id])
else { await pool.query("UPDATE servers SET password='inverse-temp' WHERE id=$1", [sid]); await pool.query("UPDATE servers SET password='NEVER_SCOPED_PASSWORD' WHERE id=$1", [sid]) }
}
const competing = await Promise.all([execute(recovery.planId).then(result => ({ ...result, planId: recovery.planId })), ...(competitor ? [execute(competitor.planId).then(result => ({ ...result, planId: competitor.planId }))] : [])])
if (competitor) assert.equal(competing.filter(result => result.state === "applied").length, 1, "Only one separately approved plan can acquire the source inverse slot")
const inverse = competing.find(result => result.state === "applied") ?? competing[0]
if (writes !== beforeRecovery + (["late-protection", "late-epoch"].includes(scenario) ? 0 : 1)) {
console.error("Scoped inverse refusal:", scenario,
(await pool.query("SELECT state,error_code FROM mcp_operations WHERE id=$1", [inverse.id])).rows,
(await pool.query("SELECT action,payload->>'phase' AS phase,payload->>'code' AS code FROM mcp_audit WHERE connection_id=$1 AND action='routing.scoped.recovery-unconfirmed' ORDER BY created_at DESC LIMIT 3", [connectionId])).rows)
}
assert.equal(writes, beforeRecovery + (["late-protection", "late-epoch"].includes(scenario) ? 0 : 1), scenario)
if ((inverse.state === "applied") !== ["normal", "concurrent", "primary-down"].includes(scenario)) {
console.error("Scoped inverse terminal refusal:", scenario,
(await pool.query("SELECT state,error_code FROM mcp_operations WHERE id=$1", [inverse.id])).rows,
(await pool.query("SELECT action,payload->>'phase' AS phase,payload->>'code' AS code FROM mcp_audit WHERE connection_id=$1 AND action='routing.scoped.recovery-unconfirmed' ORDER BY created_at DESC LIMIT 3", [connectionId])).rows)
}
assert.equal(inverse.state === "applied", ["normal", "concurrent", "primary-down"].includes(scenario), scenario)
assert.equal((await applyPlan(principal, inverse.planId, randomUUID())).id, inverse.id)
const count: number = writes
await assert.rejects(invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: original.id }), "A new inverse plan must be rejected after scenario " + scenario)
assert.equal(writes, count, "A new plan cannot acquire the same source inverse slot again")
const intent = (await pool.query("SELECT state,verified_sealed,intent_sealed,recovery_operation_id FROM mcp_scoped_recovery_intents WHERE source_operation_id=$1", [original.id])).rows
assert.equal(intent.length, 1); assert.equal(intent[0].recovery_operation_id, inverse.id)
if (["normal", "concurrent", "primary-down"].includes(scenario)) {
assert.equal(intent[0].state, "verified"); assert.ok(intent[0].verified_sealed)
const packet = JSON.parse(await decryptSecret(intent[0].verified_sealed))
assert.equal(packet.phase, "RECOVERY_VERIFIED"); assert.equal(packet.originalRule, initial)
assert.equal(filters[0].rule, initial); assert.equal(filters[1].rule, initial)
assert.equal((await reconcile(inverse.id)).matchesDesiredState, true)
await pool.query("UPDATE mcp_operation_steps SET state='applied' WHERE operation_id=$1", [inverse.id])
assert.equal((await reconcile(inverse.id)).matchesDesiredState, false)
await pool.query("UPDATE mcp_operation_steps SET state='verified' WHERE operation_id=$1", [inverse.id])
} else {
assert.notEqual(intent[0].state, "verified"); assert.equal(intent[0].verified_sealed, null)
}
if (scenario === "primary-down") assert.equal(primaryControlRequests, primaryBefore, "Sealed alternate recovery must not contact the lost primary")
primaryDown = false; mode = "normal"; pendingInversePost = false
await pool.query("DELETE FROM mcp_protections WHERE id=$1", [protection])
if (scenario === "primary-down") await pool.query("DELETE FROM mcp_management_channels WHERE server_id=$1", [sid])
}
for (const scenario of ["original-failed", "original-unknown", "deadline", "future-source-time"] as const) {
reset(); await refresh(); const { plan } = await prepare()
mode = "post-failed"; const source = await execute(plan.planId); assert.equal(source.state, "partial")
mode = scenario === "original-unknown" ? "post-unknown" : scenario === "original-failed" ? "post-failed" : "normal"
if (scenario === "deadline") await pool.query("UPDATE mcp_operations SET created_at=now()-interval '3 minutes' WHERE id=$1", [source.id])
if (scenario === "future-source-time") await pool.query("UPDATE mcp_operations SET created_at=now()+interval '1 minute' WHERE id=$1", [source.id])
const count: number = writes
await assert.rejects(invokeMcpTool(principal, "mm_operation_recovery_plan", { operationId: source.id }))
assert.equal(writes, count)
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_scoped_recovery_intents WHERE source_operation_id=$1", [source.id])).rows[0].n, 0, "Failed/unknown original path or expired authority cannot claim inverse")
}
assert.ok(inversePostChecks >= 2)
console.log("Scoped inverse SQL/HTTP: exact encrypted source/op/step, fresh A proof in B context, one durable inverse slot, exact original bytes, RECOVERY_VERIFIED, lost/ignored/collateral no replay, late protection/ABA fences, primary-down alternate and generic VERIFIED separation")
for (const scenario of ["stale", "late-epoch-aba", "late-protection", "late-pause"] as const) {
reset(); await refresh(); const { plan } = await prepare(), before = writes, protection = randomUUID()
afterIntent = async () => {
if (scenario === "stale") await pool.query("UPDATE mcp_routing_observations SET ping_at='2000-01-01' WHERE src_server_id=$1", [sid])
if (scenario === "late-epoch-aba") { await pool.query("UPDATE servers SET password='temporary' WHERE id=$1", [sid]); await pool.query("UPDATE servers SET password='NEVER_SCOPED_PASSWORD' WHERE id=$1", [sid]) }
if (scenario === "late-protection") await pool.query("INSERT INTO mcp_protections(id,object_ref,created_by) VALUES($1,$2::jsonb,$3)", [protection, JSON.stringify({ kind: "routing_filter", id: "*fa", serverId: sid, protection: "deny" }), writer.id])
if (scenario === "late-pause") await setConnectionStatus(connectionId!, "paused", writer)
}
const op = await execute(plan.planId); assert.notEqual(op.state, "applied", scenario); assert.equal(writes, before, scenario + " must be fenced AFTER durable intent BEFORE native PATCH"); assert.equal(afterIntent, undefined)
await pool.query("DELETE FROM mcp_protections WHERE id=$1", [protection]); if (scenario === "late-pause") await setConnectionStatus(connectionId!, "active", writer); await refresh()
}
{
reset(); await refresh()
const reservedProfile = routeAiProfileSchema.parse({ ...profile("a"), autoRecovery: true })
const persistent = await persistentFixture(reservedProfile), beforeWrites = writes
try {
assert.equal((await invokeMcpTool(principal, "mm_route_path_verify", { profile: reservedProfile, candidateId: "b" }) as { status: string }).status, "passed")
const beforeSlots = Number((await pool.query("SELECT count(*) AS n FROM mcp_scoped_recovery_intents r JOIN mcp_operations o ON o.id=r.source_operation_id WHERE o.connection_id=$1", [connectionId])).rows[0].n)
await assert.rejects(prepareScopedCommunitySwitch({ profile: reservedProfile, candidateId: "b" }, {
principal, connection: await effectiveMcpConnection(principal), persistentRoutingProfile: persistent,
}), (error: unknown) => error instanceof Error && error.message.includes("ROUTE_RECOVERY_EXECUTOR_UNAVAILABLE"))
assert.equal(writes, beforeWrites, "Reserved preparation guard must refuse before either forward or inverse PATCH")
assert.equal(Number((await pool.query("SELECT count(*) AS n FROM mcp_scoped_recovery_intents r JOIN mcp_operations o ON o.id=r.source_operation_id WHERE o.connection_id=$1", [connectionId])).rows[0].n), beforeSlots, "Preparation does not allocate the shared inverse slot")
persistentCases.push("reserved-context-guard-zero-patch")
} finally { await disablePersistent(persistent.id) }
}
for (const change of ["edited", "disabled", "version-aba", "expired"] as const) {
reset(); await refresh(); const persistent = await persistentFixture()
assert.equal((await invokeMcpTool(principal, "mm_route_path_verify", { profile: profile("a"), candidateId: "b" }) as { status: string }).status, "passed")
if (change === "edited") await pool.query("UPDATE mcp_routing_profiles SET config=jsonb_set(config,'{name}','\"changed before preparation\"') WHERE id=$1", [persistent.id])
if (change === "disabled") await disablePersistent(persistent.id)
if (change === "version-aba") await pool.query("UPDATE mcp_routing_profiles SET version=version+2 WHERE id=$1", [persistent.id])
if (change === "expired") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=now()-interval '1 second' WHERE id=$1", [persistent.id])
const beforeReads = requests, beforeWrites = writes
await assert.rejects(invokeMcpTool(principal, "mm_route_optimizer_plan", { profile: profile("a"), candidateId: "b" }, undefined, { persistentRoutingProfile: persistent }))
assert.equal(requests, beforeReads, `${change}: stale authority must fail before first native GET`)
assert.equal(writes, beforeWrites)
await disablePersistent(persistent.id); persistentCases.push("before-native:" + change)
}
reset(); await refresh()
{
const persistent = await persistentFixture(), { plan, input } = await prepare("a", persistent), before = writes
const checkBridge = async (operationId: string) => {
const action = { principal, connection: await effectiveMcpConnection(principal) }
const bridge = await resolveRoutingNetworkConfigurationContext(operationId, action)
const current = await consumeRoutingNetworkConfigurationContext(bridge)
assert.equal(current.operationId, operationId); assert.equal(current.securityProof, false); assert.equal(current.sharedNetworkBudgetVerified, false)
assert.equal(current.configuration.nativeId, input.nativeId); assert.equal(current.configuration.dependencyIdentity, input.dependencyProof)
const afterReads = requests
await assert.rejects(consumeRoutingNetworkConfigurationContext(bridge))
await assert.rejects(consumeRoutingNetworkConfigurationContext(JSON.parse(JSON.stringify(bridge))))
await assert.rejects(resolveRoutingNetworkConfigurationContext(operationId, action), /source already issued/)
assert.equal(requests, afterReads); assert.equal(writes, before + 1)
}
// The production forward rounds use the same one-time bridge issuance.
// Exercise the independent bridge at the committed checkpoint first.
const op = await executeWithCheckpointObservation(plan.planId, checkBridge)
assert.equal(op.state, "applied"); assert.equal(writes, before + 1)
const row = (await pool.query("SELECT c.configuration_sealed,s.result FROM mcp_operation_checkpoints c JOIN mcp_operation_steps s USING(operation_id,step_index) WHERE c.operation_id=$1", [op.id])).rows[0]
const packet = JSON.parse(await decryptSecret(row.configuration_sealed))
assert.equal(packet.revision, 2); assert.equal(row.result.proofRevision, 3)
assert.equal(fingerprint(packet.persistentProfileAuthority), fingerprint(input.persistentProfileAuthority))
assert.equal(packet.persistentProfileAuthority.packet.credentialId, principal.credentialId)
assert.equal((await reconcile(op.id)).matchesDesiredState, true)
persistentCases.push("genuine-configuration-bridge-one-consume-no-clone-grant")
await disablePersistent(persistent.id); persistentCases.push("persistent-forward-request3-checkpoint2")
}
for (const change of ["closed", "disabled", "checkpoint-tamper"] as const) {
reset(); await refresh()
const persistent = await persistentFixture(), { plan } = await prepare("a", persistent), before = writes
const op = await executeWithCheckpointObservation(plan.planId, async operationId => {
const bridge = await resolveRoutingNetworkConfigurationContext(operationId, { principal, connection: await effectiveMcpConnection(principal) })
const { rows: [checkpoint] } = await pool.query("SELECT configuration_proof FROM mcp_operation_checkpoints WHERE operation_id=$1", [operationId])
try {
if (change === "closed") closeRoutingNetworkConfigurationContext(bridge)
if (change === "disabled") await disablePersistent(persistent.id)
if (change === "checkpoint-tamper") await pool.query("UPDATE mcp_operation_checkpoints SET configuration_proof=$2 WHERE operation_id=$1", [operationId, "f".repeat(64)])
const beforeReads = requests
await assert.rejects(consumeRoutingNetworkConfigurationContext(bridge))
assert.equal(requests, beforeReads); assert.equal(writes, before + 1)
} finally {
if (change === "disabled") await pool.query("UPDATE mcp_routing_profiles SET enabled=true WHERE id=$1", [persistent.id])
if (change === "checkpoint-tamper") await pool.query("UPDATE mcp_operation_checkpoints SET configuration_proof=$2 WHERE operation_id=$1", [operationId, checkpoint.configuration_proof])
closeRoutingNetworkConfigurationContext(bridge)
}
})
assert.equal(op.state, "applied"); assert.equal(writes, before + 1)
await disablePersistent(persistent.id)
persistentCases.push("genuine-configuration-bridge-denies:" + change)
}
{
reset(); await refresh()
const persistent = await persistentFixture(), { plan } = await prepare("a", persistent), before = writes
// The production effect reader already owns this operation's native
// session. A second reader must fail before any native request.
const op = await executeWithCheckpointObservation(plan.planId, async operationId => {
assert.equal((await pool.query("SELECT plan_id FROM mcp_operations WHERE id=$1", [operationId])).rows[0]?.plan_id, plan.planId)
if(process.env.MMAPP_OWN_ROUTING_WORKER_REPRO==="true") {
await (await import("./workers.js")).tickMcpWorkers()
const profileRow=(await pool.query("SELECT enabled,state FROM mcp_routing_profiles WHERE id=$1",[persistent.id])).rows[0]
assert.equal(profileRow.enabled,true);assert.equal(profileRow.state.pausedReason,undefined)
console.log("Worker exclusion preserved exact running CONFIG_APPLIED profile")
}
const action = { principal, connection: await effectiveMcpConnection(principal) }
// Reuse the approved plan's original deadline: now +90s would extend
// its preparation-time probe horizon and mask the duplicate-owner guard.
const deadline = (await loadPlan(plan.planId)).expires_at.getTime()
const reads = requests
await assert.rejects(createRoutingNetworkNativeSession({ sourceOperationId: operationId, context: action,
authorize: async () => {}, deadline, manifest: { servers: [sid], probes: [] } }), /Source native session уже создавалась/)
assert.equal(requests, reads); assert.equal(writes, before + 1)
})
assert.equal(op.state, "applied"); assert.equal(writes, before + 1)
const { rows: [step] } = await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1 AND step_index=0", [op.id])
const effects = originalEffectReportSchema.parse(step.result.originalEffects)
assert.equal(effects.status, "observed-expected")
assert.equal(effects.rounds.length, 2)
assert.equal(effects.semanticProof, false)
assert(effects.budget && effects.budget.calls > 0 && effects.budget.calls <= 256)
assert(effects.budget.bytes > 0 && effects.budget.rows > 0)
await disablePersistent(persistent.id)
persistentCases.push("genuine-running-config-bridge-uses-one-shared-native-owner")
}
for (const change of ["edit", "pause", "credential", "grant", "channel-revoke"] as const) {
reset(); await refresh()
if (change === "channel-revoke") await registerManagementChannel({ serverId: sid, host: "127.0.0.2", port: (alternate.address() as { port: number }).port, useSsl: false, verifySsl: true,
username: "admin", password: "NEVER_SCOPED_ALTERNATE", attestation: "Synthetic persistent authority independent channel for late SQL revocation", expiresAt: new Date(Date.now() + 600000).toISOString() }, writer)
const persistent = await persistentFixture(), { plan, input } = await prepare("a", persistent), before = writes
if (change === "channel-revoke") assert.ok(input.managementBindingCiphertext)
try {
afterIntent = async () => {
if (change === "edit") await pool.query("UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late edit\"') WHERE id=$1", [persistent.id])
if (change === "pause") await pool.query("UPDATE mcp_routing_profiles SET enabled=false,state='{\"pausedReason\":\"late emergency\"}'::jsonb WHERE id=$1", [persistent.id])
if (change === "credential") await pool.query("UPDATE mcp_credentials SET revoked_at=now() WHERE id=$1", [principal.credentialId])
if (change === "grant") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=now()-interval '1 second' WHERE id=$1", [persistent.id])
if (change === "channel-revoke") await pool.query("UPDATE mcp_management_channels SET revoked_at=now() WHERE server_id=$1", [sid])
}
const op = await execute(plan.planId)
assert.notEqual(op.state, "applied"); assert.equal(afterIntent, undefined)
assert.equal(writes, before, `${change}: authority revoked after durable intent must dispatch zero PATCH`)
assert.equal((await pool.query("SELECT configuration_sealed FROM mcp_operation_checkpoints WHERE operation_id=$1", [op.id])).rows[0].configuration_sealed, null)
persistentCases.push("after-intent:" + change)
} finally {
afterIntent = undefined
if (change === "credential") await pool.query("UPDATE mcp_credentials SET revoked_at=NULL WHERE id=$1", [principal.credentialId])
if (change === "channel-revoke") await pool.query("DELETE FROM mcp_management_channels WHERE server_id=$1", [sid])
await disablePersistent(persistent.id)
}
}
await assertOriginalCredentialRevocation()
reset(); await refresh()
reset(); const { plan: drift } = await prepare(), count = writes, oldPassword = "NEVER_SCOPED_PASSWORD", beforeRequests = requests
await pool.query("UPDATE servers SET password='changed' WHERE id=$1", [sid]); await pool.query("UPDATE servers SET password=$2 WHERE id=$1", [sid, oldPassword])
assert.notEqual((await execute(drift.planId)).state, "applied"); assert.equal(writes, count); assert.equal(requests, beforeRequests)
// Refresh signed observations after the source epoch changed; old signatures cannot authorize the next plan.
const observations = (await pool.query("SELECT id FROM mcp_routing_observations WHERE src_server_id=$1", [sid])).rows
for (const observation of observations) await attestRoutingFixture(observation.id)
const abort = new AbortController(); abort.abort()
await assert.rejects(prepareScopedCommunitySwitch({ profile: profile("a"), candidateId: "b" }, { principal, connection: await effectiveMcpConnection(principal), signal: abort.signal }))
reset(); const { plan: paused } = await prepare(); await setConnectionStatus(connectionId, "paused", writer)
await assert.rejects(applyPlan(principal, paused.planId, randomUUID())); assert.equal(writes, count)
assert.deepEqual((await pool.query("SELECT * FROM filter_rules WHERE server_id=$1 ORDER BY id", [sid])).rows, dbBefore)
assert.ok(!JSON.stringify((await pool.query("SELECT result,intended_state FROM mcp_operation_steps WHERE operation_id IN(SELECT id FROM mcp_operations WHERE connection_id=$1)", [connectionId])).rows).includes("NEVER_"))
console.log("✓ Scoped Route AI SQL+HTTP: two independent non-main tables/custom chains/probe VRFs, single-field one-PATCH, comment/opaque and SQL filters preserved, verified exact receipt/read-only reconcile, ignored/lost/collateral outcomes, endpoint ABA, late-intent stale SLA/epoch ABA/protection fences, pause/cancel and public privacy")
return { accepted: true, persistentAuthorityCases: persistentCases.length, persistentCases, manualRequestRevision: 2, persistentRequestRevision: 3, persistentCheckpointRevision: 2,
realMcpDispatcher: true, realSql: true, routerOsTransport: "loopback-http-fixture", scopedControllerEnabled: false, reservedInversePositiveAccepted: false, externalNetwork: false, forwardingVerified: false }
} catch(error) {
// Awaited assertion retains the original HTTP hook cause and stack.
if(requestFailure)throw requestFailure.error
throw error
} finally {
try {
pool.query = originalQuery
if (persistentProfileIds.length) await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=ANY($1::uuid[])", [persistentProfileIds])
if (connectionId) await setConnectionStatus(connectionId, "paused", writer)
if (insertedServer) { await pool.query("DELETE FROM mcp_management_channels WHERE server_id=$1", [sid]); await pool.query("DELETE FROM servers WHERE id=$1", [sid]) }
router.closeAllConnections(); alternate.closeAllConnections()
await Promise.all([new Promise<void>((resolve, reject) => router.close(error => error ? reject(error) : resolve())), new Promise<void>((resolve, reject) => alternate.close(error => error ? reject(error) : resolve()))])
} finally {
try {
if(workerFixtureLease){
try { if(workerLocked)workerUnsafe=(await workerFixtureLease.query("SELECT pg_advisory_unlock(hashtextextended('mcp-workers',0)) AS unlocked")).rows[0].unlocked!==true }
catch(error){workerUnsafe=true;throw error}
finally {workerFixtureLease.release(workerUnsafe)}
}
} finally { if(schedulerDrained&&!schedulerWasPaused)resumeSchedulerJobs() }
}
}
}