fix(mcp): уточнить переключение маршрутов и хранение операций
This commit is contained in:
1 parent
5ce56b5168
commit
bf4b9b5443
10 files changed
+293
-30
No files matched your search
@@ -0,0 +1,18 @@
|
|||||||
|
BEGIN;
|
||||||
|
CREATE OR REPLACE FUNCTION public.mcp_ordinary_retirement_claim_immutable() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
|
||||||
|
BEGIN
|
||||||
|
IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS(
|
||||||
|
SELECT 1 FROM public.mcp_operations o JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||||
|
WHERE o.id=NEW.operation_id AND o.plan_id=NEW.plan_id AND o.connection_id=NEW.connection_id
|
||||||
|
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
|
||||||
|
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||||
|
AND (p.operation_id,p.operation_version) IN (
|
||||||
|
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
|
||||||
|
('servers.api-ping.settings.update',2),('internet-path.settings.update',1),('geoip.settings.update',1)
|
||||||
|
) AND p.state='completed'
|
||||||
|
) THEN RETURN NEW; END IF;
|
||||||
|
RAISE EXCEPTION 'ordinary operation retirement claim cannot be replaced or reopened';
|
||||||
|
END $$;
|
||||||
|
INSERT INTO public.schema_migrations(id) VALUES ('0054_mcp_geoip_retirement') ON CONFLICT(id) DO NOTHING;
|
||||||
|
COMMIT;
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
BEGIN;
|
||||||
|
CREATE OR REPLACE FUNCTION public.mcp_ordinary_retirement_claim_immutable() RETURNS trigger
|
||||||
|
LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
|
||||||
|
BEGIN
|
||||||
|
IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS(
|
||||||
|
SELECT 1 FROM public.mcp_operations o JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||||
|
WHERE o.id=NEW.operation_id AND o.plan_id=NEW.plan_id AND o.connection_id=NEW.connection_id
|
||||||
|
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
|
||||||
|
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||||
|
AND (p.operation_id,p.operation_version) IN (
|
||||||
|
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
|
||||||
|
('servers.api-ping.settings.update',2),('internet-path.settings.update',1),('geoip.settings.update',1),('backups.schedule.update',1)
|
||||||
|
) AND p.state='completed'
|
||||||
|
) THEN RETURN NEW; END IF;
|
||||||
|
RAISE EXCEPTION 'ordinary operation retirement claim cannot be replaced or reopened';
|
||||||
|
END $$;
|
||||||
|
INSERT INTO public.schema_migrations(id) VALUES ('0055_mcp_backup_schedule_retirement') ON CONFLICT(id) DO NOTHING;
|
||||||
|
COMMIT;
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import EmbeddedPostgres from 'embedded-postgres'
|
import EmbeddedPostgres from 'embedded-postgres'
|
||||||
import { createRequire } from 'node:module'
|
import { createRequire } from 'node:module'
|
||||||
import { mkdir, mkdtemp, readdir, readFile, writeFile, cp, rm } from 'node:fs/promises'
|
import { mkdir, mkdtemp, readdir, readFile, writeFile, cp, rm } from 'node:fs/promises'
|
||||||
import { resolve, join, relative, dirname } from 'node:path'
|
import { resolve, join, relative, dirname, basename } from 'node:path'
|
||||||
import { randomBytes, createHash } from 'node:crypto'
|
import { randomBytes, createHash } from 'node:crypto'
|
||||||
import { createServer } from 'node:net'
|
import { createServer } from 'node:net'
|
||||||
import { spawn } from 'node:child_process'
|
import { spawn } from 'node:child_process'
|
||||||
@@ -16,7 +16,8 @@ await readFile(join(root, 'backend/src/modules/mcp/routing-main-transition.ts'))
|
|||||||
await readFile(join(root, 'backend/src/modules/mcp/routing-controller-baseline-acceptance.ts'))
|
await readFile(join(root, 'backend/src/modules/mcp/routing-controller-baseline-acceptance.ts'))
|
||||||
if (process.argv.includes('--check-only')) { console.log(JSON.stringify({ mode, sourcePresent: true, acceptanceExecuted: false, productionApplied: false })); process.exit(0) }
|
if (process.argv.includes('--check-only')) { console.log(JSON.stringify({ mode, sourcePresent: true, acceptanceExecuted: false, productionApplied: false })); process.exit(0) }
|
||||||
await mkdir(join(root, 'tmp'), { recursive: true })
|
await mkdir(join(root, 'tmp'), { recursive: true })
|
||||||
const directory = await mkdtemp(join(root, 'tmp/routing-controller-baseline-owned-')), output = join(directory, 'dist'), sourcePins = []
|
const ownedPrefix = 'mm-mcp-test-routing-controller-baseline-'
|
||||||
|
const directory = await mkdtemp(join(root, 'tmp', ownedPrefix)), output = join(directory, 'dist'), sourcePins = []
|
||||||
const hash = buffer => createHash('sha256').update(buffer).digest('hex')
|
const hash = buffer => createHash('sha256').update(buffer).digest('hex')
|
||||||
async function compile(sourceDirectory) {
|
async function compile(sourceDirectory) {
|
||||||
for (const entry of await readdir(sourceDirectory, { withFileTypes: true })) {
|
for (const entry of await readdir(sourceDirectory, { withFileTypes: true })) {
|
||||||
@@ -59,7 +60,7 @@ try {
|
|||||||
// embedded-postgres waits for a new exit event; avoid waiting again if its
|
// embedded-postgres waits for a new exit event; avoid waiting again if its
|
||||||
// owned child has already exited. This never targets another database.
|
// owned child has already exited. This never targets another database.
|
||||||
if (postgres.process?.exitCode === null) await postgres.stop()
|
if (postgres.process?.exitCode === null) await postgres.stop()
|
||||||
if (dirname(resolve(directory)) !== resolve(root, 'tmp') || !directory.includes('routing-controller-baseline-owned-')) throw new Error('Owned routing cleanup boundary changed')
|
if (dirname(resolve(directory)) !== resolve(root, 'tmp') || !basename(directory).startsWith(ownedPrefix)) throw new Error('Owned routing cleanup boundary changed')
|
||||||
await rm(join(directory, 'pg'), { recursive: true, force: true, maxRetries: 20, retryDelay: 100 })
|
await rm(join(directory, 'pg'), { recursive: true, force: true, maxRetries: 20, retryDelay: 100 })
|
||||||
console.log('Owned routing PostgreSQL stopped and its data removed; snapshot/log evidence retained')
|
console.log('Owned routing PostgreSQL stopped and its data removed; snapshot/log evidence retained')
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,10 +12,10 @@ const reviewed = (source: string, extra: Partial<MigrationReview> = {}) => {
|
|||||||
const bytes = Buffer.from(source), file = "9000_fixture.sql"
|
const bytes = Buffer.from(source), file = "9000_fixture.sql"
|
||||||
return () => analyzeMigration(file, bytes, { file, sha256: createHash("sha256").update(bytes).digest("hex"), transaction: "owner-transaction", selfMarker: false, opaqueBodies: 0, bodyReview: [], effects: ["Explicit inert fixture review"], ...extra })
|
return () => analyzeMigration(file, bytes, { file, sha256: createHash("sha256").update(bytes).digest("hex"), transaction: "owner-transaction", selfMarker: false, opaqueBodies: 0, bodyReview: [], effects: ["Explicit inert fixture review"], ...extra })
|
||||||
}
|
}
|
||||||
await test("all 53 exact file pins and every top-level schema operation are inventoried", () => {
|
await test("all 55 exact file pins and every top-level schema operation are inventoried", () => {
|
||||||
assert.equal(inventory.length, 53); assert.equal(MIGRATION_MANIFEST.length, 53)
|
assert.equal(inventory.length, 55); assert.equal(MIGRATION_MANIFEST.length, 55)
|
||||||
assert.equal(inventory.filter(file => file.transaction === "owner-transaction").length, 17)
|
assert.equal(inventory.filter(file => file.transaction === "owner-transaction").length, 17)
|
||||||
assert.equal(inventory.filter(file => file.transaction === "self-managed-single-transaction").length, 36)
|
assert.equal(inventory.filter(file => file.transaction === "self-managed-single-transaction").length, 38)
|
||||||
assert(inventory.every(file => file.operations.length === file.statementCount && file.effects.length > 0))
|
assert(inventory.every(file => file.operations.length === file.statementCount && file.effects.length > 0))
|
||||||
let tables = new Set<string>()
|
let tables = new Set<string>()
|
||||||
for (const file of inventory) {
|
for (const file of inventory) {
|
||||||
@@ -32,8 +32,8 @@ await test("DO BEGIN and stored function BEGIN are never transaction boundaries"
|
|||||||
assert.equal(endpoint.topLevelTransactionControls.length, 2); assert.equal(endpoint.opaqueBodies.length, 2)
|
assert.equal(endpoint.topLevelTransactionControls.length, 2); assert.equal(endpoint.opaqueBodies.length, 2)
|
||||||
assert.deepEqual(endpoint.opaqueBodies.map(body => body.kind), ["stored-function-body", "executed-do-body"])
|
assert.deepEqual(endpoint.opaqueBodies.map(body => body.kind), ["stored-function-body", "executed-do-body"])
|
||||||
})
|
})
|
||||||
await test("0035 through 0053 have exact atomic file-owned terminal markers", () => {
|
await test("0035 through 0055 have exact atomic file-owned terminal markers", () => {
|
||||||
assert.deepEqual(inventory.filter(file => file.selfMarker).map(file => file.id), ["0035_server_probe_journal", "0036_diagnostic_cancellations", "0037_server_probe_storage_v2", "0038_server_poll_journal_and_writer_guards", "0039_server_poll_storage", "0040_mcp_routing_reservations", "0041_server_poll_terminal_archives", "0042_server_poll_settings_epoch", "0043_mcp_routing_forward_charges", "0044_mcp_routing_manual_inverse_charges", "0045_mcp_operation_replay_tombstones", "0046_server_poll_operation_retirement", "0047_server_poll_retirement_claims", "0048_mcp_ordinary_retirement_claims", "0049_mcp_ordinary_telemetry_retirement", "0050_server_probe_retired_sources", "0051_server_probe_retirement_claims", "0052_server_probe_retired_gc", "0053_server_probe_terminal_proofs"])
|
assert.deepEqual(inventory.filter(file => file.selfMarker).map(file => file.id), ["0035_server_probe_journal", "0036_diagnostic_cancellations", "0037_server_probe_storage_v2", "0038_server_poll_journal_and_writer_guards", "0039_server_poll_storage", "0040_mcp_routing_reservations", "0041_server_poll_terminal_archives", "0042_server_poll_settings_epoch", "0043_mcp_routing_forward_charges", "0044_mcp_routing_manual_inverse_charges", "0045_mcp_operation_replay_tombstones", "0046_server_poll_operation_retirement", "0047_server_poll_retirement_claims", "0048_mcp_ordinary_retirement_claims", "0049_mcp_ordinary_telemetry_retirement", "0050_server_probe_retired_sources", "0051_server_probe_retirement_claims", "0052_server_probe_retired_gc", "0053_server_probe_terminal_proofs", "0054_mcp_geoip_retirement", "0055_mcp_backup_schedule_retirement"])
|
||||||
for (const file of inventory.filter(file => file.selfMarker)) { assert.equal(file.selfMarker!.atomicWithAllFileEffects, true); assert.equal(file.selfMarker!.statement, file.statementCount - 2); assert.equal(file.currentOuterMarkerGap, false) }
|
for (const file of inventory.filter(file => file.selfMarker)) { assert.equal(file.selfMarker!.atomicWithAllFileEffects, true); assert.equal(file.selfMarker!.statement, file.statementCount - 2); assert.equal(file.currentOuterMarkerGap, false) }
|
||||||
assert.equal(inventory.filter(file => file.currentOuterMarkerGap).length, 34)
|
assert.equal(inventory.filter(file => file.currentOuterMarkerGap).length, 34)
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -828,6 +828,32 @@ const classifiedMigrationPins: readonly MigrationReview[] = [
|
|||||||
"effects": [
|
"effects": [
|
||||||
"Creates empty bounded private terminal proof storage. Existing payload retention, scheduler, and archive GC remain unchanged."
|
"Creates empty bounded private terminal proof storage. Existing payload retention, scheduler, and archive GC remain unchanged."
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file": "0054_mcp_geoip_retirement.sql",
|
||||||
|
"sha256": "680390f98076e3047dbb5a34eab417b253853703932d16c17e4fa3a11060accc",
|
||||||
|
"transaction": "self-managed-single-transaction",
|
||||||
|
"selfMarker": true,
|
||||||
|
"opaqueBodies": 1,
|
||||||
|
"bodyReview": [
|
||||||
|
"Adds only exact settled GeoIP settings version 1 to the existing immutable ordinary retirement claim guard."
|
||||||
|
],
|
||||||
|
"effects": [
|
||||||
|
"Replaces the ordinary claim guard function without changing settings or retiring existing rows during migration."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file": "0055_mcp_backup_schedule_retirement.sql",
|
||||||
|
"sha256": "07e63c1245ea547176d613933a81ad354e734fa64218dadced67ee8f965c01b6",
|
||||||
|
"transaction": "self-managed-single-transaction",
|
||||||
|
"selfMarker": true,
|
||||||
|
"opaqueBodies": 1,
|
||||||
|
"bodyReview": [
|
||||||
|
"Adds only exact settled backup schedule settings version 1 to the existing immutable ordinary retirement claim guard."
|
||||||
|
],
|
||||||
|
"effects": [
|
||||||
|
"Replaces the ordinary claim guard function without changing backup settings, metadata or entries during migration."
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ const {tickMcpWorkers}=await import("./workers.js")
|
|||||||
const {pauseSchedulerJobsAndWait,resumeSchedulerJobs}=await import("../../services/scheduler-running.js")
|
const {pauseSchedulerJobsAndWait,resumeSchedulerJobs}=await import("../../services/scheduler-running.js")
|
||||||
const {Client}=await import("pg")
|
const {Client}=await import("pg")
|
||||||
const {McpError}=await import("./security.js")
|
const {McpError}=await import("./security.js")
|
||||||
|
const {touchGeoipRunMeta}=await import("../../services/geoip-settings.js")
|
||||||
|
const {touchBackupScheduleRunMeta}=await import("../../services/backup-service.js")
|
||||||
const errorIs=(code:string,status:number)=>(error:unknown)=>error instanceof McpError&&error.code===code&&error.statusCode===status
|
const errorIs=(code:string,status:number)=>(error:unknown)=>error instanceof McpError&&error.code===code&&error.statusCode===status
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -25,19 +27,20 @@ try {
|
|||||||
const serverId=Number((await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned logs retention','127.0.0.1',false) RETURNING id")).rows[0].id)
|
const serverId=Number((await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned logs retention','127.0.0.1',false) RETURNING id")).rows[0].id)
|
||||||
async function access(){
|
async function access(){
|
||||||
const actor={id:randomUUID(),email:"owned-logs-retirement@example.test",name:"Owned settings retention",apps:["mm"],
|
const actor={id:randomUUID(),email:"owned-logs-retirement@example.test",name:"Owned settings retention",apps:["mm"],
|
||||||
permissions:["mm:settings:admin","mm:mcp:approve","mm:traffic:write","mm:uptime:write","mm:servers:write","mm:network:write"],
|
permissions:["mm:settings:admin","mm:mcp:approve","mm:traffic:write","mm:uptime:write","mm:servers:write","mm:network:write","mm:backups:write"],
|
||||||
issuedAt:Date.now()-1000,expiresAt:Date.now()+300000}
|
issuedAt:Date.now()-1000,expiresAt:Date.now()+300000}
|
||||||
const issued=await createMcpConnection(mcpConnectionCreateSchema.parse({name:actor.name,mode:"approval",
|
const issued=await createMcpConnection(mcpConnectionCreateSchema.parse({name:actor.name,mode:"approval",
|
||||||
permissions:actor.permissions,expiresAt:new Date(actor.expiresAt).toISOString(),
|
permissions:actor.permissions,expiresAt:new Date(actor.expiresAt).toISOString(),
|
||||||
scope:{serverIds:[serverId],operationIds:["logs.settings.update","traffic.settings.update","uptime.settings.update",
|
scope:{serverIds:[serverId],operationIds:["logs.settings.update","traffic.settings.update","uptime.settings.update",
|
||||||
"servers.api-ping.settings.update","internet-path.settings.update","operations.read","changes.apply"],allowGlobal:true},
|
"servers.api-ping.settings.update","internet-path.settings.update","geoip.settings.update","backups.schedule.update","operations.read","changes.apply"],allowGlobal:true},
|
||||||
limits:{callsPerMinute:100,mutationsPerHour:10}}),actor)
|
limits:{callsPerMinute:100,mutationsPerHour:20}}),actor)
|
||||||
return {actor,principal:await authenticateMcpToken(issued.token)}
|
return {actor,principal:await authenticateMcpToken(issued.token)}
|
||||||
}
|
}
|
||||||
const {actor,principal}=await access(),foreign=(await access()).principal
|
const {actor,principal}=await access(),foreign=(await access()).principal
|
||||||
async function operation(retentionDays:number,ageMs:number,tool="mm_logs_settings_update_plan",
|
async function operation(retentionDays:number,ageMs:number,tool="mm_logs_settings_update_plan",
|
||||||
settings:Record<string,unknown>={retentionDays}){
|
settings:Record<string,unknown>={retentionDays}){
|
||||||
const plan=await invokeMcpTool(principal,tool,{settings}) as {planId:string;state:string}
|
const plan=await invokeMcpTool(principal,tool,{settings}) as {planId:string;state:string}
|
||||||
|
if(tool==="mm_geoip_settings_update_plan"||tool==="mm_backup_schedule_update_plan")assert.equal(plan.state,"awaiting-approval")
|
||||||
if(plan.state==="awaiting-approval")await approvePlan(plan.planId,actor,true)
|
if(plan.state==="awaiting-approval")await approvePlan(plan.planId,actor,true)
|
||||||
const key=randomUUID(),applied=await applyPlan(principal,plan.planId,key)
|
const key=randomUUID(),applied=await applyPlan(principal,plan.planId,key)
|
||||||
let terminal
|
let terminal
|
||||||
@@ -141,8 +144,83 @@ try {
|
|||||||
await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
|
await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
|
||||||
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:record.id}),errorIs("NOT_FOUND",404))
|
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:record.id}),errorIs("NOT_FOUND",404))
|
||||||
}
|
}
|
||||||
|
// Keep the owned fixture offline even if a product scheduler refresh occurs.
|
||||||
|
await pool.query("UPDATE geoip_settings SET enabled=false WHERE id=1")
|
||||||
|
await touchGeoipRunMeta({lastCheckAt:new Date().toISOString(),lastError:"owned GeoIP metadata",etags:{"owned-test":"unchanged"}})
|
||||||
|
const geoip=await operation(0,age+60000,"mm_geoip_settings_update_plan",{enabled:false,updateIntervalSec:86400})
|
||||||
|
const geoipState=async()=>(await pool.query("SELECT to_jsonb(g) AS row FROM geoip_settings g WHERE id=1")).rows[0].row
|
||||||
|
const geoipBefore=await geoipState()
|
||||||
|
assert.equal(geoipBefore.enabled,false);assert.equal(geoipBefore.update_interval_sec,86400)
|
||||||
|
assert.equal(geoipBefore.last_error,"owned GeoIP metadata");assert.deepEqual(geoipBefore.etags_json,{"owned-test":"unchanged"})
|
||||||
|
const {rows:[geoipStep]}=await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1",[geoip.id])
|
||||||
|
const assertGeoipUnclaimed=async()=>{
|
||||||
|
assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[geoip.id])).rows.length,0)
|
||||||
|
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[geoip.id])).rows.length,1)
|
||||||
|
assert.deepEqual(await geoipState(),geoipBefore)
|
||||||
|
}
|
||||||
|
await pool.query("UPDATE mcp_operation_steps SET result=jsonb_set(result,'{settings,updateIntervalSec}','172800') WHERE operation_id=$1",[geoip.id])
|
||||||
|
await tickMcpWorkers();await tickMcpWorkers();await assertGeoipUnclaimed()
|
||||||
|
await pool.query("UPDATE mcp_operation_steps SET result=$2 WHERE operation_id=$1",[geoip.id,geoipStep.result])
|
||||||
|
const geoipHold=randomUUID()
|
||||||
|
await pool.query("INSERT INTO mcp_drift(id,operation_id,expected,observed) VALUES($1,$2,'{}','{}')",[geoipHold,geoip.id])
|
||||||
|
await tickMcpWorkers();await assertGeoipUnclaimed()
|
||||||
|
await pool.query("DELETE FROM mcp_drift WHERE id=$1",[geoipHold])
|
||||||
|
await tickMcpWorkers();await tickMcpWorkers()
|
||||||
|
const {rows:[geoipTombstone]}=await pool.query(`SELECT t.action_id,t.target_global,t.target_server_ids::text[] AS server_ids,
|
||||||
|
NOT EXISTS(SELECT 1 FROM mcp_operations WHERE id=$1) AS operation_gone,
|
||||||
|
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan_gone,
|
||||||
|
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step_gone,
|
||||||
|
NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint_gone
|
||||||
|
FROM mcp_operation_replay_tombstones t WHERE t.operation_id=$1 AND t.plan_id=$2`,[geoip.id,geoip.planId])
|
||||||
|
assert.deepEqual(geoipTombstone,{action_id:"geoip.settings.update",target_global:true,server_ids:[String(serverId)],
|
||||||
|
operation_gone:true,plan_gone:true,step_gone:true,checkpoint_gone:true})
|
||||||
|
await assert.rejects(applyPlan(principal,geoip.planId,geoip.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
|
||||||
|
await assert.rejects(applyPlan(principal,randomUUID(),geoip.key),errorIs("STATE_CONFLICT",409))
|
||||||
|
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:geoip.id}),errorIs("NOT_FOUND",404))
|
||||||
|
assert.deepEqual(await geoipState(),geoipBefore)
|
||||||
|
await pool.query("UPDATE backup_schedule_settings SET enabled=false,server_ids_json='[]' WHERE id=1")
|
||||||
|
await touchBackupScheduleRunMeta({lastRunAt:new Date().toISOString(),lastDurationMs:123,lastError:"owned backup scheduler metadata"})
|
||||||
|
await pool.query(`INSERT INTO backup_entries(id,server_id,server_name,filename,size_bytes,kind,notes,created_at)
|
||||||
|
VALUES($1,$2,'owned fixture','owned-retention.rsc',123,'auto','keep',clock_timestamp())`,[randomUUID(),serverId])
|
||||||
|
const backupState=async()=>(await pool.query(`SELECT to_jsonb(s) AS schedule,
|
||||||
|
(SELECT jsonb_agg(to_jsonb(b) ORDER BY id) FROM backup_entries b) AS entries
|
||||||
|
FROM backup_schedule_settings s WHERE id=1`)).rows[0]
|
||||||
|
const backupOutside=await operation(0,age+120000,"mm_backup_schedule_update_plan",{enabled:false,serverIds:[String(serverId+1000)]})
|
||||||
|
const outsideState=await backupState()
|
||||||
|
await tickMcpWorkers();await tickMcpWorkers()
|
||||||
|
assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[backupOutside.id])).rows.length,0)
|
||||||
|
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[backupOutside.id])).rows.length,1)
|
||||||
|
assert.deepEqual(await backupState(),outsideState)
|
||||||
|
await pool.query("UPDATE backup_schedule_settings SET server_ids_json='[]' WHERE id=1")
|
||||||
|
const backup=await operation(0,age+60000,"mm_backup_schedule_update_plan",{enabled:false,keepCount:9,serverIds:[String(serverId)]})
|
||||||
|
const backupBefore=await backupState()
|
||||||
|
assert.equal(backupBefore.schedule.enabled,false);assert.equal(backupBefore.schedule.keep_count,9)
|
||||||
|
assert.deepEqual(backupBefore.schedule.server_ids_json,[String(serverId)])
|
||||||
|
assert.equal(backupBefore.schedule.last_duration_ms,123);assert.equal(backupBefore.schedule.last_error,"owned backup scheduler metadata")
|
||||||
|
assert.deepEqual(backupBefore.entries,outsideState.entries)
|
||||||
|
const {rows:[backupStep]}=await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1",[backup.id])
|
||||||
|
await pool.query("UPDATE mcp_operation_steps SET result=jsonb_set(result,'{keepCount}','10') WHERE operation_id=$1",[backup.id])
|
||||||
|
await tickMcpWorkers();await tickMcpWorkers()
|
||||||
|
assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[backup.id])).rows.length,0)
|
||||||
|
assert.deepEqual(await backupState(),backupBefore)
|
||||||
|
await pool.query("UPDATE mcp_operation_steps SET result=$2 WHERE operation_id=$1",[backup.id,backupStep.result])
|
||||||
|
await tickMcpWorkers();await tickMcpWorkers()
|
||||||
|
const {rows:[backupTombstone]}=await pool.query(`SELECT t.action_id,t.target_global,t.target_server_ids::text[] AS server_ids,
|
||||||
|
NOT EXISTS(SELECT 1 FROM mcp_operations WHERE id=$1) AS operation_gone,
|
||||||
|
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan_gone,
|
||||||
|
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step_gone,
|
||||||
|
NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint_gone
|
||||||
|
FROM mcp_operation_replay_tombstones t WHERE t.operation_id=$1 AND t.plan_id=$2`,[backup.id,backup.planId])
|
||||||
|
assert.deepEqual(backupTombstone,{action_id:"backups.schedule.update",target_global:true,server_ids:[String(serverId)],
|
||||||
|
operation_gone:true,plan_gone:true,step_gone:true,checkpoint_gone:true})
|
||||||
|
await assert.rejects(applyPlan(principal,backup.planId,backup.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
|
||||||
|
await assert.rejects(applyPlan(principal,randomUUID(),backup.key),errorIs("STATE_CONFLICT",409))
|
||||||
|
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:backup.id}),errorIs("NOT_FOUND",404))
|
||||||
|
assert.deepEqual(await backupState(),backupBefore)
|
||||||
await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{scope,serverIds}','[]') WHERE id=$1",[principal.connectionId])
|
await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{scope,serverIds}','[]') WHERE id=$1",[principal.connectionId])
|
||||||
await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("NOT_FOUND",404))
|
await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("NOT_FOUND",404))
|
||||||
for(const {record} of telemetry)await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("NOT_FOUND",404))
|
for(const {record} of telemetry)await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("NOT_FOUND",404))
|
||||||
console.log("Owned settings operations: five exact families, authenticated apply, physical 180-day purge, preserved settings and scoped replay")
|
await assert.rejects(applyPlan(principal,geoip.planId,geoip.key),errorIs("NOT_FOUND",404))
|
||||||
|
await assert.rejects(applyPlan(principal,backup.planId,backup.key),errorIs("NOT_FOUND",404))
|
||||||
|
console.log("Owned settings operations: seven exact families, authenticated apply, physical 180-day purge, preserved settings and scoped replay")
|
||||||
}finally{resumeSchedulerJobs();await closePool()}
|
}finally{resumeSchedulerJobs();await closePool()}
|
||||||
@@ -1,5 +1,7 @@
|
|||||||
import { randomUUID } from "node:crypto"
|
import { randomUUID } from "node:crypto"
|
||||||
import { logSettingsPatchSchema, logSettingsSchema } from "@mmapp/contracts"
|
import { logSettingsPatchSchema, logSettingsSchema } from "@mmapp/contracts"
|
||||||
|
import { geoipSettingsDtoSchema, geoipSettingsPatchSchema } from "@mmapp/contracts/geoip"
|
||||||
|
import { backupScheduleSettingsDtoSchema, putBackupScheduleSettingsSchema } from "@mmapp/contracts/backups"
|
||||||
import { z } from "zod"
|
import { z } from "zod"
|
||||||
import { connectProbeStorage } from "../../services/server-probe-storage-connection.js"
|
import { connectProbeStorage } from "../../services/server-probe-storage-connection.js"
|
||||||
import type { ProbeStorageSql } from "../../services/server-probe-journal-maintenance.js"
|
import type { ProbeStorageSql } from "../../services/server-probe-journal-maintenance.js"
|
||||||
@@ -15,6 +17,10 @@ const ids = z.array(positive).max(1000)
|
|||||||
const retention = z.number().int().min(1).max(365)
|
const retention = z.number().int().min(1).max(365)
|
||||||
const bool = z.boolean()
|
const bool = z.boolean()
|
||||||
const interval = (min:number,max:number)=>z.number().int().min(min).max(max)
|
const interval = (min:number,max:number)=>z.number().int().min(min).max(max)
|
||||||
|
const backupServerIds=z.array(z.string().regex(/^[1-9]\d{0,15}$/).refine(value=>Number.isSafeInteger(Number(value))))
|
||||||
|
.max(1000).refine(values=>new Set(values).size===values.length)
|
||||||
|
const backupSchedule=backupScheduleSettingsDtoSchema.extend({serverIds:backupServerIds}).strict()
|
||||||
|
const backupBefore=z.object({value:backupSchedule,serverIds:ids}).strict()
|
||||||
// Drizzle's string-mode timestamptz keeps PostgreSQL's space and hour-only
|
// Drizzle's string-mode timestamptz keeps PostgreSQL's space and hour-only
|
||||||
// offset. Accept that exact wire form as well as an ISO timestamptz.
|
// offset. Accept that exact wire form as well as an ISO timestamptz.
|
||||||
const timestamp = z.string().regex(/^\d{4}-\d\d-\d\d[ T]\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d(?::\d\d)?)$/)
|
const timestamp = z.string().regex(/^\d{4}-\d\d-\d\d[ T]\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d(?::\d\d)?)$/)
|
||||||
@@ -37,6 +43,18 @@ const families={
|
|||||||
speedEnabled:bool,intervalSec:interval(5,86400),probeIntervalSec:interval(5,86400),speedIntervalSec:interval(10,86400),retentionDays:retention}),true,true),
|
speedEnabled:bool,intervalSec:interval(5,86400),probeIntervalSec:interval(5,86400),speedIntervalSec:interval(10,86400),retentionDays:retention}),true,true),
|
||||||
"servers.api-ping.settings.update":telemetryFamily(2,"servers-api-ping-settings",z.object({enabled:bool,intervalSec:interval(10,86400)}),false,true),
|
"servers.api-ping.settings.update":telemetryFamily(2,"servers-api-ping-settings",z.object({enabled:bool,intervalSec:interval(10,86400)}),false,true),
|
||||||
"internet-path.settings.update":telemetryFamily(1,"internet-path-settings",z.object({enabled:bool,intervalSec:interval(30,86400),retentionDays:retention}),true,false),
|
"internet-path.settings.update":telemetryFamily(1,"internet-path-settings",z.object({enabled:bool,intervalSec:interval(30,86400),retentionDays:retention}),true,false),
|
||||||
|
"geoip.settings.update":{
|
||||||
|
version:1,domain:"geoip-settings",
|
||||||
|
patch:geoipSettingsPatchSchema.strict().refine(value=>Object.keys(value).length>0),
|
||||||
|
before:z.object({value:z.object({enabled:bool,updateIntervalSec:positive}).strict(),serverIds:ids}).strict(),
|
||||||
|
effect:z.object({settings:geoipSettingsDtoSchema.strict(),verificationScope:z.literal("configuration-persistence")})
|
||||||
|
.strict().transform(result=>result.settings),
|
||||||
|
},
|
||||||
|
"backups.schedule.update":{
|
||||||
|
version:1,domain:"backup-schedule",
|
||||||
|
patch:putBackupScheduleSettingsSchema.extend({serverIds:backupServerIds.optional()}).strict().refine(value=>Object.keys(value).length>0),
|
||||||
|
before:backupBefore,effect:backupSchedule,
|
||||||
|
},
|
||||||
} as const
|
} as const
|
||||||
const planEvidence = z.object({
|
const planEvidence = z.object({
|
||||||
target:z.object({scope:z.literal("global")}).strict(),
|
target:z.object({scope:z.literal("global")}).strict(),
|
||||||
@@ -64,7 +82,7 @@ const ELIGIBLE = `FROM public.mcp_operations o
|
|||||||
JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=0
|
JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=0
|
||||||
WHERE (p.operation_id,p.operation_version) IN (
|
WHERE (p.operation_id,p.operation_version) IN (
|
||||||
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
|
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
|
||||||
('servers.api-ping.settings.update',2),('internet-path.settings.update',1)
|
('servers.api-ping.settings.update',2),('internet-path.settings.update',1),('geoip.settings.update',1),('backups.schedule.update',1)
|
||||||
) AND p.state='completed'
|
) AND p.state='completed'
|
||||||
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
|
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
|
||||||
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||||
@@ -128,7 +146,7 @@ async function prepare(sql:ProbeStorageSql, operationId:string, connectionId:str
|
|||||||
requireThat(fingerprint(step.intended_state)===fingerprint(redact(service)))
|
requireThat(fingerprint(step.intended_state)===fingerprint(redact(service)))
|
||||||
const patch=object.parse(family.patch.parse(service.input.settings))
|
const patch=object.parse(family.patch.parse(service.input.settings))
|
||||||
const effect=object.parse(family.effect.parse(step.result))
|
const effect=object.parse(family.effect.parse(step.result))
|
||||||
requireThat(Object.entries(patch).every(([key,value])=>effect[key]===value))
|
requireThat(Object.entries(patch).every(([key,value])=>fingerprint(effect[key])===fingerprint(value)))
|
||||||
requireThat(fingerprint(operation.result)===fingerprint({confirmedWrites:1,verificationScope:"configuration"}))
|
requireThat(fingerprint(operation.result)===fingerprint({confirmedWrites:1,verificationScope:"configuration"}))
|
||||||
for(const value of [checkpoint.before_sealed,checkpoint.after_sealed]){
|
for(const value of [checkpoint.before_sealed,checkpoint.after_sealed]){
|
||||||
requireThat(typeof value==="string"&&value.length<=1024&&fingerprint(await open({sealed:value}))===fingerprint([]))
|
requireThat(typeof value==="string"&&value.length<=1024&&fingerprint(await open({sealed:value}))===fingerprint([]))
|
||||||
@@ -145,6 +163,10 @@ async function prepare(sql:ProbeStorageSql, operationId:string, connectionId:str
|
|||||||
&&domainObjects.length===1&&domainObjects[0].kind===family.domain&&domainObjects[0].id==="1"&&domainObjects[0].serverId===undefined)
|
&&domainObjects.length===1&&domainObjects[0].kind===family.domain&&domainObjects[0].id==="1"&&domainObjects[0].serverId===undefined)
|
||||||
requireThat("serverIds" in before&&fingerprint(before.serverIds)===fingerprint(serverIds))
|
requireThat("serverIds" in before&&fingerprint(before.serverIds)===fingerprint(serverIds))
|
||||||
}
|
}
|
||||||
|
if(action==="backups.schedule.update"){
|
||||||
|
const scope=new Set(serverIds.map(String))
|
||||||
|
requireThat([...backupBefore.parse(before).value.serverIds,...backupSchedule.parse(effect).serverIds].every(id=>scope.has(id)))
|
||||||
|
}
|
||||||
const signer=await privateFingerprinterExisting("mcp-ordinary-retirement-source-v1")
|
const signer=await privateFingerprinterExisting("mcp-ordinary-retirement-source-v1")
|
||||||
return {planId:uuid.parse(plan.id),action,serverIds,key:z.string().min(1).max(200).parse(operation.idempotency_key),proof:signer(fingerprint(raw))}
|
return {planId:uuid.parse(plan.id),action,serverIds,key:z.string().min(1).max(200).parse(operation.idempotency_key),proof:signer(fingerprint(raw))}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,14 +43,43 @@ test("new unknown attempts block older successful data independently for each me
|
|||||||
test("failover requires three distinct failures and a freshly verified eligible standby", () => {
|
test("failover requires three distinct failures and a freshly verified eligible standby", () => {
|
||||||
const ranking = rankRoutingCandidates(profile, [...a, sample("b1", 1, 4, "wan-b"), sample("b2", 4, 3, "exit-b")], now)
|
const ranking = rankRoutingCandidates(profile, [...a, sample("b1", 1, 4, "wan-b"), sample("b2", 4, 3, "exit-b")], now)
|
||||||
let state: ControllerState = {}
|
let state: ControllerState = {}
|
||||||
const health = (id: string, status: PathHealth["status"] = "failed"): PathHealth[] => [{ id, candidateId: "a", status, observedAt: at }, { id: "reserve", candidateId: "b", status: "passed", observedAt: at }]
|
const health = (id: string, time = now, status: PathHealth["status"] = "failed"): PathHealth[] => {
|
||||||
|
const observedAt = new Date(time).toISOString()
|
||||||
|
return [{ id, candidateId: "a", status, observedAt }, { id: "reserve", candidateId: "b", status: "passed", observedAt }]
|
||||||
|
}
|
||||||
for (let i = 0; i < 5; i++) { const next = decideRoutingSwitch(profile, state, ranking, "a", now, health("one")); state = next.state; assert.equal(next.switchTo, null) }
|
for (let i = 0; i < 5; i++) { const next = decideRoutingSwitch(profile, state, ranking, "a", now, health("one")); state = next.state; assert.equal(next.switchTo, null) }
|
||||||
assert.equal(state.failureCount, 1)
|
assert.equal(state.failureCount, 1)
|
||||||
const second = decideRoutingSwitch(profile, state, ranking, "a", now, health("two")); assert.equal(second.switchTo, null)
|
const secondAt = now + 60000, thirdAt = now + 120000
|
||||||
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", now, health("three")).switchTo, "b")
|
const second = decideRoutingSwitch(profile, state, ranking, "a", secondAt, health("two", secondAt)); assert.equal(second.switchTo, null)
|
||||||
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", now, health("three", "unknown")).switchTo, null)
|
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", thirdAt, health("three", thirdAt)).switchTo, "b")
|
||||||
assert.equal(decideRoutingSwitch(profile, { ...second.state, switches: [at, at, at] }, ranking, "a", now, health("three")).switchTo, null)
|
const unknown = decideRoutingSwitch(profile, second.state, ranking, "a", thirdAt, health("three", thirdAt, "unknown"))
|
||||||
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", now + 100000, health("three")).switchTo, null)
|
assert.equal(unknown.switchTo, null)
|
||||||
|
assert.equal(unknown.state.failureObservedAt, undefined)
|
||||||
|
assert.equal(decideRoutingSwitch(profile, { ...second.state, switches: [at, at, at] }, ranking, "a", thirdAt, health("three", thirdAt)).switchTo, null)
|
||||||
|
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", thirdAt + 100000, health("three", thirdAt)).switchTo, null)
|
||||||
|
})
|
||||||
|
test("failover starts a fresh series after downtime or legacy state and ignores reordered failures", () => {
|
||||||
|
const ranking = rankRoutingCandidates(profile, [...a, sample("b1", 1, 4, "wan-b"), sample("b2", 4, 3, "exit-b")], now)
|
||||||
|
const last = now + 60000
|
||||||
|
const persisted: ControllerState = JSON.parse(JSON.stringify({ failureCount: 2, failureKey: "two",
|
||||||
|
failureObservedAt: new Date(last).toISOString(), failingCandidateId: "a" }))
|
||||||
|
const check = (state: ControllerState, id: string, time: number, clock = time) => {
|
||||||
|
const observedAt = new Date(time).toISOString()
|
||||||
|
return decideRoutingSwitch(profile, state, ranking, "a", clock,
|
||||||
|
[{ id, candidateId: "a", status: "failed", observedAt }, { id: "reserve", candidateId: "b", status: "passed", observedAt }])
|
||||||
|
}
|
||||||
|
for (const result of [check(persisted, "two", last + 60000), check(persisted, "older", last - 1000, last),
|
||||||
|
check(persisted, "clock-rollback", last - 1000), check(persisted, "equal-time", last)]) {
|
||||||
|
assert.equal(result.switchTo, null)
|
||||||
|
assert.equal(result.state.failureCount, 2)
|
||||||
|
assert.equal(result.state.failureObservedAt, persisted.failureObservedAt)
|
||||||
|
}
|
||||||
|
for (const [state, time] of [[persisted, now + 4 * 3600000], [{ ...persisted, failureObservedAt: undefined }, last + 60000]] as const) {
|
||||||
|
const result = check(state, "next", time)
|
||||||
|
assert.equal(result.switchTo, null)
|
||||||
|
assert.equal(result.state.failureCount, 1)
|
||||||
|
assert.equal(result.state.failureObservedAt, new Date(time).toISOString())
|
||||||
|
}
|
||||||
})
|
})
|
||||||
test("routing combines RTT, independent losses and bottleneck speed, never claims an observed path", () => {
|
test("routing combines RTT, independent losses and bottleneck speed, never claims an observed path", () => {
|
||||||
const ranked = rankRoutingCandidates(profile, a, now).find(row => row.candidate.id === "a")!
|
const ranked = rankRoutingCandidates(profile, a, now).find(row => row.candidate.id === "a")!
|
||||||
@@ -92,3 +121,63 @@ test("controller does not count polling the same sample as consecutive confirmat
|
|||||||
assert.equal(decideRoutingSwitch(profile, { ...state, switches: [at, at, at] }, ranks, "a", now).switchTo, null)
|
assert.equal(decideRoutingSwitch(profile, { ...state, switches: [at, at, at] }, ranks, "a", now).switchTo, null)
|
||||||
assert.equal(decideRoutingSwitch(profile, state, ranks, null, now).switchTo, null)
|
assert.equal(decideRoutingSwitch(profile, state, ranks, null, now).switchTo, null)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("hysteresis retains pending confirmation without counting sub-threshold observations or releasing cooldown", () => {
|
||||||
|
const ranked = rankRoutingCandidates(profile, [...a, sample("b1", 1, 4, "wan-b"), sample("b2", 4, 3, "exit-b")], now)
|
||||||
|
const compare = (state: ControllerState, gain: number, observation: string, time = now) => {
|
||||||
|
const ranks = ["b", "a"].map(id => ({ ...ranked.find(row => row.candidate.id === id)!,
|
||||||
|
score: id === "a" ? 50 : 50 + gain, observationIds: [id + observation] }))
|
||||||
|
return decideRoutingSwitch(profile, state, ranks, "a", time)
|
||||||
|
}
|
||||||
|
|
||||||
|
const initial = compare({}, 15, "1").state
|
||||||
|
assert.equal(initial.consecutive, 1)
|
||||||
|
for (const [gain, observation] of [[6, "2"], [5, "3"]] as const) {
|
||||||
|
const result = compare(initial, gain, observation)
|
||||||
|
assert.equal(result.switchTo, null)
|
||||||
|
assert.deepEqual(result.state, initial)
|
||||||
|
}
|
||||||
|
const reset = compare(initial, 4, "4")
|
||||||
|
assert.equal(reset.state.pendingCandidateId, undefined)
|
||||||
|
assert.equal(reset.state.consecutive, 0)
|
||||||
|
|
||||||
|
const second = compare(initial, 15, "5")
|
||||||
|
assert.equal(second.switchTo, null)
|
||||||
|
const third = compare({ ...second.state, lastSwitchAt: at }, 15, "6")
|
||||||
|
assert.equal(third.state.consecutive, 3)
|
||||||
|
assert.equal(third.switchTo, null)
|
||||||
|
|
||||||
|
const afterCooldown = now + profile.cooldownSec * 1000
|
||||||
|
assert.equal(compare(third.state, 6, "7", afterCooldown).switchTo, null)
|
||||||
|
assert.equal(compare(third.state, 15, "8", afterCooldown).switchTo, "b")
|
||||||
|
})
|
||||||
|
|
||||||
|
test("a different pending candidate cannot borrow confirmations from a shared observation key", () => {
|
||||||
|
const ranks = rankRoutingCandidates(profile, [...a, sample("b1", 1, 4, "wan-b"), sample("b2", 4, 3, "exit-b")], now)
|
||||||
|
.map(row => ({ ...row, score: row.candidate.id === "a" ? 50 : 75 }))
|
||||||
|
.sort((left, right) => right.score - left.score)
|
||||||
|
const observationKey = ranks.flatMap(row => row.observationIds).sort().join(":")
|
||||||
|
const persisted: ControllerState = { pendingCandidateId: "previous-candidate", consecutive: 3, observationKey }
|
||||||
|
const selected = decideRoutingSwitch(profile, persisted, ranks, "a", now)
|
||||||
|
assert.equal(selected.switchTo, null)
|
||||||
|
assert.equal(selected.state.pendingCandidateId, "b")
|
||||||
|
assert.equal(selected.state.consecutive, 0)
|
||||||
|
assert.equal(selected.state.observationKey, observationKey)
|
||||||
|
|
||||||
|
const replay = decideRoutingSwitch(profile, selected.state, ranks, "a", now)
|
||||||
|
assert.equal(replay.switchTo, null)
|
||||||
|
assert.equal(replay.state.consecutive, 0)
|
||||||
|
|
||||||
|
const freshRanks = [{ ...ranks[0], observationIds: ["new-b-1", "new-b-2"] }, ranks[1]]
|
||||||
|
const fresh = decideRoutingSwitch(profile, replay.state, freshRanks, "a", now)
|
||||||
|
assert.equal(fresh.state.consecutive, 1)
|
||||||
|
assert.equal(fresh.switchTo, null)
|
||||||
|
|
||||||
|
const belowFloor = [{ ...freshRanks[0], score: 54 }, freshRanks[1]]
|
||||||
|
const reset = decideRoutingSwitch(profile, fresh.state, belowFloor, "a", now)
|
||||||
|
assert.equal(reset.state.pendingCandidateId, undefined)
|
||||||
|
assert.equal(reset.state.consecutive, 0)
|
||||||
|
const replayAfterReset = decideRoutingSwitch(profile, reset.state, freshRanks, "a", now)
|
||||||
|
assert.equal(replayAfterReset.switchTo, null)
|
||||||
|
assert.equal(replayAfterReset.state.consecutive, 0)
|
||||||
|
})
|
||||||
@@ -51,36 +51,47 @@ export function rankRoutingCandidates(profile: RouteAiProfile, observations: Rou
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface PathHealth { id: string; candidateId: string; status: "passed" | "failed" | "unknown"; observedAt: string }
|
export interface PathHealth { id: string; candidateId: string; status: "passed" | "failed" | "unknown"; observedAt: string }
|
||||||
export interface ControllerState { pendingCandidateId?: string; consecutive?: number; observationKey?: string; lastSwitchAt?: string; switches?: string[]; operationId?: string; pausedReason?: string; failureCount?: number; failureKey?: string; failingCandidateId?: string }
|
export interface ControllerState { pendingCandidateId?: string; consecutive?: number; observationKey?: string; lastSwitchAt?: string; switches?: string[]; operationId?: string; pausedReason?: string; failureCount?: number; failureKey?: string; failureObservedAt?: string; failingCandidateId?: string }
|
||||||
|
const pathHealthFreshnessSec = 90
|
||||||
export function decideRoutingSwitch(profile: RouteAiProfile, state: ControllerState, ranking: ReturnType<typeof rankRoutingCandidates>, currentCandidateId: string | null, now = Date.now(), health: PathHealth[] = [], chargedSwitchesPerHour?: number) {
|
export function decideRoutingSwitch(profile: RouteAiProfile, state: ControllerState, ranking: ReturnType<typeof rankRoutingCandidates>, currentCandidateId: string | null, now = Date.now(), health: PathHealth[] = [], chargedSwitchesPerHour?: number) {
|
||||||
const next: ControllerState = { ...state, switches: (state.switches ?? []).filter(at => now - Date.parse(at) < 3600000) }
|
const next: ControllerState = { ...state, switches: (state.switches ?? []).filter(at => now - Date.parse(at) < 3600000) }
|
||||||
if (chargedSwitchesPerHour !== undefined && (!Number.isSafeInteger(chargedSwitchesPerHour) || chargedSwitchesPerHour < next.switches!.length || chargedSwitchesPerHour > 4096)) throw Error("Invalid common routing charge count")
|
if (chargedSwitchesPerHour !== undefined && (!Number.isSafeInteger(chargedSwitchesPerHour) || chargedSwitchesPerHour < next.switches!.length || chargedSwitchesPerHour > 4096)) throw Error("Invalid common routing charge count")
|
||||||
const switchCount = chargedSwitchesPerHour ?? next.switches!.length
|
const switchCount = chargedSwitchesPerHour ?? next.switches!.length
|
||||||
const best = ranking.find(candidate => candidate.eligible), current = ranking.find(candidate => candidate.candidate.id === currentCandidateId)
|
const best = ranking.find(candidate => candidate.eligible), current = ranking.find(candidate => candidate.candidate.id === currentCandidateId)
|
||||||
const currentHealth = health.find(value => value.candidateId === currentCandidateId && fresh(value.observedAt, 90, now))
|
const currentHealth = health.find(value => value.candidateId === currentCandidateId && fresh(value.observedAt, pathHealthFreshnessSec, now))
|
||||||
if (currentHealth?.status === "failed" && currentCandidateId && !state.pausedReason) {
|
if (currentHealth?.status === "failed" && currentCandidateId && !state.pausedReason) {
|
||||||
if (state.failureKey !== currentHealth.id) {
|
const observed = Date.parse(currentHealth.observedAt), previous = Date.parse(state.failureObservedAt ?? "")
|
||||||
next.failureCount = state.failingCandidateId === currentCandidateId ? (state.failureCount ?? 0) + 1 : 1
|
const sameCandidate = state.failingCandidateId === currentCandidateId
|
||||||
next.failureKey = currentHealth.id; next.failingCandidateId = currentCandidateId
|
if (state.failureKey === currentHealth.id) {
|
||||||
|
if (!sameCandidate || observed !== previous) return { state: next, switchTo: null, reason: "Ожидание новой проверки отказа" }
|
||||||
|
} else {
|
||||||
|
if (observed <= previous) return { state: next, switchTo: null, reason: "Ожидание новой проверки отказа" }
|
||||||
|
const continues = sameCandidate && fresh(state.failureObservedAt ?? null, pathHealthFreshnessSec, observed)
|
||||||
|
next.failureCount = continues ? (state.failureCount ?? 0) + 1 : 1
|
||||||
|
next.failureKey = currentHealth.id; next.failureObservedAt = currentHealth.observedAt; next.failingCandidateId = currentCandidateId
|
||||||
}
|
}
|
||||||
const reserve = ranking.find(value => value.eligible && value.candidate.id !== currentCandidateId && health.some(check => check.candidateId === value.candidate.id && check.status === "passed" && fresh(check.observedAt, 90, now)))
|
const reserve = ranking.find(value => value.eligible && value.candidate.id !== currentCandidateId && health.some(check => check.candidateId === value.candidate.id && check.status === "passed" && fresh(check.observedAt, pathHealthFreshnessSec, now)))
|
||||||
if ((next.failureCount ?? 0) < 3 || !reserve) return { state: next, switchTo: null, reason: "Failover ожидает три отдельных отказа и подтверждённый резерв" }
|
if ((next.failureCount ?? 0) < 3 || !reserve) return { state: next, switchTo: null, reason: "Failover ожидает три отдельных отказа и подтверждённый резерв" }
|
||||||
if (switchCount >= profile.maxSwitchesPerHour) return { state: { ...next, pausedReason: "Лимит переключений" }, switchTo: null, reason: "Исчерпан часовой бюджет, включая failover" }
|
if (switchCount >= profile.maxSwitchesPerHour) return { state: { ...next, pausedReason: "Лимит переключений" }, switchTo: null, reason: "Исчерпан часовой бюджет, включая failover" }
|
||||||
if (next.lastSwitchAt && now - Date.parse(next.lastSwitchAt) < profile.cooldownSec * 1000) return { state: next, switchTo: null, reason: "Failover ожидает окончания cooldown" }
|
if (next.lastSwitchAt && now - Date.parse(next.lastSwitchAt) < profile.cooldownSec * 1000) return { state: next, switchTo: null, reason: "Failover ожидает окончания cooldown" }
|
||||||
return { state: { ...next, failureCount: 0 }, switchTo: reserve.candidate.id, reason: "Failover: три подтверждённых отказа и проверенный резерв" }
|
return { state: { ...next, failureCount: 0 }, switchTo: reserve.candidate.id, reason: "Failover: три подтверждённых отказа и проверенный резерв" }
|
||||||
}
|
}
|
||||||
next.failureCount = 0; next.failureKey = undefined; next.failingCandidateId = undefined
|
next.failureCount = 0; next.failureKey = undefined; next.failureObservedAt = undefined; next.failingCandidateId = undefined
|
||||||
if (state.pausedReason || !currentCandidateId || !best || best.candidate.id === currentCandidateId) return { state: { ...next, pendingCandidateId: undefined, consecutive: 0 }, switchTo: null, reason: "Нет подтверждённого улучшения или текущего пути" }
|
if (state.pausedReason || !currentCandidateId || !best || best.candidate.id === currentCandidateId) return { state: { ...next, pendingCandidateId: undefined, consecutive: 0 }, switchTo: null, reason: "Нет подтверждённого улучшения или текущего пути" }
|
||||||
if (!current?.eligible || current.score === null) return { state: next, switchTo: null, reason: "Состояние текущего пути неизвестно; failover требует отдельных подтверждённых проверок отказа" }
|
if (!current?.eligible || current.score === null) return { state: next, switchTo: null, reason: "Состояние текущего пути неизвестно; failover требует отдельных подтверждённых проверок отказа" }
|
||||||
const gain = best.score! - current.score
|
const gain = best.score! - current.score
|
||||||
if (gain < Math.max(0, profile.switchThreshold - profile.hysteresis)) return { state: { ...next, consecutive: 0, pendingCandidateId: undefined }, switchTo: null, reason: "Сработал гистерезис" }
|
if (gain < Math.max(0, profile.switchThreshold - profile.hysteresis)) return { state: { ...next, consecutive: 0, pendingCandidateId: undefined }, switchTo: null, reason: "Сработал гистерезис" }
|
||||||
if (gain < profile.switchThreshold && next.pendingCandidateId !== best.candidate.id) return { state: next, switchTo: null, reason: "Порог улучшения не достигнут" }
|
if (gain < profile.switchThreshold) return { state: next, switchTo: null, reason: "Порог улучшения не достигнут" }
|
||||||
const observationKey = [...best.observationIds, ...current.observationIds].sort().join(":")
|
const observationKey = [...best.observationIds, ...current.observationIds].sort().join(":")
|
||||||
|
if (next.pendingCandidateId !== undefined && next.pendingCandidateId !== best.candidate.id) {
|
||||||
|
next.pendingCandidateId = best.candidate.id
|
||||||
|
next.consecutive = 0
|
||||||
|
}
|
||||||
if (observationKey !== next.observationKey) {
|
if (observationKey !== next.observationKey) {
|
||||||
next.consecutive = next.pendingCandidateId === best.candidate.id ? (next.consecutive ?? 0) + 1 : 1
|
next.consecutive = next.pendingCandidateId === best.candidate.id ? (next.consecutive ?? 0) + 1 : 1
|
||||||
next.pendingCandidateId = best.candidate.id; next.observationKey = observationKey
|
next.pendingCandidateId = best.candidate.id; next.observationKey = observationKey
|
||||||
}
|
}
|
||||||
if ((next.consecutive ?? 0) < profile.confirmationSamples) return { state: next, switchTo: null, reason: "Ожидание новых измерений" }
|
if (next.pendingCandidateId !== best.candidate.id || (next.consecutive ?? 0) < profile.confirmationSamples) return { state: next, switchTo: null, reason: "Ожидание новых измерений" }
|
||||||
if (next.lastSwitchAt && now - Date.parse(next.lastSwitchAt) < profile.cooldownSec * 1000) return { state: next, switchTo: null, reason: "Cooldown" }
|
if (next.lastSwitchAt && now - Date.parse(next.lastSwitchAt) < profile.cooldownSec * 1000) return { state: next, switchTo: null, reason: "Cooldown" }
|
||||||
if (switchCount >= profile.maxSwitchesPerHour) return { state: { ...next, pausedReason: "Лимит переключений" }, switchTo: null, reason: "Исчерпан часовой бюджет переключений" }
|
if (switchCount >= profile.maxSwitchesPerHour) return { state: { ...next, pausedReason: "Лимит переключений" }, switchTo: null, reason: "Исчерпан часовой бюджет переключений" }
|
||||||
return { state: next, switchTo: best.candidate.id, reason: "Улучшение подтверждено свежими наблюдениями" }
|
return { state: next, switchTo: best.candidate.id, reason: "Улучшение подтверждено свежими наблюдениями" }
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ async function main() {
|
|||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
await applySqlMigrations(pool)
|
await applySqlMigrations(pool)
|
||||||
assert.equal(MIGRATION_MANIFEST.at(-1)?.file, "0053_server_probe_terminal_proofs.sql")
|
assert.equal(MIGRATION_MANIFEST.at(-1)?.file, "0055_mcp_backup_schedule_retirement.sql")
|
||||||
assert.equal(MIGRATION_MANIFEST.find(pin=>pin.file==="0041_server_poll_terminal_archives.sql")?.sha256,"fbb8935ce977cfc507801f01beb8f303d4817ed63c6bdd4c05799768dfaac9d4")
|
assert.equal(MIGRATION_MANIFEST.find(pin=>pin.file==="0041_server_poll_terminal_archives.sql")?.sha256,"fbb8935ce977cfc507801f01beb8f303d4817ed63c6bdd4c05799768dfaac9d4")
|
||||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM schema_migrations")).rows[0].n, MIGRATION_MANIFEST.length)
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM schema_migrations")).rows[0].n, MIGRATION_MANIFEST.length)
|
||||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM server_poll_storage_admissions")).rows[0].n, 0)
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM server_poll_storage_admissions")).rows[0].n, 0)
|
||||||
|
|||||||
Reference in new issue
Block a user